Update docs

apps/ui/src/components/UserSettings.vue

@@ -57,7 +57,7 @@
             </div>
 
             <div class="form-group">
-              <label>Rewrite rules</label>
+              <label>Markdown Links Rewrite Rules</label>
               <CodeEditor v-model="user_config.rewrite_rules_yaml" lang="yaml" />
             </div>
           </form>

src/containers/google_folder/UploadContainer.ts

@@ -4,20 +4,20 @@ import * as htmlparser2 from 'htmlparser2';
 import {ElementType} from 'htmlparser2';
 import * as domutils from 'domutils';
 import {Element, Text} from 'domhandler';
-
 import render from 'dom-serializer';
-import {FileId} from '../../model/model';
-import {MimeTypes} from '../../model/GoogleFile';
-import {LocalFile} from '../../model/LocalFile';
-import {Container, ContainerConfig, ContainerConfigArr, ContainerEngine} from '../../ContainerEngine';
-import {GoogleDriveService} from '../../google/GoogleDriveService';
-import {UserConfigService} from './UserConfigService';
-import {FileContentService} from '../../utils/FileContentService';
-import {getContentFileService} from '../transform/utils';
-import {DirectoryScanner} from '../transform/DirectoryScanner';
-import {getDesiredPath} from '../transform/LocalFilesGenerator';
-import {markdownToHtml} from '../../google/markdownToHtml';
-import {convertToAbsolutePath} from '../../LinkTranslator';
+
+import {FileId} from '../../model/model.ts';
+import {MimeTypes} from '../../model/GoogleFile.ts';
+import {LocalFile} from '../../model/LocalFile.ts';
+import {Container, ContainerConfig, ContainerConfigArr, ContainerEngine} from '../../ContainerEngine.ts';
+import {GoogleDriveService} from '../../google/GoogleDriveService.ts';
+import {UserConfigService} from './UserConfigService.ts';
+import {FileContentService} from '../../utils/FileContentService.ts';
+import {getContentFileService} from '../transform/utils.ts';
+import {DirectoryScanner} from '../transform/DirectoryScanner.ts';
+import {getDesiredPath} from '../transform/LocalFilesGenerator.ts';
+import {markdownToHtml} from '../../google/markdownToHtml.ts';
+import {convertToAbsolutePath} from '../../LinkTranslator.ts';
 
 const __filename = fileURLToPath(import.meta.url);
 
@@ -85,6 +85,7 @@ export class UploadContainer extends Container {
           case MimeTypes.MARKDOWN:
             if (file.id === 'TO_FILL') {
               file.id = ids.splice(0, 1)[0];
+              file['isNewId'] = true;
             }
             break;
         }
@@ -133,15 +134,27 @@ export class UploadContainer extends Container {
               const html = await markdownToHtml(content);
               entry.performRewrite = html;
               const buffer = Buffer.from(new TextEncoder().encode(html));
-              const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, MimeTypes.HTML, buffer);
-              file.id = response.id;
+              if (file['isNewId']) {
+                const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, MimeTypes.HTML, buffer); // generatedIds not supported to gdocs
+                file.id = response.id;
+                delete file['isNewId'];
+              } else {
+                const response = await this.googleDriveService.update(access_token, entry.parent, file.title, MimeTypes.HTML, buffer, file.id);
+                file.id = response.id;
+              }
             }
             break;
           case MimeTypes.IMAGE_SVG:
             if (file.id && file.id !== 'TO_FILL') {
               const content = await dirFileService.readBuffer(entry.path);
-              const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
-              file.id = response.id;
+              if (file['isNewId']) {
+                const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
+                file.id = response.id;
+                delete file['isNewId'];
+              } else {
+                const response = await this.googleDriveService.update(access_token, entry.parent, file.title, file.mimeType, content, file.id);
+                file.id = response.id;
+              }
             }
             break;
         }
@@ -194,23 +207,26 @@ export class UploadContainer extends Container {
         case MimeTypes.MARKDOWN:
           if (map[getDesiredPath(file.title)]) {
             file.id = map[getDesiredPath(file.title)].id;
+          } else { // Upload only missing
+            file.id = 'TO_FILL';
+            retVal.push({
+              path: parentPath + '/' + file.fileName,
+              file,
+              parent: folderId
+            });
           }
-          retVal.push({
-            path: parentPath + '/' + file.fileName,
-            file,
-            parent: folderId
-          });
           break;
         case MimeTypes.IMAGE_SVG:
           if (map[getDesiredPath(file.title)]) {
             file.id = map[getDesiredPath(file.title)].id;
+          } else { // Upload only missing
+            file.id = 'TO_FILL';
+            retVal.push({
+              path: parentPath + '/' + file.fileName,
+              file,
+              parent: folderId
+            });
           }
-
-          retVal.push({
-            path: parentPath + '/' + file.fileName,
-            file,
-            parent: folderId
-          });
           break;
       }
 

src/containers/google_folder/UserConfigService.ts

@@ -55,6 +55,11 @@ const DEFAULT_CONFIG: UserConfig = {
 };
 
 const DEFAULT_REWRITE_RULES = [
+  {
+    tag: 'A',
+    match: '$alt',
+    template: '$href'
+  },
   {
     match: '(?:https?:\\/\\/)?(?:www\\.)?(?:youtube\\.com\\/(?:[^\\/\\n\\s]+\\/\\S+\\/|(?:v|e(?:mbed)?)\\/|\\S*?[?&]v=)|youtu\\.be\\/)([a-zA-Z0-9_-]{11})',
     replace: '(?:https?:\\/\\/)?(?:www\\.)?(?:youtube\\.com\\/(?:[^\\/\\n\\s]+\\/\\S+\\/|(?:v|e(?:mbed)?)\\/|\\S*?[?&]v=)|youtu\\.be\\/)([a-zA-Z0-9_-]{11})',

src/google/GoogleDriveService.ts

@@ -348,10 +348,17 @@ export class GoogleDriveService {
     formData.append('Metadata', new Blob([JSON.stringify(metadata)], { type: 'application/json; charset=UTF-8' }) );
     formData.append('Media', new Blob([buffer], { type: mimeType }), name);
 
-    return await driveFetchMultipart(this.quotaLimiter, accessToken, 'POST', url, {
-      uploadType: 'multipart',
-      supportsAllDrives: true
-    }, formData);
+    try {
+      return await driveFetchMultipart(this.quotaLimiter, accessToken, 'POST', url, {
+        uploadType: 'multipart',
+        supportsAllDrives: true
+      }, formData);
+    } catch (err) {
+      if (409 === parseInt(err.status)) {
+        this.logger.error(`Conflict on uploading: ${id} ${name}`);
+      }
+      throw err;
+    }
   }
 
   async update(accessToken: string, folderId: FileId, name: string, mimeType: string, buffer: Buffer, fileId: FileId) {

src/odt/applyRewriteRule.ts

@@ -18,11 +18,17 @@ export function applyRewriteRule(rule: RewriteRule, chunk: Chunk) {
   const href = chunk.href || '';
   const basename = path.basename(href);
   const alt = chunk.alt || '';
+  const chunkTag = chunk.tag || '';
 
-  if (rule.tag && chunk.tag !== rule.tag) {
+  if (rule.tag && chunkTag.replaceAll('/', '') !== rule.tag.replaceAll('/', '')) {
     return { shouldBreak: false };
   }
 
+  if (rule.match && rule.match === '$alt') {
+    if (href !== alt) {
+      return { shouldBreak: false };
+    }
+  } else
   if (rule.match) {
     const matchRegExp = new RegExp(rule.match);
     if (!matchRegExp.test(href)) {

website/_index.md

@@ -15,7 +15,7 @@ WikiGDrive is a node app that uses the [Google Drive API](https://developers.goo
 
 [Google Drive Notes](https://docs.google.com/document/d/1H6vwfQXIexdg4ldfaoPUjhOZPnSkNn6h29WD6Fi-SBY/edit#)
 | [Github Project](https://github.com/mieweb/wikiGDrive/projects)
-| [Github Developer Notes](docs/developer.md)
+| [Github Developer Notes](docs/developer_guide.md)
 
 With a "Shared Drive" as the key, WikiGDrive:
 
@@ -30,7 +30,7 @@ WikiGDrive scans for changes in the drive and then refresh the local converted f
 
 ## Developer Documentation
 
-* [Developer README](docs/developer.md)
+* [Developer README](docs/developer_guide.md)
 * [Internals](docs/internals.md)
 
 ## Install from NPM

website/docs/_index.md

@@ -1,5 +1,5 @@
 ---
-title: Introduction
+title: _index
 navWeight: 1000 # Upper weight gets higher precedence, optional.
 ---
 # wikiGDrive
@@ -16,23 +16,23 @@ WikiGDrive is a node app that uses the [Google Drive API](https://developers.goo
 
 [Google Drive Notes](https://docs.google.com/document/d/1H6vwfQXIexdg4ldfaoPUjhOZPnSkNn6h29WD6Fi-SBY/edit#)
 | [Github Project](https://github.com/mieweb/wikiGDrive/projects)
-| [Github Developer Notes](docs/developer.md)
+| [Github Developer Notes](./developer_guide.md)
 
 With a "Shared Drive" as the key, WikiGDrive:
 
 * Reads all the files from a Google "Shared Drive"
 * Builds a map of the driveId (URL) to the pathname in the "Shared Drive"
 * For each Google Document:
-    * Converts to a Markdown file with the path (instead of the driveId for the file)
-    * Changes driveId to the path (eg: 12lvdxKgGsD.../edit would be changed to /filename
-    * Support diagrams as SVG (and map the URLs in the diagram)
+   * Converts to a Markdown file with the path (instead of the driveId for the file)
+   * Changes driveId to the path (eg: 12lvdxKgGsD.../edit would be changed to /filename
+   * Support diagrams as SVG (and map the URLs in the diagram)
 
 WikiGDrive scans for changes in the drive and then refresh the local converted files.
 
 ## Developer Documentation
 
-* [Developer README](docs/developer.md)
-* [Internals](docs/internals.md)
+* [Developer README](./developer_guide.md)
+* [Internals](./internals.md)
 
 ## Usage and options
 

website/docs/developer.md → website/docs/developer_guide.md

@@ -6,7 +6,7 @@ navWeight: -15
 
 See [Node setup on the system](#Node-setup-on-the-system) for prereq.
 
-[Example Google Drive Shared Folder](https://drive.google.com/drive/folders/0AIkOKXbzWCtSUk9PVA)
+[Example Google Drive Shared Folder](https://drive.google.com/open?id=0AIkOKXbzWCtSUk9PVA)
 
 # Node setup on the system
 
@@ -70,8 +70,8 @@ docker rm -f wikigdrive
 ## Authentication
 
 ### Client ID for the Web Application Add-On / Authentication
-Link to production's OAUTH configuration:
-https://console.cloud.google.com/apis/credentials/oauthclient/762352378313-3u5pagjnk24g9640a5j1bmlsvobtlq2k.apps.googleusercontent.com?project=wikigdrive
+
+Link to production's OAUTH configuration: https://console.cloud.google.com/apis/credentials/oauthclient/762352378313-3u5pagjnk24g9640a5j1bmlsvobtlq2k.apps.googleusercontent.com?project=wikigdrive
 
 Stored in /home/wikigdrive/env.prod
 
@@ -84,7 +84,7 @@ https://console.cloud.google.com/iam-admin/serviceaccounts/details/1031846960952
 
 This is for configuring Google Apps and their Console to permit the Google Marketplace to the store.
 
-See [app_script](../../apps/app-script)
+See folder `/apps/app-script` in the sources
 
 
 ## Runner
@@ -114,7 +114,7 @@ root@wgd-dev:~# docker exec -it wikigdrive-prod bash
 wikigdrive --service_account /service_account.json  drives
 ```
 
-![Code Diagram](https://docs.google.com/drawings/d/e/2PACX-1vREcniLAig0DiPqSxu5QRqgiGHWL5INKfjMlqSvXK9vTbas3JqorzbuONLeTrNOD0MBPC7QB3Gd_NY7/pub?w=960&h=720) [src](https://docs.google.com/drawings/d/1LSveM3s_Fmi9411FW9Z-NA50fbNHHW2y_PQo3NSUPAI/edit)
+![Code Diagram](https://docs.google.com/drawings/d/e/2PACX-1vREcniLAig0DiPqSxu5QRqgiGHWL5INKfjMlqSvXK9vTbas3JqorzbuONLeTrNOD0MBPC7QB3Gd_NY7/pub?w=960&h=720) [src](https://docs.google.com/drawings/d/1LSveM3s_Fmi9411FW9Z-NA50fbNHHW2y_PQo3NSUPAI/edit)
 
 Cool trick to watch changes as they happen in a document:
 
@@ -148,12 +148,18 @@ ZIPKIN_URL=http://localhost:9411
 
 Chrome
 
-    Go to `chrome://inspect`
+```
+Go to `chrome://inspect`
+```
 
 Visual Studio Code 1.10+
 
-    In the Debug panel, click the settings icon to open .vscode/launch.json. Select "Node.js" for initial setup.
+```
+In the Debug panel, click the settings icon to open .vscode/launch.json. Select "Node.js" for initial setup.
+```
 
 JetBrains WebStorm and other JetBrains IDEs
 
-    Create a new Node.js debug configuration and hit Debug. --inspect will be used by default for Node.js 7+. To disable uncheck js.debugger.node.use.inspect in the IDE Registry. To learn more about running and debugging Node.js in WebStorm and other JetBrains IDEs, check out WebStorm online help.
+```
+Create a new Node.js debug configuration and hit Debug. --inspect will be used by default for Node.js 7+. To disable uncheck js.debugger.node.use.inspect in the IDE Registry. To learn more about running and debugging Node.js in WebStorm and other JetBrains IDEs, check out WebStorm online help.
+```

website/docs/install.md

@@ -1,5 +1,5 @@
 ---
-title: Installation Guide
+title: Install
 navWeight: 990 # Upper weight gets higher precedence, optional.
 ---
 ## Install from NPM

website/docs/internals.md

@@ -11,7 +11,7 @@ navWeight: -15
 
 * folders.json - a listing of each google shared folder
 * One folder for each drive
-    * Second folder with the same name with `_transform` on the end to hold markdown version
+   * Second folder with the same name with `_transform` on the end to hold markdown version
 * quota.json - google throttle for limited rate
 
 ```
@@ -81,12 +81,12 @@ navWeight: -15
 
 ## Note this is going away.  Will be replacing this single database with a multi-file version for scale.
 
-- id - Google's fileId
-- name - Title set inside google docs. It is not unique
-- mimeType - Google's mime type or 'conflict' or 'redirect'
-- modifiedTime - Server-size mtime
-- localPath - real local path, unique with handled conflicts and redirects (in case of title rename)
-- lastAuthor - Google's last author if available
+* id - Google's fileId
+* name - Title set inside google docs. It is not unique
+* mimeType - Google's mime type or 'conflict' or 'redirect'
+* modifiedTime - Server-size mtime
+* localPath - real local path, unique with handled conflicts and redirects (in case of title rename)
+* lastAuthor - Google's last author if available
 
 ```
 {
@@ -128,11 +128,11 @@ navWeight: -15
 
 ### local_files.json is indexed with file id
 
-- desiredLocalPath - slugified name. It is not unique, wikigdrive handles redirects so it is NOT real path in local system
-- dirty - file needs to be downloaded
-- conflicting - array of fileIds when mimeType = 'conflict'
-- localPath - path to transformed markdown file
-- modifiedTime - fetched from google server
+* desiredLocalPath - slugified name. It is not unique, wikigdrive handles redirects so it is NOT real path in local system
+* dirty - file needs to be downloaded
+* conflicting - array of fileIds when mimeType = 'conflict'
+* localPath - path to transformed markdown file
+* modifiedTime - fetched from google server
 
 ```
 {
@@ -152,13 +152,13 @@ navWeight: -15
 
 ### Transform stage:
 
-0. Get files to transform (does not exist in local_files.json, have different modifiedTime, are trashed), generate desireLocalPaths based on parents
-1. If file is removed - remove .md file, remove images
-2. If file is new (not exists in local_files.json) - add to localFiles, schedule for generation
-3. If file exists but with different desireLocalPath:
-   3.1. Remove old .md, remove old images
-   3.2. Schedule for generation
-   3.3. Generate redir with old localPath
-4. Remove dangling redirects
-5. Check if there are any conflicts (same desireLocalPath)
-6. Check if any conflicts can be removed
+1. Get files to transform (does not exist in local_files.json, have different modifiedTime, are trashed), generate desireLocalPaths based on parents
+2. If file is removed - remove .md file, remove images
+3. If file is new (not exists in local_files.json) - add to localFiles, schedule for generation
+4. If file exists but with different desireLocalPath:
+  * Remove old .md, remove old images
+  * Schedule for generation
+  * Generate redir with old localPath
+5. Remove dangling redirects
+6. Check if there are any conflicts (same desireLocalPath)
+7. Check if any conflicts can be removed