Rewrite links after google upload

mieweb · Sep 19, 2023 · 599ef14 · 599ef14
1 parent b30dda3
commit 599ef14
Show file tree

Hide file tree

Showing 7 changed files with 162 additions and 44 deletions.
diff --git a/src/LinkTranslator.ts b/src/LinkTranslator.ts
@@ -63,6 +63,9 @@ export function convertToRelativeSvgPath(localPath, basePath) {
 }
 
 export function convertToAbsolutePath(fullPath: string, relativePath: string) {
+  if (relativePath.indexOf('://') > -1) {
+    return '';
+  }
   if (fullPath.startsWith('/')) {
     fullPath = '/' + fullPath;
   }

diff --git a/src/containers/google_folder/UploadContainer.ts b/src/containers/google_folder/UploadContainer.ts
@@ -1,19 +1,27 @@
 import winston from 'winston';
 import {fileURLToPath} from 'url';
+import htmlparser2 from 'htmlparser2';
+import domutils from 'domutils';
+import {Element} from 'domhandler';
+import render from 'dom-serializer';
+
+import {FileId} from '../../model/model';
 import {MimeTypes} from '../../model/GoogleFile';
+import {LocalFile} from '../../model/LocalFile';
 import {Container, ContainerConfig, ContainerConfigArr, ContainerEngine} from '../../ContainerEngine';
 import {GoogleDriveService} from '../../google/GoogleDriveService';
 import {UserConfigService} from './UserConfigService';
 import {FileContentService} from '../../utils/FileContentService';
 import {getContentFileService} from '../transform/utils';
 import {DirectoryScanner} from '../transform/DirectoryScanner';
 import {getDesiredPath} from '../transform/LocalFilesGenerator';
-import {FileId} from '../../model/model';
-import {LocalFile} from '../../model/LocalFile';
+import {markdownToHtml} from '../../google/markdownToHtml';
+import {convertToAbsolutePath} from '../../LinkTranslator';
 
 const __filename = fileURLToPath(import.meta.url);
 
 interface FileToUpload {
+  performRewrite?: string;
   path: string;
   file: LocalFile;
   parent: FileId;
@@ -27,6 +35,8 @@ export class UploadContainer extends Container {
   private userConfigService: UserConfigService;
   private generatedFileService: FileContentService;
 
+  private pathToIdMap = {};
+
   constructor(public readonly params: ContainerConfig, public readonly paramsArr: ContainerConfigArr = {}) {
     super(params, paramsArr);
   }
@@ -55,18 +65,50 @@ export class UploadContainer extends Container {
     let cnt = 0;
     for (const entry of files) {
       const file = entry.file;
-      if (file.id === 'TO_FILL') {
-        cnt++;
+      switch (file.mimeType) {
+        case MimeTypes.IMAGE_SVG:
+        case MimeTypes.MARKDOWN:
+          if (file.id === 'TO_FILL') {
+            cnt++;
+          }
+          break;
       }
     }
 
     if (cnt > 0) {
       const ids = await this.googleDriveService.generateIds(access_token, cnt);
       for (const entry of files) {
         const file = entry.file;
-        if (file.id === 'TO_FILL') {
-          file.id = ids.splice(0, 1)[0];
+        switch (file.mimeType) {
+          case MimeTypes.IMAGE_SVG:
+          case MimeTypes.MARKDOWN:
+            if (file.id === 'TO_FILL') {
+              file.id = ids.splice(0, 1)[0];
+            }
+            break;
+        }
+      }
+    }
+  }
+
+  async updateLinks(driveIdId: FileId, dirFileService: FileContentService, files: FileToUpload[]) {
+    // Second pass because of error: Generated IDs are not supported for Docs Editors formats
+    const access_token = this.params.access_token;
+
+    for (const entry of files) {
+      const file = entry.file;
+
+      try {
+        switch (file.mimeType) {
+          case MimeTypes.MARKDOWN:
+            if (file.id && file.id !== 'TO_FILL' && entry.performRewrite) {
+              const rewrittenHtml = await this.rewriteLinks(entry.performRewrite, entry.path);
+              const response = await this.googleDriveService.update(access_token, entry.parent, file.title, MimeTypes.HTML, rewrittenHtml, file.id);
+            }
+            break;
         }
+      } catch (err) {
+        this.logger.error('[' + entry.path + ']: ' + err.message);
       }
     }
   }
@@ -77,21 +119,34 @@ export class UploadContainer extends Container {
     for (const entry of files) {
       const file = entry.file;
 
-      switch (file.mimeType) {
-        case MimeTypes.FOLDER_MIME:
-          break;
-        case MimeTypes.MARKDOWN:
-          if (file.id && file.id !== 'TO_FILL') {
-            const content = await dirFileService.readBuffer(entry.path);
-            const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
-          }
-          break;
-        case MimeTypes.IMAGE_SVG:
-          if (file.id && file.id !== 'TO_FILL') {
-            const content = await dirFileService.readBuffer(entry.path);
-            const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
-          }
-          break;
+      try {
+        switch (file.mimeType) {
+          case MimeTypes.FOLDER_MIME:
+            break;
+          case MimeTypes.MARKDOWN:
+            if (file.id && file.id !== 'TO_FILL') {
+              const content = await dirFileService.readBuffer(entry.path);
+              const html = await markdownToHtml(content);
+              entry.performRewrite = html;
+              const buffer = Buffer.from(new TextEncoder().encode(html));
+              const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, MimeTypes.HTML, buffer);
+              file.id = response.id;
+            }
+            break;
+          case MimeTypes.IMAGE_SVG:
+            if (file.id && file.id !== 'TO_FILL') {
+              const content = await dirFileService.readBuffer(entry.path);
+              const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
+              file.id = response.id;
+            }
+            break;
+        }
+      } catch (err) {
+        this.logger.error('[' + entry.path + ']: ' + err.message);
+      }
+
+      if (file.id && file.id !== 'TO_FILL') {
+        this.pathToIdMap[entry.path] = file.id;
       }
     }
   }
@@ -119,6 +174,7 @@ export class UploadContainer extends Container {
     }
 
     for (const file of Object.values(files)) {
+      const fullPath = parentPath + '/' + file.fileName;
       switch (file.mimeType) {
         case MimeTypes.FOLDER_MIME:
           if (file.id === 'TO_FILL') {
@@ -129,17 +185,17 @@ export class UploadContainer extends Container {
               file.id = map[file.title].id;
             }
           }
-          await this.uploadDir(file.id, await dirFileService.getSubFileService(file.fileName), parentPath + '/' + file.fileName);
+          await this.uploadDir(file.id, await dirFileService.getSubFileService(file.fileName), fullPath);
           break;
         case MimeTypes.MARKDOWN:
           if (map[getDesiredPath(file.title)]) {
             file.id = map[getDesiredPath(file.title)].id;
           }
-          {
-            const content = await dirFileService.readBuffer(file.fileName);
-            const response = await this.googleDriveService.upload(access_token, folderId, file.title, file.mimeType, content);
-            file.id = response.id;
-          }
+          retVal.push({
+            path: parentPath + '/' + file.fileName,
+            file,
+            parent: folderId
+          });
           break;
         case MimeTypes.IMAGE_SVG:
           if (map[getDesiredPath(file.title)]) {
@@ -151,14 +207,11 @@ export class UploadContainer extends Container {
             file,
             parent: folderId
           });
-
-          {
-            const content = await dirFileService.readBuffer(file.fileName);
-            const response = await this.googleDriveService.upload(access_token, folderId, file.title, file.mimeType, content);
-            file.id = response.id;
-          }
           break;
+      }
 
+      if (file.id && file.id !== 'TO_FILL') {
+        this.pathToIdMap[fullPath] = file.id;
       }
     }
 
@@ -171,13 +224,46 @@ export class UploadContainer extends Container {
       throw new Error('Content subdirectory must be set and start with /');
     }
 
+    this.pathToIdMap = {};
+
     const contentFileService = await getContentFileService(this.generatedFileService, this.userConfigService);
     const files = await this.uploadDir(this.params.folderId, contentFileService, '');
     await this.addIds(this.params.folderId, contentFileService, files);
     await this.uploadFiles(this.params.folderId, contentFileService, files);
+    await this.updateLinks(this.params.folderId, contentFileService, files);
   }
 
   onProgressNotify(callback: ({total, completed, warnings}: { total?: number; completed?: number; warnings?: number }) => void) {
     this.progressNotifyCallback = callback;
   }
+
+  private async rewriteLinks(html: string, entryPath: string): Promise<Buffer> {
+    const dom = htmlparser2.parseDocument(html);
+
+    const links = domutils.findAll((elem: Element) => {
+      return ['a'].includes(elem.tagName) && !!elem.attribs?.href;
+    }, dom.childNodes);
+    const images = domutils.findAll((elem: Element) => {
+      return ['img'].includes(elem.tagName) && !!elem.attribs?.src;
+    }, dom.childNodes);
+
+    for (const elem of links) {
+      const targetPath = convertToAbsolutePath(entryPath, elem.attribs.href);
+      if (!targetPath) {
+        continue;
+      }
+      if (this.pathToIdMap[targetPath]) {
+        elem.attribs.href = 'https://drive.google.com/open?id=' + this.pathToIdMap[targetPath];
+      }
+    }
+    for (const elem of images) {
+      const targetPath = convertToAbsolutePath(entryPath, elem.attribs.src);
+      if (this.pathToIdMap[targetPath]) {
+        elem.attribs.src = 'https://drive.google.com/open?id=' + this.pathToIdMap[targetPath];
+      }
+    }
+
+    const rewrittenHtml = render(dom);
+    return Buffer.from(new TextEncoder().encode(rewrittenHtml));
+  }
 }
diff --git a/src/containers/server/routes/GoogleDriveController.ts b/src/containers/server/routes/GoogleDriveController.ts
@@ -46,7 +46,6 @@ export class GoogleDriveController extends Controller {
   async getUpload(@RouteParamUser() user, @RouteParamPath('driveId') driveId: string) {
     const serverUrl = process.env.DOMAIN;
 
-    console.log('driveId', driveId);
     const state = new URLSearchParams(filterParams({
       uploadDrive: 1,
       driveId: driveId !== 'none' ? (driveId || '') : ''

diff --git a/src/google/GoogleDriveService.ts b/src/google/GoogleDriveService.ts
@@ -332,10 +332,8 @@ export class GoogleDriveService {
         // 'mimeType': MimeTypes.DRAWING_MIME, // Error: Bad Request
         googleMimeType = MimeTypes.IMAGE_SVG;
         break;
-      case MimeTypes.MARKDOWN:
+      case MimeTypes.HTML:
         googleMimeType = MimeTypes.DOCUMENT_MIME;
-        buffer = Buffer.from(new TextEncoder().encode(await markdownToHtml(buffer)));
-        mimeType = 'text/html';
         break;
     }
 
@@ -356,4 +354,35 @@ export class GoogleDriveService {
       supportsAllDrives: true
     }, formData);
   }
+
+  async update(accessToken: string, folderId: FileId, name: string, mimeType: string, buffer: Buffer, fileId: FileId) {
+    const url = `https://www.googleapis.com/upload/drive/v3/files/${fileId}`;
+
+    let googleMimeType = 'application/octet-stream';
+    switch (mimeType) {
+      case MimeTypes.IMAGE_SVG:
+        // 'mimeType': MimeTypes.DRAWING_MIME, // Error: Bad Request
+        googleMimeType = MimeTypes.IMAGE_SVG;
+        break;
+      case MimeTypes.HTML:
+        googleMimeType = MimeTypes.DOCUMENT_MIME;
+        break;
+    }
+
+    const metadata = {
+      name,
+      mimeType: googleMimeType,
+      // parents: [folderId],
+      fields: '*'
+    };
+
+    const formData  = new FormData();
+    formData.append('Metadata', new Blob([JSON.stringify(metadata)], { type: 'application/json; charset=UTF-8' }) );
+    formData.append('Media', new Blob([buffer], { type: mimeType }), name);
+
+    return await driveFetchMultipart(this.quotaLimiter, accessToken, 'PATCH', url, {
+      uploadType: 'multipart',
+      supportsAllDrives: true
+    }, formData);
+  }
 }
diff --git a/src/google/driveFetch.ts b/src/google/driveFetch.ts
@@ -223,15 +223,15 @@ export async function driveFetchMultipart(quotaLimiter: QuotaLimiter, accessToke
     }
   }
 
-  const after = `\r\n--${boundary}--`;
+  const after = `\n--${boundary}--`;
   function generateMultipart(image: ArrayBuffer, mimetype) {
     const source = new Uint8Array(image); // Wrap in view to get data
 
     const before = [
-      `\r\n--${boundary}\r\n`,
-      `Content-Type: ${mimetype}\r\n`,
-      `Content-Length: ${source.byteLength}\r\n`,
-      '\r\n'
+      `\n--${boundary}\n`,
+      `Content-Type: ${mimetype}\n`,
+      `Content-Length: ${source.byteLength}\n`,
+      '\n'
     ].join('');
 
     const size = before.length + source.byteLength;

diff --git a/src/model/GoogleFile.ts b/src/model/GoogleFile.ts
@@ -32,7 +32,8 @@ export const MimeTypes = {
   APPS_SCRIPT: 'application/vnd.google-apps.script',
   MARKDOWN: 'text/x-markdown',
   SHORTCUT: 'application/vnd.google-apps.shortcut',
-  IMAGE_SVG: 'image/svg+xml'
+  IMAGE_SVG: 'image/svg+xml',
+  HTML: 'text/html'
 };
 export const MimeToExt = {
   'image/svg+xml': '.svg',

diff --git a/test/markdownToHtmlTest.ts b/test/markdownToHtmlTest.ts
@@ -33,7 +33,7 @@ describe('markdownToHtml', () => {
 
     assert.equal(map['./Doc1.svg'], '/aaa/bbb/Doc1.svg');
     assert.equal(map['Doc2.svg'], '/aaa/bbb/Doc2.svg');
-    assert.equal(map['../subdir/Diagram.svgg'], '/aaa/subdir/Diagram.svg');
+    assert.equal(map['../subdir/Diagram.svg'], '/aaa/subdir/Diagram.svg');
 
     const serilzd = render(dom);
     console.log(serilzd);