From be02d09658071f37327b7c49f6e1347c547e3574 Mon Sep 17 00:00:00 2001
From: Grzegorz Godlewski <gg@gitgis.com>
Date: Wed, 24 Jul 2024 19:29:48 +0200
Subject: [PATCH] Use experimental google markdown support

---
 apps/ui/src/components/UserSettings.vue       |   8 +
 src/ContainerEngine.ts                        |   7 +-
 .../google_folder/GoogleFolderContainer.ts    |   8 +
 .../google_folder/TaskFetchFolder.ts          |  56 +++--
 .../google_folder/UserConfigService.ts        |   1 +
 src/containers/job/JobManagerContainer.ts     |   3 +
 .../server/routes/ConfigController.ts         |  14 +-
 src/containers/transform/LocalLinks.ts        |   1 +
 .../transform/TaskGoogleMarkdownTransform.ts  | 202 ++++++++++++++++++
 .../transform/TransformContainer.ts           |  45 ++--
 10 files changed, 307 insertions(+), 38 deletions(-)
 create mode 100644 src/containers/transform/TaskGoogleMarkdownTransform.ts

diff --git a/apps/ui/src/components/UserSettings.vue b/apps/ui/src/components/UserSettings.vue
index d9468356..1b6d17ea 100644
--- a/apps/ui/src/components/UserSettings.vue
+++ b/apps/ui/src/components/UserSettings.vue
@@ -35,6 +35,14 @@
               <img v-if="user_config.hugo_theme.preview_img" :src="user_config.hugo_theme.preview_img" style="height: 250px;" :alt="user_config.hugo_theme.id" />
             </div>
 
+            <div class="form-group">
+              <label>Use Google Markdowns</label>
+              <select class="form-control" @change="user_config.use_google_markdowns = !user_config.use_google_markdowns">
+                <option :selected="!user_config.use_google_markdowns" value="">Disabled</option>
+                <option :selected="user_config.use_google_markdowns" value="enabled">Enabled</option>
+              </select>
+            </div>
+
             <div class="form-group">
               <label>Autosync</label>
               <select class="form-control" @change="user_config.auto_sync = !user_config.auto_sync">
diff --git a/src/ContainerEngine.ts b/src/ContainerEngine.ts
index 3f7ae99b..4cdfdcb1 100644
--- a/src/ContainerEngine.ts
+++ b/src/ContainerEngine.ts
@@ -1,9 +1,10 @@
 import casual from 'casual';
 import winston from 'winston';
-import {FileContentService} from './utils/FileContentService';
 import {QueueObject} from 'async';
-import {QueueTask} from './containers/google_folder/QueueTask';
-import {FileId} from './model/model';
+
+import {FileContentService} from './utils/FileContentService.ts';
+import {QueueTask} from './containers/google_folder/QueueTask.ts';
+import {FileId} from './model/model.ts';
 
 export interface ContainerConfig {
   name?: string;
diff --git a/src/containers/google_folder/GoogleFolderContainer.ts b/src/containers/google_folder/GoogleFolderContainer.ts
index 15f80a07..c72dd6e7 100644
--- a/src/containers/google_folder/GoogleFolderContainer.ts
+++ b/src/containers/google_folder/GoogleFolderContainer.ts
@@ -11,6 +11,7 @@ import {DateISO, FileId} from '../../model/model.ts';
 import {FolderRegistryContainer} from '../folder_registry/FolderRegistryContainer.ts';
 import {GoogleTreeProcessor} from './GoogleTreeProcessor.ts';
 import {HasAccessToken} from '../../google/AuthClient.ts';
+import {UserConfigService} from './UserConfigService.ts';
 
 const __filename = fileURLToPath(import.meta.url);
 
@@ -81,6 +82,13 @@ export class GoogleFolderContainer extends Container {
           this.forceDownloadFilters,
           {filterFilesIds: this.filterFilesIds, filterFoldersIds}
         );
+
+        const folderId = this.params.name;
+        const googleFileSystem = await this.filesService.getSubFileService(folderId, '/');
+        const userConfigService = new UserConfigService(googleFileSystem);
+        await userConfigService.load();
+
+        taskFetchFolder.setUseGoogleMarkdowns(userConfigService.config.use_google_markdowns);
         downloader.addTask(taskFetchFolder);
       }
     }
diff --git a/src/containers/google_folder/TaskFetchFolder.ts b/src/containers/google_folder/TaskFetchFolder.ts
index e9e1d066..a19fa450 100644
--- a/src/containers/google_folder/TaskFetchFolder.ts
+++ b/src/containers/google_folder/TaskFetchFolder.ts
@@ -18,6 +18,8 @@ interface Filters {
 
 export class TaskFetchFolder extends QueueTask {
 
+  private useGoogleMarkdowns = false;
+
   constructor(protected logger: winston.Logger,
               private googleDriveService: GoogleDriveService,
               private auth: HasAccessToken,
@@ -28,6 +30,10 @@ export class TaskFetchFolder extends QueueTask {
     super(logger);
   }
 
+  setUseGoogleMarkdowns(value: boolean) {
+    this.useGoogleMarkdowns = value;
+  }
+
   async run(): Promise<QueueTask[]> {
     if (this.filters.filterFoldersIds.length > 0) {
       if (this.filters.filterFoldersIds.indexOf(this.file.id) === -1) {
@@ -88,15 +94,19 @@ export class TaskFetchFolder extends QueueTask {
 
         switch (file.mimeType) {
           case MimeTypes.FOLDER_MIME:
-            tasks.push(new TaskFetchFolder(
-              this.logger,
-              this.googleDriveService,
-              this.auth,
-              await this.fileService.getSubFileService(file.id),
-              file,
-              this.forceDownloadFilters,
-              this.filters
-            ));
+            {
+              const task = new TaskFetchFolder(
+                this.logger,
+                this.googleDriveService,
+                this.auth,
+                await this.fileService.getSubFileService(file.id),
+                file,
+                this.forceDownloadFilters,
+                this.filters
+              );
+              task.setUseGoogleMarkdowns(this.useGoogleMarkdowns);
+              tasks.push(task);
+            }
             break;
 
           case MimeTypes.DRAWING_MIME:
@@ -111,14 +121,26 @@ export class TaskFetchFolder extends QueueTask {
             break;
 
           case MimeTypes.DOCUMENT_MIME:
-            tasks.push(new TaskFetchDocument(
-              this.logger,
-              this.googleDriveService,
-              this.auth,
-              await this.fileService,
-              file,
-              forceDownload
-            ));
+            if (!this.useGoogleMarkdowns) {
+              tasks.push(new TaskFetchDocument(
+                this.logger,
+                this.googleDriveService,
+                this.auth,
+                await this.fileService,
+                file,
+                forceDownload
+              ));
+            } else {
+              tasks.push(new TaskFetchBinary(
+                this.logger,
+                this.googleDriveService,
+                this.auth,
+                await this.fileService,
+                file,
+                forceDownload,
+                MimeTypes.MARKDOWN, 'md'
+              ));
+            }
             break;
 
           case MimeTypes.SPREADSHEET_MIME:
diff --git a/src/containers/google_folder/UserConfigService.ts b/src/containers/google_folder/UserConfigService.ts
index 904fc416..fc15fde2 100644
--- a/src/containers/google_folder/UserConfigService.ts
+++ b/src/containers/google_folder/UserConfigService.ts
@@ -38,6 +38,7 @@ export class UserConfig {
   hugo_theme?: HugoTheme;
   config_toml?: string;
   transform_subdir?: string;
+  use_google_markdowns?: boolean;
   auto_sync?: boolean;
   fm_without_version?: boolean;
   actions_yaml?: string;
diff --git a/src/containers/job/JobManagerContainer.ts b/src/containers/job/JobManagerContainer.ts
index 05e134d0..be01585f 100644
--- a/src/containers/job/JobManagerContainer.ts
+++ b/src/containers/job/JobManagerContainer.ts
@@ -468,6 +468,9 @@ export class JobManagerContainer extends Container {
 
     const userConfigService = new UserConfigService(googleFileSystem);
     await userConfigService.load();
+
+    transformContainer.setUseGoogleMarkdowns(userConfigService.config.use_google_markdowns);
+
     transformContainer.onProgressNotify(({ completed, total, warnings, failed }) => {
       if (!this.driveJobsMap[folderId]) {
         return;
diff --git a/src/containers/server/routes/ConfigController.ts b/src/containers/server/routes/ConfigController.ts
index 4807f48f..5d8d09fa 100644
--- a/src/containers/server/routes/ConfigController.ts
+++ b/src/containers/server/routes/ConfigController.ts
@@ -1,11 +1,11 @@
 import yaml from 'js-yaml';
 
-import {Controller, RouteGet, RouteParamBody, RouteParamPath, RoutePost, RoutePut} from './Controller';
-import {FileContentService} from '../../../utils/FileContentService';
-import {GitScanner} from '../../../git/GitScanner';
-import {UserConfigService} from '../../google_folder/UserConfigService';
-import {FolderRegistryContainer} from '../../folder_registry/FolderRegistryContainer';
-import {ContainerEngine} from '../../../ContainerEngine';
+import {Controller, RouteGet, RouteParamBody, RouteParamPath, RoutePost, RoutePut} from './Controller.ts';
+import {FileContentService} from '../../../utils/FileContentService.ts';
+import {GitScanner} from '../../../git/GitScanner.ts';
+import {UserConfigService} from '../../google_folder/UserConfigService.ts';
+import {FolderRegistryContainer} from '../../folder_registry/FolderRegistryContainer.ts';
+import {ContainerEngine} from '../../../ContainerEngine.ts';
 
 export interface ConfigBody {
   config: {
@@ -15,6 +15,7 @@ export interface ConfigBody {
     rewrite_rules_yaml?: string;
     hugo_theme: HugoTheme;
     auto_sync: boolean;
+    use_google_markdowns: boolean;
     fm_without_version: boolean;
     actions_yaml?: string;
   };
@@ -111,6 +112,7 @@ export class ConfigController extends Controller {
       userConfigService.config.actions_yaml = body.config?.actions_yaml;
     }
     userConfigService.config.auto_sync = !!body.config?.auto_sync;
+    userConfigService.config.use_google_markdowns = !!body.config?.use_google_markdowns;
     userConfigService.config.fm_without_version = !!body.config?.fm_without_version;
 
     await userConfigService.save();
diff --git a/src/containers/transform/LocalLinks.ts b/src/containers/transform/LocalLinks.ts
index 4e96cf3a..b82fe7cb 100644
--- a/src/containers/transform/LocalLinks.ts
+++ b/src/containers/transform/LocalLinks.ts
@@ -1,4 +1,5 @@
 import {FileContentService} from '../../utils/FileContentService.ts';
+import {FileId} from '../../model/model.ts';
 
 interface Link {
   fileId: string;
diff --git a/src/containers/transform/TaskGoogleMarkdownTransform.ts b/src/containers/transform/TaskGoogleMarkdownTransform.ts
new file mode 100644
index 00000000..28d2a9ae
--- /dev/null
+++ b/src/containers/transform/TaskGoogleMarkdownTransform.ts
@@ -0,0 +1,202 @@
+import fs from 'fs';
+import winston from 'winston';
+
+import {QueueTask} from '../google_folder/QueueTask.ts';
+import {JobManagerContainer} from '../job/JobManagerContainer.ts';
+import {FileContentService} from '../../utils/FileContentService.ts';
+import {GoogleFile} from '../../model/GoogleFile.ts';
+import {BinaryFile, DrawingFile, LocalFile, MdFile} from '../../model/LocalFile.ts';
+import {LocalLinks} from './LocalLinks.ts';
+import {UserConfig} from '../google_folder/UserConfigService.ts';
+import {SvgTransform} from '../../SvgTransform.ts';
+import {generateDocumentFrontMatter} from './frontmatters/generateDocumentFrontMatter.ts';
+import {generateConflictMarkdown} from './frontmatters/generateConflictMarkdown.ts';
+import {googleMimeToExt} from './TaskLocalFileTransform.ts';
+import {getUrlHash, urlToFolderId} from '../../utils/idParsers.js';
+
+export class TaskGoogleMarkdownTransform extends QueueTask {
+  constructor(protected logger: winston.Logger,
+              private jobManagerContainer: JobManagerContainer,
+              private realFileName: string,
+              private googleFolder: FileContentService,
+              private googleFile: GoogleFile,
+              private destinationDirectory: FileContentService,
+              private localFile: LocalFile,
+              private localLinks: LocalLinks,
+              private userConfig: UserConfig
+  ) {
+    super(logger);
+    this.retries = 0;
+
+    if (!this.localFile.fileName) {
+      throw new Error(`No fileName for: ${this.localFile.id}`);
+    }
+  }
+
+  async run(): Promise<QueueTask[]> {
+    await this.generate(this.localFile);
+
+    return [];
+  }
+
+  async generateBinary(binaryFile: BinaryFile) {
+    await new Promise<void>((resolve, reject) => {
+      try {
+        const dest = this.destinationDirectory.createWriteStream(this.realFileName);
+
+        dest.on('error', err => {
+          reject(err);
+        });
+
+        const ext = googleMimeToExt(this.googleFile.mimeType, this.googleFile.name);
+        const stream = this.googleFolder.createReadStream(`${binaryFile.id}${ext ? '.' + ext : ''}`)
+          .on('error', err => {
+            reject(err);
+          })
+          .pipe(dest);
+
+        stream.on('finish', () => {
+          resolve();
+        });
+        stream.on('error', err => {
+          reject(err);
+        });
+      } catch (err) {
+        reject(err);
+      }
+    });
+  }
+
+  async generateDrawing(drawingFile: DrawingFile) {
+    await new Promise<void>((resolve, reject) => {
+      try {
+        // await this.destinationDirectory.mkdir(getFileDir(targetPath));
+        const dest = this.destinationDirectory.createWriteStream(this.realFileName);
+        const svgTransform = new SvgTransform(drawingFile.fileName);
+        // const svgPath = this.googleScanner.getFilePathPrefix(drawingFile.id) + '.svg';
+
+        dest.on('error', err => {
+          reject(err);
+        });
+
+        const stream = this.googleFolder.createReadStream(`${drawingFile.id}.svg`)
+          .on('error', err => {
+            reject(err);
+          })
+          .pipe(svgTransform)
+          .pipe(dest);
+
+        stream.on('finish', () => {
+          this.localLinks.append(drawingFile.id, drawingFile.fileName, Array.from(svgTransform.links));
+          resolve();
+        });
+        stream.on('error', err => {
+          reject(err);
+        });
+      } catch (err) {
+        reject(err);
+      }
+    });
+  }
+
+  async generateDocument(localFile: MdFile) {
+    const links = new Set<string>();
+
+    const mdPath = this.googleFolder.getRealPath() + '/' + localFile.id + '.md';
+
+    const input: Buffer = fs.readFileSync(mdPath);
+
+    const originalMarkdown = new TextDecoder().decode(input);
+
+    const markdownLinkRegex = /\[([^\]]+)\]\(([^)]+?)\)/g;
+
+    function replaceMarkdownLinks(markdown, replacerFunction) {
+      return markdown.replace(markdownLinkRegex, (match, linkText, url, title) => {
+        // Call the replacer function with the link text, URL, and title
+        return replacerFunction(linkText, url, title);
+      });
+    }
+
+    function customReplacer(linkText, href) {
+      href = href.replaceAll('\\', '');
+
+      const id = urlToFolderId(href);
+      const hash = getUrlHash(href);
+      if (id) {
+        href = 'gdoc:' + id + hash;
+      }
+      if (href && !href.startsWith('#') && href.indexOf(':') > -1) {
+        links.add(href);
+      }
+
+      return `[${linkText}](${href})`;
+    }
+
+    const markdownRewrittenLinks = replaceMarkdownLinks(originalMarkdown, customReplacer);
+
+    const pattern = /\*\{\{%\s+.*?\s+%\}\}\*/g;
+
+    const markdown = markdownRewrittenLinks.replace(pattern, (match) => {
+      // Remove the surrounding asterisks
+      return match.slice(1, -1);
+    });
+
+    // links = Array.from(converter.links);
+    const frontMatter = generateDocumentFrontMatter(localFile, Array.from(links), this.userConfig.fm_without_version);
+    const errors = [];
+    this.warnings = errors.length;
+
+    for (const errorMsg of errors) {
+      this.logger.warn('Error in: ['+ this.localFile.fileName +'](' + this.localFile.fileName + ') ' + errorMsg, {
+        errorMdFile: this.localFile.fileName,
+        errorMdMsg: errorMsg
+      });
+    }
+
+    await this.destinationDirectory.writeFile(this.realFileName, frontMatter + markdown);
+    this.localLinks.append(localFile.id, localFile.fileName, Array.from(links));
+  }
+
+  async generate(localFile: LocalFile): Promise<void> {
+    try {
+      const verStr = this.localFile.version ? ' #' + this.localFile.version : ' ';
+      if (localFile.type === 'conflict') {
+        this.logger.info('Transforming conflict: ' + this.localFile.fileName);
+        const md = generateConflictMarkdown(localFile);
+        await this.destinationDirectory.writeFile(this.realFileName, md);
+      } else if (localFile.type === 'redir') { // TODO
+        this.logger.info('Transforming redir: ' + this.localFile.fileName);
+        // const redirectToFile = this.toGenerate.find(f => f.id === localFile.redirectTo);
+        // const redirectToFile = this.generatedScanner.getFileById(localFile.redirectTo);
+        // const md = generateRedirectMarkdown(localFile, redirectToFile, this.linkTranslator);
+        // await this.destinationDirectory.mkdir(getFileDir(targetPath));
+        // await this.destinationDirectory.writeFile(targetPath, md);
+        // await this.generatedScanner.update(targetPath, md);
+      } else if (localFile.type === 'md') {
+        this.logger.info('Transforming markdown: ' + this.localFile.fileName + verStr);
+        // const googleFile = await this.googleScanner.getFileById(localFile.id);
+        // const downloadFile = await this.downloadFilesStorage.findFile(f => f.id === localFile.id);
+        if (this.googleFile) { // && downloadFile
+          await this.generateDocument(localFile);
+        }
+      } else if (localFile.type === 'drawing') {
+        this.logger.info('Transforming drawing: ' + this.localFile.fileName + verStr);
+        // const googleFile = await this.googleScanner.getFileById(localFile.id);
+        // const downloadFile = await this.downloadFilesStorage.findFile(f => f.id === localFile.id);
+        if (this.googleFile) { // && downloadFile
+          await this.generateDrawing(localFile);
+        }
+      } else if (localFile.type === 'binary') {
+        this.logger.info('Transforming binary: ' + this.localFile.fileName + verStr);
+        if (this.googleFile) { // && downloadFile
+          await this.generateBinary(localFile);
+        }
+      }
+      this.logger.info('Transformed: ' + this.localFile.fileName + verStr);
+    } catch (err) {
+      this.logger.error('Error transforming ' + localFile.fileName + ' ' + err.stack ? err.stack : err.message);
+      throw err;
+    }
+  }
+
+}
diff --git a/src/containers/transform/TransformContainer.ts b/src/containers/transform/TransformContainer.ts
index cae4c7c5..b9655166 100644
--- a/src/containers/transform/TransformContainer.ts
+++ b/src/containers/transform/TransformContainer.ts
@@ -25,6 +25,7 @@ import {LunrIndexer} from '../search/LunrIndexer.ts';
 import {JobManagerContainer} from '../job/JobManagerContainer.ts';
 import {UserConfigService} from '../google_folder/UserConfigService.ts';
 import {getUrlHash} from '../../utils/idParsers.ts';
+import {TaskGoogleMarkdownTransform} from './TaskGoogleMarkdownTransform.ts';
 
 const __filename = fileURLToPath(import.meta.url);
 
@@ -208,6 +209,7 @@ export class TransformContainer extends Container {
   private progressNotifyCallback: ({total, completed, warnings, failed}: { total?: number; completed?: number; warnings?: number; failed?: number }) => void;
   private transformLog: TransformLog;
   private isFailed = false;
+  private useGoogleMarkdowns = false;
 
   constructor(public readonly params: ContainerConfig, public readonly paramsArr: ContainerConfigArr = {}) {
     super(params, paramsArr);
@@ -287,18 +289,33 @@ export class TransformContainer extends Container {
 
       const jobManagerContainer = <JobManagerContainer>this.engine.getContainer('job_manager');
 
-      const task = new TaskLocalFileTransform(
-        this.logger,
-        jobManagerContainer,
-        realFileName,
-        googleFolder,
-        googleFile,
-        destinationDirectory,
-        localFile,
-        this.localLinks,
-        this.userConfigService.config
-      );
-      queueTransformer.addTask(task);
+      if (!this.useGoogleMarkdowns) {
+        const task = new TaskLocalFileTransform(
+          this.logger,
+          jobManagerContainer,
+          realFileName,
+          googleFolder,
+          googleFile,
+          destinationDirectory,
+          localFile,
+          this.localLinks,
+          this.userConfigService.config
+        );
+        queueTransformer.addTask(task);
+      } else {
+        const task = new TaskGoogleMarkdownTransform(
+          this.logger,
+          jobManagerContainer,
+          realFileName,
+          googleFolder,
+          googleFile,
+          destinationDirectory,
+          localFile,
+          this.localLinks,
+          this.userConfigService.config
+        );
+        queueTransformer.addTask(task);
+      }
     }
 
     const dirNames = destinationDirectory.getVirtualPath().replace(/\/$/, '').split('/');
@@ -519,4 +536,8 @@ export class TransformContainer extends Container {
   onProgressNotify(callback: ({total, completed, warnings, failed}: { total?: number; completed?: number, warnings?: number, failed?: number }) => void) {
     this.progressNotifyCallback = callback;
   }
+
+  setUseGoogleMarkdowns(value: boolean) {
+    this.useGoogleMarkdowns = value;
+  }
 }