Add rewrite rules for markdown

Should allow inserting custom video players and image previews

See: #399

apps/ui/src/components/UserSettings.vue

@@ -55,6 +55,11 @@
               <label>Config.toml for preview</label>
               <CodeEditor v-model="user_config.config_toml" lang="toml" />
             </div>
+
+            <div class="form-group">
+              <label>Rewrite rules</label>
+              <CodeEditor v-model="user_config.rewrite_rules_yaml" lang="yaml" />
+            </div>
           </form>
           <br/>
           <button class="btn btn-primary" type="button" @click="save">Save</button>

apps/ui/src/services/CachedFileClientService.ts

@@ -1,7 +1,7 @@
-import {FileClientService} from './FileClientService.js';
+import {FileClientService} from './FileClientService.ts';
 
 export class CachedFileClientService extends FileClientService {
-  private cache: any;
+  private cache: unknown;
 
   constructor(authenticatedClient) {
     super(authenticatedClient);

apps/ui/tsconfig.json

@@ -4,6 +4,7 @@
     "module": "ESNext",
     "moduleResolution": "node",
     "target": "ESNext",
+    "allowImportingTsExtensions": true,
     "resolveJsonModule": true,
     "esModuleInterop": true,
     "sourceMap": true,

src/containers/google_folder/UserConfigService.ts

@@ -2,10 +2,11 @@ import path from 'path';
 import fs from 'fs';
 import {exec} from 'child_process';
 import yaml from 'js-yaml';
-import {FileContentService} from '../../utils/FileContentService';
-import {HugoTheme} from '../server/routes/ConfigController';
-import {FRONTMATTER_DUMP_OPTS} from '../transform/frontmatters/frontmatter';
-import {DEFAULT_ACTIONS} from '../action/ActionRunnerContainer';
+import {FileContentService} from '../../utils/FileContentService.ts';
+import {HugoTheme} from '../server/routes/ConfigController.ts';
+import {FRONTMATTER_DUMP_OPTS} from '../transform/frontmatters/frontmatter.ts';
+import {DEFAULT_ACTIONS} from '../action/ActionRunnerContainer.ts';
+import {RewriteRule} from '../../odt/applyRewriteRule.js';
 
 async function execAsync(command: string) {
   const err = new Error();
@@ -40,6 +41,7 @@ export class UserConfig {
   auto_sync?: boolean;
   fm_without_version?: boolean;
   actions_yaml?: string;
+  rewrite_rules?: RewriteRule[];
 }
 
 const DEFAULT_CONFIG: UserConfig = {
@@ -52,6 +54,14 @@ const DEFAULT_CONFIG: UserConfig = {
   }
 };
 
+const DEFAULT_REWRITE_RULES = [
+  {
+    match: '(?:https?:\\/\\/)?(?:www\\.)?(?:youtube\\.com\\/(?:[^\\/\\n\\s]+\\/\\S+\\/|(?:v|e(?:mbed)?)\\/|\\S*?[?&]v=)|youtu\\.be\\/)([a-zA-Z0-9_-]{11})',
+    replace: '(?:https?:\\/\\/)?(?:www\\.)?(?:youtube\\.com\\/(?:[^\\/\\n\\s]+\\/\\S+\\/|(?:v|e(?:mbed)?)\\/|\\S*?[?&]v=)|youtu\\.be\\/)([a-zA-Z0-9_-]{11})',
+    template: '[$label](https://youtube.be/$value)'
+  }
+];
+
 export class UserConfigService {
 
   public config: UserConfig;
@@ -69,6 +79,9 @@ export class UserConfigService {
     if (!this.config.actions_yaml) {
       this.config.actions_yaml = yaml.dump(DEFAULT_ACTIONS, FRONTMATTER_DUMP_OPTS);
     }
+    if (!this.config.rewrite_rules || this.config.rewrite_rules.length === 0) {
+      this.config.rewrite_rules = DEFAULT_REWRITE_RULES;
+    }
 
     return this.config;
   }

src/containers/server/routes/ConfigController.ts

@@ -1,3 +1,5 @@
+import yaml from 'js-yaml';
+
 import {Controller, RouteGet, RouteParamBody, RouteParamPath, RoutePost, RoutePut} from './Controller';
 import {FileContentService} from '../../../utils/FileContentService';
 import {GitScanner} from '../../../git/GitScanner';
@@ -10,6 +12,7 @@ export interface ConfigBody {
     remote_branch: string;
     config_toml?: string;
     transform_subdir?: string;
+    rewrite_rules_yaml?: string;
     hugo_theme: HugoTheme;
     auto_sync: boolean;
     fm_without_version: boolean;
@@ -47,7 +50,7 @@ export class ConfigController extends Controller {
     const hugo_themes = await loadHugoThemes(this.filesService);
 
     return {
-      config: userConfigService.config,
+      config: { ...userConfigService.config, rewrite_rules_yaml: yaml.dump(userConfigService.config.rewrite_rules || []) },
       public_key: await userConfigService.getDeployKey(),
       hugo_themes
     };
@@ -90,6 +93,9 @@ export class ConfigController extends Controller {
     if (body.config?.config_toml) {
       userConfigService.config.config_toml = body.config?.config_toml;
     }
+    if (body.config?.rewrite_rules_yaml) {
+      userConfigService.config.rewrite_rules = yaml.load(body.config?.rewrite_rules_yaml);
+    }
     let modified = false;
     if ('string' === typeof body.config?.transform_subdir) {
       let trimmed = body.config?.transform_subdir.trim();

src/containers/transform/TaskLocalFileTransform.ts

@@ -1,19 +1,20 @@
-import {QueueTask} from '../google_folder/QueueTask';
 import winston from 'winston';
-import {FileContentService} from '../../utils/FileContentService';
-import {GoogleFile, MimeTypes} from '../../model/GoogleFile';
-import {BinaryFile, DrawingFile, LocalFile, MdFile} from '../../model/LocalFile';
-import {SvgTransform} from '../../SvgTransform';
-import {generateDocumentFrontMatter} from './frontmatters/generateDocumentFrontMatter';
-import {generateConflictMarkdown} from './frontmatters/generateConflictMarkdown';
-import {OdtProcessor} from '../../odt/OdtProcessor';
-import {UnMarshaller} from '../../odt/UnMarshaller';
-import {DocumentStyles, LIBREOFFICE_CLASSES} from '../../odt/LibreOffice';
-import {OdtToMarkdown} from '../../odt/OdtToMarkdown';
-import {LocalLinks} from './LocalLinks';
-import {SINGLE_THREADED_TRANSFORM} from './QueueTransformer';
-import {JobManagerContainer} from '../job/JobManagerContainer';
-import {UserConfig} from '../google_folder/UserConfigService';
+
+import {QueueTask} from '../google_folder/QueueTask.ts';
+import {FileContentService} from '../../utils/FileContentService.ts';
+import {GoogleFile, MimeTypes} from '../../model/GoogleFile.ts';
+import {BinaryFile, DrawingFile, LocalFile, MdFile} from '../../model/LocalFile.ts';
+import {SvgTransform} from '../../SvgTransform.ts';
+import {generateDocumentFrontMatter} from './frontmatters/generateDocumentFrontMatter.ts';
+import {generateConflictMarkdown} from './frontmatters/generateConflictMarkdown.ts';
+import {OdtProcessor} from '../../odt/OdtProcessor.ts';
+import {UnMarshaller} from '../../odt/UnMarshaller.ts';
+import {DocumentStyles, LIBREOFFICE_CLASSES} from '../../odt/LibreOffice.ts';
+import {OdtToMarkdown} from '../../odt/OdtToMarkdown.ts';
+import {LocalLinks} from './LocalLinks.ts';
+import {SINGLE_THREADED_TRANSFORM} from './QueueTransformer.ts';
+import {JobManagerContainer} from '../job/JobManagerContainer.ts';
+import {UserConfig} from '../google_folder/UserConfigService.ts';
 
 export function googleMimeToExt(mimeType: string, fileName: string) {
   switch (mimeType) {
@@ -132,6 +133,8 @@ export class TaskLocalFileTransform extends QueueTask {
     const odtPath = this.googleFolder.getRealPath() + '/' + localFile.id + '.odt';
     const destinationPath = this.destinationDirectory.getRealPath();
 
+    const rewriteRules = this.userConfig.rewrite_rules || [];
+
     if (SINGLE_THREADED_TRANSFORM) {
       const processor = new OdtProcessor(odtPath, true);
       await processor.load();
@@ -150,6 +153,7 @@ export class TaskLocalFileTransform extends QueueTask {
       }
 
       const converter = new OdtToMarkdown(document, styles, fileNameMap);
+      converter.setRewriteRules(rewriteRules);
       if (this.realFileName === '_index.md') {
         converter.setPicturesDir('./' + this.realFileName.replace(/.md$/, '.assets/'));
       } else {
@@ -173,6 +177,7 @@ export class TaskLocalFileTransform extends QueueTask {
         realFileName: this.realFileName,
         odtPath,
         destinationPath,
+        rewriteRules,
         fm_without_version: this.userConfig.fm_without_version
       });
 

src/odt/MarkdownChunks.ts

@@ -1,5 +1,6 @@
-import {ListStyle, Style, TextProperty} from './LibreOffice';
-import {inchesToMm, inchesToPixels} from './utils';
+import {ListStyle, Style, TextProperty} from './LibreOffice.ts';
+import {inchesToPixels} from './utils.ts';
+import {applyRewriteRule, RewriteRule} from './applyRewriteRule.ts';
 
 export type OutputMode = 'md' | 'html' | 'raw';
 
@@ -56,6 +57,7 @@ export interface MarkdownTagChunk {
 
 type MarkdownChunk = MarkdownTextChunk | MarkdownTagChunk;
 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 function debugChunkToText(chunk: MarkdownChunk) {
   if (chunk.isTag === false) {
     return chunk.text;
@@ -312,6 +314,80 @@ function chunkToText(chunk: MarkdownChunk) {
 
 }
 
+
+function chunksToText(chunks: MarkdownChunk[], rules: RewriteRule[]) {
+  const retVal = [];
+
+  for (let chunkNo = 0; chunkNo < chunks.length; chunkNo++) {
+    const chunk = chunks[chunkNo];
+
+    if ('tag' in chunk && ['SVG/', 'IMG/'].includes(chunk.tag)) {
+      let broke = false;
+      for (const rule of rules) {
+        const { shouldBreak, text } = applyRewriteRule(rule, {
+          ...chunk,
+          href: 'payload' in chunk ? chunk.payload?.href : undefined,
+          alt: 'payload' in chunk ? chunk.payload?.alt : undefined
+        });
+
+        if (shouldBreak) {
+          retVal.push(text);
+          broke = true;
+          break;
+        }
+      }
+
+      if (broke) {
+        continue;
+      }
+    }
+
+    if ('tag' in chunk && 'A' === chunk.tag) {
+      let matchingNo = -1;
+
+      for (let idx = chunkNo + 1; idx < chunks.length; idx++) {
+        const chunkEnd = chunks[idx];
+        if ('tag' in chunkEnd && chunkEnd.tag === '/A') {
+          matchingNo = idx;
+          break;
+        }
+      }
+
+      if (matchingNo !== -1) {
+        const alt = chunksToText(chunks.slice(chunkNo + 1, matchingNo).filter(i => !i.isTag), rules).join('');
+        let broke = false;
+        for (const rule of rules) {
+          const { shouldBreak, text } = applyRewriteRule(rule, {
+            ...chunk,
+            href: 'payload' in chunk ? chunk.payload?.href : undefined,
+            alt
+          });
+
+          if (shouldBreak) {
+            retVal.push(text);
+            broke = true;
+            break;
+          }
+        }
+
+        if (broke) {
+          chunks.splice(chunkNo, matchingNo - chunkNo);
+          continue;
+        }
+      }
+    }
+
+    retVal.push(chunkToText(chunk));
+  }
+
+  // chunks.map(c => chunkToText(c));
+/*
+*/
+
+  return retVal;
+}
+
+
 export class MarkdownChunks {
   chunks: MarkdownChunk[] = [];
 
@@ -323,13 +399,13 @@ export class MarkdownChunks {
     this.chunks.push(s);
   }
 
-  toString() {
+  toString(rules: RewriteRule[] = []) {
     // console.log(this.chunks.map(c => debugChunkToText(c)).join('\n'));
-    return this.chunks.map(c => chunkToText(c)).join('');
+    return chunksToText(this.chunks, rules).join('');
   }
 
-  extractText(start: number, end: number) {
-    const slice = this.chunks.slice(start, end).filter(i => !i.isTag).map(c => chunkToText(c)).join('');
+  extractText(start: number, end: number, rules: RewriteRule[] = []) {
+    const slice = chunksToText(this.chunks.slice(start, end).filter(i => !i.isTag), rules).join('');
     return slice;
   }
 

src/odt/OdtToMarkdown.ts

@@ -15,13 +15,14 @@ import {
   TextProperty,
   TextSpace,
   TextSpan
-} from './LibreOffice';
-import {urlToFolderId} from '../utils/idParsers';
-import {MarkdownChunks} from './MarkdownChunks';
-import {isMarkdownMacro, StateMachine} from './StateMachine';
-import {inchesToPixels, inchesToSpaces, spaces} from './utils';
-import {extractPath} from './extractPath';
-import {mergeDeep} from './mergeDeep';
+} from './LibreOffice.ts';
+import {urlToFolderId} from '../utils/idParsers.ts';
+import {MarkdownChunks} from './MarkdownChunks.ts';
+import {isMarkdownMacro, StateMachine} from './StateMachine.ts';
+import {inchesToPixels, inchesToSpaces, spaces} from './utils.ts';
+import {extractPath} from './extractPath.ts';
+import {mergeDeep} from './mergeDeep.ts';
+import {RewriteRule} from './applyRewriteRule.ts';
 
 function getBaseFileName(fileName) {
   return fileName.replace(/.*\//, '');
@@ -62,6 +63,7 @@ export class OdtToMarkdown {
   public readonly links: Set<string> = new Set<string>();
   private readonly chunks: MarkdownChunks = new MarkdownChunks();
   private picturesDir = '';
+  private rewriteRules: RewriteRule[] = [];
 
   constructor(private document: DocumentContent, private documentStyles: DocumentStyles, private fileNameMap: FileNameMap = {}) {
     this.stateMachine = new StateMachine(this.chunks);
@@ -115,7 +117,7 @@ export class OdtToMarkdown {
 
     this.stateMachine.postProcess();
 
-    const markdown = this.chunks.toString();
+    const markdown = this.chunks.toString(this.rewriteRules);
     const trimmed = this.trimBreaks(markdown);
     return await this.rewriteHeaders(trimmed);
   }
@@ -326,7 +328,7 @@ export class OdtToMarkdown {
   async drawGToText(drawG: DrawG) {
     this.stateMachine.pushTag('HTML_MODE/');
 
-    const style = this.getStyle(drawG.styleName);
+    this.getStyle(drawG.styleName);
 
     let maxx = 0;
     let maxy = 0;
@@ -682,4 +684,9 @@ export class OdtToMarkdown {
   setPicturesDir(picturesDir: string) {
     this.picturesDir = picturesDir;
   }
+
+  setRewriteRules(rewriteRules: RewriteRule[]) {
+    this.rewriteRules = rewriteRules;
+    this.stateMachine.setRewriteRules(rewriteRules);
+  }
 }