Make hash links nicer

src/LinkTranslator.ts

@@ -40,7 +40,7 @@ export function convertExtension(localPath: string, mode?: LinkMode) {
   return dirName + parts.join('.');
 }
 
-export function convertToRelativeMarkDownPath(localPath, basePath) {
+export function convertToRelativeMarkDownPath(localPath: string, basePath: string) {
   if (localPath.startsWith('https://')) return localPath;
   if (localPath.startsWith('http://')) return localPath;
   if (basePath === localPath) return '.';
@@ -51,7 +51,7 @@ export function convertToRelativeMarkDownPath(localPath, basePath) {
   })));
 }
 
-export function convertToRelativeSvgPath(localPath, basePath) {
+export function convertToRelativeSvgPath(localPath: string, basePath: string) {
   if (localPath.startsWith('https://')) return localPath;
   if (localPath.startsWith('http://')) return localPath;
   if (basePath === localPath) return '.';

src/containers/server/ServerContainer.ts

@@ -44,7 +44,7 @@ import {GoogleTreeProcessor} from '../google_folder/GoogleTreeProcessor.ts';
 import {initStaticDistPages} from './static.ts';
 import {initUiServer} from './vuejs.ts';
 import {initErrorHandler} from './error.ts';
-import {WebHookController} from './routes/WebHookController.js';
+import {WebHookController} from './routes/WebHookController.ts';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);

src/containers/transform/TaskGoogleMarkdownTransform.ts

@@ -12,7 +12,7 @@ import {SvgTransform} from '../../SvgTransform.ts';
 import {generateDocumentFrontMatter} from './frontmatters/generateDocumentFrontMatter.ts';
 import {generateConflictMarkdown} from './frontmatters/generateConflictMarkdown.ts';
 import {googleMimeToExt} from './TaskLocalFileTransform.ts';
-import {getUrlHash, urlToFolderId} from '../../utils/idParsers.js';
+import {getUrlHash, urlToFolderId} from '../../utils/idParsers.ts';
 
 export class TaskGoogleMarkdownTransform extends QueueTask {
   constructor(protected logger: winston.Logger,

src/containers/transform/TaskLocalFileTransform.ts

@@ -49,7 +49,8 @@ export class TaskLocalFileTransform extends QueueTask {
               private destinationDirectory: FileContentService,
               private localFile: LocalFile,
               private localLinks: LocalLinks,
-              private userConfig: UserConfig
+              private userConfig: UserConfig,
+              private globalHeadersMap: {[key: string]: string},
               ) {
     super(logger);
     this.retries = 0;
@@ -128,6 +129,7 @@ export class TaskLocalFileTransform extends QueueTask {
   async generateDocument(localFile: MdFile) {
     let frontMatter;
     let markdown;
+    let headersMap = {};
     let links = [];
     let errors = [];
 
@@ -163,6 +165,7 @@ export class TaskLocalFileTransform extends QueueTask {
       } else {
         converter.setPicturesDir('../' + this.realFileName.replace(/.md$/, '.assets/'), picturesDirAbsolute);
       }
+      headersMap = converter.getHeadersMap();
       markdown = await converter.convert();
       links = Array.from(converter.links);
       frontMatter = generateDocumentFrontMatter(localFile, links, this.userConfig.fm_without_version);
@@ -174,6 +177,7 @@ export class TaskLocalFileTransform extends QueueTask {
         frontMatter: string;
         markdown: string;
         errors: Array<string>;
+        headersMap: {[key: string]: string};
       }
 
       const workerResult: WorkerResult = <WorkerResult>await this.jobManagerContainer.scheduleWorker('OdtToMarkdown', {
@@ -190,6 +194,7 @@ export class TaskLocalFileTransform extends QueueTask {
       frontMatter = workerResult.frontMatter;
       markdown = workerResult.markdown;
       errors = workerResult.errors;
+      headersMap = workerResult.headersMap;
       this.warnings = errors.length;
     }
 
@@ -202,6 +207,9 @@ export class TaskLocalFileTransform extends QueueTask {
 
     await this.destinationDirectory.writeFile(this.realFileName, frontMatter + markdown);
     this.localLinks.append(localFile.id, localFile.fileName, links);
+    for (const k in headersMap) {
+      this.globalHeadersMap['gdoc:' + localFile.id + k] = 'gdoc:' + localFile.id + headersMap[k];
+    }
   }
 
   async generate(localFile: LocalFile): Promise<void> {

src/containers/transform/TransformContainer.ts

@@ -9,7 +9,6 @@ import {GoogleFilesScanner} from './GoogleFilesScanner.ts';
 import {convertToRelativeMarkDownPath, convertToRelativeSvgPath} from '../../LinkTranslator.ts';
 import {LocalFilesGenerator} from './LocalFilesGenerator.ts';
 import {QueueTransformer} from './QueueTransformer.ts';
-import {NavigationHierarchy} from './generateNavigationHierarchy.ts';
 import {ConflictFile, LocalFile, RedirFile} from '../../model/LocalFile.ts';
 import {TaskLocalFileTransform} from './TaskLocalFileTransform.ts';
 import {MimeTypes} from '../../model/GoogleFile.ts';
@@ -200,7 +199,6 @@ export class TransformLog extends Transport {
 export class TransformContainer extends Container {
   private logger: winston.Logger;
   private generatedFileService: FileContentService;
-  private hierarchy: NavigationHierarchy = {};
   private localLog: LocalLog;
   private localLinks: LocalLinks;
   private filterFilesIds: FileId[];
@@ -210,6 +208,7 @@ export class TransformContainer extends Container {
   private transformLog: TransformLog;
   private isFailed = false;
   private useGoogleMarkdowns = false;
+  private globalHeadersMap: {[key: string]: string} = {};
 
   constructor(public readonly params: ContainerConfig, public readonly paramsArr: ContainerConfigArr = {}) {
     super(params, paramsArr);
@@ -299,7 +298,8 @@ export class TransformContainer extends Container {
           destinationDirectory,
           localFile,
           this.localLinks,
-          this.userConfigService.config
+          this.userConfigService.config,
+          this.globalHeadersMap
         );
         queueTransformer.addTask(task);
       } else {
@@ -443,15 +443,22 @@ export class TransformContainer extends Container {
 
       if (fileName.endsWith('.md') || fileName.endsWith('.svg')) {
         const content = await destinationDirectory.readFile(fileName);
-        const newContent = content.replace(/(gdoc:[A-Z0-9_-]+)/ig, (str: string) => {
-          const fileId = str.substring('gdoc:'.length).replace(/#.*/, '');
-          const hash = getUrlHash(str);
+        const newContent = content.replace(/(gdoc:[A-Z0-9_-]+)(#[^'")\s]*)?/ig, (str: string) => {
+          let fileId = str.substring('gdoc:'.length).replace(/#.*/, '');
+          let hash = getUrlHash(str) || '';
+          if (hash && this.globalHeadersMap[str]) {
+            const idx = this.globalHeadersMap[str].indexOf('#');
+            if (idx >= 0) {
+              fileId = this.globalHeadersMap[str].substring('gdoc:'.length, idx);
+              hash = this.globalHeadersMap[str].substring(idx);
+            }
+          }
           const lastLog = this.localLog.findLastFile(fileId);
           if (lastLog && lastLog.event !== 'removed') {
             if (fileName.endsWith('.svg')) {
               return convertToRelativeSvgPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName);
             } else {
-              return convertToRelativeMarkDownPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName);
+              return convertToRelativeMarkDownPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName) + hash;
             }
           } else {
             return 'https://drive.google.com/open?id=' + fileId + hash.replace('#_', '#heading=h.');

src/odt/OdtToMarkdown.ts

@@ -68,6 +68,7 @@ export class OdtToMarkdown {
   private picturesDir = '';
   private picturesDirAbsolute = '';
   private rewriteRules: RewriteRule[] = [];
+  private headersMap: { [p: string]: string } = {};
 
   constructor(private document: DocumentContent, private documentStyles: DocumentStyles, private fileNameMap: StringToStringMap = {}, private xmlMap: StringToStringMap = {}) {
   }
@@ -131,7 +132,8 @@ export class OdtToMarkdown {
     // text = this.processMacros(text);
     // text = this.fixBlockMacros(text);
 
-    await postProcess(this.chunks, this.rewriteRules);
+    const { headersMap } = await postProcess(this.chunks, this.rewriteRules);
+    this.headersMap = headersMap;
 
     const markdown = this.chunks.toString();
     return this.trimBreaks(markdown);
@@ -223,7 +225,7 @@ export class OdtToMarkdown {
 
   addLink(href: string) {
     if (href && !href.startsWith('#') && href.indexOf(':') > -1) {
-      this.links.add(href);
+      this.links.add(href.replace(/#.*$/, ''));
     }
   }
 
@@ -701,4 +703,8 @@ export class OdtToMarkdown {
     this.errors.push(error);
   }
 
+  getHeadersMap() {
+    return this.headersMap;
+  }
+
 }

src/odt/executeOdtToMarkdown.ts

@@ -42,5 +42,7 @@ export async function executeOdtToMarkdown(workerData) {
     fs.writeFileSync(path.join(workerData.destinationPath, workerData.realFileName.replace(/.md$/, '.debug.xml')), markdown);
   }
 
-  return { links, frontMatter, markdown, errors };
+  const headersMap = converter.getHeadersMap();
+
+  return { links, frontMatter, markdown, errors, headersMap };
 }

src/odt/postprocess/postProcess.ts

@@ -40,7 +40,7 @@ export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRu
   convertMathMl(chunks);
 
   trimParagraphs(chunks);
-  await rewriteHeaders(chunks);
+  const { headersMap} = await rewriteHeaders(chunks);
   trimParagraphs(chunks);
   addEmptyLinesAfterParas(chunks);
   removeTdParas(chunks); // Requires: addEmptyLinesAfterParas
@@ -67,4 +67,6 @@ export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRu
   if (process.env.DEBUG_COLORS) {
     dump(chunks.body);
   }
+
+  return { headersMap };
 }

src/odt/postprocess/rewriteHeaders.ts

@@ -1,7 +1,10 @@
-import {walkRecursiveAsync} from '../markdownNodesUtils.ts';
+import slugify from 'slugify';
+import {extractText, walkRecursiveAsync, walkRecursiveSync} from '../markdownNodesUtils.ts';
 import {MarkdownNodes, MarkdownTextNode} from '../MarkdownNodes.ts';
 
-export async function rewriteHeaders(markdownChunks: MarkdownNodes) {
+export async function rewriteHeaders(markdownChunks: MarkdownNodes): Promise<{ headersMap: {[key: string]: string} }> {
+  const headersMap = {};
+
   let inPre = false;
   await walkRecursiveAsync(markdownChunks.body, async (chunk, ctx: { nodeIdx: number }) => {
     if (chunk.isTag && 'PRE' === chunk.tag) {
@@ -21,6 +24,9 @@ export async function rewriteHeaders(markdownChunks: MarkdownNodes) {
     }
 
     if (chunk.isTag && ['H1', 'H2', 'H3', 'H4'].includes(chunk.tag)) {
+      const innerTxt = extractText(chunk);
+      const slug = slugify(innerTxt.trim(), { replacement: '-', lower: true, remove: /[#*+~.,^()'"!:@]/g });
+
       if (chunk.children.length === 1) {
         const child = chunk.children[0];
         if (child.isTag && child.tag === 'BOOKMARK/') {
@@ -32,7 +38,11 @@ export async function rewriteHeaders(markdownChunks: MarkdownNodes) {
         const child = chunk.children[j];
         if (child.isTag && child.tag === 'BOOKMARK/') {
           const toMove = chunk.children.splice(j, 1);
-          chunk.children.splice(chunk.children.length, 0, ...toMove);
+          if (slug && !headersMap['#' + child.payload.id]) {
+            headersMap['#' + child.payload.id] = '#' + slug;
+          } else {
+            chunk.children.splice(chunk.children.length, 0, ...toMove);
+          }
           break;
         }
       }
@@ -57,4 +67,16 @@ export async function rewriteHeaders(markdownChunks: MarkdownNodes) {
       return { nodeIdx: ctx.nodeIdx + 1 };
     }
   });
+
+  if (Object.keys(headersMap).length > 0) {
+    walkRecursiveSync(markdownChunks.body, (chunk) => {
+      if (chunk.isTag === true && chunk.payload?.href) {
+        if (headersMap[chunk.payload.href]) {
+          chunk.payload.href = headersMap[chunk.payload.href];
+        }
+      }
+    });
+  }
+
+  return { headersMap };
 }

test/git/RebaseTest.ts

@@ -382,8 +382,6 @@ describe('RebaseTest', function () {
       {
         const history = await scannerLocal.history('');
 
-        console.log('historyhistory', history);
-
         assert.equal(2, history.length);
         assert.equal('Change on second repo', history[0].message);
 

test/odt_md/bullets.md

@@ -1,20 +1,20 @@
-## Bullet List <a id="_80qew9aprjq8"></a>
+## Bullet List
 
 * Capability to print standard leave forms from the patient portal
 * Pre-loaded standard email templates
 * Absence Management encounter
 * Quick View Absence Management Worklist
 * Leave Types Report
 
-## Ordered List <a id="_k2lx86ardtkr"></a>
+## Ordered List
 
 1. Capability to print standard leave forms from the patient portal
 2. Pre-loaded standard email templates
 3. Absence Management encounter
 4. Quick View Absence Management Worklist
 5. Leave Types Report
 
-## Mixed Lists <a id="_xnxyy4ptrcvu"></a>
+## Mixed Lists
 
 Item before  list
 

test/odt_md/confluence.md

@@ -1,14 +1,14 @@
-# Confluence to Google Docs Conversion Notes <a id="_77510dqpcwl"></a>
+# Confluence to Google Docs Conversion Notes
 
-## Goal <a id="_10c0ntqm6mev"></a>
+## Goal
 
 Convert Confluence Documents in to Google Documents for the purpose of using WikiGDrive to publish them.
 
-## Delivery <a id="_i96xic41327t"></a>
+## Delivery
 
 A new github repo with a node.js script specific to this conversion.
 
-## High level Process <a id="_wrw8sf3g3pc3"></a>
+## High level Process
 
 * Scan all of the documents in a Confluence Space
 * Make google documents in a shared drive (two passes will be required so links between documents can be known as content is added).
@@ -30,13 +30,13 @@ A new github repo with a node.js script specific to this conversion.
     * Embedded Video should be converted to an image with a hyperlink
     * Formatting is not required to be converted.
 
-## Proposed Instructions <a id="_m98292szff3y"></a>
+## Proposed Instructions
 
 ```
 confluence2google <path to space> <path to google shared drive>
 ```
 
-## Links and Possible Approaches <a id="_sdx7522zhby"></a>
+## Links and Possible Approaches
 
 1. Use REST API
     1. [Confluence Cloud REST API](https://developer.atlassian.com/cloud/confluence/rest/)
@@ -52,7 +52,7 @@ confluence2google <path to space> <path to google shared drive>
 
 <a id="ggsym7lvzx37"></a>
 
-## Examples <a id="_rsmqmchgeq17"></a>
+## Examples
 
 Simple - [https://confluence.example.com/display/DOCS/Sample](https://confluence.example.com/display/DOCS/Sample)