From d66402d4724cd7ed77907314283cf75f4f68a821 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 16 Aug 2024 18:28:10 +0200 Subject: [PATCH] Remove unused anchor bookmarks See #472 --- .../transform/TaskLocalFileTransform.ts | 8 ++++++ .../transform/TransformContainer.ts | 27 +++++++++++++++++-- .../frontmatters/generateDirectoryYaml.ts | 6 ++--- src/odt/OdtToMarkdown.ts | 8 +++++- src/odt/executeOdtToMarkdown.ts | 3 ++- src/odt/postprocess/postProcess.ts | 4 +-- src/odt/postprocess/rewriteHeaders.ts | 9 +++++-- website/ui.md | 3 +++ 8 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 website/ui.md diff --git a/src/containers/transform/TaskLocalFileTransform.ts b/src/containers/transform/TaskLocalFileTransform.ts index 59309c3c..5174cb64 100644 --- a/src/containers/transform/TaskLocalFileTransform.ts +++ b/src/containers/transform/TaskLocalFileTransform.ts @@ -51,6 +51,7 @@ export class TaskLocalFileTransform extends QueueTask { private localLinks: LocalLinks, private userConfig: UserConfig, private globalHeadersMap: {[key: string]: string}, + private globalInvisibleBookmarks: {[key: number]: number}, ) { super(logger); this.retries = 0; @@ -130,6 +131,7 @@ export class TaskLocalFileTransform extends QueueTask { let frontMatter; let markdown; let headersMap = {}; + let invisibleBookmarks = {}; let links = []; let errors = []; @@ -166,6 +168,7 @@ export class TaskLocalFileTransform extends QueueTask { converter.setPicturesDir('../' + this.realFileName.replace(/.md$/, '.assets/'), picturesDirAbsolute); } headersMap = converter.getHeadersMap(); + invisibleBookmarks = converter.getInvisibleBookmarks(); markdown = await converter.convert(); links = Array.from(converter.links); frontMatter = generateDocumentFrontMatter(localFile, links, this.userConfig.fm_without_version); @@ -178,6 +181,7 @@ export class TaskLocalFileTransform extends QueueTask { markdown: string; errors: Array; headersMap: {[key: string]: string}; + invisibleBookmarks: {[key: string]: string}; } const workerResult: WorkerResult = await this.jobManagerContainer.scheduleWorker('OdtToMarkdown', { @@ -195,6 +199,7 @@ export class TaskLocalFileTransform extends QueueTask { markdown = workerResult.markdown; errors = workerResult.errors; headersMap = workerResult.headersMap; + invisibleBookmarks = workerResult.invisibleBookmarks; this.warnings = errors.length; } @@ -210,6 +215,9 @@ export class TaskLocalFileTransform extends QueueTask { for (const k in headersMap) { this.globalHeadersMap['gdoc:' + localFile.id + k] = 'gdoc:' + localFile.id + headersMap[k]; } + for (const k in invisibleBookmarks) { + this.globalInvisibleBookmarks['gdoc:' + localFile.id + k] = invisibleBookmarks[k]; + } } async generate(localFile: LocalFile): Promise { diff --git a/src/containers/transform/TransformContainer.ts b/src/containers/transform/TransformContainer.ts index 16606699..ec396526 100644 --- a/src/containers/transform/TransformContainer.ts +++ b/src/containers/transform/TransformContainer.ts @@ -25,6 +25,7 @@ import {JobManagerContainer} from '../job/JobManagerContainer.ts'; import {UserConfigService} from '../google_folder/UserConfigService.ts'; import {getUrlHash} from '../../utils/idParsers.ts'; import {TaskGoogleMarkdownTransform} from './TaskGoogleMarkdownTransform.ts'; +import {frontmatter} from './frontmatters/frontmatter.js'; const __filename = fileURLToPath(import.meta.url); @@ -209,6 +210,7 @@ export class TransformContainer extends Container { private isFailed = false; private useGoogleMarkdowns = false; private globalHeadersMap: {[key: string]: string} = {}; + private globalInvisibleBookmarks: {[key: string]: number} = {}; constructor(public readonly params: ContainerConfig, public readonly paramsArr: ContainerConfigArr = {}) { super(params, paramsArr); @@ -299,7 +301,8 @@ export class TransformContainer extends Container { localFile, this.localLinks, this.userConfigService.config, - this.globalHeadersMap + this.globalHeadersMap, + this.globalInvisibleBookmarks ); queueTransformer.addTask(task); } else { @@ -435,6 +438,7 @@ export class TransformContainer extends Container { async rewriteLinks(destinationDirectory: FileContentService) { const files = await destinationDirectory.list(); + for (const fileName of files) { if (await destinationDirectory.isDirectory(fileName)) { await this.rewriteLinks(await destinationDirectory.getSubFileService(fileName)); @@ -443,7 +447,26 @@ export class TransformContainer extends Container { if (fileName.endsWith('.md') || fileName.endsWith('.svg')) { const content = await destinationDirectory.readFile(fileName); - const newContent = content.replace(/(gdoc:[A-Z0-9_-]+)(#[^'")\s]*)?/ig, (str: string) => { + + const parsed = frontmatter(content); + const props = parsed.data; + let newContent = content; + if (props?.id) { + const unusedHashes = Object + .keys(this.globalInvisibleBookmarks) + .filter(hash => hash.startsWith('gdoc:' + props.id)); + + for (const hash of unusedHashes) { + if (!this.globalHeadersMap[hash]) { + const postHash = hash.substring(hash.indexOf('#') + 1); + newContent = newContent + .replaceAll(` `, '') + .replaceAll(``, ''); + } + } + } + + newContent = newContent.replace(/(gdoc:[A-Z0-9_-]+)(#[^'")\s]*)?/ig, (str: string) => { let fileId = str.substring('gdoc:'.length).replace(/#.*/, ''); let hash = getUrlHash(str) || ''; if (hash && this.globalHeadersMap[str]) { diff --git a/src/containers/transform/frontmatters/generateDirectoryYaml.ts b/src/containers/transform/frontmatters/generateDirectoryYaml.ts index a98db50c..ab30d783 100644 --- a/src/containers/transform/frontmatters/generateDirectoryYaml.ts +++ b/src/containers/transform/frontmatters/generateDirectoryYaml.ts @@ -1,8 +1,8 @@ import yaml from 'js-yaml'; -import {FRONTMATTER_DUMP_OPTS} from './frontmatter'; -import {GoogleFile, MimeTypes} from '../../../model/GoogleFile'; -import {LocalFile} from '../../../model/LocalFile'; +import {FRONTMATTER_DUMP_OPTS} from './frontmatter.ts'; +import {GoogleFile, MimeTypes} from '../../../model/GoogleFile.ts'; +import {LocalFile} from '../../../model/LocalFile.ts'; export function generateDirectoryYaml(fileName: string, directory: GoogleFile, realFileNameToGenerated: { [realFileName: string]: LocalFile }) { return yaml.dump({ diff --git a/src/odt/OdtToMarkdown.ts b/src/odt/OdtToMarkdown.ts index 80306c2d..7348fc73 100644 --- a/src/odt/OdtToMarkdown.ts +++ b/src/odt/OdtToMarkdown.ts @@ -69,6 +69,7 @@ export class OdtToMarkdown { private picturesDirAbsolute = ''; private rewriteRules: RewriteRule[] = []; private headersMap: { [p: string]: string } = {}; + private invisibleBookmarks: { [p: string]: number } = {}; constructor(private document: DocumentContent, private documentStyles: DocumentStyles, private fileNameMap: StringToStringMap = {}, private xmlMap: StringToStringMap = {}) { } @@ -132,8 +133,9 @@ export class OdtToMarkdown { // text = this.processMacros(text); // text = this.fixBlockMacros(text); - const { headersMap } = await postProcess(this.chunks, this.rewriteRules); + const { headersMap, invisibleBookmarks } = await postProcess(this.chunks, this.rewriteRules); this.headersMap = headersMap; + this.invisibleBookmarks = invisibleBookmarks; const markdown = this.chunks.toString(); return this.trimBreaks(markdown); @@ -707,4 +709,8 @@ export class OdtToMarkdown { return this.headersMap; } + getInvisibleBookmarks() { + return this.invisibleBookmarks; + } + } diff --git a/src/odt/executeOdtToMarkdown.ts b/src/odt/executeOdtToMarkdown.ts index de1d0348..7139df1b 100644 --- a/src/odt/executeOdtToMarkdown.ts +++ b/src/odt/executeOdtToMarkdown.ts @@ -43,6 +43,7 @@ export async function executeOdtToMarkdown(workerData) { } const headersMap = converter.getHeadersMap(); + const invisibleBookmarks = converter.getInvisibleBookmarks(); - return { links, frontMatter, markdown, errors, headersMap }; + return { links, frontMatter, markdown, errors, headersMap, invisibleBookmarks }; } diff --git a/src/odt/postprocess/postProcess.ts b/src/odt/postprocess/postProcess.ts index cd125669..e184982d 100644 --- a/src/odt/postprocess/postProcess.ts +++ b/src/odt/postprocess/postProcess.ts @@ -40,7 +40,7 @@ export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRu convertMathMl(chunks); trimParagraphs(chunks); - const { headersMap} = await rewriteHeaders(chunks); + const { headersMap, invisibleBookmarks} = await rewriteHeaders(chunks); trimParagraphs(chunks); addEmptyLinesAfterParas(chunks); removeTdParas(chunks); // Requires: addEmptyLinesAfterParas @@ -68,5 +68,5 @@ export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRu dump(chunks.body); } - return { headersMap }; + return { headersMap, invisibleBookmarks }; } diff --git a/src/odt/postprocess/rewriteHeaders.ts b/src/odt/postprocess/rewriteHeaders.ts index 12900974..352ac0a7 100644 --- a/src/odt/postprocess/rewriteHeaders.ts +++ b/src/odt/postprocess/rewriteHeaders.ts @@ -2,8 +2,9 @@ import slugify from 'slugify'; import {extractText, walkRecursiveAsync, walkRecursiveSync} from '../markdownNodesUtils.ts'; import {MarkdownNodes, MarkdownTextNode} from '../MarkdownNodes.ts'; -export async function rewriteHeaders(markdownChunks: MarkdownNodes): Promise<{ headersMap: {[key: string]: string} }> { +export async function rewriteHeaders(markdownChunks: MarkdownNodes): Promise<{ headersMap: {[key: string]: string}, invisibleBookmarks: {[key: string]: number} }> { const headersMap = {}; + const invisibleBookmarks = {}; let inPre = false; await walkRecursiveAsync(markdownChunks.body, async (chunk, ctx: { nodeIdx: number }) => { @@ -55,6 +56,10 @@ export async function rewriteHeaders(markdownChunks: MarkdownNodes): Promise<{ h await walkRecursiveAsync(markdownChunks.body, async (chunk, ctx: { nodeIdx: number }) => { if (chunk.isTag && 'BOOKMARK/' === chunk.tag) { + if (!headersMap['#' + chunk.payload.id]) { + invisibleBookmarks['#' + chunk.payload.id] = 1; + } + if (chunk.parent.children.length < 2) { return; } @@ -78,5 +83,5 @@ export async function rewriteHeaders(markdownChunks: MarkdownNodes): Promise<{ h }); } - return { headersMap }; + return { headersMap, invisibleBookmarks }; } diff --git a/website/ui.md b/website/ui.md new file mode 100644 index 00000000..a03497ab --- /dev/null +++ b/website/ui.md @@ -0,0 +1,3 @@ +--- +comment: This file is used to generate empty ui/index.html to perform SSR serving +---