-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use experimental google markdown support
- Loading branch information
1 parent
adbd690
commit be02d09
Showing
10 changed files
with
307 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
202 changes: 202 additions & 0 deletions
202
src/containers/transform/TaskGoogleMarkdownTransform.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| import fs from 'fs'; | ||
| import winston from 'winston'; | ||
|
|
||
| import {QueueTask} from '../google_folder/QueueTask.ts'; | ||
| import {JobManagerContainer} from '../job/JobManagerContainer.ts'; | ||
| import {FileContentService} from '../../utils/FileContentService.ts'; | ||
| import {GoogleFile} from '../../model/GoogleFile.ts'; | ||
| import {BinaryFile, DrawingFile, LocalFile, MdFile} from '../../model/LocalFile.ts'; | ||
| import {LocalLinks} from './LocalLinks.ts'; | ||
| import {UserConfig} from '../google_folder/UserConfigService.ts'; | ||
| import {SvgTransform} from '../../SvgTransform.ts'; | ||
| import {generateDocumentFrontMatter} from './frontmatters/generateDocumentFrontMatter.ts'; | ||
| import {generateConflictMarkdown} from './frontmatters/generateConflictMarkdown.ts'; | ||
| import {googleMimeToExt} from './TaskLocalFileTransform.ts'; | ||
| import {getUrlHash, urlToFolderId} from '../../utils/idParsers.js'; | ||
|
|
||
| export class TaskGoogleMarkdownTransform extends QueueTask { | ||
| constructor(protected logger: winston.Logger, | ||
| private jobManagerContainer: JobManagerContainer, | ||
| private realFileName: string, | ||
| private googleFolder: FileContentService, | ||
| private googleFile: GoogleFile, | ||
| private destinationDirectory: FileContentService, | ||
| private localFile: LocalFile, | ||
| private localLinks: LocalLinks, | ||
| private userConfig: UserConfig | ||
| ) { | ||
| super(logger); | ||
| this.retries = 0; | ||
|
|
||
| if (!this.localFile.fileName) { | ||
| throw new Error(`No fileName for: ${this.localFile.id}`); | ||
| } | ||
| } | ||
|
|
||
| async run(): Promise<QueueTask[]> { | ||
| await this.generate(this.localFile); | ||
|
|
||
| return []; | ||
| } | ||
|
|
||
| async generateBinary(binaryFile: BinaryFile) { | ||
| await new Promise<void>((resolve, reject) => { | ||
| try { | ||
| const dest = this.destinationDirectory.createWriteStream(this.realFileName); | ||
|
|
||
| dest.on('error', err => { | ||
| reject(err); | ||
| }); | ||
|
|
||
| const ext = googleMimeToExt(this.googleFile.mimeType, this.googleFile.name); | ||
| const stream = this.googleFolder.createReadStream(`${binaryFile.id}${ext ? '.' + ext : ''}`) | ||
| .on('error', err => { | ||
| reject(err); | ||
| }) | ||
| .pipe(dest); | ||
|
|
||
| stream.on('finish', () => { | ||
| resolve(); | ||
| }); | ||
| stream.on('error', err => { | ||
| reject(err); | ||
| }); | ||
| } catch (err) { | ||
| reject(err); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| async generateDrawing(drawingFile: DrawingFile) { | ||
| await new Promise<void>((resolve, reject) => { | ||
| try { | ||
| // await this.destinationDirectory.mkdir(getFileDir(targetPath)); | ||
| const dest = this.destinationDirectory.createWriteStream(this.realFileName); | ||
| const svgTransform = new SvgTransform(drawingFile.fileName); | ||
| // const svgPath = this.googleScanner.getFilePathPrefix(drawingFile.id) + '.svg'; | ||
|
|
||
| dest.on('error', err => { | ||
| reject(err); | ||
| }); | ||
|
|
||
| const stream = this.googleFolder.createReadStream(`${drawingFile.id}.svg`) | ||
| .on('error', err => { | ||
| reject(err); | ||
| }) | ||
| .pipe(svgTransform) | ||
| .pipe(dest); | ||
|
|
||
| stream.on('finish', () => { | ||
| this.localLinks.append(drawingFile.id, drawingFile.fileName, Array.from(svgTransform.links)); | ||
| resolve(); | ||
| }); | ||
| stream.on('error', err => { | ||
| reject(err); | ||
| }); | ||
| } catch (err) { | ||
| reject(err); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| async generateDocument(localFile: MdFile) { | ||
| const links = new Set<string>(); | ||
|
|
||
| const mdPath = this.googleFolder.getRealPath() + '/' + localFile.id + '.md'; | ||
|
|
||
| const input: Buffer = fs.readFileSync(mdPath); | ||
|
|
||
| const originalMarkdown = new TextDecoder().decode(input); | ||
|
|
||
| const markdownLinkRegex = /\[([^\]]+)\]\(([^)]+?)\)/g; | ||
|
|
||
| function replaceMarkdownLinks(markdown, replacerFunction) { | ||
| return markdown.replace(markdownLinkRegex, (match, linkText, url, title) => { | ||
| // Call the replacer function with the link text, URL, and title | ||
| return replacerFunction(linkText, url, title); | ||
|
Check warning Code scanning / CodeQL Superfluous trailing arguments Warning
Superfluous argument passed to
function customReplacer Error loading related location Loading |
||
| }); | ||
| } | ||
|
|
||
| function customReplacer(linkText, href) { | ||
| href = href.replaceAll('\\', ''); | ||
|
|
||
| const id = urlToFolderId(href); | ||
| const hash = getUrlHash(href); | ||
| if (id) { | ||
| href = 'gdoc:' + id + hash; | ||
| } | ||
| if (href && !href.startsWith('#') && href.indexOf(':') > -1) { | ||
| links.add(href); | ||
| } | ||
|
|
||
| return `[${linkText}](${href})`; | ||
| } | ||
|
|
||
| const markdownRewrittenLinks = replaceMarkdownLinks(originalMarkdown, customReplacer); | ||
|
|
||
| const pattern = /\*\{\{%\s+.*?\s+%\}\}\*/g; | ||
|
|
||
| const markdown = markdownRewrittenLinks.replace(pattern, (match) => { | ||
| // Remove the surrounding asterisks | ||
| return match.slice(1, -1); | ||
| }); | ||
|
|
||
| // links = Array.from(converter.links); | ||
| const frontMatter = generateDocumentFrontMatter(localFile, Array.from(links), this.userConfig.fm_without_version); | ||
| const errors = []; | ||
| this.warnings = errors.length; | ||
|
|
||
| for (const errorMsg of errors) { | ||
| this.logger.warn('Error in: ['+ this.localFile.fileName +'](' + this.localFile.fileName + ') ' + errorMsg, { | ||
| errorMdFile: this.localFile.fileName, | ||
| errorMdMsg: errorMsg | ||
| }); | ||
| } | ||
|
|
||
| await this.destinationDirectory.writeFile(this.realFileName, frontMatter + markdown); | ||
| this.localLinks.append(localFile.id, localFile.fileName, Array.from(links)); | ||
| } | ||
|
|
||
| async generate(localFile: LocalFile): Promise<void> { | ||
| try { | ||
| const verStr = this.localFile.version ? ' #' + this.localFile.version : ' '; | ||
| if (localFile.type === 'conflict') { | ||
| this.logger.info('Transforming conflict: ' + this.localFile.fileName); | ||
| const md = generateConflictMarkdown(localFile); | ||
| await this.destinationDirectory.writeFile(this.realFileName, md); | ||
| } else if (localFile.type === 'redir') { // TODO | ||
| this.logger.info('Transforming redir: ' + this.localFile.fileName); | ||
| // const redirectToFile = this.toGenerate.find(f => f.id === localFile.redirectTo); | ||
| // const redirectToFile = this.generatedScanner.getFileById(localFile.redirectTo); | ||
| // const md = generateRedirectMarkdown(localFile, redirectToFile, this.linkTranslator); | ||
| // await this.destinationDirectory.mkdir(getFileDir(targetPath)); | ||
| // await this.destinationDirectory.writeFile(targetPath, md); | ||
| // await this.generatedScanner.update(targetPath, md); | ||
| } else if (localFile.type === 'md') { | ||
| this.logger.info('Transforming markdown: ' + this.localFile.fileName + verStr); | ||
| // const googleFile = await this.googleScanner.getFileById(localFile.id); | ||
| // const downloadFile = await this.downloadFilesStorage.findFile(f => f.id === localFile.id); | ||
| if (this.googleFile) { // && downloadFile | ||
| await this.generateDocument(localFile); | ||
| } | ||
| } else if (localFile.type === 'drawing') { | ||
| this.logger.info('Transforming drawing: ' + this.localFile.fileName + verStr); | ||
| // const googleFile = await this.googleScanner.getFileById(localFile.id); | ||
| // const downloadFile = await this.downloadFilesStorage.findFile(f => f.id === localFile.id); | ||
| if (this.googleFile) { // && downloadFile | ||
| await this.generateDrawing(localFile); | ||
| } | ||
| } else if (localFile.type === 'binary') { | ||
| this.logger.info('Transforming binary: ' + this.localFile.fileName + verStr); | ||
| if (this.googleFile) { // && downloadFile | ||
| await this.generateBinary(localFile); | ||
| } | ||
| } | ||
| this.logger.info('Transformed: ' + this.localFile.fileName + verStr); | ||
| } catch (err) { | ||
| this.logger.error('Error transforming ' + localFile.fileName + ' ' + err.stack ? err.stack : err.message); | ||
| throw err; | ||
| } | ||
| } | ||
|
|
||
| } | ||
Oops, something went wrong.