diff --git a/.env.example b/.env.example index fca07307..0f08e425 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,4 @@ CLIENT_ID= CLIENT_SECRET= -RENDER_IMAGE=hugo_render -VOLUME_DATA=/home//wikigdrive -VOLUME_PREVIEW=/home//wikigdrive-preview -DOMAIN=localhost +DOMAIN=http://localhost:3000 diff --git a/.github/workflows/DevelopServerDeploy.yml b/.github/workflows/DevelopServerDeploy.yml index 81676206..af9adfa4 100644 --- a/.github/workflows/DevelopServerDeploy.yml +++ b/.github/workflows/DevelopServerDeploy.yml @@ -57,12 +57,11 @@ jobs: - name: Copy index for vite run: mkdir -p ${GITHUB_WORKSPACE}/dist/hugo && cp -rf /var/www/dev.wikigdrive.com/* ${GITHUB_WORKSPACE}/dist/hugo - - uses: whoan/docker-build-with-cache-action@v5 + - uses: docker/build-push-action@v6 with: - image_name: "wikigdrive-develop" - image_tag: "${{ github.sha }},latest" - push_image_and_stages: false - build_extra_args: "{'--build-arg': 'GIT_SHA=${{ github.sha }}'}" + tags: "wikigdrive-develop:${{ github.sha }},wikigdrive-develop:latest" + push: false + build-args: "{'--build-arg': 'GIT_SHA=${{ github.sha }}'}" - name: Stop and remove run: docker stop wikigdrive-develop ; docker rm wikigdrive-develop @@ -73,6 +72,7 @@ jobs: docker run -d --name wikigdrive-develop \ --restart unless-stopped \ --network nginx \ + --tmpfs /tmp \ -v wikiGDriveDevelop:/data \ -v /home/wikigdrive/service_account.json:/service_account.json \ -v /home/wikigdrive/env.develop:/usr/src/app/.env \ @@ -81,7 +81,6 @@ jobs: -e "GIT_SHA=${GITHUB_SHA}" \ -e "ZIPKIN_URL=https://dev.wikigdrive.com/zipkin" \ -e "ZIPKIN_SERVICE=wikigdrive-develop" \ - --link=zipkin:zipkin \ --publish 127.0.0.1:4000:3000 \ "wikigdrive-develop:${GITHUB_SHA}" wikigdrive \ --service_account /service_account.json \ diff --git a/.github/workflows/ProdServerDeploy.yml b/.github/workflows/ProdServerDeploy.yml index 2095e66b..0b20b672 100644 --- a/.github/workflows/ProdServerDeploy.yml +++ b/.github/workflows/ProdServerDeploy.yml @@ -58,12 +58,11 @@ jobs: - name: Copy index for vite run: mkdir -p ${GITHUB_WORKSPACE}/dist/hugo && cp -rf /var/www/wikigdrive.com/* ${GITHUB_WORKSPACE}/dist/hugo - - uses: whoan/docker-build-with-cache-action@v5 + - uses: docker/build-push-action@v6 with: - image_name: "wikigdrive-prod" - image_tag: "${{ github.sha }},latest" - push_image_and_stages: false - build_extra_args: "{'--build-arg': 'GIT_SHA=${{ github.sha }}'}" + tags: "wikigdrive-prod:${{ github.sha }},wikigdrive-prod:latest" + push: false + build-args: "{'--build-arg': 'GIT_SHA=${{ github.sha }}'}" - name: Stop and remove run: docker stop wikigdrive-prod ; docker rm wikigdrive-prod @@ -72,6 +71,7 @@ jobs: - name: Start run: | docker run -d --name wikigdrive-prod \ + --tmpfs /tmp \ -v wikiGDriveProd:/data \ -v /home/wikigdrive/service_account.json:/service_account.json \ -v /home/wikigdrive/env.prod:/usr/src/app/.env \ diff --git a/.github/workflows/feat-deploy.yml b/.github/workflows/feat-deploy.yml index 8bfb37e2..94e20986 100644 --- a/.github/workflows/feat-deploy.yml +++ b/.github/workflows/feat-deploy.yml @@ -32,7 +32,7 @@ jobs: build: needs: test - runs-on: wgd-dev + runs-on: ubuntu-latest steps: - name: Create pull request diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index b4cd649a..addbba36 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -2,11 +2,13 @@ name: Pull request created on: pull_request: - branches: [ master ] types: [edited, synchronize] + paths-ignore: + - 'develop/**' jobs: test: + if: github.event.pull_request.head.ref != 'develop' && contains( github.event.pull_request.labels.*.name, 'deploy-pr') runs-on: ubuntu-latest steps: @@ -31,70 +33,76 @@ jobs: run: npm run test build: + if: github.event.pull_request.head.ref != 'develop' && contains( github.event.pull_request.labels.*.name, 'deploy-pr') needs: test runs-on: wgd-dev steps: - - name: Test - run: echo "${{ github.event.number }}" - - - uses: actions/checkout@v4 - - - name: Use Node.js 20.x - uses: actions/setup-node@v3 - with: - node-version: 20 - cache: npm - - - name: Build action runner - run: docker build -t "wgd-action-runner:pr-${{ github.event.number }}" --build-arg "GIT_SHA=${{ github.sha }}" apps/wgd-action-runner - - - name: Build hugo docs - run: | - docker run \ - -v "${GITHUB_WORKSPACE}/hugo:/site" \ - -v "${GITHUB_WORKSPACE}/website:/website" \ - -v "/var/www/pr-${{ github.event.number }}.wikigdrive.com:/dist/hugo" \ - --env CONFIG_TOML="/site/config/_default/config.toml" --env BASE_URL="https://pr-${{ github.event.number }}.wikigdrive.com" \ - wgd-action-runner:pr-${{ github.event.number }} /steps/step_render_hugo - - - name: Copy index for vite - run: mkdir -p ${GITHUB_WORKSPACE}/dist/hugo && cp -rf /var/www/pr-${{ github.event.number }}.wikigdrive.com/* ${GITHUB_WORKSPACE}/dist/hugo - - - name: build - uses: whoan/docker-build-with-cache-action@v5 - with: - image_name: "wikigdrive-feature" - image_tag: "${{ github.sha }}" - push_image_and_stages: false - build_extra_args: "{'--build-arg': 'GIT_SHA=${{ github.sha }}'}" + - uses: actions/checkout@v4 + + - name: Use Node.js 20.x + uses: actions/setup-node@v3 + with: + node-version: 20 + cache: npm + + - name: Build action runner + run: docker build -t "wgd-action-runner:pr-${{ github.event.number }}" --build-arg "GIT_SHA=${{ github.sha }}" apps/wgd-action-runner + + - name: Build hugo docs + run: | + docker run \ + -v "${GITHUB_WORKSPACE}/hugo:/site" \ + -v "${GITHUB_WORKSPACE}/website:/website" \ + -v "/var/www/pr-${{ github.event.number }}.wikigdrive.com:/dist/hugo" \ + --env CONFIG_TOML="/site/config/_default/config.toml" --env BASE_URL="https://pr-${{ github.event.number }}.wikigdrive.com" \ + wgd-action-runner:pr-${{ github.event.number }} /steps/step_render_hugo + + - name: Copy index for vite + run: mkdir -p ${GITHUB_WORKSPACE}/dist/hugo && cp -rf /var/www/pr-${{ github.event.number }}.wikigdrive.com/* ${GITHUB_WORKSPACE}/dist/hugo + + - name: build + uses: docker/build-push-action@v6 + with: + tags: "wikigdrive-feature:${{ github.sha }}" + push: false + build-args: "{'--build-arg': 'GIT_SHA=${{ github.sha }}'}" + + - name: Stop and remove + run: docker stop "pr-${{ github.event.number }}" ; docker rm "pr-${{ github.event.number }}" + continue-on-error: true + + - name: "Create empty volume" + run: docker volume rm -f "pr-${{ github.event.number }}" ; docker volume create "pr-${{ github.event.number }}" + + - name: Start + run: | + docker run -d --name "pr-${{ github.event.number }}" \ + --restart unless-stopped \ + --network nginx \ + --tmpfs /tmp \ + -v "pr-${{ github.event.number }}":/data \ + -v /home/wikigdrive/service_account.json:/service_account.json \ + -v /home/wikigdrive/env.develop:/usr/src/app/.env \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "/var/www/pr-${{ github.event.number }}.wikigdrive.com:/usr/src/app/dist/hugo" \ + -e "GIT_SHA=${{ github.sha }}" \ + -e "ZIPKIN_URL=https://pr-${{ github.event.number }}.wikigdrive.com/zipkin" \ + -e "ZIPKIN_SERVICE=pr-${{ github.event.number }}" \ + -e "AUTH_DOMAIN=https://dev.wikigdrive.com" \ + -e "AUTH_INSTANCE=pr-${{ github.event.number }}" \ + -e "DOMAIN=https://pr-${{ github.event.number }}.wikigdrive.com" \ + "wikigdrive-feature:${{ github.sha }}" wikigdrive \ + --service_account /service_account.json \ + --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com \ + --workdir /data \ + server 3000 + + remove: + if: github.event.pull_request.head.ref != 'develop' && !contains( github.event.pull_request.labels.*.name, 'deploy-pr') + runs-on: wgd-dev + steps: - name: Stop and remove run: docker stop "pr-${{ github.event.number }}" ; docker rm "pr-${{ github.event.number }}" continue-on-error: true - - - name: "Create empty volume" - run: docker volume rm -f "pr-${{ github.event.number }}" ; docker volume create "pr-${{ github.event.number }}" - - - name: Start - run: | - docker run -d --name "pr-${{ github.event.number }}" \ - --restart unless-stopped \ - --network nginx \ - -v "pr-${{ github.event.number }}":/data \ - -v /home/wikigdrive/service_account.json:/service_account.json \ - -v /home/wikigdrive/env.develop:/usr/src/app/.env \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v "/var/www/pr-${{ github.event.number }}.wikigdrive.com:/usr/src/app/dist/hugo" \ - -e "GIT_SHA=${{ github.sha }}" \ - -e "ZIPKIN_URL=https://pr-${{ github.event.number }}.wikigdrive.com/zipkin" \ - -e "ZIPKIN_SERVICE=pr-${{ github.event.number }}" \ - -e "AUTH_DOMAIN=https://dev.wikigdrive.com" \ - -e "AUTH_INSTANCE=pr-${{ github.event.number }}" \ - -e "DOMAIN=https://pr-${{ github.event.number }}.wikigdrive.com" \ - --link=zipkin:zipkin \ - "wikigdrive-feature:${{ github.sha }}" wikigdrive \ - --service_account /service_account.json \ - --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com \ - --workdir /data \ - server 3000 diff --git a/apps/ui/assets/main.css b/apps/ui/assets/main.css index 0da6f1d1..1e36073d 100644 --- a/apps/ui/assets/main.css +++ b/apps/ui/assets/main.css @@ -53,7 +53,7 @@ var(--bs-table-hover-bg); .files-list__item > i { display: inline-block; - width: 0px; + width: 0; padding-right: 20px; } @@ -202,3 +202,13 @@ a[disabled] { border-left: 3px solid var(--bs-gray-500); padding: 2em 0 2em 3em; } + +.prism-editor__editor, .prism-editor-wrapper { + font-size: 16px; + font-family: monospace; +} +pre.prism-editor__editor { + overflow-x: scroll; /* Adds a horizontal scrollbar when necessary */ + white-space: pre !important; /* Ensures that text does not wrap */ + word-wrap: normal !important; /* Ensures that long words don't break */ +} diff --git a/apps/ui/src/components/BackLinks.vue b/apps/ui/src/components/BackLinks.vue index d0f9ffda..6a5490e0 100644 --- a/apps/ui/src/components/BackLinks.vue +++ b/apps/ui/src/components/BackLinks.vue @@ -1,5 +1,9 @@ @@ -37,6 +38,7 @@ diff --git a/apps/ui/src/components/UserSettings.vue b/apps/ui/src/components/UserSettings.vue index d9468356..f3fd02d5 100644 --- a/apps/ui/src/components/UserSettings.vue +++ b/apps/ui/src/components/UserSettings.vue @@ -1,5 +1,5 @@ - + @@ -117,7 +118,6 @@ import ZipkinViewer from '../components/ZipkinViewer.vue'; import ChangesViewer from '../components/ChangesViewer.vue'; import JobsViewer from '../components/JobsViewer.vue'; import UserSettings from '../components/UserSettings.vue'; -import DangerSettings from '../components/DangerSettings.vue'; import GitLog from '../components/GitLog.vue'; import GitCommit from '../components/GitCommit.vue'; import GitInfo from '../components/GitInfo.vue'; @@ -125,10 +125,12 @@ import DriveTools from '../components/DriveTools.vue'; import NavBar from '../components/NavBar.vue'; import GitSettings from '../components/GitSettings.vue'; import WorkflowsEditor from '../components/WorkflowsEditor.vue'; +import SettingsTab from '../components/SettingsTab.vue'; export default { name: 'FolderView', components: { + SettingsTab, GitSettings, NavBar, DriveTools, @@ -147,7 +149,6 @@ export default { ChangesViewer, JobsViewer, UserSettings, - DangerSettings, WorkflowsEditor, GitLog, GitCommit, diff --git a/package-lock.json b/package-lock.json index 610fbeb3..5f91293a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mieweb/wikigdrive", - "version": "2.0.0-alpha", + "version": "2.12.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@mieweb/wikigdrive", - "version": "2.0.0-alpha", + "version": "2.12.1", "license": "ISC", "workspaces": [ "apps/ui" @@ -65,11 +65,12 @@ "vite": "4.5.3", "winston": "3.8.2", "winston-transport": "4.5.0", - "ws": "8.2.3", + "ws": "8.18.0", "xml-js": "1.6.11", "xmldoc": "^1.1.2" }, "bin": { + "odt2md": "src/odt2md.sh", "wgd": "src/wikigdrive.sh", "wikigdrive": "src/wikigdrive.sh", "wikigdrivectl": "src/wikigdrivectl.sh" @@ -82,20 +83,20 @@ "@types/dockerode": "3.3.28", "@types/express": "4.17.13", "@types/lunr": "2.3.4", - "@types/mocha": "10.0.2", + "@types/mocha": "10.0.7", "@types/node": "20.10.0", "@types/passport": "1.0.9", "@types/relateurl": "0.2.29", "@types/ws": "8.5.3", "@types/xmldoc": "^1.1.5", - "@typescript-eslint/eslint-plugin": "6.12.0", - "@typescript-eslint/parser": "6.12.0", + "@typescript-eslint/eslint-plugin": "7.16.0", + "@typescript-eslint/parser": "7.16.0", "chai": "5.0.0", "diff": "5.2.0", - "eslint": "8.54.0", + "eslint": "8.57.0", "husky": "7.0.4", "jshint": "2.13.4", - "mocha": "10.2.0", + "mocha": "10.7.0", "sinon": "13.0.1" }, "engines": { @@ -552,9 +553,9 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.3.tgz", - "integrity": "sha512-yZzuIG+jnVu6hNSzFEN07e8BxF3uAzYtQb6uDkaYZLo6oYZDCq454c5kB8zxnzfCYyP4MIuyBn10L0DqwujTmA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, "dependencies": { "ajv": "^6.12.4", @@ -587,22 +588,23 @@ } }, "node_modules/@eslint/js": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.54.0.tgz", - "integrity": "sha512-ut5V+D+fOoWPgGGNj83GGjnntO39xDy6DWxO0wb7Jp3DcMX0TfIqdzHF85VTQkerdyGmuuMD9AKAo5KiNlf/AQ==", + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", + "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.13", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", - "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", + "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "deprecated": "Use @eslint/config-array instead", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^2.0.1", - "debug": "^4.1.1", + "@humanwhocodes/object-schema": "^2.0.2", + "debug": "^4.3.1", "minimatch": "^3.0.5" }, "engines": { @@ -623,9 +625,10 @@ } }, "node_modules/@humanwhocodes/object-schema": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", - "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", "dev": true }, "node_modules/@jridgewell/resolve-uri": { @@ -1322,12 +1325,6 @@ "@types/send": "*" } }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true - }, "node_modules/@types/jsonwebtoken": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz", @@ -1349,9 +1346,9 @@ "dev": true }, "node_modules/@types/mocha": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.2.tgz", - "integrity": "sha512-NaHL0+0lLNhX6d9rs+NSt97WH/gIlRHmszXbQ/8/MV/eVcFNdeJ/GYhrFuUc8K7WuPhRhTSdMkCp8VMzhUq85w==", + "version": "10.0.7", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.7.tgz", + "integrity": "sha512-GN8yJ1mNTcFcah/wKEFIJckJx9iJLoMSzWcfRRuxz/Jk+U6KQNnml+etbtxFK8lPjzOw3zp4Ha/kjSst9fsHYw==", "dev": true }, "node_modules/@types/node": { @@ -1389,12 +1386,6 @@ "integrity": "sha512-QSvevZ+IRww2ldtfv1QskYsqVVVwCKQf1XbwtcyyoRvLIQzfyPhj/C+3+PKzSDRdiyejaiLgnq//XTkleorpLg==", "dev": true }, - "node_modules/@types/semver": { - "version": "7.5.6", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz", - "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", - "dev": true - }, "node_modules/@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -1440,33 +1431,31 @@ "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.12.0.tgz", - "integrity": "sha512-XOpZ3IyJUIV1b15M7HVOpgQxPPF7lGXgsfcEIu3yDxFPaf/xZKt7s9QO/pbk7vpWQyVulpJbu4E5LwpZiQo4kA==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.16.0.tgz", + "integrity": "sha512-py1miT6iQpJcs1BiJjm54AMzeuMPBSPuKPlnT8HlfudbcS5rYeX5jajpLf3mrdRh9dA/Ec2FVUY0ifeVNDIhZw==", "dev": true, "dependencies": { - "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.12.0", - "@typescript-eslint/type-utils": "6.12.0", - "@typescript-eslint/utils": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0", - "debug": "^4.3.4", + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "7.16.0", + "@typescript-eslint/type-utils": "7.16.0", + "@typescript-eslint/utils": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0", "graphemer": "^1.4.0", - "ignore": "^5.2.4", + "ignore": "^5.3.1", "natural-compare": "^1.4.0", - "semver": "^7.5.4", - "ts-api-utils": "^1.0.1" + "ts-api-utils": "^1.3.0" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^6.0.0 || ^6.0.0-alpha", - "eslint": "^7.0.0 || ^8.0.0" + "@typescript-eslint/parser": "^7.0.0", + "eslint": "^8.56.0" }, "peerDependenciesMeta": { "typescript": { @@ -1475,26 +1464,26 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.12.0.tgz", - "integrity": "sha512-s8/jNFPKPNRmXEnNXfuo1gemBdVmpQsK1pcu+QIvuNJuhFzGrpD7WjOcvDc/+uEdfzSYpNu7U/+MmbScjoQ6vg==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.16.0.tgz", + "integrity": "sha512-ar9E+k7CU8rWi2e5ErzQiC93KKEFAXA2Kky0scAlPcxYblLt8+XZuHUZwlyfXILyQa95P6lQg+eZgh/dDs3+Vw==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "6.12.0", - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/typescript-estree": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0", + "@typescript-eslint/scope-manager": "7.16.0", + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/typescript-estree": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0", "debug": "^4.3.4" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^7.0.0 || ^8.0.0" + "eslint": "^8.56.0" }, "peerDependenciesMeta": { "typescript": { @@ -1503,16 +1492,16 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.12.0.tgz", - "integrity": "sha512-5gUvjg+XdSj8pcetdL9eXJzQNTl3RD7LgUiYTl8Aabdi8hFkaGSYnaS6BLc0BGNaDH+tVzVwmKtWvu0jLgWVbw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.16.0.tgz", + "integrity": "sha512-8gVv3kW6n01Q6TrI1cmTZ9YMFi3ucDT7i7aI5lEikk2ebk1AEjrwX8MDTdaX5D7fPXMBLvnsaa0IFTAu+jcfOw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0" + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", @@ -1520,25 +1509,25 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.12.0.tgz", - "integrity": "sha512-WWmRXxhm1X8Wlquj+MhsAG4dU/Blvf1xDgGaYCzfvStP2NwPQh6KBvCDbiOEvaE0filhranjIlK/2fSTVwtBng==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.16.0.tgz", + "integrity": "sha512-j0fuUswUjDHfqV/UdW6mLtOQQseORqfdmoBNDFOqs9rvNVR2e+cmu6zJu/Ku4SDuqiJko6YnhwcL8x45r8Oqxg==", "dev": true, "dependencies": { - "@typescript-eslint/typescript-estree": "6.12.0", - "@typescript-eslint/utils": "6.12.0", + "@typescript-eslint/typescript-estree": "7.16.0", + "@typescript-eslint/utils": "7.16.0", "debug": "^4.3.4", - "ts-api-utils": "^1.0.1" + "ts-api-utils": "^1.3.0" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^7.0.0 || ^8.0.0" + "eslint": "^8.56.0" }, "peerDependenciesMeta": { "typescript": { @@ -1547,12 +1536,12 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.12.0.tgz", - "integrity": "sha512-MA16p/+WxM5JG/F3RTpRIcuOghWO30//VEOvzubM8zuOOBYXsP+IfjoCXXiIfy2Ta8FRh9+IO9QLlaFQUU+10Q==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.16.0.tgz", + "integrity": "sha512-fecuH15Y+TzlUutvUl9Cc2XJxqdLr7+93SQIbcZfd4XRGGKoxyljK27b+kxKamjRkU7FYC6RrbSCg0ALcZn/xw==", "dev": true, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", @@ -1560,21 +1549,22 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.12.0.tgz", - "integrity": "sha512-vw9E2P9+3UUWzhgjyyVczLWxZ3GuQNT7QpnIY3o5OMeLO/c8oHljGc8ZpryBMIyympiAAaKgw9e5Hl9dCWFOYw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.16.0.tgz", + "integrity": "sha512-a5NTvk51ZndFuOLCh5OaJBELYc2O3Zqxfl3Js78VFE1zE46J2AaVuW+rEbVkQznjkmlzWsUI15BG5tQMixzZLw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0", + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", - "semver": "^7.5.4", - "ts-api-utils": "^1.0.1" + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "ts-api-utils": "^1.3.0" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", @@ -1586,42 +1576,63 @@ } } }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@typescript-eslint/utils": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.12.0.tgz", - "integrity": "sha512-LywPm8h3tGEbgfyjYnu3dauZ0U7R60m+miXgKcZS8c7QALO9uWJdvNoP+duKTk2XMWc7/Q3d/QiCuLN9X6SWyQ==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.16.0.tgz", + "integrity": "sha512-PqP4kP3hb4r7Jav+NiRCntlVzhxBNWq6ZQ+zQwII1y/G/1gdIPeYDCKr2+dH6049yJQsWZiHU6RlwvIFBXXGNA==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", - "@types/json-schema": "^7.0.12", - "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.12.0", - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/typescript-estree": "6.12.0", - "semver": "^7.5.4" + "@typescript-eslint/scope-manager": "7.16.0", + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/typescript-estree": "7.16.0" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^7.0.0 || ^8.0.0" + "eslint": "^8.56.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.12.0.tgz", - "integrity": "sha512-rg3BizTZHF1k3ipn8gfrzDXXSFKyOEB5zxYXInQ6z0hUvmQlhaZQzK+YmHmNViMA9HzW5Q9+bPPt90bU6GQwyw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.16.0.tgz", + "integrity": "sha512-rMo01uPy9C7XxG7AFsxa8zLnWXTF8N3PYclekWSrurvhwiw1eW88mrKiAYe6s53AUY57nTRz8dJsuuXdkAhzCg==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.12.0", - "eslint-visitor-keys": "^3.4.1" + "@typescript-eslint/types": "7.16.0", + "eslint-visitor-keys": "^3.4.3" }, "engines": { - "node": "^16.0.0 || >=18.0.0" + "node": "^18.18.0 || >=20.0.0" }, "funding": { "type": "opencollective", @@ -1830,9 +1841,9 @@ } }, "node_modules/ansi-colors": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", - "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", "engines": { "node": ">=6" } @@ -2086,12 +2097,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -2596,9 +2607,9 @@ "integrity": "sha512-ER7EjqVAMkRRsxNCC5YqJ9d9VQYuWdGt7aiH2qA5R5wt8ZmWaP2dLUSIK6y/kVzLMlmh1Tvu5xUf4M/wdGJ5KA==" }, "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", "dependencies": { "ms": "2.1.2" }, @@ -3010,16 +3021,16 @@ } }, "node_modules/eslint": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.54.0.tgz", - "integrity": "sha512-NY0DfAkM8BIZDVl6PgSa1ttZbx3xHgJzSNJKYcQglem6CppHyMhRIQkBVSSMaSRnLhig3jsDbEzOjwCVt4AmmA==", + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", + "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.3", - "@eslint/js": "8.54.0", - "@humanwhocodes/config-array": "^0.11.13", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.0", + "@humanwhocodes/config-array": "^0.11.14", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -3422,9 +3433,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -3661,9 +3672,9 @@ } }, "node_modules/globals": { - "version": "13.23.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz", - "integrity": "sha512-XAmF0RjlrjY23MA51q3HltdlGxUpXPvg0GioKiD9X6HD28iMjo2dKC8Vqwm7lne4GNr78+RHTfliktR6ZH09wA==", + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -3869,9 +3880,9 @@ ] }, "node_modules/ignore": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", - "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", "dev": true, "engines": { "node": ">= 4" @@ -4465,17 +4476,6 @@ "get-func-name": "^2.0.1" } }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/lunr": { "version": "2.3.9", "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", @@ -4576,12 +4576,12 @@ } }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", + "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", "dev": true, "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -4665,32 +4665,31 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/mocha": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", - "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", - "dev": true, - "dependencies": { - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.4", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "5.0.1", - "ms": "2.1.3", - "nanoid": "3.3.3", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "workerpool": "6.2.1", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.7.0.tgz", + "integrity": "sha512-v8/rBWr2VO5YkspYINnvu81inSz2y3ODJrhO175/Exzor1RcEZZkizgE2A+w/CAXXoESS8Kys5E62dOHGHzULA==", + "dev": true, + "dependencies": { + "ansi-colors": "^4.1.3", + "browser-stdout": "^1.3.1", + "chokidar": "^3.5.3", + "debug": "^4.3.5", + "diff": "^5.2.0", + "escape-string-regexp": "^4.0.0", + "find-up": "^5.0.0", + "glob": "^8.1.0", + "he": "^1.2.0", + "js-yaml": "^4.1.0", + "log-symbols": "^4.1.0", + "minimatch": "^5.1.6", + "ms": "^2.1.3", + "serialize-javascript": "^6.0.2", + "strip-json-comments": "^3.1.1", + "supports-color": "^8.1.1", + "workerpool": "^6.5.1", + "yargs": "^16.2.0", + "yargs-parser": "^20.2.9", + "yargs-unparser": "^2.0.0" }, "bin": { "_mocha": "bin/_mocha", @@ -4698,10 +4697,6 @@ }, "engines": { "node": ">= 14.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/mochajs" } }, "node_modules/mocha/node_modules/brace-expansion": { @@ -4713,19 +4708,30 @@ "balanced-match": "^1.0.0" } }, - "node_modules/mocha/node_modules/diff": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", - "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "node_modules/mocha/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "deprecated": "Glob versions prior to v9 are no longer supported", "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, "engines": { - "node": ">=0.3.1" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/mocha/node_modules/minimatch": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", - "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dev": true, "dependencies": { "brace-expansion": "^2.0.1" @@ -4779,18 +4785,6 @@ "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==", "optional": true }, - "node_modules/nanoid": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", - "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", - "dev": true, - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -5536,12 +5530,9 @@ "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", "bin": { "semver": "bin/semver.js" }, @@ -5591,9 +5582,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/serialize-javascript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", - "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", "dev": true, "dependencies": { "randombytes": "^2.1.0" @@ -6102,12 +6093,12 @@ "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, "node_modules/ts-api-utils": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.3.tgz", - "integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", + "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", "dev": true, "engines": { - "node": ">=16.13.0" + "node": ">=16" }, "peerDependencies": { "typescript": ">=4.2.0" @@ -6488,9 +6479,9 @@ } }, "node_modules/workerpool": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", - "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", + "integrity": "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==", "dev": true }, "node_modules/wrap-ansi": { @@ -6515,15 +6506,15 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "node_modules/ws": { - "version": "8.2.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", - "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "engines": { "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" + "utf-8-validate": ">=5.0.2" }, "peerDependenciesMeta": { "bufferutil": { @@ -6562,11 +6553,6 @@ "node": ">=10" } }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -6586,9 +6572,9 @@ } }, "node_modules/yargs-parser": { - "version": "20.2.4", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", - "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, "engines": { "node": ">=10" @@ -6834,9 +6820,9 @@ "dev": true }, "@eslint/eslintrc": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.3.tgz", - "integrity": "sha512-yZzuIG+jnVu6hNSzFEN07e8BxF3uAzYtQb6uDkaYZLo6oYZDCq454c5kB8zxnzfCYyP4MIuyBn10L0DqwujTmA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, "requires": { "ajv": "^6.12.4", @@ -6862,19 +6848,19 @@ } }, "@eslint/js": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.54.0.tgz", - "integrity": "sha512-ut5V+D+fOoWPgGGNj83GGjnntO39xDy6DWxO0wb7Jp3DcMX0TfIqdzHF85VTQkerdyGmuuMD9AKAo5KiNlf/AQ==", + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", + "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", "dev": true }, "@humanwhocodes/config-array": { - "version": "0.11.13", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", - "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", + "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", "dev": true, "requires": { - "@humanwhocodes/object-schema": "^2.0.1", - "debug": "^4.1.1", + "@humanwhocodes/object-schema": "^2.0.2", + "debug": "^4.3.1", "minimatch": "^3.0.5" } }, @@ -6885,9 +6871,9 @@ "dev": true }, "@humanwhocodes/object-schema": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", - "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", "dev": true }, "@jridgewell/resolve-uri": { @@ -7379,12 +7365,6 @@ "@types/send": "*" } }, - "@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true - }, "@types/jsonwebtoken": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz", @@ -7406,9 +7386,9 @@ "dev": true }, "@types/mocha": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.2.tgz", - "integrity": "sha512-NaHL0+0lLNhX6d9rs+NSt97WH/gIlRHmszXbQ/8/MV/eVcFNdeJ/GYhrFuUc8K7WuPhRhTSdMkCp8VMzhUq85w==", + "version": "10.0.7", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.7.tgz", + "integrity": "sha512-GN8yJ1mNTcFcah/wKEFIJckJx9iJLoMSzWcfRRuxz/Jk+U6KQNnml+etbtxFK8lPjzOw3zp4Ha/kjSst9fsHYw==", "dev": true }, "@types/node": { @@ -7446,12 +7426,6 @@ "integrity": "sha512-QSvevZ+IRww2ldtfv1QskYsqVVVwCKQf1XbwtcyyoRvLIQzfyPhj/C+3+PKzSDRdiyejaiLgnq//XTkleorpLg==", "dev": true }, - "@types/semver": { - "version": "7.5.6", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz", - "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", - "dev": true - }, "@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -7497,103 +7471,119 @@ "dev": true }, "@typescript-eslint/eslint-plugin": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.12.0.tgz", - "integrity": "sha512-XOpZ3IyJUIV1b15M7HVOpgQxPPF7lGXgsfcEIu3yDxFPaf/xZKt7s9QO/pbk7vpWQyVulpJbu4E5LwpZiQo4kA==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.16.0.tgz", + "integrity": "sha512-py1miT6iQpJcs1BiJjm54AMzeuMPBSPuKPlnT8HlfudbcS5rYeX5jajpLf3mrdRh9dA/Ec2FVUY0ifeVNDIhZw==", "dev": true, "requires": { - "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.12.0", - "@typescript-eslint/type-utils": "6.12.0", - "@typescript-eslint/utils": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0", - "debug": "^4.3.4", + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "7.16.0", + "@typescript-eslint/type-utils": "7.16.0", + "@typescript-eslint/utils": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0", "graphemer": "^1.4.0", - "ignore": "^5.2.4", + "ignore": "^5.3.1", "natural-compare": "^1.4.0", - "semver": "^7.5.4", - "ts-api-utils": "^1.0.1" + "ts-api-utils": "^1.3.0" } }, "@typescript-eslint/parser": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.12.0.tgz", - "integrity": "sha512-s8/jNFPKPNRmXEnNXfuo1gemBdVmpQsK1pcu+QIvuNJuhFzGrpD7WjOcvDc/+uEdfzSYpNu7U/+MmbScjoQ6vg==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.16.0.tgz", + "integrity": "sha512-ar9E+k7CU8rWi2e5ErzQiC93KKEFAXA2Kky0scAlPcxYblLt8+XZuHUZwlyfXILyQa95P6lQg+eZgh/dDs3+Vw==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "6.12.0", - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/typescript-estree": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0", + "@typescript-eslint/scope-manager": "7.16.0", + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/typescript-estree": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0", "debug": "^4.3.4" } }, "@typescript-eslint/scope-manager": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.12.0.tgz", - "integrity": "sha512-5gUvjg+XdSj8pcetdL9eXJzQNTl3RD7LgUiYTl8Aabdi8hFkaGSYnaS6BLc0BGNaDH+tVzVwmKtWvu0jLgWVbw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.16.0.tgz", + "integrity": "sha512-8gVv3kW6n01Q6TrI1cmTZ9YMFi3ucDT7i7aI5lEikk2ebk1AEjrwX8MDTdaX5D7fPXMBLvnsaa0IFTAu+jcfOw==", "dev": true, "requires": { - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0" + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0" } }, "@typescript-eslint/type-utils": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.12.0.tgz", - "integrity": "sha512-WWmRXxhm1X8Wlquj+MhsAG4dU/Blvf1xDgGaYCzfvStP2NwPQh6KBvCDbiOEvaE0filhranjIlK/2fSTVwtBng==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.16.0.tgz", + "integrity": "sha512-j0fuUswUjDHfqV/UdW6mLtOQQseORqfdmoBNDFOqs9rvNVR2e+cmu6zJu/Ku4SDuqiJko6YnhwcL8x45r8Oqxg==", "dev": true, "requires": { - "@typescript-eslint/typescript-estree": "6.12.0", - "@typescript-eslint/utils": "6.12.0", + "@typescript-eslint/typescript-estree": "7.16.0", + "@typescript-eslint/utils": "7.16.0", "debug": "^4.3.4", - "ts-api-utils": "^1.0.1" + "ts-api-utils": "^1.3.0" } }, "@typescript-eslint/types": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.12.0.tgz", - "integrity": "sha512-MA16p/+WxM5JG/F3RTpRIcuOghWO30//VEOvzubM8zuOOBYXsP+IfjoCXXiIfy2Ta8FRh9+IO9QLlaFQUU+10Q==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.16.0.tgz", + "integrity": "sha512-fecuH15Y+TzlUutvUl9Cc2XJxqdLr7+93SQIbcZfd4XRGGKoxyljK27b+kxKamjRkU7FYC6RrbSCg0ALcZn/xw==", "dev": true }, "@typescript-eslint/typescript-estree": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.12.0.tgz", - "integrity": "sha512-vw9E2P9+3UUWzhgjyyVczLWxZ3GuQNT7QpnIY3o5OMeLO/c8oHljGc8ZpryBMIyympiAAaKgw9e5Hl9dCWFOYw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.16.0.tgz", + "integrity": "sha512-a5NTvk51ZndFuOLCh5OaJBELYc2O3Zqxfl3Js78VFE1zE46J2AaVuW+rEbVkQznjkmlzWsUI15BG5tQMixzZLw==", "dev": true, "requires": { - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/visitor-keys": "6.12.0", + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/visitor-keys": "7.16.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", - "semver": "^7.5.4", - "ts-api-utils": "^1.0.1" + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "ts-api-utils": "^1.3.0" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + }, + "minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + } + } } }, "@typescript-eslint/utils": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.12.0.tgz", - "integrity": "sha512-LywPm8h3tGEbgfyjYnu3dauZ0U7R60m+miXgKcZS8c7QALO9uWJdvNoP+duKTk2XMWc7/Q3d/QiCuLN9X6SWyQ==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.16.0.tgz", + "integrity": "sha512-PqP4kP3hb4r7Jav+NiRCntlVzhxBNWq6ZQ+zQwII1y/G/1gdIPeYDCKr2+dH6049yJQsWZiHU6RlwvIFBXXGNA==", "dev": true, "requires": { "@eslint-community/eslint-utils": "^4.4.0", - "@types/json-schema": "^7.0.12", - "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.12.0", - "@typescript-eslint/types": "6.12.0", - "@typescript-eslint/typescript-estree": "6.12.0", - "semver": "^7.5.4" + "@typescript-eslint/scope-manager": "7.16.0", + "@typescript-eslint/types": "7.16.0", + "@typescript-eslint/typescript-estree": "7.16.0" } }, "@typescript-eslint/visitor-keys": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.12.0.tgz", - "integrity": "sha512-rg3BizTZHF1k3ipn8gfrzDXXSFKyOEB5zxYXInQ6z0hUvmQlhaZQzK+YmHmNViMA9HzW5Q9+bPPt90bU6GQwyw==", + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.16.0.tgz", + "integrity": "sha512-rMo01uPy9C7XxG7AFsxa8zLnWXTF8N3PYclekWSrurvhwiw1eW88mrKiAYe6s53AUY57nTRz8dJsuuXdkAhzCg==", "dev": true, "requires": { - "@typescript-eslint/types": "6.12.0", - "eslint-visitor-keys": "^3.4.1" + "@typescript-eslint/types": "7.16.0", + "eslint-visitor-keys": "^3.4.3" } }, "@ungap/structured-clone": { @@ -7765,9 +7755,9 @@ } }, "ansi-colors": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", - "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==" + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==" }, "ansi-escapes": { "version": "4.3.1", @@ -7971,12 +7961,12 @@ } }, "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "requires": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" } }, "browser-stdout": { @@ -8361,9 +8351,9 @@ "integrity": "sha512-ER7EjqVAMkRRsxNCC5YqJ9d9VQYuWdGt7aiH2qA5R5wt8ZmWaP2dLUSIK6y/kVzLMlmh1Tvu5xUf4M/wdGJ5KA==" }, "debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", "requires": { "ms": "2.1.2" } @@ -8661,16 +8651,16 @@ "dev": true }, "eslint": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.54.0.tgz", - "integrity": "sha512-NY0DfAkM8BIZDVl6PgSa1ttZbx3xHgJzSNJKYcQglem6CppHyMhRIQkBVSSMaSRnLhig3jsDbEzOjwCVt4AmmA==", + "version": "8.57.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", + "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", "dev": true, "requires": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.3", - "@eslint/js": "8.54.0", - "@humanwhocodes/config-array": "^0.11.13", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.0", + "@humanwhocodes/config-array": "^0.11.14", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -8984,9 +8974,9 @@ } }, "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "requires": { "to-regex-range": "^5.0.1" @@ -9159,9 +9149,9 @@ } }, "globals": { - "version": "13.23.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz", - "integrity": "sha512-XAmF0RjlrjY23MA51q3HltdlGxUpXPvg0GioKiD9X6HD28iMjo2dKC8Vqwm7lne4GNr78+RHTfliktR6ZH09wA==", + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -9289,9 +9279,9 @@ "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, "ignore": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", - "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", "dev": true }, "immediate": { @@ -9769,14 +9759,6 @@ "get-func-name": "^2.0.1" } }, - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, "lunr": { "version": "2.3.9", "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", @@ -9856,12 +9838,12 @@ "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" }, "micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", + "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", "dev": true, "requires": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" } }, @@ -9918,32 +9900,31 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "mocha": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", - "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", - "dev": true, - "requires": { - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.4", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "5.0.1", - "ms": "2.1.3", - "nanoid": "3.3.3", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "workerpool": "6.2.1", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.7.0.tgz", + "integrity": "sha512-v8/rBWr2VO5YkspYINnvu81inSz2y3ODJrhO175/Exzor1RcEZZkizgE2A+w/CAXXoESS8Kys5E62dOHGHzULA==", + "dev": true, + "requires": { + "ansi-colors": "^4.1.3", + "browser-stdout": "^1.3.1", + "chokidar": "^3.5.3", + "debug": "^4.3.5", + "diff": "^5.2.0", + "escape-string-regexp": "^4.0.0", + "find-up": "^5.0.0", + "glob": "^8.1.0", + "he": "^1.2.0", + "js-yaml": "^4.1.0", + "log-symbols": "^4.1.0", + "minimatch": "^5.1.6", + "ms": "^2.1.3", + "serialize-javascript": "^6.0.2", + "strip-json-comments": "^3.1.1", + "supports-color": "^8.1.1", + "workerpool": "^6.5.1", + "yargs": "^16.2.0", + "yargs-parser": "^20.2.9", + "yargs-unparser": "^2.0.0" }, "dependencies": { "brace-expansion": { @@ -9955,16 +9936,23 @@ "balanced-match": "^1.0.0" } }, - "diff": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", - "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", - "dev": true + "glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + } }, "minimatch": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", - "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dev": true, "requires": { "brace-expansion": "^2.0.1" @@ -10008,12 +9996,6 @@ "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==", "optional": true }, - "nanoid": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", - "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", - "dev": true - }, "napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -10537,12 +10519,9 @@ "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" }, "semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "requires": { - "lru-cache": "^6.0.0" - } + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==" }, "send": { "version": "0.18.0", @@ -10587,9 +10566,9 @@ } }, "serialize-javascript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", - "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", "dev": true, "requires": { "randombytes": "^2.1.0" @@ -10979,9 +10958,9 @@ "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, "ts-api-utils": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.3.tgz", - "integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", + "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", "dev": true, "requires": {} }, @@ -11223,9 +11202,9 @@ } }, "workerpool": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", - "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", + "integrity": "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==", "dev": true }, "wrap-ansi": { @@ -11244,9 +11223,9 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { - "version": "8.2.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", - "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "requires": {} }, "xml-js": { @@ -11271,11 +11250,6 @@ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "dev": true }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -11292,9 +11266,9 @@ } }, "yargs-parser": { - "version": "20.2.4", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", - "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true }, "yargs-unparser": { diff --git a/package.json b/package.json index bf84fdc9..cb0ae0f5 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,7 @@ "vite": "4.5.3", "winston": "3.8.2", "winston-transport": "4.5.0", - "ws": "8.2.3", + "ws": "8.18.0", "xml-js": "1.6.11", "xmldoc": "^1.1.2" }, @@ -130,20 +130,20 @@ "@types/dockerode": "3.3.28", "@types/express": "4.17.13", "@types/lunr": "2.3.4", - "@types/mocha": "10.0.2", + "@types/mocha": "10.0.7", "@types/node": "20.10.0", "@types/passport": "1.0.9", "@types/relateurl": "0.2.29", "@types/ws": "8.5.3", "@types/xmldoc": "^1.1.5", - "@typescript-eslint/eslint-plugin": "6.12.0", - "@typescript-eslint/parser": "6.12.0", + "@typescript-eslint/eslint-plugin": "7.16.0", + "@typescript-eslint/parser": "7.16.0", "chai": "5.0.0", "diff": "5.2.0", - "eslint": "8.54.0", + "eslint": "8.57.0", "husky": "7.0.4", "jshint": "2.13.4", - "mocha": "10.2.0", + "mocha": "10.7.0", "sinon": "13.0.1" }, "husky": { diff --git a/src/ContainerEngine.ts b/src/ContainerEngine.ts index 3f7ae99b..4cdfdcb1 100644 --- a/src/ContainerEngine.ts +++ b/src/ContainerEngine.ts @@ -1,9 +1,10 @@ import casual from 'casual'; import winston from 'winston'; -import {FileContentService} from './utils/FileContentService'; import {QueueObject} from 'async'; -import {QueueTask} from './containers/google_folder/QueueTask'; -import {FileId} from './model/model'; + +import {FileContentService} from './utils/FileContentService.ts'; +import {QueueTask} from './containers/google_folder/QueueTask.ts'; +import {FileId} from './model/model.ts'; export interface ContainerConfig { name?: string; diff --git a/src/LinkTranslator.ts b/src/LinkTranslator.ts index 310ce343..58029ee3 100644 --- a/src/LinkTranslator.ts +++ b/src/LinkTranslator.ts @@ -40,7 +40,7 @@ export function convertExtension(localPath: string, mode?: LinkMode) { return dirName + parts.join('.'); } -export function convertToRelativeMarkDownPath(localPath, basePath) { +export function convertToRelativeMarkDownPath(localPath: string, basePath: string) { if (localPath.startsWith('https://')) return localPath; if (localPath.startsWith('http://')) return localPath; if (basePath === localPath) return '.'; @@ -51,7 +51,7 @@ export function convertToRelativeMarkDownPath(localPath, basePath) { }))); } -export function convertToRelativeSvgPath(localPath, basePath) { +export function convertToRelativeSvgPath(localPath: string, basePath: string) { if (localPath.startsWith('https://')) return localPath; if (localPath.startsWith('http://')) return localPath; if (basePath === localPath) return '.'; diff --git a/src/containers/action/ActionRunnerContainer.ts b/src/containers/action/ActionRunnerContainer.ts index b3f0ef57..97587d96 100644 --- a/src/containers/action/ActionRunnerContainer.ts +++ b/src/containers/action/ActionRunnerContainer.ts @@ -183,27 +183,13 @@ export class ActionRunnerContainer extends Container { result = await docker.run(process.env.ACTION_IMAGE, [`/steps/step_${step.uses}`], writable, { HostConfig: { + Binds: [ // Unlike Mounts those are created if not existing in the host + `${process.env.VOLUME_PREVIEW}/${driveId}/${themeId}:/site/public:rw`, + `${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro`, + `${process.env.VOLUME_DATA}${contentDir}:/site/content:ro`, + `${process.env.VOLUME_DATA}/${driveId}/tmp_dir:/site/tmp_dir:rw`, + ], Mounts: [ - { - Source: `${process.env.VOLUME_DATA}/${driveId}_transform`, - Target: '/repo', - Type: 'bind' - }, - { - Source: `${process.env.VOLUME_DATA}${contentDir}`, - Target: '/site/content', - Type: 'bind' - }, - { - Source: `${process.env.VOLUME_PREVIEW}/${driveId}/${themeId}`, - Target: '/site/public', - Type: 'bind' - }, - { - Source: `${process.env.VOLUME_DATA}/${driveId}/tmp_dir`, - Target: '/site/tmp_dir', - Type: 'bind' - }, { Source: '', Target: '/site/resources', @@ -236,39 +222,21 @@ export class ActionRunnerContainer extends Container { -v "${process.env.VOLUME_DATA}${contentDir}:/site/content" \\ -v "${process.env.VOLUME_PREVIEW}/${driveId}/_manual:/site/public" \\ -v "${process.env.VOLUME_DATA}/${driveId}/tmp_dir:/site/tmp_dir" \\ - --mount type=tmpfs,destination=/site/resources" \\ + --mount "type=tmpfs,destination=/site/resources" \\ ${Object.keys(env).map(key => `--env ${key}="${env[key]}"`).join(' ')} \\ ${process.env.ACTION_IMAGE} /steps/step_${step.uses} `); result = await docker.run(process.env.ACTION_IMAGE, [`/steps/step_${step.uses}`], writable, { HostConfig: { + Binds: [ // Unlike Mounts those are created if not existing in the host + `${process.env.VOLUME_PREVIEW}/${driveId}/_manual:/site/public:rw`, + `${process.env.VOLUME_DATA}/${driveId}_transform:/repo:ro`, + `${process.env.VOLUME_DATA}/${driveIdTransform}:/site:rw`, + `${process.env.VOLUME_DATA}${contentDir}:/site/content:rw`, + `${process.env.VOLUME_DATA}/${driveId}/tmp_dir:/site/tmp_dir:rw`, + ], Mounts: [ - { - Source: `${process.env.VOLUME_DATA}/${driveId}_transform`, - Target: '/repo', - Type: 'bind' - }, - { - Source: `${process.env.VOLUME_DATA}/${driveIdTransform}`, - Target: '/site', - Type: 'bind', - }, - { - Source: `${process.env.VOLUME_DATA}${contentDir}`, - Target: '/site/content', - Type: 'bind' - }, - { - Source: `${process.env.VOLUME_PREVIEW}/${driveId}/_manual`, - Target: '/site/public', - Type: 'bind' - }, - { - Source: `${process.env.VOLUME_DATA}/${driveId}/tmp_dir`, - Target: '/site/tmp_dir', - Type: 'bind' - }, { Source: '', Target: '/site/resources', @@ -368,7 +336,7 @@ export class ActionRunnerContainer extends Container { private payloadToEnv() { const additionalEnv = {}; - additionalEnv['REMOTE_BRANCH'] = this.userConfigService.config?.remote_branch || 'master'; + additionalEnv['REMOTE_BRANCH'] = this.userConfigService.config?.remote_branch || 'main'; if (this.params.payload && this.params.payload.startsWith('{')) { try { diff --git a/src/containers/changes/WatchChangesContainer.ts b/src/containers/changes/WatchChangesContainer.ts index 388af46a..d48630a6 100644 --- a/src/containers/changes/WatchChangesContainer.ts +++ b/src/containers/changes/WatchChangesContainer.ts @@ -1,16 +1,17 @@ import winston from 'winston'; -import {Container, ContainerEngine} from '../../ContainerEngine'; -import {GoogleApiContainer} from '../google_api/GoogleApiContainer'; -import {GoogleDriveService} from '../../google/GoogleDriveService'; import {fileURLToPath} from 'url'; -import {FolderRegistryContainer} from '../folder_registry/FolderRegistryContainer'; -import {GoogleFile} from '../../model/GoogleFile'; -import {GoogleTreeProcessor} from '../google_folder/GoogleTreeProcessor'; -import {initJob, JobManagerContainer} from '../job/JobManagerContainer'; -import {UserConfigService} from '../google_folder/UserConfigService'; -import {type FileId} from '../../model/model'; -import {TelemetryClass, TelemetryMethod, TelemetryMethodDisable} from '../../telemetry'; -import {HasAccessToken} from '../../google/AuthClient'; + +import {Container, ContainerEngine} from '../../ContainerEngine.ts'; +import {GoogleApiContainer} from '../google_api/GoogleApiContainer.ts'; +import {GoogleDriveService} from '../../google/GoogleDriveService.ts'; +import {FolderRegistryContainer} from '../folder_registry/FolderRegistryContainer.ts'; +import {GoogleFile} from '../../model/GoogleFile.ts'; +import {GoogleTreeProcessor} from '../google_folder/GoogleTreeProcessor.ts'; +import {initJob, JobManagerContainer} from '../job/JobManagerContainer.ts'; +import {UserConfigService} from '../google_folder/UserConfigService.ts'; +import {type FileId} from '../../model/model.ts'; +import {TelemetryClass, TelemetryMethod, TelemetryMethodDisable} from '../../telemetry.ts'; +import {HasAccessToken} from '../../google/AuthClient.ts'; const __filename = fileURLToPath(import.meta.url); @@ -20,7 +21,8 @@ export class WatchChangesContainer extends Container { private auth: HasAccessToken; private googleDriveService: GoogleDriveService; private lastToken: { [driveId: string]: string } = {}; - private intervals: { [driveId: string]: NodeJS.Timer } = {}; + private intervals: { [driveId: string]: NodeJS.Timeout } = {}; + private working: { [driveId: string]: boolean } = {}; @TelemetryMethodDisable() async init(engine: ContainerEngine): Promise { @@ -111,10 +113,21 @@ export class WatchChangesContainer extends Container { if (this.intervals[driveId]) { return; } + + this.logger.info('Starting watching: ' + driveId); + this.intervals[driveId] = setInterval(async () => { + if (!this.auth) { + return; + } + if (this.working[driveId]) { + return; + } + this.working[driveId] = true; try { if (!this.lastToken[driveId]) { this.lastToken[driveId] = await this.googleDriveService.getStartTrackToken(this.auth, driveId); + // await this.googleDriveService.setupWatchChannel(this.auth, this.lastToken[driveId], driveId); return; } @@ -125,6 +138,8 @@ export class WatchChangesContainer extends Container { const folderRegistryContainer = this.engine.getContainer('folder_registry'); await folderRegistryContainer.refreshDrives(); } + } finally { + delete this.working[driveId]; } }, 3000); } diff --git a/src/containers/folder_registry/FolderRegistryContainer.ts b/src/containers/folder_registry/FolderRegistryContainer.ts index 0354bba2..1412327d 100644 --- a/src/containers/folder_registry/FolderRegistryContainer.ts +++ b/src/containers/folder_registry/FolderRegistryContainer.ts @@ -62,21 +62,29 @@ export class FolderRegistryContainer extends Container { const oldDrives = Object.values(await this.getFolders()); const apiContainer: GoogleApiContainer = this.engine.getContainer('google_api'); - const drives = await apiContainer.listDrives(); - - for (const newDrive of drives) { - if (!oldDrives.find(oldDrive => oldDrive.id === newDrive.id)) { - try { - await this.registerFolder(newDrive.id); - } catch (err) { - this.logger.error(err.stack ? err.stack : err.message); + try { + const drives = await apiContainer.listDrives(); + + for (const newDrive of drives) { + if (!oldDrives.find(oldDrive => oldDrive.id === newDrive.id)) { + try { + await this.registerFolder(newDrive.id); + } catch (err) { + this.logger.error(err.stack ? err.stack : err.message); + } } } - } - for (const oldDrive of oldDrives) { - if (!drives.find(newDrive => newDrive.id === oldDrive.id)) { - await this.unregisterFolder(oldDrive.id); + for (const oldDrive of oldDrives) { + if (!drives.find(newDrive => newDrive.id === oldDrive.id)) { + await this.unregisterFolder(oldDrive.id); + } + } + } catch (err) { + if (401 === err?.status) { + this.logger.warn('Not authenticated to Google API. Skipping drives refresh.'); + return; } + throw err; } } diff --git a/src/containers/google_api/GoogleApiContainer.ts b/src/containers/google_api/GoogleApiContainer.ts index 1718606d..3e6b0f2e 100644 --- a/src/containers/google_api/GoogleApiContainer.ts +++ b/src/containers/google_api/GoogleApiContainer.ts @@ -10,6 +10,7 @@ import {GoogleFile} from '../../model/GoogleFile.ts'; import {GoogleAuth, HasAccessToken, UserAuthClient, ServiceAuthClient, getCliCode} from '../../google/AuthClient.ts'; import {fileURLToPath} from 'url'; +import {AuthError} from '../server/auth.ts'; const __filename = fileURLToPath(import.meta.url); @@ -70,23 +71,39 @@ export class GoogleApiContainer extends Container { } async listDrives(): Promise { + if (!this.auth) { + throw new AuthError('Not authenticated', 401); + } + const googleDriveService = new GoogleDriveService(this.logger, this.quotaLimiter); const accessToken = await this.auth.getAccessToken(); return await googleDriveService.listDrives(accessToken); } async getDrive(driveId: FileId): Promise { + if (!this.auth) { + throw new AuthError('Not authenticated', 401); + } + const googleDriveService = new GoogleDriveService(this.logger, this.quotaLimiter); const accessToken = await this.auth.getAccessToken(); return await googleDriveService.getDrive(accessToken, driveId); } async getFolder(fileId: FileId): Promise { + if (!this.auth) { + throw new AuthError('Not authenticated', 401); + } + const googleDriveService = new GoogleDriveService(this.logger, this.quotaLimiter); return await googleDriveService.getFile(this.auth, fileId); } async shareDrive(driveId: string, shareEmail: string): Promise { + if (!this.auth) { + throw new AuthError('Not authenticated', 401); + } + const googleDriveService = new GoogleDriveService(this.logger, this.quotaLimiter); const accessToken = await this.auth.getAccessToken(); return await googleDriveService.shareDrive(accessToken, driveId, shareEmail); diff --git a/src/containers/google_folder/GoogleFolderContainer.ts b/src/containers/google_folder/GoogleFolderContainer.ts index 15f80a07..c72dd6e7 100644 --- a/src/containers/google_folder/GoogleFolderContainer.ts +++ b/src/containers/google_folder/GoogleFolderContainer.ts @@ -11,6 +11,7 @@ import {DateISO, FileId} from '../../model/model.ts'; import {FolderRegistryContainer} from '../folder_registry/FolderRegistryContainer.ts'; import {GoogleTreeProcessor} from './GoogleTreeProcessor.ts'; import {HasAccessToken} from '../../google/AuthClient.ts'; +import {UserConfigService} from './UserConfigService.ts'; const __filename = fileURLToPath(import.meta.url); @@ -81,6 +82,13 @@ export class GoogleFolderContainer extends Container { this.forceDownloadFilters, {filterFilesIds: this.filterFilesIds, filterFoldersIds} ); + + const folderId = this.params.name; + const googleFileSystem = await this.filesService.getSubFileService(folderId, '/'); + const userConfigService = new UserConfigService(googleFileSystem); + await userConfigService.load(); + + taskFetchFolder.setUseGoogleMarkdowns(userConfigService.config.use_google_markdowns); downloader.addTask(taskFetchFolder); } } diff --git a/src/containers/google_folder/TaskFetchAsset.ts b/src/containers/google_folder/TaskFetchAsset.ts index a1659bd0..186b5742 100644 --- a/src/containers/google_folder/TaskFetchAsset.ts +++ b/src/containers/google_folder/TaskFetchAsset.ts @@ -1,10 +1,10 @@ -import {QueueTask} from './QueueTask'; import winston from 'winston'; -import {GoogleDriveService} from '../../google/GoogleDriveService'; -import {FileContentService} from '../../utils/FileContentService'; -import {GoogleFile} from '../../model/GoogleFile'; -import {googleMimeToExt} from '../transform/TaskLocalFileTransform'; -import {HasAccessToken} from '../../google/AuthClient'; +import {QueueTask} from './QueueTask.ts'; +import {GoogleDriveService} from '../../google/GoogleDriveService.ts'; +import {FileContentService} from '../../utils/FileContentService.ts'; +import {GoogleFile} from '../../model/GoogleFile.ts'; +import {googleMimeToExt} from '../transform/TaskLocalFileTransform.ts'; +import {HasAccessToken} from '../../google/AuthClient.ts'; export class TaskFetchAsset extends QueueTask { diff --git a/src/containers/google_folder/TaskFetchBinary.ts b/src/containers/google_folder/TaskFetchBinary.ts index 30cd0a76..94426ba0 100644 --- a/src/containers/google_folder/TaskFetchBinary.ts +++ b/src/containers/google_folder/TaskFetchBinary.ts @@ -1,9 +1,9 @@ -import {QueueTask} from './QueueTask'; import winston from 'winston'; -import {GoogleDriveService} from '../../google/GoogleDriveService'; -import {FileContentService} from '../../utils/FileContentService'; -import {SimpleFile} from '../../model/GoogleFile'; -import {HasAccessToken} from '../../google/AuthClient'; +import {QueueTask} from './QueueTask.ts'; +import {GoogleDriveService} from '../../google/GoogleDriveService.ts'; +import {FileContentService} from '../../utils/FileContentService.ts'; +import {SimpleFile} from '../../model/GoogleFile.ts'; +import {HasAccessToken} from '../../google/AuthClient.ts'; export class TaskFetchBinary extends QueueTask { diff --git a/src/containers/google_folder/TaskFetchDiagram.ts b/src/containers/google_folder/TaskFetchDiagram.ts index 736c0211..3b2c98e3 100644 --- a/src/containers/google_folder/TaskFetchDiagram.ts +++ b/src/containers/google_folder/TaskFetchDiagram.ts @@ -1,9 +1,9 @@ -import {QueueTask} from './QueueTask'; import winston from 'winston'; -import {GoogleDriveService} from '../../google/GoogleDriveService'; -import {FileContentService} from '../../utils/FileContentService'; -import {SimpleFile} from '../../model/GoogleFile'; -import {HasAccessToken} from '../../google/AuthClient'; +import {QueueTask} from './QueueTask.ts'; +import {GoogleDriveService} from '../../google/GoogleDriveService.ts'; +import {FileContentService} from '../../utils/FileContentService.ts'; +import {SimpleFile} from '../../model/GoogleFile.ts'; +import {HasAccessToken} from '../../google/AuthClient.ts'; export class TaskFetchDiagram extends QueueTask { diff --git a/src/containers/google_folder/TaskFetchDocument.ts b/src/containers/google_folder/TaskFetchDocument.ts index 8bb3040c..5dae2af3 100644 --- a/src/containers/google_folder/TaskFetchDocument.ts +++ b/src/containers/google_folder/TaskFetchDocument.ts @@ -1,10 +1,10 @@ -import {QueueTask} from './QueueTask'; import winston from 'winston'; -import {GoogleDriveService} from '../../google/GoogleDriveService'; -import {FileContentService} from '../../utils/FileContentService'; -import {BufferWritable} from '../../utils/BufferWritable'; -import {SimpleFile} from '../../model/GoogleFile'; -import {HasAccessToken} from '../../google/AuthClient'; +import {QueueTask} from './QueueTask.ts'; +import {GoogleDriveService} from '../../google/GoogleDriveService.ts'; +import {FileContentService} from '../../utils/FileContentService.ts'; +import {BufferWritable} from '../../utils/BufferWritable.ts'; +import {SimpleFile} from '../../model/GoogleFile.ts'; +import {HasAccessToken} from '../../google/AuthClient.ts'; export class TaskFetchDocument extends QueueTask { constructor(protected logger: winston.Logger, diff --git a/src/containers/google_folder/TaskFetchFolder.ts b/src/containers/google_folder/TaskFetchFolder.ts index e6f7d4c2..a19fa450 100644 --- a/src/containers/google_folder/TaskFetchFolder.ts +++ b/src/containers/google_folder/TaskFetchFolder.ts @@ -1,14 +1,15 @@ -import {GoogleDriveService} from '../../google/GoogleDriveService'; -import {FileContentService} from '../../utils/FileContentService'; -import {INITIAL_RETRIES, QueueTask} from './QueueTask'; import winston from 'winston'; -import {TaskFetchDiagram} from './TaskFetchDiagram'; -import {TaskFetchDocument} from './TaskFetchDocument'; -import {TaskFetchBinary} from './TaskFetchBinary'; -import {TaskFetchAsset} from './TaskFetchAsset'; -import {MimeTypes, SimpleFile} from '../../model/GoogleFile'; -import {FileId} from '../../model/model'; -import {HasAccessToken} from '../../google/AuthClient'; +import {GoogleDriveService} from '../../google/GoogleDriveService.ts'; +import {FileContentService} from '../../utils/FileContentService.ts'; +import {INITIAL_RETRIES, QueueTask} from './QueueTask.ts'; +import {TaskFetchDiagram} from './TaskFetchDiagram.ts'; +import {TaskFetchDocument} from './TaskFetchDocument.ts'; +import {TaskFetchBinary} from './TaskFetchBinary.ts'; +import {TaskFetchAsset} from './TaskFetchAsset.ts'; +import {MimeTypes, SimpleFile} from '../../model/GoogleFile.ts'; +import {FileId} from '../../model/model.ts'; +import {HasAccessToken} from '../../google/AuthClient.ts'; +import {StopWatch} from '../../utils/StopWatch.ts'; interface Filters { filterFoldersIds: FileId[]; @@ -17,6 +18,8 @@ interface Filters { export class TaskFetchFolder extends QueueTask { + private useGoogleMarkdowns = false; + constructor(protected logger: winston.Logger, private googleDriveService: GoogleDriveService, private auth: HasAccessToken, @@ -27,6 +30,10 @@ export class TaskFetchFolder extends QueueTask { super(logger); } + setUseGoogleMarkdowns(value: boolean) { + this.useGoogleMarkdowns = value; + } + async run(): Promise { if (this.filters.filterFoldersIds.length > 0) { if (this.filters.filterFoldersIds.indexOf(this.file.id) === -1) { @@ -34,6 +41,8 @@ export class TaskFetchFolder extends QueueTask { } } + const stopWatch = new StopWatch(); + if (this.retries < INITIAL_RETRIES) { await new Promise(resolve => setTimeout(resolve, 1000)); this.logger.info('Listening (retry): ' + this.file.id); @@ -85,15 +94,19 @@ export class TaskFetchFolder extends QueueTask { switch (file.mimeType) { case MimeTypes.FOLDER_MIME: - tasks.push(new TaskFetchFolder( - this.logger, - this.googleDriveService, - this.auth, - await this.fileService.getSubFileService(file.id), - file, - this.forceDownloadFilters, - this.filters - )); + { + const task = new TaskFetchFolder( + this.logger, + this.googleDriveService, + this.auth, + await this.fileService.getSubFileService(file.id), + file, + this.forceDownloadFilters, + this.filters + ); + task.setUseGoogleMarkdowns(this.useGoogleMarkdowns); + tasks.push(task); + } break; case MimeTypes.DRAWING_MIME: @@ -108,14 +121,26 @@ export class TaskFetchFolder extends QueueTask { break; case MimeTypes.DOCUMENT_MIME: - tasks.push(new TaskFetchDocument( - this.logger, - this.googleDriveService, - this.auth, - await this.fileService, - file, - forceDownload - )); + if (!this.useGoogleMarkdowns) { + tasks.push(new TaskFetchDocument( + this.logger, + this.googleDriveService, + this.auth, + await this.fileService, + file, + forceDownload + )); + } else { + tasks.push(new TaskFetchBinary( + this.logger, + this.googleDriveService, + this.auth, + await this.fileService, + file, + forceDownload, + MimeTypes.MARKDOWN, 'md' + )); + } break; case MimeTypes.SPREADSHEET_MIME: @@ -202,6 +227,11 @@ export class TaskFetchFolder extends QueueTask { await this.fileService.writeJson('.folder-files.json', filesToSave); } + const timeString = stopWatch.toString(1000); + if (timeString) { + this.logger.info('Slow listening: ' + this.file.id + ' ' + timeString); + } + return tasks; } diff --git a/src/containers/google_folder/UserConfigService.ts b/src/containers/google_folder/UserConfigService.ts index 904fc416..67234f3f 100644 --- a/src/containers/google_folder/UserConfigService.ts +++ b/src/containers/google_folder/UserConfigService.ts @@ -38,6 +38,7 @@ export class UserConfig { hugo_theme?: HugoTheme; config_toml?: string; transform_subdir?: string; + use_google_markdowns?: boolean; auto_sync?: boolean; fm_without_version?: boolean; actions_yaml?: string; @@ -45,7 +46,7 @@ export class UserConfig { } const DEFAULT_CONFIG: UserConfig = { - remote_branch: 'master', + remote_branch: 'main', hugo_theme: { id: 'ananke', name: 'Anake', diff --git a/src/containers/job/JobManagerContainer.ts b/src/containers/job/JobManagerContainer.ts index 05e134d0..be01585f 100644 --- a/src/containers/job/JobManagerContainer.ts +++ b/src/containers/job/JobManagerContainer.ts @@ -468,6 +468,9 @@ export class JobManagerContainer extends Container { const userConfigService = new UserConfigService(googleFileSystem); await userConfigService.load(); + + transformContainer.setUseGoogleMarkdowns(userConfigService.config.use_google_markdowns); + transformContainer.onProgressNotify(({ completed, total, warnings, failed }) => { if (!this.driveJobsMap[folderId]) { return; diff --git a/src/containers/server/ServerContainer.ts b/src/containers/server/ServerContainer.ts index fbf89385..4f9df7f7 100644 --- a/src/containers/server/ServerContainer.ts +++ b/src/containers/server/ServerContainer.ts @@ -44,6 +44,7 @@ import {GoogleTreeProcessor} from '../google_folder/GoogleTreeProcessor.ts'; import {initStaticDistPages} from './static.ts'; import {initUiServer} from './vuejs.ts'; import {initErrorHandler} from './error.ts'; +import {WebHookController} from './routes/WebHookController.ts'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -256,6 +257,9 @@ export class ServerContainer extends Container { const driveUiController = new DriveUiController('/driveui', this.logger, this.filesService, this.authContainer); app.use('/driveui', await driveUiController.getRouter()); + const webHookController = new WebHookController('/webhook', this.logger); + app.use('/webhook', await webHookController.getRouter()); + app.use('/api/share-token', authenticate(this.logger), (req, res) => { if ('POST' !== req.method) { throw new Error('Incorrect method'); diff --git a/src/containers/server/auth.ts b/src/containers/server/auth.ts index 1edb03cd..cf2c63e7 100644 --- a/src/containers/server/auth.ts +++ b/src/containers/server/auth.ts @@ -61,24 +61,24 @@ interface JwtDecryptedPayload extends GoogleUser { driveId: string; } -export function signToken(payload: JwtDecryptedPayload): string { - const expiresIn = 365 * 24 * 3600; // process.env.JWT_ACCESS_TOKEN_EXPIRATION_TIME || +function signToken(payload: JwtDecryptedPayload, jwtSecret: string): string { + const expiresIn = 365 * 24 * 3600; const encrypted: JwtEncryptedPayload = { sub: payload.id, name: payload.name, email: payload.email, - gat: encrypt(payload.google_access_token, process.env.JWT_SECRET), - grt: payload.google_refresh_token ? encrypt(payload.google_refresh_token, process.env.JWT_SECRET) : undefined, + gat: encrypt(payload.google_access_token, jwtSecret), + grt: payload.google_refresh_token ? encrypt(payload.google_refresh_token, jwtSecret) : undefined, ged: payload.google_expiry_date, driveId: payload.driveId }; - return jsonwebtoken.sign(encrypted, process.env.JWT_SECRET, { expiresIn }); + return jsonwebtoken.sign(encrypted, jwtSecret, { expiresIn }); } -export function verifyToken(accessCookie: string): JwtDecryptedPayload { - const encrypted: JwtEncryptedPayload = jsonwebtoken.verify(accessCookie, process.env.JWT_SECRET); +function verifyToken(accessCookie: string, jwtSecret: string): JwtDecryptedPayload { + const encrypted: JwtEncryptedPayload = jsonwebtoken.verify(accessCookie, jwtSecret); return { id: encrypted.sub, @@ -240,6 +240,8 @@ export async function getAuth(req: Request, res: Response, next) { const googleDriveService = new GoogleDriveService(this.logger, null); const googleUser: GoogleUser = await authClient.getUser(await authClient.getAccessToken()); + const jwtSecret = process.env.JWT_SECRET; + if (driveId) { const drive = await googleDriveService.getDrive(await authClient.getAccessToken(), driveId); if (drive.id) { @@ -247,7 +249,7 @@ export async function getAuth(req: Request, res: Response, next) { ...googleUser, ...await authClient.getAuthData(), driveId: driveId - }); + }, jwtSecret); setAccessCookie(res, accessToken); res.redirect(redirectTo || '/'); return; @@ -257,7 +259,7 @@ export async function getAuth(req: Request, res: Response, next) { ...googleUser, ...await authClient.getAuthData(), driveId: driveId - }); + }, jwtSecret); setAccessCookie(res, accessToken); res.redirect(redirectTo || '/drive'); return; @@ -289,8 +291,10 @@ async function decodeAuthenticateInfo(req, res, next) { return; } + const jwtSecret = process.env.JWT_SECRET; + try { - const decoded = verifyToken(req.cookies.accessToken); + const decoded = verifyToken(req.cookies.accessToken, jwtSecret); if (!decoded.id) { return next(redirError(req, 'No jwt.sub')); } @@ -319,7 +323,7 @@ async function decodeAuthenticateInfo(req, res, next) { ...decoded, ...await authClient.getAuthData(), driveId: driveId - }); + }, jwtSecret); setAccessCookie(res, accessToken); } diff --git a/src/containers/server/loadRunningInstance.ts b/src/containers/server/loadRunningInstance.ts index 9c95faa7..ba9be2e7 100644 --- a/src/containers/server/loadRunningInstance.ts +++ b/src/containers/server/loadRunningInstance.ts @@ -11,8 +11,10 @@ export async function loadRunningInstance() { const json = JSON.parse(content); if (json.pid > 0) { try { - // sending the signal 0 to a given PID just checks if any process with the given PID is running, and you have the permission to send a signal to it. - process.kill(json.pid, 0); + // kill -0 seems not to work on alpine for some reason + if (!fs.existsSync('/proc/' + json.pid)) { + return null; + } } catch (err) { return null; } diff --git a/src/containers/server/routes/BackLinksController.ts b/src/containers/server/routes/BackLinksController.ts index 3b6d4f02..214496fa 100644 --- a/src/containers/server/routes/BackLinksController.ts +++ b/src/containers/server/routes/BackLinksController.ts @@ -25,14 +25,20 @@ export class BackLinksController extends Controller { const localLinks = new LocalLinks(contentFileService); await localLinks.load(); - const linkFileIds = localLinks.getLinks(fileId); + const linksArr = localLinks.getLinks(fileId); + if (linksArr === false) { + return { backlinks: [], links: [], notGenerated: true }; + } + const links = []; - for (const linkFileId of linkFileIds) { - const [file] = await markdownTreeProcessor.findById(linkFileId); + for (const linkObj of linksArr) { + const { fileId, linksCount } = linkObj; + const [file] = await markdownTreeProcessor.findById(fileId); if (file) { links.push({ folderId: file.parentId, - fileId: linkFileId, + fileId: fileId, + linksCount, path: file.path, name: file.fileName }); @@ -41,12 +47,14 @@ export class BackLinksController extends Controller { const backLinkFileIds = localLinks.getBackLinks(fileId); const backlinks = []; - for (const backLinkFileId of backLinkFileIds) { - const [file] = await markdownTreeProcessor.findById(backLinkFileId); + for (const backLinkObj of backLinkFileIds) { + const { fileId, linksCount } = backLinkObj; + const [file] = await markdownTreeProcessor.findById(fileId); if (file) { backlinks.push({ folderId: file.parentId, - fileId: backLinkFileId, + fileId: fileId, + linksCount, path: file.path, name: file.fileName }); diff --git a/src/containers/server/routes/ConfigController.ts b/src/containers/server/routes/ConfigController.ts index 4807f48f..d201dad9 100644 --- a/src/containers/server/routes/ConfigController.ts +++ b/src/containers/server/routes/ConfigController.ts @@ -1,11 +1,11 @@ import yaml from 'js-yaml'; -import {Controller, RouteGet, RouteParamBody, RouteParamPath, RoutePost, RoutePut} from './Controller'; -import {FileContentService} from '../../../utils/FileContentService'; -import {GitScanner} from '../../../git/GitScanner'; -import {UserConfigService} from '../../google_folder/UserConfigService'; -import {FolderRegistryContainer} from '../../folder_registry/FolderRegistryContainer'; -import {ContainerEngine} from '../../../ContainerEngine'; +import {Controller, RouteGet, RouteParamBody, RouteParamPath, RoutePost, RoutePut} from './Controller.ts'; +import {FileContentService} from '../../../utils/FileContentService.ts'; +import {GitScanner} from '../../../git/GitScanner.ts'; +import {UserConfigService} from '../../google_folder/UserConfigService.ts'; +import {FolderRegistryContainer} from '../../folder_registry/FolderRegistryContainer.ts'; +import {ContainerEngine} from '../../../ContainerEngine.ts'; export interface ConfigBody { config: { @@ -15,6 +15,7 @@ export interface ConfigBody { rewrite_rules_yaml?: string; hugo_theme: HugoTheme; auto_sync: boolean; + use_google_markdowns: boolean; fm_without_version: boolean; actions_yaml?: string; }; @@ -79,13 +80,14 @@ export class ConfigController extends Controller { const gitScanner = new GitScanner(this.logger, transformedFileSystem.getRealPath(), 'wikigdrive@wikigdrive.com'); await gitScanner.initialize(); + await gitScanner.setSafeDirectory(); const googleFileSystem = await this.filesService.getSubFileService(driveId, ''); const userConfigService = new UserConfigService(googleFileSystem); await userConfigService.load(); if (body.config?.remote_branch) { - userConfigService.config.remote_branch = body.config?.remote_branch || 'master'; + userConfigService.config.remote_branch = body.config?.remote_branch || 'main'; } if (body.config?.hugo_theme) { userConfigService.config.hugo_theme = body.config?.hugo_theme; @@ -111,6 +113,7 @@ export class ConfigController extends Controller { userConfigService.config.actions_yaml = body.config?.actions_yaml; } userConfigService.config.auto_sync = !!body.config?.auto_sync; + userConfigService.config.use_google_markdowns = !!body.config?.use_google_markdowns; userConfigService.config.fm_without_version = !!body.config?.fm_without_version; await userConfigService.save(); diff --git a/src/containers/server/routes/Controller.ts b/src/containers/server/routes/Controller.ts index 3ae57ca2..23498fc6 100644 --- a/src/containers/server/routes/Controller.ts +++ b/src/containers/server/routes/Controller.ts @@ -30,6 +30,12 @@ export interface ControllerRouteParamBody { docs?: RouteDoc; } +export interface ControllerRouteParamHeaders { + type: 'headers'; + parameterIndex: number; + docs?: RouteDoc; +} + export interface ControllerRouteParamStream { type: 'stream'; parameterIndex: number; @@ -70,7 +76,7 @@ export interface ControllerRouteParamPath { } type ControllerRouteParam = ControllerRouteParamGetAll | ControllerRouteParamQuery - | ControllerRouteParamBody | ControllerRouteParamPath | ControllerRouteParamStream + | ControllerRouteParamBody | ControllerRouteParamHeaders | ControllerRouteParamPath | ControllerRouteParamStream | ControllerRouteParamRelated | ControllerRouteParamUser | ControllerRouteParamMethod; export interface RouteDoc { @@ -245,6 +251,12 @@ export class Controller implements ControllerCallContext { args[param.parameterIndex] = body; } break; + case 'headers': + { + const headers = req.headers; + args[param.parameterIndex] = headers; + } + break; case 'stream': args[param.parameterIndex] = req; break; @@ -450,6 +462,18 @@ export function RouteParamBody(docs: RouteDoc = {}) { }; } +export function RouteParamHeaders(docs: RouteDoc = {}) { + return function (targetClass: Controller, methodProp: string, parameterIndex: number) { + const route = targetClass.getRoute(targetClass, methodProp); + const param: ControllerRouteParamHeaders = { + type: 'headers', + parameterIndex, + docs + }; + route.params.push(param); + }; +} + export function RouteParamStream(docs: RouteDoc = {}) { return function (targetClass: Controller, methodProp: string, parameterIndex: number) { const route = targetClass.getRoute(targetClass, methodProp); diff --git a/src/containers/server/routes/DriveController.ts b/src/containers/server/routes/DriveController.ts index 48c4acba..cc7cf319 100644 --- a/src/containers/server/routes/DriveController.ts +++ b/src/containers/server/routes/DriveController.ts @@ -1,16 +1,16 @@ -import {Controller, RouteGet, RouteParamPath, RouteParamUser, RouteResponse} from './Controller'; -import {GitScanner} from '../../../git/GitScanner'; -import {FolderRegistryContainer} from '../../folder_registry/FolderRegistryContainer'; -import {UserConfigService} from '../../google_folder/UserConfigService'; -import {FileContentService} from '../../../utils/FileContentService'; -import {GoogleDriveService} from '../../../google/GoogleDriveService'; -import {MarkdownTreeProcessor} from '../../transform/MarkdownTreeProcessor'; -import {AuthConfig} from '../../../model/AccountJson'; -import {googleMimeToExt} from '../../transform/TaskLocalFileTransform'; -import {Container} from '../../../ContainerEngine'; -import {GoogleTreeProcessor} from '../../google_folder/GoogleTreeProcessor'; -import {getContentFileService} from '../../transform/utils'; -import {redirError} from '../auth'; +import {Controller, RouteGet, RouteParamPath, RouteParamUser, RouteResponse} from './Controller.ts'; +import {GitScanner} from '../../../git/GitScanner.ts'; +import {FolderRegistryContainer} from '../../folder_registry/FolderRegistryContainer.ts'; +import {UserConfigService} from '../../google_folder/UserConfigService.ts'; +import {FileContentService} from '../../../utils/FileContentService.ts'; +import {GoogleDriveService} from '../../../google/GoogleDriveService.ts'; +import {MarkdownTreeProcessor} from '../../transform/MarkdownTreeProcessor.ts'; +import {AuthConfig} from '../../../model/AccountJson.ts'; +import {googleMimeToExt} from '../../transform/TaskLocalFileTransform.ts'; +import {Container} from '../../../ContainerEngine.ts'; +import {GoogleTreeProcessor} from '../../google_folder/GoogleTreeProcessor.ts'; +import {getContentFileService} from '../../transform/utils.ts'; +import {redirError} from '../auth.ts'; export class DriveController extends Controller { constructor(subPath: string, diff --git a/src/containers/server/routes/GitController.ts b/src/containers/server/routes/GitController.ts index 39b1cb10..ba059e05 100644 --- a/src/containers/server/routes/GitController.ts +++ b/src/containers/server/routes/GitController.ts @@ -15,6 +15,10 @@ interface CmdPost { cmd: string; } +interface RemovePath { + filePath: string; +} + export default class GitController extends Controller { constructor(subPath: string, private readonly filesService: FileContentService, @@ -166,4 +170,22 @@ export default class GitController extends Controller { } } + @RoutePost('/:driveId/remove_cached') + async removeCached(@RouteParamPath('driveId') driveId: string, @RouteParamBody() body: RemovePath) { + try { + const transformedFileSystem = await this.filesService.getSubFileService(driveId + '_transform', ''); + const gitScanner = new GitScanner(this.logger, transformedFileSystem.getRealPath(), 'wikigdrive@wikigdrive.com'); + await gitScanner.initialize(); + await gitScanner.removeCached(body.filePath); + + return {}; + } catch (err) { + this.logger.error(err.stack ? err.stack : err.message); + if (err.message.indexOf('Failed to retrieve list of SSH authentication methods') > -1) { + return { error: 'Failed to authenticate' }; + } + throw err; + } + } + } diff --git a/src/containers/server/routes/SearchController.ts b/src/containers/server/routes/SearchController.ts index a972a0e3..0688c91e 100644 --- a/src/containers/server/routes/SearchController.ts +++ b/src/containers/server/routes/SearchController.ts @@ -6,10 +6,10 @@ import { RouteGet, RouteParamPath, RouteParamQuery -} from './Controller'; -import {FileContentService} from '../../../utils/FileContentService'; -import {ShareErrorHandler} from './FolderController'; -import {UserConfigService} from '../../google_folder/UserConfigService'; +} from './Controller.ts'; +import {FileContentService} from '../../../utils/FileContentService.ts'; +import {ShareErrorHandler} from './FolderController.ts'; +import {UserConfigService} from '../../google_folder/UserConfigService.ts'; export class SearchController extends Controller { constructor(subPath: string, private filesService: FileContentService) { diff --git a/src/containers/server/routes/WebHookController.ts b/src/containers/server/routes/WebHookController.ts new file mode 100644 index 00000000..1d5523ac --- /dev/null +++ b/src/containers/server/routes/WebHookController.ts @@ -0,0 +1,19 @@ +import {Logger} from 'winston'; + +import { + Controller, RouteParamBody, RouteParamHeaders, RoutePost, +} from './Controller.ts'; + +export class WebHookController extends Controller { + + constructor(subPath: string, private readonly queryLogger: Logger) { + super(subPath); + } + + @RoutePost('/') + async postEvent(@RouteParamBody() body: undefined, @RouteParamHeaders() headers: undefined) { + this.queryLogger.info(`WebHookController.postEvent ${JSON.stringify(headers)} ${JSON.stringify(body)}`); + + return {}; + } +} diff --git a/src/containers/transform/LocalLinks.ts b/src/containers/transform/LocalLinks.ts index 0884f9b6..b82fe7cb 100644 --- a/src/containers/transform/LocalLinks.ts +++ b/src/containers/transform/LocalLinks.ts @@ -1,4 +1,5 @@ import {FileContentService} from '../../utils/FileContentService.ts'; +import {FileId} from '../../model/model.ts'; interface Link { fileId: string; @@ -10,11 +11,13 @@ export const LINKS_NAME = '.wgd-local-links.csv'; export class LocalLinks { private links: Link[]; + private notGenerated = true; constructor(private transformFileService: FileContentService) { } async load() { if (!await this.transformFileService.exists(LINKS_NAME)) { + this.notGenerated = true; this.links = []; return; } @@ -33,10 +36,11 @@ export class LocalLinks { } groups[cells[0]].links.push(cells[2]); } + this.notGenerated = false; this.links = Object.values(groups); } - append(fileId: string, fileName: string, links: string[]) { + append(fileId: FileId, fileName: string, links: string[]) { const link = this.links.find(l => l.fileId === fileId); if (link) { link.fileName = fileName; @@ -48,7 +52,7 @@ export class LocalLinks { } } - getBackLinks(fileId) { + getBackLinks(fileId: FileId) { const retVal = new Set(); for (const link of this.links) { for (const targetLink of link.links) { @@ -57,17 +61,29 @@ export class LocalLinks { } } } - return Array.from(retVal); + + return Array.from(retVal) + .map(fileId => ({ + fileId, + linksCount: this.links.find(link => link.fileId === fileId)?.links.length || 0 + })); } - getLinks(fileId) { + getLinks(fileId: FileId) { + if (this.notGenerated) { + return false; + } + for (const link of this.links) { if (link.fileId === fileId) { const links = link.links .filter(link => link.startsWith('gdoc:')) - .map(link => link.substring('gdoc:'.length)); + .map(link => link.substring('gdoc:'.length).replace(/#.*/, '')); - return links; + return links.map(fileId => ({ + fileId, + linksCount: this.links.find(link => link.fileId === fileId)?.links.length || 0 + })); } } return []; diff --git a/src/containers/transform/TaskGoogleMarkdownTransform.ts b/src/containers/transform/TaskGoogleMarkdownTransform.ts new file mode 100644 index 00000000..37d2ca82 --- /dev/null +++ b/src/containers/transform/TaskGoogleMarkdownTransform.ts @@ -0,0 +1,202 @@ +import fs from 'fs'; +import winston from 'winston'; + +import {QueueTask} from '../google_folder/QueueTask.ts'; +import {JobManagerContainer} from '../job/JobManagerContainer.ts'; +import {FileContentService} from '../../utils/FileContentService.ts'; +import {GoogleFile} from '../../model/GoogleFile.ts'; +import {BinaryFile, DrawingFile, LocalFile, MdFile} from '../../model/LocalFile.ts'; +import {LocalLinks} from './LocalLinks.ts'; +import {UserConfig} from '../google_folder/UserConfigService.ts'; +import {SvgTransform} from '../../SvgTransform.ts'; +import {generateDocumentFrontMatter} from './frontmatters/generateDocumentFrontMatter.ts'; +import {generateConflictMarkdown} from './frontmatters/generateConflictMarkdown.ts'; +import {googleMimeToExt} from './TaskLocalFileTransform.ts'; +import {getUrlHash, urlToFolderId} from '../../utils/idParsers.ts'; + +export class TaskGoogleMarkdownTransform extends QueueTask { + constructor(protected logger: winston.Logger, + private jobManagerContainer: JobManagerContainer, + private realFileName: string, + private googleFolder: FileContentService, + private googleFile: GoogleFile, + private destinationDirectory: FileContentService, + private localFile: LocalFile, + private localLinks: LocalLinks, + private userConfig: UserConfig + ) { + super(logger); + this.retries = 0; + + if (!this.localFile.fileName) { + throw new Error(`No fileName for: ${this.localFile.id}`); + } + } + + async run(): Promise { + await this.generate(this.localFile); + + return []; + } + + async generateBinary(binaryFile: BinaryFile) { + await new Promise((resolve, reject) => { + try { + const dest = this.destinationDirectory.createWriteStream(this.realFileName); + + dest.on('error', err => { + reject(err); + }); + + const ext = googleMimeToExt(this.googleFile.mimeType, this.googleFile.name); + const stream = this.googleFolder.createReadStream(`${binaryFile.id}${ext ? '.' + ext : ''}`) + .on('error', err => { + reject(err); + }) + .pipe(dest); + + stream.on('finish', () => { + resolve(); + }); + stream.on('error', err => { + reject(err); + }); + } catch (err) { + reject(err); + } + }); + } + + async generateDrawing(drawingFile: DrawingFile) { + await new Promise((resolve, reject) => { + try { + // await this.destinationDirectory.mkdir(getFileDir(targetPath)); + const dest = this.destinationDirectory.createWriteStream(this.realFileName); + const svgTransform = new SvgTransform(drawingFile.fileName); + // const svgPath = this.googleScanner.getFilePathPrefix(drawingFile.id) + '.svg'; + + dest.on('error', err => { + reject(err); + }); + + const stream = this.googleFolder.createReadStream(`${drawingFile.id}.svg`) + .on('error', err => { + reject(err); + }) + .pipe(svgTransform) + .pipe(dest); + + stream.on('finish', () => { + this.localLinks.append(drawingFile.id, drawingFile.fileName, Array.from(svgTransform.links)); + resolve(); + }); + stream.on('error', err => { + reject(err); + }); + } catch (err) { + reject(err); + } + }); + } + + async generateDocument(localFile: MdFile) { + const links = new Set(); + + const mdPath = this.googleFolder.getRealPath() + '/' + localFile.id + '.md'; + + const input: Buffer = fs.readFileSync(mdPath); + + const originalMarkdown = new TextDecoder().decode(input); + + const markdownLinkRegex = /\[([^\]]+)\]\(([^)]+?)\)/g; + + function replaceMarkdownLinks(markdown, replacerFunction) { + return markdown.replace(markdownLinkRegex, (match, linkText, url) => { + // Call the replacer function with the link text, URL, and title + return replacerFunction(linkText, url); + }); + } + + function customReplacer(linkText, href) { + href = href.replaceAll('\\', ''); + + const id = urlToFolderId(href); + const hash = getUrlHash(href); + if (id) { + href = 'gdoc:' + id + hash; + } + if (href && !href.startsWith('#') && href.indexOf(':') > -1) { + links.add(href); + } + + return `[${linkText}](${href})`; + } + + const markdownRewrittenLinks = replaceMarkdownLinks(originalMarkdown, customReplacer); + + const pattern = /\*\{\{%\s+.*?\s+%\}\}\*/g; + + const markdown = markdownRewrittenLinks.replace(pattern, (match) => { + // Remove the surrounding asterisks + return match.slice(1, -1); + }); + + // links = Array.from(converter.links); + const frontMatter = generateDocumentFrontMatter(localFile, Array.from(links), this.userConfig.fm_without_version); + const errors = []; + this.warnings = errors.length; + + for (const errorMsg of errors) { + this.logger.warn('Error in: ['+ this.localFile.fileName +'](' + this.localFile.fileName + ') ' + errorMsg, { + errorMdFile: this.localFile.fileName, + errorMdMsg: errorMsg + }); + } + + await this.destinationDirectory.writeFile(this.realFileName, frontMatter + markdown); + this.localLinks.append(localFile.id, localFile.fileName, Array.from(links)); + } + + async generate(localFile: LocalFile): Promise { + try { + const verStr = this.localFile.version ? ' #' + this.localFile.version : ' '; + if (localFile.type === 'conflict') { + this.logger.info('Transforming conflict: ' + this.localFile.fileName); + const md = generateConflictMarkdown(localFile); + await this.destinationDirectory.writeFile(this.realFileName, md); + } else if (localFile.type === 'redir') { // TODO + this.logger.info('Transforming redir: ' + this.localFile.fileName); + // const redirectToFile = this.toGenerate.find(f => f.id === localFile.redirectTo); + // const redirectToFile = this.generatedScanner.getFileById(localFile.redirectTo); + // const md = generateRedirectMarkdown(localFile, redirectToFile, this.linkTranslator); + // await this.destinationDirectory.mkdir(getFileDir(targetPath)); + // await this.destinationDirectory.writeFile(targetPath, md); + // await this.generatedScanner.update(targetPath, md); + } else if (localFile.type === 'md') { + this.logger.info('Transforming markdown: ' + this.localFile.fileName + verStr); + // const googleFile = await this.googleScanner.getFileById(localFile.id); + // const downloadFile = await this.downloadFilesStorage.findFile(f => f.id === localFile.id); + if (this.googleFile) { // && downloadFile + await this.generateDocument(localFile); + } + } else if (localFile.type === 'drawing') { + this.logger.info('Transforming drawing: ' + this.localFile.fileName + verStr); + // const googleFile = await this.googleScanner.getFileById(localFile.id); + // const downloadFile = await this.downloadFilesStorage.findFile(f => f.id === localFile.id); + if (this.googleFile) { // && downloadFile + await this.generateDrawing(localFile); + } + } else if (localFile.type === 'binary') { + this.logger.info('Transforming binary: ' + this.localFile.fileName + verStr); + if (this.googleFile) { // && downloadFile + await this.generateBinary(localFile); + } + } + this.logger.info('Transformed: ' + this.localFile.fileName + verStr); + } catch (err) { + this.logger.error('Error transforming ' + localFile.fileName + ' ' + err.stack ? err.stack : err.message); + throw err; + } + } + +} diff --git a/src/containers/transform/TaskLocalFileTransform.ts b/src/containers/transform/TaskLocalFileTransform.ts index 23214da0..5174cb64 100644 --- a/src/containers/transform/TaskLocalFileTransform.ts +++ b/src/containers/transform/TaskLocalFileTransform.ts @@ -49,7 +49,9 @@ export class TaskLocalFileTransform extends QueueTask { private destinationDirectory: FileContentService, private localFile: LocalFile, private localLinks: LocalLinks, - private userConfig: UserConfig + private userConfig: UserConfig, + private globalHeadersMap: {[key: string]: string}, + private globalInvisibleBookmarks: {[key: number]: number}, ) { super(logger); this.retries = 0; @@ -128,6 +130,8 @@ export class TaskLocalFileTransform extends QueueTask { async generateDocument(localFile: MdFile) { let frontMatter; let markdown; + let headersMap = {}; + let invisibleBookmarks = {}; let links = []; let errors = []; @@ -163,6 +167,8 @@ export class TaskLocalFileTransform extends QueueTask { } else { converter.setPicturesDir('../' + this.realFileName.replace(/.md$/, '.assets/'), picturesDirAbsolute); } + headersMap = converter.getHeadersMap(); + invisibleBookmarks = converter.getInvisibleBookmarks(); markdown = await converter.convert(); links = Array.from(converter.links); frontMatter = generateDocumentFrontMatter(localFile, links, this.userConfig.fm_without_version); @@ -174,6 +180,8 @@ export class TaskLocalFileTransform extends QueueTask { frontMatter: string; markdown: string; errors: Array; + headersMap: {[key: string]: string}; + invisibleBookmarks: {[key: string]: string}; } const workerResult: WorkerResult = await this.jobManagerContainer.scheduleWorker('OdtToMarkdown', { @@ -190,6 +198,8 @@ export class TaskLocalFileTransform extends QueueTask { frontMatter = workerResult.frontMatter; markdown = workerResult.markdown; errors = workerResult.errors; + headersMap = workerResult.headersMap; + invisibleBookmarks = workerResult.invisibleBookmarks; this.warnings = errors.length; } @@ -202,6 +212,12 @@ export class TaskLocalFileTransform extends QueueTask { await this.destinationDirectory.writeFile(this.realFileName, frontMatter + markdown); this.localLinks.append(localFile.id, localFile.fileName, links); + for (const k in headersMap) { + this.globalHeadersMap['gdoc:' + localFile.id + k] = 'gdoc:' + localFile.id + headersMap[k]; + } + for (const k in invisibleBookmarks) { + this.globalInvisibleBookmarks['gdoc:' + localFile.id + k] = invisibleBookmarks[k]; + } } async generate(localFile: LocalFile): Promise { diff --git a/src/containers/transform/TransformContainer.ts b/src/containers/transform/TransformContainer.ts index 8f82d516..e9d78231 100644 --- a/src/containers/transform/TransformContainer.ts +++ b/src/containers/transform/TransformContainer.ts @@ -9,7 +9,6 @@ import {GoogleFilesScanner} from './GoogleFilesScanner.ts'; import {convertToRelativeMarkDownPath, convertToRelativeSvgPath} from '../../LinkTranslator.ts'; import {LocalFilesGenerator} from './LocalFilesGenerator.ts'; import {QueueTransformer} from './QueueTransformer.ts'; -import {NavigationHierarchy} from './generateNavigationHierarchy.ts'; import {ConflictFile, LocalFile, RedirFile} from '../../model/LocalFile.ts'; import {TaskLocalFileTransform} from './TaskLocalFileTransform.ts'; import {MimeTypes} from '../../model/GoogleFile.ts'; @@ -24,6 +23,9 @@ import {MarkdownTreeProcessor} from './MarkdownTreeProcessor.ts'; import {LunrIndexer} from '../search/LunrIndexer.ts'; import {JobManagerContainer} from '../job/JobManagerContainer.ts'; import {UserConfigService} from '../google_folder/UserConfigService.ts'; +import {getUrlHash} from '../../utils/idParsers.ts'; +import {TaskGoogleMarkdownTransform} from './TaskGoogleMarkdownTransform.ts'; +import {frontmatter} from './frontmatters/frontmatter.js'; const __filename = fileURLToPath(import.meta.url); @@ -198,7 +200,6 @@ export class TransformLog extends Transport { export class TransformContainer extends Container { private logger: winston.Logger; private generatedFileService: FileContentService; - private hierarchy: NavigationHierarchy = {}; private localLog: LocalLog; private localLinks: LocalLinks; private filterFilesIds: FileId[]; @@ -207,6 +208,9 @@ export class TransformContainer extends Container { private progressNotifyCallback: ({total, completed, warnings, failed}: { total?: number; completed?: number; warnings?: number; failed?: number }) => void; private transformLog: TransformLog; private isFailed = false; + private useGoogleMarkdowns = false; + private globalHeadersMap: {[key: string]: string} = {}; + private globalInvisibleBookmarks: {[key: string]: number} = {}; constructor(public readonly params: ContainerConfig, public readonly paramsArr: ContainerConfigArr = {}) { super(params, paramsArr); @@ -286,18 +290,35 @@ export class TransformContainer extends Container { const jobManagerContainer = this.engine.getContainer('job_manager'); - const task = new TaskLocalFileTransform( - this.logger, - jobManagerContainer, - realFileName, - googleFolder, - googleFile, - destinationDirectory, - localFile, - this.localLinks, - this.userConfigService.config - ); - queueTransformer.addTask(task); + if (!this.useGoogleMarkdowns) { + const task = new TaskLocalFileTransform( + this.logger, + jobManagerContainer, + realFileName, + googleFolder, + googleFile, + destinationDirectory, + localFile, + this.localLinks, + this.userConfigService.config, + this.globalHeadersMap, + this.globalInvisibleBookmarks + ); + queueTransformer.addTask(task); + } else { + const task = new TaskGoogleMarkdownTransform( + this.logger, + jobManagerContainer, + realFileName, + googleFolder, + googleFile, + destinationDirectory, + localFile, + this.localLinks, + this.userConfigService.config + ); + queueTransformer.addTask(task); + } } const dirNames = destinationDirectory.getVirtualPath().replace(/\/$/, '').split('/'); @@ -342,10 +363,10 @@ export class TransformContainer extends Container { processed.add(fileId); const backLinks = this.localLinks.getBackLinks(fileId); for (const backLink of backLinks) { - if (processed.has(backLink)) { + if (processed.has(backLink.fileId)) { continue; } - filterFilesIds.add(backLink); + filterFilesIds.add(backLink.fileId); } } if (filterFilesIds.size > 0) { @@ -417,6 +438,7 @@ export class TransformContainer extends Container { async rewriteLinks(destinationDirectory: FileContentService) { const files = await destinationDirectory.list(); + for (const fileName of files) { if (await destinationDirectory.isDirectory(fileName)) { await this.rewriteLinks(await destinationDirectory.getSubFileService(fileName)); @@ -425,17 +447,55 @@ export class TransformContainer extends Container { if (fileName.endsWith('.md') || fileName.endsWith('.svg')) { const content = await destinationDirectory.readFile(fileName); - const newContent = content.replace(/(gdoc:[A-Z0-9_-]+)/ig, (str: string) => { - const fileId = str.substring('gdoc:'.length); + + const parsed = frontmatter(content); + const props = parsed.data; + let newContent = content; + if (props?.id) { + newContent = newContent.replace(/\n? ?<\/a>\n?/igm, (str: string, hash: string) => { + const fullLink = 'gdoc:' + props.id + '#' + hash; + if (this.globalInvisibleBookmarks[fullLink]) { + const retVal = str.replace(``, ''); + if (retVal === '\n \n') { + return '\n'; + } + if (retVal === '\n\n') { + return '\n'; + } + if (retVal.endsWith(' \n')) { + return retVal.substring(0, retVal.length - 2) + '\n'; + } + if (retVal.startsWith('\n ')) { + return '\n' + retVal.substring(1); + } + if (retVal === ' ') { + return ''; + } + return retVal; + } + return str; + }); + } + + newContent = newContent.replace(/(gdoc:[A-Z0-9_-]+)(#[^'")\s]*)?/ig, (str: string) => { + let fileId = str.substring('gdoc:'.length).replace(/#.*/, ''); + let hash = getUrlHash(str) || ''; + if (hash && this.globalHeadersMap[str]) { + const idx = this.globalHeadersMap[str].indexOf('#'); + if (idx >= 0) { + fileId = this.globalHeadersMap[str].substring('gdoc:'.length, idx); + hash = this.globalHeadersMap[str].substring(idx); + } + } const lastLog = this.localLog.findLastFile(fileId); if (lastLog && lastLog.event !== 'removed') { if (fileName.endsWith('.svg')) { return convertToRelativeSvgPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName); } else { - return convertToRelativeMarkDownPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName); + return convertToRelativeMarkDownPath(lastLog.filePath, destinationDirectory.getVirtualPath() + fileName) + hash; } } else { - return 'https://drive.google.com/open?id=' + fileId; + return 'https://drive.google.com/open?id=' + fileId + hash.replace('#_', '#heading=h.'); } }); if (content !== newContent) { @@ -517,4 +577,8 @@ export class TransformContainer extends Container { onProgressNotify(callback: ({total, completed, warnings, failed}: { total?: number; completed?: number, warnings?: number, failed?: number }) => void) { this.progressNotifyCallback = callback; } + + setUseGoogleMarkdowns(value: boolean) { + this.useGoogleMarkdowns = value; + } } diff --git a/src/containers/transform/frontmatters/generateDirectoryYaml.ts b/src/containers/transform/frontmatters/generateDirectoryYaml.ts index a98db50c..ab30d783 100644 --- a/src/containers/transform/frontmatters/generateDirectoryYaml.ts +++ b/src/containers/transform/frontmatters/generateDirectoryYaml.ts @@ -1,8 +1,8 @@ import yaml from 'js-yaml'; -import {FRONTMATTER_DUMP_OPTS} from './frontmatter'; -import {GoogleFile, MimeTypes} from '../../../model/GoogleFile'; -import {LocalFile} from '../../../model/LocalFile'; +import {FRONTMATTER_DUMP_OPTS} from './frontmatter.ts'; +import {GoogleFile, MimeTypes} from '../../../model/GoogleFile.ts'; +import {LocalFile} from '../../../model/LocalFile.ts'; export function generateDirectoryYaml(fileName: string, directory: GoogleFile, realFileNameToGenerated: { [realFileName: string]: LocalFile }) { return yaml.dump({ diff --git a/src/containers/transform/frontmatters/generateDocumentFrontMatter.ts b/src/containers/transform/frontmatters/generateDocumentFrontMatter.ts index 7de0500f..066f48c4 100644 --- a/src/containers/transform/frontmatters/generateDocumentFrontMatter.ts +++ b/src/containers/transform/frontmatters/generateDocumentFrontMatter.ts @@ -1,7 +1,7 @@ import yaml from 'js-yaml'; -import {MdFile} from '../../../model/LocalFile'; -import {FRONTMATTER_DUMP_OPTS} from './frontmatter'; +import {MdFile} from '../../../model/LocalFile.ts'; +import {FRONTMATTER_DUMP_OPTS} from './frontmatter.ts'; export function generateDocumentFrontMatter(localFile: MdFile, links: string[], fm_without_version = false) { diff --git a/src/git/GitScanner.ts b/src/git/GitScanner.ts index cefef44d..af8655a7 100644 --- a/src/git/GitScanner.ts +++ b/src/git/GitScanner.ts @@ -48,6 +48,13 @@ export class GitScanner { let [ stdout, stderr ] = [ null, null ]; + if (!opts.env) { + opts.env = {}; + } + if (!opts.env['HOME']) { + opts.env['HOME'] = process.env.HOME; + } + try { await new Promise((resolve, reject) => { exec(command, { cwd: this.rootPath, env: opts.env, maxBuffer: 1024 * 1024 }, (error, stdoutResult, stderrResult) => { @@ -188,7 +195,13 @@ export class GitScanner { const chunk = removedFiles.splice(0, 400); const rmParam = chunk.map(fileName => `"${sanitize(fileName)}"`).join(' '); if (rmParam) { - await this.exec(`git rm -r ${rmParam}`); + try { + await this.exec(`git rm -r ${rmParam}`); + } catch (err) { + if (err.message.indexOf('did not match any files') === -1) { + throw err; + } + } } } @@ -208,11 +221,20 @@ export class GitScanner { async pullBranch(remoteBranch: string, sshParams?: SshParams) { if (!remoteBranch) { - remoteBranch = 'master'; + remoteBranch = 'main'; } - await this.exec(`git pull --autostash --rebase origin ${remoteBranch}:master`, { + const committer = { + name: 'WikiGDrive', + email: this.email + }; + + await this.exec(`git pull --rebase origin ${remoteBranch}`, { env: { + GIT_AUTHOR_NAME: committer.name, + GIT_AUTHOR_EMAIL: committer.email, + GIT_COMMITTER_NAME: committer.name, + GIT_COMMITTER_EMAIL: committer.email, GIT_SSH_COMMAND: sshParams?.privateKeyFile ? `ssh -i ${sanitize(sshParams.privateKeyFile)} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes` : undefined } }); @@ -230,12 +252,12 @@ export class GitScanner { await this.exec(`git clone ${this.rootPath} ${dir}`, { skipLogger: true }); } - async pushBranch(remoteBranch: string, sshParams?: SshParams, localBranch = 'master') { + async pushBranch(remoteBranch: string, sshParams?: SshParams, localBranch = 'main') { if (!remoteBranch) { - remoteBranch = 'master'; + remoteBranch = 'main'; } - if (localBranch !== 'master') { + if (localBranch !== 'main') { await this.exec(`git push --force origin ${localBranch}:${remoteBranch}`, { env: { GIT_SSH_COMMAND: sshParams?.privateKeyFile ? `ssh -i ${sanitize(sshParams.privateKeyFile)} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes` : undefined @@ -250,7 +272,7 @@ export class GitScanner { }; try { - await this.exec(`git push origin master:${remoteBranch}`, { + await this.exec(`git push origin main:${remoteBranch}`, { env: { GIT_SSH_COMMAND: sshParams?.privateKeyFile ? `ssh -i ${sanitize(sshParams.privateKeyFile)} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes` : undefined } @@ -281,7 +303,7 @@ export class GitScanner { throw err; } - await this.exec(`git push origin master:${remoteBranch}`, { + await this.exec(`git push origin main:${remoteBranch}`, { env: { GIT_SSH_COMMAND: sshParams?.privateKeyFile ? `ssh -i ${sanitize(sshParams.privateKeyFile)} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes` : undefined } @@ -294,7 +316,10 @@ export class GitScanner { } async resetToLocal(sshParams?: SshParams) { - await this.exec('git checkout master --force', {}); + await this.exec('git checkout main --force', {}); + try { + await this.exec('git rebase --abort', {}); + } catch (ignoredError) { /* empty */ } await this.exec('git reset --hard HEAD', { env: { GIT_SSH_COMMAND: sshParams?.privateKeyFile ? `ssh -i ${sanitize(sshParams.privateKeyFile)} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes` : undefined @@ -305,7 +330,7 @@ export class GitScanner { async resetToRemote(remoteBranch: string, sshParams?: SshParams) { if (!remoteBranch) { - remoteBranch = 'master'; + remoteBranch = 'main'; } await this.exec('git fetch origin', { @@ -600,10 +625,14 @@ export class GitScanner { } if (!await this.isRepo()) { - await this.exec('git init -b master', { skipLogger: true }); + await this.exec('git init -b main', { skipLogger: true }); } } + async setSafeDirectory() { + await this.exec('git config --global --add safe.directory ' + this.rootPath); + } + async getBranchCommit(branch: string): Promise { try { const res = await this.exec(`git rev-parse ${branch}`, { skipLogger: true }); @@ -881,7 +910,7 @@ export class GitScanner { } async cmd(cmd: string) { - if (!['status', 'remote -v'].includes(cmd)) { + if (!['status', 'remote -v', 'ls-files --stage'].includes(cmd)) { throw new Error('Forbidden command'); } @@ -889,4 +918,8 @@ export class GitScanner { return { stdout: result.stdout, stderr: result.stderr }; } + + async removeCached(filePath: string) { + await this.exec(`git rm --cached ${filePath}`); + } } diff --git a/src/google/GoogleDriveService.ts b/src/google/GoogleDriveService.ts index 951de7d6..762022ea 100644 --- a/src/google/GoogleDriveService.ts +++ b/src/google/GoogleDriveService.ts @@ -4,12 +4,15 @@ import {Logger} from 'winston'; import {Writable} from 'stream'; -import {GoogleFile, MimeToExt, MimeTypes, SimpleFile} from '../model/GoogleFile'; -import {Drive, Permission} from '../containers/folder_registry/FolderRegistryContainer'; -import {FileId} from '../model/model'; -import {driveFetch, driveFetchMultipart, driveFetchStream} from './driveFetch'; -import {QuotaLimiter} from './QuotaLimiter'; -import {HasAccessToken} from './AuthClient'; +import crypto from 'crypto'; + +import {GoogleFile, MimeToExt, MimeTypes, SimpleFile} from '../model/GoogleFile.ts'; +import {FileId} from '../model/model.ts'; +import {Drive, Permission} from '../containers/folder_registry/FolderRegistryContainer.ts'; +import {driveFetch, driveFetchMultipart, driveFetchStream} from './driveFetch.ts'; +import {QuotaLimiter} from './QuotaLimiter.ts'; +import {HasAccessToken} from './AuthClient.ts'; +import {StopWatch} from '../utils/StopWatch.ts'; export interface Changes { token: string; @@ -45,6 +48,40 @@ export class GoogleDriveService { constructor(private logger: Logger, private quotaLimiter: QuotaLimiter) { } + async setupWatchChannel(auth: HasAccessToken, startPageToken: string, driveId: string) { + // This API does not work as intended, no webhook is executed on change + + const hexstring = crypto.randomBytes(16).toString('hex'); + const uuid = hexstring.substring(0,8) + '-' + hexstring.substring(8,12) + '-' + hexstring.substring(12,16) + '-' + hexstring.substring(16,20) + '-' + hexstring.substring(20); + + const params = { + pageToken: startPageToken, + supportsAllDrives: true, + includeItemsFromAllDrives: true, + // fields: '*', // file(id, name, mimeType, modifiedTime, size, md5Checksum, lastModifyingUser, parents, version, exportLinks, trashed)', + includeRemoved: true, + driveId: driveId ? driveId : undefined + }; + + const body = { + id: uuid, + type: 'web_hook', + address: process.env.DOMAIN + '/webhook', // Your receiving URL. + expiration: String(Date.now() + 10 * 60 * 1000) + }; + + await driveFetch(this.quotaLimiter, await auth.getAccessToken(), 'POST', 'https://www.googleapis.com/drive/v3/changes/watch', params, body); +/* + { + kind: 'api#channel', + id: '1873c104-3f34-07e2-086e-c97e8c23cb55', + resourceId: 'VZoPsZrgUX6TNl0BxbV2rN_zUIU', + resourceUri: 'https://www.googleapis.com/drive/v3/changes?alt=json&driveId=0AI7ud-sa0EAJUk9PVA&fields=*&includeItemsFromAllDrives=true&includeRemoved=true&pageToken=78&supportsAllDrives=true', + expiration: '1727025863000' + } +*/ + } + async getStartTrackToken(auth: HasAccessToken, driveId?: string): Promise { const params = { supportsAllDrives: true, @@ -59,23 +96,6 @@ export class GoogleDriveService { return res.startPageToken; } - async subscribeWatch(auth: HasAccessToken, pageToken: string, driveId?: string) { - try { - const params = { - pageToken, - supportsAllDrives: true, - includeItemsFromAllDrives: true, - fields: 'newStartPageToken, nextPageToken, changes( file(id, name, mimeType, modifiedTime, size, md5Checksum, lastModifyingUser, parents, version, exportLinks, trashed), removed)', - includeRemoved: true, - driveId: driveId ? driveId : undefined - }; - return await driveFetch(this.quotaLimiter, await auth.getAccessToken(), 'POST', 'https://www.googleapis.com/drive/v3/changes/watch', params); - } catch (err) { - err.message = 'Error watching: ' + err.message; - throw err; - } - } - async watchChanges(auth: HasAccessToken, pageToken: string, driveId?: string): Promise { try { const params = { @@ -103,7 +123,7 @@ export class GoogleDriveService { files: files }; } catch (err) { - err.message = 'Error watching changes: ' + err.message; + err.message = `Error [${err.status}] watching changes: ${err.message} on drive ${driveId}`; throw err; } } @@ -196,9 +216,10 @@ export class GoogleDriveService { // includeItemsFromAllDrives: true, // supportsAllDrives: true }; + const stopWatch = new StopWatch(); const res = await driveFetchStream(this.quotaLimiter, await auth.getAccessToken(), 'GET', `https://www.googleapis.com/drive/v3/files/${file.id}/export`, params); await res.pipeTo(Writable.toWeb(dest)); - this.logger.info('Exported document: ' + file.id + ext + ' [' + file.name + ']'); + this.logger.info('Exported document: ' + file.id + ext + ' [' + file.name + '] ' + stopWatch.toString()); } catch (err) { if (!err.isQuotaError && err?.code != 404) { this.logger.error(err.stack ? err.stack : err.message); diff --git a/src/google/driveFetch.ts b/src/google/driveFetch.ts index 1184d966..23a7b288 100644 --- a/src/google/driveFetch.ts +++ b/src/google/driveFetch.ts @@ -1,20 +1,7 @@ -import {Readable} from 'stream'; -import {SimpleFile} from '../model/GoogleFile'; import opentelemetry from '@opentelemetry/api'; -import {QuotaLimiter} from './QuotaLimiter'; -import {instrumentFunction} from '../telemetry'; - -async function handleReadable(obj): Promise { - if (obj instanceof Readable) { - const chunks = []; - for await (const chunk of obj) { - chunks.push(chunk); - } - return Buffer.concat(chunks).toString(); - } else { - return obj; - } -} +import {SimpleFile} from '../model/GoogleFile.ts'; +import {QuotaLimiter} from './QuotaLimiter.ts'; +import {instrumentFunction} from '../telemetry.ts'; interface GoogleDriveServiceErrorParams { isQuotaError: boolean; @@ -30,7 +17,7 @@ export class GoogleDriveServiceError extends Error { private folderId: string; private isQuotaError: boolean; - constructor(msg, params?: GoogleDriveServiceErrorParams) { + constructor(msg: string, params?: GoogleDriveServiceErrorParams) { super(msg); if (params) { this.status = params.status; @@ -85,7 +72,7 @@ function jsonToErrorMessage(json): string { } } -export function filterParams(params) { +export function filterParams(params: Record): Record { const retVal = {}; for (const key of Object.keys(params)) { @@ -129,9 +116,9 @@ export async function convertResponseToError(response) { }); } -async function driveRequest(quotaLimiter: QuotaLimiter, accessToken: string, method, requestUrl, params, body?): Promise { - params = filterParams(params); - const url = requestUrl + '?' + new URLSearchParams(params).toString(); +async function driveRequest(quotaLimiter: QuotaLimiter, accessToken: string, method: string, requestUrl: string, params: Record, body?: unknown): Promise { + const filteredParams = filterParams(params); + const url = requestUrl + '?' + new URLSearchParams(filteredParams).toString(); let traceparent; if (process.env.ZIPKIN_URL) { @@ -147,6 +134,7 @@ async function driveRequest(quotaLimiter: QuotaLimiter, accessToken: string, met method, headers: { Authorization: 'Bearer ' + accessToken, + 'Accept-Encoding': 'gzip', 'Content-type': body ? 'application/json' : undefined, traceparent }, @@ -196,7 +184,7 @@ async function driveRequest(quotaLimiter: QuotaLimiter, accessToken: string, met }); } -export async function driveFetch(quotaLimiter: QuotaLimiter, accessToken: string, method, url, params, bodyReq?) { +export async function driveFetch(quotaLimiter: QuotaLimiter, accessToken: string, method: string, url: string, params: Record, bodyReq?: unknown) { const response = await driveRequest(quotaLimiter, accessToken, method, url, params, bodyReq); let bodyResp = ''; try { @@ -207,18 +195,18 @@ export async function driveFetch(quotaLimiter: QuotaLimiter, accessToken: string } } -export async function driveFetchStream(quotaLimiter: QuotaLimiter, accessToken: string, method, url, params): Promise> { +export async function driveFetchStream(quotaLimiter: QuotaLimiter, accessToken: string, method: string, url: string, params: Record): Promise> { const response = await driveRequest(quotaLimiter, accessToken, method, url, params); return response.body; } const boundary = '-------314159265358979323846'; -export async function driveFetchMultipart(quotaLimiter: QuotaLimiter, accessToken: string, method, requestUrl, params, formData: FormData): Promise { - params = filterParams(params); - const url = requestUrl + '?' + new URLSearchParams(params).toString(); +export async function driveFetchMultipart(quotaLimiter: QuotaLimiter, accessToken: string, method: string, requestUrl: string, params: Record, formData: FormData): Promise { + const filteredParams = filterParams(params); + const url = requestUrl + '?' + new URLSearchParams(filteredParams).toString(); - let traceparent; + let traceparent: string; if (process.env.ZIPKIN_URL) { const span = opentelemetry.trace.getActiveSpan(); if (span) { @@ -227,7 +215,7 @@ export async function driveFetchMultipart(quotaLimiter: QuotaLimiter, accessToke } const after = `\n--${boundary}--`; - function generateMultipart(image: ArrayBuffer, mimetype) { + function generateMultipart(image: ArrayBuffer, mimetype: string) { const source = new Uint8Array(image); // Wrap in view to get data const before = [ diff --git a/src/odt/LibreOffice.ts b/src/odt/LibreOffice.ts index b38243c7..f071589e 100644 --- a/src/odt/LibreOffice.ts +++ b/src/odt/LibreOffice.ts @@ -76,7 +76,8 @@ export class TextLink implements ParagraphSection { @XmlElement() @XmlAttribute('text:name', 'name') -export class TextBookmark { +export class TextBookmark implements ParagraphSection { + type = 'bookmark'; name: string; } @@ -200,7 +201,7 @@ export class TextChangeEnd { @XmlElement() @XmlText('list', {isArray: true}) @XmlAttribute('text:style-name', 'styleName') -@XmlElementChild('text:bookmark', 'bookmark', 'TextBookmark') +@XmlElementChild('text:bookmark', 'list', 'TextBookmark', {isArray: true}) @XmlElementChild('text:a', 'list', 'TextLink', {isArray: true}) @XmlElementChild('text:span', 'list', 'TextSpan', {isArray: true}) @XmlElementChild('draw:rect', 'list', 'DrawRect', {isArray: true}) @@ -215,8 +216,7 @@ export class TextChangeEnd { @XmlElementChild('text:change-end', 'list', 'TextChangeEnd', {isArray: true}) export class TextParagraph implements TextSection { type = 'paragraph'; - bookmark: TextBookmark; - list: Array = []; + list: Array = []; annotations: OfficeAnnotation[] = []; styleName: string; } diff --git a/src/odt/MarkdownNodes.ts b/src/odt/MarkdownNodes.ts index fe8679b1..557ecf9f 100644 --- a/src/odt/MarkdownNodes.ts +++ b/src/odt/MarkdownNodes.ts @@ -16,7 +16,7 @@ export type TAG = 'BODY' | 'HR/' | 'B' | 'I' | 'BI' | 'BLANK/' | // | '/B' | '/I 'EMB_SVG' | 'EMB_SVG_G' | 'EMB_SVG_P/' | 'EMB_SVG_TEXT' | // | '/EMB_SVG' | '/EMB_SVG_G' | '/EMB_SVG_TEXT' 'EMB_SVG_TSPAN' | // | '/EMB_SVG_TSPAN' 'MATHML' | - 'CHANGE_START' | 'CHANGE_END' | 'RAW_MODE/' | 'HTML_MODE/' | 'MD_MODE/' | 'MACRO_MODE/' | 'COMMENT'; + 'CHANGE_START' | 'CHANGE_END' | 'RAW_MODE/' | 'HTML_MODE/' | 'MD_MODE/' | 'MACRO_MODE/' | 'COMMENT' | 'BOOKMARK/'; export interface TagPayload { lang?: string; @@ -34,7 +34,6 @@ export interface TagPayload { listStyle?: ListStyle; continueNumbering?: boolean; listLevel?: number; - bookmarkName?: string; pathD?: string; x?: number; diff --git a/src/odt/OdtToMarkdown.ts b/src/odt/OdtToMarkdown.ts index 4c2c633e..7348fc73 100644 --- a/src/odt/OdtToMarkdown.ts +++ b/src/odt/OdtToMarkdown.ts @@ -10,7 +10,7 @@ import { TableCell, TableOfContent, TableRow, - TableTable, + TableTable, TextBookmark, TextLink, TextList, TextParagraph, @@ -18,7 +18,7 @@ import { TextSpace, TextSpan } from './LibreOffice.ts'; -import {urlToFolderId} from '../utils/idParsers.ts'; +import {getUrlHash, urlToFolderId} from '../utils/idParsers.ts'; import {MarkdownNodes, MarkdownTagNode} from './MarkdownNodes.ts'; import {inchesToPixels, inchesToSpaces, spaces} from './utils.ts'; import {extractPath} from './extractPath.ts'; @@ -68,6 +68,8 @@ export class OdtToMarkdown { private picturesDir = ''; private picturesDirAbsolute = ''; private rewriteRules: RewriteRule[] = []; + private headersMap: { [p: string]: string } = {}; + private invisibleBookmarks: { [p: string]: number } = {}; constructor(private document: DocumentContent, private documentStyles: DocumentStyles, private fileNameMap: StringToStringMap = {}, private xmlMap: StringToStringMap = {}) { } @@ -131,7 +133,9 @@ export class OdtToMarkdown { // text = this.processMacros(text); // text = this.fixBlockMacros(text); - await postProcess(this.chunks, this.rewriteRules); + const { headersMap, invisibleBookmarks } = await postProcess(this.chunks, this.rewriteRules); + this.headersMap = headersMap; + this.invisibleBookmarks = invisibleBookmarks; const markdown = this.chunks.toString(); return this.trimBreaks(markdown); @@ -223,15 +227,16 @@ export class OdtToMarkdown { addLink(href: string) { if (href && !href.startsWith('#') && href.indexOf(':') > -1) { - this.links.add(href); + this.links.add(href.replace(/#.*$/, '')); } } async linkToText(currentTagNode: MarkdownTagNode, link: TextLink): Promise { let href = link.href; const id = urlToFolderId(href); + const hash = getUrlHash(link.href); if (id) { - href = 'gdoc:' + id; + href = 'gdoc:' + id + hash; } this.addLink(href); @@ -458,38 +463,36 @@ export class OdtToMarkdown { return false; } - async paragraphToText(currentTagNode: MarkdownTagNode, paragraph: TextParagraph): Promise { const style = this.getStyle(paragraph.styleName); const listStyle = this.getListStyle(style.listStyleName); - const bookmarkName = paragraph.bookmark?.name || null; if (this.hasStyle(paragraph, 'Heading_20_1')) { - const header = this.chunks.createNode('H1', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle, bookmarkName }); + const header = this.chunks.createNode('H1', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle }); this.chunks.append(currentTagNode, header); currentTagNode = header; } else if (this.hasStyle(paragraph, 'Heading_20_2')) { - const header = this.chunks.createNode('H2', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle, bookmarkName }); + const header = this.chunks.createNode('H2', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle }); this.chunks.append(currentTagNode, header); currentTagNode = header; } else if (this.hasStyle(paragraph, 'Heading_20_3')) { - const header = this.chunks.createNode('H3', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle, bookmarkName }); + const header = this.chunks.createNode('H3', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle }); this.chunks.append(currentTagNode, header); currentTagNode = header; } else if (this.hasStyle(paragraph, 'Heading_20_4')) { - const header = this.chunks.createNode('H4', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle, bookmarkName }); + const header = this.chunks.createNode('H4', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle }); this.chunks.append(currentTagNode, header); currentTagNode = header; } else if (this.isCourier(paragraph.styleName)) { - const block = this.chunks.createNode('PRE', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle, bookmarkName }); + const block = this.chunks.createNode('PRE', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle }); this.chunks.append(currentTagNode, block); currentTagNode = block; } else { - const block = this.chunks.createNode('P', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle, bookmarkName }); + const block = this.chunks.createNode('P', { marginLeft: inchesToSpaces(style.paragraphProperties?.marginLeft), style, listStyle }); this.chunks.append(currentTagNode, block); currentTagNode = block; } @@ -596,6 +599,12 @@ export class OdtToMarkdown { case 'change_end': this.chunks.append(currentTagNode, this.chunks.createNode('CHANGE_END')); break; + case 'bookmark': + { + const bookmark = child; + this.chunks.append(currentTagNode, this.chunks.createNode('BOOKMARK/', { id: bookmark.name })); + } + break; } } } @@ -696,4 +705,12 @@ export class OdtToMarkdown { this.errors.push(error); } + getHeadersMap() { + return this.headersMap; + } + + getInvisibleBookmarks() { + return this.invisibleBookmarks; + } + } diff --git a/src/odt/executeOdtToMarkdown.ts b/src/odt/executeOdtToMarkdown.ts index 2d6c71c4..7139df1b 100644 --- a/src/odt/executeOdtToMarkdown.ts +++ b/src/odt/executeOdtToMarkdown.ts @@ -42,5 +42,8 @@ export async function executeOdtToMarkdown(workerData) { fs.writeFileSync(path.join(workerData.destinationPath, workerData.realFileName.replace(/.md$/, '.debug.xml')), markdown); } - return { links, frontMatter, markdown, errors }; + const headersMap = converter.getHeadersMap(); + const invisibleBookmarks = converter.getInvisibleBookmarks(); + + return { links, frontMatter, markdown, errors, headersMap, invisibleBookmarks }; } diff --git a/src/odt/markdownNodesUtils.ts b/src/odt/markdownNodesUtils.ts index 1fc3c8da..5c0d385b 100644 --- a/src/odt/markdownNodesUtils.ts +++ b/src/odt/markdownNodesUtils.ts @@ -221,6 +221,8 @@ function chunkToText(chunk: MarkdownNode, ctx: ToTextContext) { return addLiNumbers(chunk, ctx, chunksToText(chunk.children, { ...ctx, inListItem: true, parentLevel: chunk.payload.listLevel })); case 'TOC': return chunksToText(chunk.children, ctx); // TODO + case 'BOOKMARK/': + return ``; } return chunksToText(chunk.children, ctx); case 'html': @@ -296,6 +298,8 @@ function chunkToText(chunk: MarkdownNode, ctx: ToTextContext) { const fontSize = inchesToPixels(chunk.payload.style?.textProperties.fontSize); return `` + chunksToText(chunk.children, ctx) + '\n'; } + case 'BOOKMARK/': + return ``; } return chunksToText(chunk.children, ctx); default: @@ -320,7 +324,7 @@ export async function walkRecursiveAsync(node: MarkdownNode, callback: (node: Ma const subCtx = await callback(node, ctx) || Object.assign({}, ctx); for (let nodeIdx = 0; nodeIdx < node.children.length; nodeIdx++) { const child = node.children[nodeIdx]; - const retVal = await walkRecursiveAsync(child, callback, { ...subCtx, nodeIdx }); + const retVal = await walkRecursiveAsync(child, callback, { ...subCtx, nodeIdx }, callbackEnd); if (retVal && 'nodeIdx' in retVal && typeof retVal.nodeIdx === 'number') { nodeIdx = retVal.nodeIdx; } diff --git a/src/odt/postprocess/convertCodeBlockParagraphs.ts b/src/odt/postprocess/convertCodeBlockParagraphs.ts index f67b42ac..b4e98603 100644 --- a/src/odt/postprocess/convertCodeBlockParagraphs.ts +++ b/src/odt/postprocess/convertCodeBlockParagraphs.ts @@ -27,7 +27,6 @@ export function convertCodeBlockParagraphs(markdownChunks: MarkdownNodes) { for (let nodeIdx2 = ctx.nodeIdx + 1; nodeIdx2 < node.parent.children.length; nodeIdx2++) { const node2 = node.parent.children[nodeIdx2]; if (isCodeBlockPara(node2, CODEBLOCK_END)) { - console.log('hhh', ctx.nodeIdx, nodeIdx2); const inner = node.parent.children.splice(ctx.nodeIdx + 1, nodeIdx2 - (ctx.nodeIdx + 1)); const toInsert = inner.map((part, idx) => { diff --git a/src/odt/postprocess/postProcess.ts b/src/odt/postprocess/postProcess.ts index 6b6d97ea..e184982d 100644 --- a/src/odt/postprocess/postProcess.ts +++ b/src/odt/postprocess/postProcess.ts @@ -39,12 +39,13 @@ export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRu convertCodeBlockParagraphs(chunks); convertMathMl(chunks); + trimParagraphs(chunks); + const { headersMap, invisibleBookmarks} = await rewriteHeaders(chunks); trimParagraphs(chunks); addEmptyLinesAfterParas(chunks); removeTdParas(chunks); // Requires: addEmptyLinesAfterParas mergeTexts(chunks); - await rewriteHeaders(chunks); mergeParagraphs(chunks); unwrapEmptyPre(chunks); @@ -66,4 +67,6 @@ export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRu if (process.env.DEBUG_COLORS) { dump(chunks.body); } + + return { headersMap, invisibleBookmarks }; } diff --git a/src/odt/postprocess/rewriteHeaders.ts b/src/odt/postprocess/rewriteHeaders.ts index 0a5ba35d..352ac0a7 100644 --- a/src/odt/postprocess/rewriteHeaders.ts +++ b/src/odt/postprocess/rewriteHeaders.ts @@ -1,27 +1,87 @@ import slugify from 'slugify'; import {extractText, walkRecursiveAsync, walkRecursiveSync} from '../markdownNodesUtils.ts'; -import {MarkdownNodes} from '../MarkdownNodes.ts'; +import {MarkdownNodes, MarkdownTextNode} from '../MarkdownNodes.ts'; -export async function rewriteHeaders(markdownChunks: MarkdownNodes) { +export async function rewriteHeaders(markdownChunks: MarkdownNodes): Promise<{ headersMap: {[key: string]: string}, invisibleBookmarks: {[key: string]: number} }> { const headersMap = {}; + const invisibleBookmarks = {}; - await walkRecursiveAsync(markdownChunks.body, async (chunk) => { - if (chunk.isTag === true && ['H1', 'H2', 'H3', 'H4'].includes(chunk.tag)) { // && 'md' === this.currentMode) { - if (chunk.payload.bookmarkName) { - const innerTxt = extractText(chunk); - const slug = slugify(innerTxt.trim(), { replacement: '-', lower: true, remove: /[#*+~.()'"!:@]/g }); - if (slug) { - headersMap['#' + chunk.payload.bookmarkName] = '#' + slug; + let inPre = false; + await walkRecursiveAsync(markdownChunks.body, async (chunk, ctx: { nodeIdx: number }) => { + if (chunk.isTag && 'PRE' === chunk.tag) { + inPre = true; + } + + if (inPre) { + if (chunk.isTag) { + for (let j = chunk.children.length - 1; j >= 0; j--) { + const child = chunk.children[j]; + if (child.isTag && child.tag === 'BOOKMARK/') { + chunk.children.splice(j, 1); + break; + } + } + } + } + + if (chunk.isTag && ['H1', 'H2', 'H3', 'H4'].includes(chunk.tag)) { + const innerTxt = extractText(chunk); + const slug = slugify(innerTxt.trim(), { replacement: '-', lower: true, remove: /[#*+~.,^()'"!:@]/g }); + + if (chunk.children.length === 1) { + const child = chunk.children[0]; + if (child.isTag && child.tag === 'BOOKMARK/') { + chunk.parent.children.splice(ctx.nodeIdx, 1, child); + return; + } + } + for (let j = 0; j < chunk.children.length - 1; j++) { + const child = chunk.children[j]; + if (child.isTag && child.tag === 'BOOKMARK/') { + const toMove = chunk.children.splice(j, 1); + if (slug && !headersMap['#' + child.payload.id]) { + headersMap['#' + child.payload.id] = '#' + slug; + } else { + chunk.children.splice(chunk.children.length, 0, ...toMove); + } + break; } } } + }, {}, async (chunk) => { + if (chunk.isTag && 'PRE' === chunk.tag) { + inPre = false; + } }); - walkRecursiveSync(markdownChunks.body, (chunk) => { - if (chunk.isTag === true && chunk.payload?.href) { - if (headersMap[chunk.payload.href]) { - chunk.payload.href = headersMap[chunk.payload.href]; + await walkRecursiveAsync(markdownChunks.body, async (chunk, ctx: { nodeIdx: number }) => { + if (chunk.isTag && 'BOOKMARK/' === chunk.tag) { + if (!headersMap['#' + chunk.payload.id]) { + invisibleBookmarks['#' + chunk.payload.id] = 1; + } + + if (chunk.parent.children.length < 2) { + return; } + const space: MarkdownTextNode = { + isTag: false, + text: ' ', + comment: 'rewriteHeaders.ts: space before anchor' + }; + chunk.parent.children.splice(ctx.nodeIdx, 0, space); + return { nodeIdx: ctx.nodeIdx + 1 }; } }); + + if (Object.keys(headersMap).length > 0) { + walkRecursiveSync(markdownChunks.body, (chunk) => { + if (chunk.isTag === true && chunk.payload?.href) { + if (headersMap[chunk.payload.href]) { + chunk.payload.href = headersMap[chunk.payload.href]; + } + } + }); + } + + return { headersMap, invisibleBookmarks }; } diff --git a/src/odt/postprocess/trimParagraphs.ts b/src/odt/postprocess/trimParagraphs.ts index 31621e04..0e88d0ec 100644 --- a/src/odt/postprocess/trimParagraphs.ts +++ b/src/odt/postprocess/trimParagraphs.ts @@ -26,7 +26,7 @@ export function trimParagraphs(markdownChunks: MarkdownNodes) { while (chunk.children.length > 0) { const firstChunk = chunk.children[0]; if (firstChunk.isTag === false) { - let origText = firstChunk.text; + const origText = firstChunk.text; firstChunk.text = firstChunk.text.replace(/^\s+/, ''); if (firstChunk.text === '') { diff --git a/src/utils/StopWatch.ts b/src/utils/StopWatch.ts new file mode 100644 index 00000000..a48f38eb --- /dev/null +++ b/src/utils/StopWatch.ts @@ -0,0 +1,28 @@ +export class StopWatch { + private start: number; + private end: number; + + constructor() { + this.start = Date.now(); + } + + stop() { + this.end = Date.now(); + } + + toString(limitMs = 500) { + if (!this.end) { + this.stop(); + } + const delta = this.end - this.start; + if (delta > limitMs) { + if (delta > 1000) { + return `${Math.round(delta / 100) / 10}s`; + } + return `${delta}ms`; + } + + return ''; + } + +} diff --git a/src/utils/idParsers.ts b/src/utils/idParsers.ts index 9a7ab0f2..a58018db 100644 --- a/src/utils/idParsers.ts +++ b/src/utils/idParsers.ts @@ -71,3 +71,11 @@ export function urlToFolderId(url: string): string | null { return null; } + +export function getUrlHash(url: string): string { + const idx = url.indexOf('#'); + if (idx >= 0 && idx < url.length - 1) { + return url.substring(idx).replace('#heading=h.', '#_'); + } + return ''; +} diff --git a/src/wikigdrive.sh b/src/wikigdrive.sh index 448aa299..819bb4fe 100755 --- a/src/wikigdrive.sh +++ b/src/wikigdrive.sh @@ -9,16 +9,21 @@ cd $MAIN_DIR POSITIONAL_ARGS=() CMD="" -INSPECT="" +OPTS="" ORIG_ARGS=$@ while [[ $# -gt 0 ]]; do case $1 in - --inspect) - INSPECT="$1" + --inspect | watch) + OPTS="$OPTS $1" shift # past argument ;; + --watch-path) + OPTS="$OPTS $1 $2" + shift # past argument + shift # past value + ;; --link_mode | --workdir | --drive | --debug | --client_id | --client_secret | --service_account | --share_email | --server_port | --transform_subdir) POSITIONAL_ARGS+=("$1") # save positional arg1 POSITIONAL_ARGS+=("$2") # save positional arg2 @@ -43,8 +48,4 @@ if [[ ! -f "$MAIN_DIR/src/cli/wikigdrive-$CMD.ts" ]]; then CMD="usage" fi -if test "$INSPECT" = "--inspect"; then - /usr/bin/env node --inspect --no-warnings --enable-source-maps --experimental-specifier-resolution=node --loader ts-node/esm $MAIN_DIR/src/cli/wikigdrive-$CMD.ts $ORIG_ARGS -else - /usr/bin/env node --no-warnings --enable-source-maps --experimental-specifier-resolution=node --loader ts-node/esm $MAIN_DIR/src/cli/wikigdrive-$CMD.ts $ORIG_ARGS -fi +/usr/bin/env node $OPTS --no-warnings --enable-source-maps --experimental-specifier-resolution=node --loader ts-node/esm $MAIN_DIR/src/cli/wikigdrive-$CMD.ts $ORIG_ARGS diff --git a/test/git/GitTest.ts b/test/git/GitTest.ts index 82330383..298d7f21 100644 --- a/test/git/GitTest.ts +++ b/test/git/GitTest.ts @@ -295,9 +295,8 @@ describe('GitTest', function () { const scannerLocal = new GitScanner(logger, localRepoDir, COMMITER1.email); await scannerLocal.initialize(); - fs.writeFileSync(path.join(localRepoDir, 'file1.md'), 'Initial content'); - { + fs.writeFileSync(path.join(localRepoDir, 'file1.md'), 'Initial content'); const changes = await scannerLocal.changes(); assert.equal(2, (await scannerLocal.changes()).length); await scannerLocal.commit('First commit', changes.map(change => change.path), [], COMMITER1); @@ -313,13 +312,71 @@ describe('GitTest', function () { const scannerSecond = new GitScanner(logger, secondRepoDir, COMMITER2.email); await scannerSecond.initialize(); - fs.unlinkSync(secondRepoDir + '/.gitignore'); await scannerSecond.setRemoteUrl(githubRepoDir); - await scannerSecond.pullBranch('main'); + + // fs.mkdirSync(path.join(secondRepoDir, 'content')); + // fs.writeFileSync(path.join(secondRepoDir, '_errors.md'), '---\ntype: \'page\'\n---\nChange local'); + + await scannerSecond.resetToRemote('main'); + // await scannerSecond.pullBranch('main'); const history = await scannerSecond.history('/'); assert.equal(history.length, 1); assert.equal(history[0].author_name, 'John '); + + { + fs.writeFileSync(path.join(localRepoDir, 'file2.md'), 'Second change'); + const changes = await scannerLocal.changes(); + assert.equal(1, (await scannerLocal.changes()).length); + await scannerLocal.commit('Second commit', changes.map(change => change.path), [], COMMITER2); + } + + await scannerLocal.pushBranch('main'); + + { + // fs.writeFileSync(path.join(localRepoDir, '_errors1.md'), 'Change local'); + // const changes = await scannerLocal.changes(); + // assert.equal(1, (await scannerLocal.changes()).length); + // await scannerLocal.commit('Local commit', changes.map(change => change.path), [], COMMITER1); + } + + const fd = fs.openSync(path.join(secondRepoDir, '_errors.md'), 'w'); + // fs.writeSync(fd, ''); + + // fs.writeFileSync(path.join(secondRepoDir, '_errors.md'), ''); + // await scannerSecond.cmd('add -N _errors.md'); + // fs.chmodSync(path.join(secondRepoDir, '_errors.md'), 0o771); + + + await scannerSecond.autoCommit(); + + console.info(await scannerSecond.cmd('ls-files --stage')); + + fs.writeSync(fd, 'test'); + // fs.closeSync(fd); + + console.info(await scannerSecond.cmd('ls-files --stage')); + + // fs.unlinkSync(path.join(secondRepoDir, '_errors.md')); + // fs.writeFileSync(path.join(secondRepoDir, '_errors.md'), '---\ntype: \'page\'\n---\nChange local'); + + // await scannerSecond.autoCommit(); + + // fs.writeFileSync(path.join(secondRepoDir, 'untracked.md'), 'untracked'); + + console.info(await scannerSecond.cmd('ls-files --stage')); + + await scannerSecond.resetToLocal(); + await scannerSecond.pullBranch('main'); + + console.info(await scannerSecond.cmd('status')); + + { + const history = await scannerSecond.history('/'); + assert.equal(history.length, 2); + assert.equal(history[0].author_name, 'Bob '); + assert.equal(history[1].author_name, 'John '); + } } finally { fs.rmSync(localRepoDir, { recursive: true, force: true }); fs.rmSync(githubRepoDir, { recursive: true, force: true }); @@ -484,7 +541,7 @@ describe('GitTest', function () { await scannerLocal.commit('Third commit', ['file1.md'], [], COMMITER1); const headCommit = await scannerLocal.getBranchCommit('HEAD'); - const masterCommit = await scannerLocal.getBranchCommit('master'); + const masterCommit = await scannerLocal.getBranchCommit('main'); const remoteCommit = await scannerLocal.getBranchCommit('refs/remotes/origin/main'); const remoteCommit2 = await scannerSecond.getBranchCommit('HEAD'); diff --git a/test/git/RebaseTest.ts b/test/git/RebaseTest.ts index b83a5d0d..73346b21 100644 --- a/test/git/RebaseTest.ts +++ b/test/git/RebaseTest.ts @@ -1,11 +1,12 @@ -import {GitScanner} from '../../src/git/GitScanner'; -import winston from 'winston'; -import {createTmpDir} from '../utils'; import fs from 'fs'; import path from 'path'; -import {assert} from 'chai'; import {execSync} from 'child_process'; -import {instrumentLogger} from '../../src/utils/logger/logger'; +import winston from 'winston'; +import {assert} from 'chai'; + +import {GitScanner} from '../../src/git/GitScanner.ts'; +import {createTmpDir} from '../utils.ts'; +import {instrumentLogger} from '../../src/utils/logger/logger.ts'; const COMMITER1 = { name: 'John', email: 'john@example.tld' @@ -71,7 +72,7 @@ describe('RebaseTest', function () { } const headCommit = await scannerLocal.getBranchCommit('HEAD'); - const masterCommit = await scannerLocal.getBranchCommit('master'); + const masterCommit = await scannerLocal.getBranchCommit('main'); const remoteCommit = await scannerLocal.getBranchCommit('refs/remotes/origin/main'); assert.equal(headCommit, masterCommit); assert.equal(headCommit, remoteCommit); @@ -140,7 +141,7 @@ describe('RebaseTest', function () { } const headCommit = await scannerLocal.getBranchCommit('HEAD'); - const masterCommit = await scannerLocal.getBranchCommit('master'); + const masterCommit = await scannerLocal.getBranchCommit('main'); const remoteCommit = await scannerLocal.getBranchCommit('refs/remotes/origin/main'); assert.equal(headCommit, masterCommit); assert.equal(headCommit, remoteCommit); @@ -206,7 +207,7 @@ describe('RebaseTest', function () { } const headCommit = await scannerSecond.getBranchCommit('HEAD'); - const masterCommit = await scannerSecond.getBranchCommit('master'); + const masterCommit = await scannerSecond.getBranchCommit('main'); const remoteCommit = await scannerSecond.getBranchCommit('refs/remotes/origin/main'); assert.equal(headCommit, masterCommit); assert.equal(headCommit, remoteCommit); @@ -298,7 +299,7 @@ describe('RebaseTest', function () { } const headCommit = await scannerSecond.getBranchCommit('HEAD'); - const masterCommit = await scannerSecond.getBranchCommit('master'); + const masterCommit = await scannerSecond.getBranchCommit('main'); const remoteCommit = await scannerSecond.getBranchCommit('refs/remotes/origin/main'); assert.equal(headCommit, masterCommit); assert.equal(headCommit, remoteCommit); @@ -373,7 +374,7 @@ describe('RebaseTest', function () { { const headCommit = await scannerLocal.getBranchCommit('HEAD'); - const masterCommit = await scannerLocal.getBranchCommit('master'); + const masterCommit = await scannerLocal.getBranchCommit('main'); const remoteCommit = await scannerLocal.getBranchCommit('refs/remotes/origin/main'); assert.equal(headCommit, masterCommit); assert.equal(headCommit, remoteCommit); @@ -382,8 +383,6 @@ describe('RebaseTest', function () { { const history = await scannerLocal.history(''); - console.log('historyhistory', history); - assert.equal(2, history.length); assert.equal('Change on second repo', history[0].message); @@ -395,7 +394,7 @@ describe('RebaseTest', function () { } const headCommit = await scannerSecond.getBranchCommit('HEAD'); - const masterCommit = await scannerSecond.getBranchCommit('master'); + const masterCommit = await scannerSecond.getBranchCommit('main'); const remoteCommit = await scannerSecond.getBranchCommit('refs/remotes/origin/main'); assert.equal(headCommit, masterCommit); assert.equal(headCommit, remoteCommit); diff --git a/test/odt_md/confluence.md b/test/odt_md/confluence.md index 4b961fb1..22e93b1f 100644 --- a/test/odt_md/confluence.md +++ b/test/odt_md/confluence.md @@ -18,7 +18,7 @@ A new github repo with a node.js script specific to this conversion. * Import content from Confluence Page into Google Documents * Heading should be headings * Paragraphs should be paragraphs - * Images (which are attachments in Confluence) should be embedded in google docs + * Images (which are attachments in Confluence) should be embedded in google docs * Macros can be wrapped in {{% curlies %}} and dumped as text * Example Block Macro: {{% macroname propertyname='value' propertyname='value' %}} @@ -50,6 +50,8 @@ confluence2google 3. Use HTML 1. [Confluence Export](https://confluence.atlassian.com/confcloud/import-a-confluence-space-724765531.html) that makes an HTML file + + ## Examples Simple - [https://confluence.example.com/display/DOCS/Sample](https://confluence.example.com/display/DOCS/Sample) diff --git a/test/odt_md/header-link.md b/test/odt_md/header-link.md index 2dda8f47..cc5752fe 100644 --- a/test/odt_md/header-link.md +++ b/test/odt_md/header-link.md @@ -4,6 +4,8 @@ See [Node setup on the system](#node-setup-on-the-system) for prereq. [Example Google Drive Shared Folder](gdoc:0AIkOKXbzWCtSUk9PVA) + + # Node setup on the system ## using OS diff --git a/test/odt_md/project-overview.md b/test/odt_md/project-overview.md index 10736ba2..f63d7fbe 100644 --- a/test/odt_md/project-overview.md +++ b/test/odt_md/project-overview.md @@ -106,7 +106,7 @@ Options: wikigdrive keeps a local JSON config file in the dest directory with state from prior runs. The config contains a map of URL driveIds to the local filenames along with metadata about each file. -## Renames and Redirecting +## Renames and Redirecting When a Document is renamed or moved in the shared drive the driveId says the same, but its place in the filesystem changes. For example a document named "Carbon" would be created as Carbon.md. Sometime later its renamed to "Carbon Fiber" then a new file "Carbon Fiber.md" would be made with the content and the old "Carbon.md" is changed to: @@ -161,6 +161,8 @@ The contents of Carbon.md would show each of the conflicting references: * [Carbon](Carbon-2.md) + + ## Table of Contents and Index In the root of the local filesystem two files will be created: the toc.md and index.md @@ -181,7 +183,8 @@ The index is a listing of all of the defined terms and their references in the d ![](10000201000005480000004BB83F3F8B5F0C77BD.png) * Italics/bold in an unordered list: ([issue](https://github.com/mieweb/wikiGDrive/issues/16)) Italics are not being rendered if in a list item. We may need to find these and replace the */** with em/strong tags. Example is rendered in browser next to [Google Doc](gdoc:108WScoxxGKKKOsGWF7UNZ4rLRanGXu6BPdJ-axjVn5s). ![](1000020100000243000000F28AB7617254FDBB3A.png) - + + ## Images Two kinds of images exist within Google Docs: 1) Embedded images stored within the document and 2) images that are referenced to another "Drawing" on the google drive. WikiGDrive processes images by placing them in a folder named with a similar name to the page. (eg: index.md would result in a folder index.images with each embedded image in that folder). diff --git a/website/_index.md b/website/_index.md index 0a36593d..7c019dce 100644 --- a/website/_index.md +++ b/website/_index.md @@ -15,8 +15,8 @@ WikiGDrive is a node app that uses the [Google Drive API](https://developers.goo ![Diagram](docs/diagram.svg) [Google Drive Notes](https://docs.google.com/document/d/1H6vwfQXIexdg4ldfaoPUjhOZPnSkNn6h29WD6Fi-SBY/edit#) -| [Github Project](https://github.com/mieweb/wikiGDrive/projects) -| [Github Developer Notes](docs/developer-guide.md) +| [GitHub Project](https://github.com/mieweb/wikiGDrive/projects) +| [GitHub Developer Notes](docs/developer-guide.md) With a "Shared Drive" as the key, WikiGDrive: diff --git a/website/docs/_index.md b/website/docs/_index.md index 1e07b3a5..3110f8b4 100644 --- a/website/docs/_index.md +++ b/website/docs/_index.md @@ -1,5 +1,5 @@ --- -title: _index +title: Usage navWeight: 1000 # Upper weight gets higher precedence, optional. --- # wikiGDrive diff --git a/website/docs/developer-guide.md b/website/docs/developer-guide.md index 6e1556b1..b66cfa32 100644 --- a/website/docs/developer-guide.md +++ b/website/docs/developer-guide.md @@ -13,7 +13,7 @@ See [Node setup on the system](#node-setup-on-the-system) for prereq. ## using OS ``` -curl -sL https://deb.nodesource.com/setup_16.x | sudo bash - +curl -sL https://deb.nodesource.com/setup_20.x | sudo bash - sudo apt install nodejs ``` @@ -21,7 +21,7 @@ sudo apt install nodejs ``` sudo npm install -g n -sudo n 16.15.0 +sudo n 20.10.0 ``` ## Version Strategy @@ -44,23 +44,49 @@ wikigdrive --workdir ~/wikigdrive --service_account ~/workspaces/mieweb/wikigdri ## Running locally with docker ``` +export VOLUME_DATA=~/wikigdrive +export VOLUME_PREVIEW=~/wikigdrive_html + +# Create some dir for wikigdrive data +mkdir -p $VOLUME_DATA +# Create some dir for wikigdrive rendered html files +mkdir -p $VOLUME_PREVIEW + +# Running zipkin is an option docker run --name zipkin -d -p 9411:9411 --restart unless-stopped openzipkin/zipkin +# Build action runner docker build -t wgd-action-runner apps/wgd-action-runner -docker build -t wikigdrive . +# Build hugo docs +docker run \ + -v ~/workspaces/mieweb/wikiGDrive/hugo:/site \ + -v ~/workspaces/mieweb/wikiGDrive/website:/website \ + -v $VOLUME_PREVIEW/docs:/dist/hugo \ + --env CONFIG_TOML="/site/config/_default/config.toml" --env BASE_URL="https://localhost:3000" \ + wgd-action-runner /steps/step_render_hugo -docker run --rm --user=$(id -u) -it \ - -v /data/wikigdrive:/data \ - -v ~/workspaces/mieweb/wikigdrive-with-service-account.json:/service_account.json \ - -v ~/workspaces/mieweb/wikiGDrive:/usr/src/app \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --link zipkin:zipkin \ - --publish 127.0.0.1:3000:3000 \ - --publish 127.0.0.1:24678:24678 \ - wikigdrive \ - ./src/wikigdrived.sh --service_account /service_account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com --workdir /data server 3000 +# Build wikigdrive +docker build -t wikigdrive . +# Run wikigdrive +docker run --rm --user=$(id -u):$(getent group docker | cut -d: -f3) -it \ + -v $VOLUME_DATA:/srv/wikigdrive \ + -v $VOLUME_PREVIEW:$VOLUME_PREVIEW \ + -v $VOLUME_PREVIEW/docs:/usr/src/app/dist/hugo \ + -v ~/workspaces/mieweb/wikigdrive-with-service-account.json:/service_account.json \ + -v ~/workspaces/mieweb/wikiGDrive:/usr/src/app \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e VOLUME_DATA=$VOLUME_DATA \ + -e VOLUME_PREVIEW=$VOLUME_PREVIEW \ + --link zipkin:zipkin \ + --publish 127.0.0.1:3000:3000 \ + --publish 127.0.0.1:24678:24678 \ + --name wikigdrive-develop \ + wikigdrive \ + ./src/wikigdrive.sh --watch-path /usr/src/app/src --service_account /service_account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com --workdir /srv/wikigdrive server 3000 + +# Stop wikigdrive docker rm -f wikigdrive # 24678 - vite hot reload port @@ -148,7 +174,7 @@ ZIPKIN_URL=http://localhost:9411 ## Debugging ``` -./src/wikigdrived.sh --inspect --workdir ~/wikigdrive --service_account ~/workspaces/mieweb/wikigdrive-with-service-account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com server 3000 +./src/wikigdrive.sh --inspect --workdir ~/wikigdrive --service_account ~/workspaces/mieweb/wikigdrive-with-service-account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com server 3000 ``` Chrome diff --git a/website/docs/install.md b/website/docs/install.md index fd6b4b04..29abbcb3 100644 --- a/website/docs/install.md +++ b/website/docs/install.md @@ -17,8 +17,17 @@ npm i -g @mieweb/wikigdrive ## App setup +Across few years Google changed this process several times. + +Here are rough guidelines where to click. They will be probably changed :(. + 1. Go to [console](https://console.developers.google.com/) -2. Create New Project -3. Enable Apis -> add Google Drive API -4. Enable Apis -> Add Google Docs API -5. Credentials -> Create Credentials (OAuth Client ID) -> Other ( see authorization section ) +2. Under project download click ![New project](./install/new_project.png) +3. Add Google Drive API ![Enable Apis](./install/enable_api.png) +4. Go to ![Credentials](./install/create_credentials3.png) +5. Click ![Create Credentials](./install/create_credentials.png) +6. Select type of data to access ![](./install/credentials.png) +7. Create ![Service account](./install/service_account.png) +8. Download ![Service account key](./install/service_account_key.png) +8. Create ![Consent screen](./install/consent.png) +9. Create ![Oauth Client ID](./install/oauth.png) diff --git a/website/docs/install/consent.png b/website/docs/install/consent.png new file mode 100644 index 00000000..ef17460f Binary files /dev/null and b/website/docs/install/consent.png differ diff --git a/website/docs/install/create_credentials.png b/website/docs/install/create_credentials.png new file mode 100644 index 00000000..773b1bfa Binary files /dev/null and b/website/docs/install/create_credentials.png differ diff --git a/website/docs/install/create_credentials3.png b/website/docs/install/create_credentials3.png new file mode 100644 index 00000000..b34f085d Binary files /dev/null and b/website/docs/install/create_credentials3.png differ diff --git a/website/docs/install/credentials.png b/website/docs/install/credentials.png new file mode 100644 index 00000000..bfaa8d0d Binary files /dev/null and b/website/docs/install/credentials.png differ diff --git a/website/docs/install/enable_api.png b/website/docs/install/enable_api.png new file mode 100644 index 00000000..d45cc867 Binary files /dev/null and b/website/docs/install/enable_api.png differ diff --git a/website/docs/install/enable_api_2.png b/website/docs/install/enable_api_2.png new file mode 100644 index 00000000..a23b89e7 Binary files /dev/null and b/website/docs/install/enable_api_2.png differ diff --git a/website/docs/install/new_project.png b/website/docs/install/new_project.png new file mode 100644 index 00000000..b72d8b81 Binary files /dev/null and b/website/docs/install/new_project.png differ diff --git a/website/docs/install/oauth.png b/website/docs/install/oauth.png new file mode 100644 index 00000000..4ebcbd8d Binary files /dev/null and b/website/docs/install/oauth.png differ diff --git a/website/docs/install/service_account.png b/website/docs/install/service_account.png new file mode 100644 index 00000000..8b6d3799 Binary files /dev/null and b/website/docs/install/service_account.png differ diff --git a/website/docs/install/service_account_key.png b/website/docs/install/service_account_key.png new file mode 100644 index 00000000..07916dac Binary files /dev/null and b/website/docs/install/service_account_key.png differ diff --git a/website/ui.md b/website/ui.md new file mode 100644 index 00000000..a03497ab --- /dev/null +++ b/website/ui.md @@ -0,0 +1,3 @@ +--- +comment: This file is used to generate empty ui/index.html to perform SSR serving +---