From 170466542ce0c3fc3a99a73bbca8de570369ffe8 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Mon, 9 Oct 2023 21:39:08 +0200 Subject: [PATCH] Refactor cli commands --- .github/workflows/DevelopServerDeploy.yml | 2 +- .github/workflows/ProdServerDeploy.yml | 2 +- Dockerfile | 2 +- doc/_index.md | 21 +- doc/developer.md | 4 +- doc/internals.md | 1 - package-lock.json | 6 +- package.json | 10 +- src/ContainerEngine.ts | 4 + src/MainService.ts | 317 ------------------ src/cli.sh | 14 - src/cli/getAuthConfig.ts | 32 ++ src/cli/usage.ts | 49 +++ src/cli/wikigdrive-config.ts | 55 +++ src/cli/wikigdrive-drives.ts | 62 ++++ src/cli/wikigdrive-prune.ts | 90 +++++ src/cli/wikigdrive-pull.ts | 118 +++++++ src/cli/wikigdrive-register.ts | 93 +++++ src/cli/wikigdrive-server.ts | 195 +++++++++++ src/cli/wikigdrive-transform.ts | 102 ++++++ src/cli/wikigdrive-unregister.ts | 90 +++++ src/cli/wikigdrive-usage.ts | 7 + src/cli/wikigdrivectl-inspect.ts | 46 +++ src/cli/wikigdrivectl-ps.ts | 39 +++ src/cli/wikigdrivectl-usage.ts | 7 + .../FolderRegistryContainer.ts | 5 + src/containers/server/loadRunningInstance.ts | 1 + .../server/routes/ConfigController.ts | 2 +- src/main.ts | 116 ------- src/model/CliParams.ts | 4 - src/utils/logger/logger.ts | 4 +- src/wikigdrive.sh | 50 +++ src/wikigdrivectl.sh | 50 +++ 33 files changed, 1112 insertions(+), 488 deletions(-) delete mode 100644 src/MainService.ts delete mode 100755 src/cli.sh create mode 100644 src/cli/getAuthConfig.ts create mode 100644 src/cli/usage.ts create mode 100644 src/cli/wikigdrive-config.ts create mode 100644 src/cli/wikigdrive-drives.ts create mode 100644 src/cli/wikigdrive-prune.ts create mode 100644 src/cli/wikigdrive-pull.ts create mode 100644 src/cli/wikigdrive-register.ts create mode 100644 src/cli/wikigdrive-server.ts create mode 100644 src/cli/wikigdrive-transform.ts create mode 100644 src/cli/wikigdrive-unregister.ts create mode 100644 src/cli/wikigdrive-usage.ts create mode 100644 src/cli/wikigdrivectl-inspect.ts create mode 100644 src/cli/wikigdrivectl-ps.ts create mode 100644 src/cli/wikigdrivectl-usage.ts create mode 100755 src/wikigdrive.sh create mode 100755 src/wikigdrivectl.sh diff --git a/.github/workflows/DevelopServerDeploy.yml b/.github/workflows/DevelopServerDeploy.yml index 08bc73a3..c1695602 100644 --- a/.github/workflows/DevelopServerDeploy.yml +++ b/.github/workflows/DevelopServerDeploy.yml @@ -78,7 +78,7 @@ jobs: -e "ZIPKIN_SERVICE=wikigdrive-develop" \ --link=zipkin:zipkin \ --publish 127.0.0.1:4000:3000 \ - "wikigdrive-develop:${GITHUB_SHA}" wikigdrive-ts \ + "wikigdrive-develop:${GITHUB_SHA}" wikigdrive \ --service_account /service_account.json \ --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com \ --workdir /data \ diff --git a/.github/workflows/ProdServerDeploy.yml b/.github/workflows/ProdServerDeploy.yml index 8c1ccd94..f346d0f6 100644 --- a/.github/workflows/ProdServerDeploy.yml +++ b/.github/workflows/ProdServerDeploy.yml @@ -76,7 +76,7 @@ jobs: -e "GIT_SHA=${GITHUB_SHA}" \ --publish 127.0.0.1:3000:3000 \ --restart unless-stopped \ - "wikigdrive-prod:${GITHUB_SHA}" wikigdrive-ts \ + "wikigdrive-prod:${GITHUB_SHA}" wikigdrive \ --service_account /service_account.json \ --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com \ --workdir /data \ diff --git a/Dockerfile b/Dockerfile index fa84ce59..58d14957 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,4 +17,4 @@ RUN cd /usr/src/app/apps/ui && npm install && npm run build WORKDIR "/usr/src/app" -CMD [ "sh", "-c", "wikigdrive-ts --workdir /data server 3000" ] +CMD [ "sh", "-c", "wikigdrive --workdir /data server 3000" ] diff --git a/doc/_index.md b/doc/_index.md index e432b096..3a8dc247 100644 --- a/doc/_index.md +++ b/doc/_index.md @@ -62,8 +62,6 @@ wikigdrive init --drive "https://drive.google.com/drive/folders/FOLDER_ID" --config /location/of/.wgd - Location of config file --dest /location/of/downloaded/content - Destination for downloaded and converted markdown files ---drive_id - An ID of the drive - --client_id - ID of google app, alternatively can be passed in .env or through environment variable CLIENT_ID; --client_secret - Secret of google app, alternatively can be passed in .env or through environment variable CLIENT_SECRET; @@ -71,8 +69,6 @@ wikigdrive init --drive "https://drive.google.com/drive/folders/FOLDER_ID" --link_mode mdURLs - `/filename.md` --link_mode dirURLs - `/filename/` --link_mode uglyURLs - `/filename.html` - see https://gohugo.io/getting-started/configuration/ - ---without-folder-structure Download documents into single, flat folder ``` List available drive ids that wikigdrive has access to on Google: @@ -87,22 +83,7 @@ Run one time documents pull wikigdrive pull ``` -Run continuous documents watch - -``` -wikigdrive watch --git_update_delay=10 - ---watch - Run program in loop, watch for gdrive changes ---git_update_delay=x - trigger git update hook after x minutes -``` - -Status - -``` -wikigdrive status -``` - -Run server mode for webhooks support (TODO: not implemented yet) +Run server mode ``` wikigdrive server diff --git a/doc/developer.md b/doc/developer.md index dab2ed1e..b910798c 100644 --- a/doc/developer.md +++ b/doc/developer.md @@ -50,7 +50,7 @@ docker run --rm --user=$(id -u) -it \ --publish 127.0.0.1:3000:3000 \ --publish 127.0.0.1:24678:24678 \ wikigdrive \ - ./src/cli.sh --service_account /service_account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com --workdir /data server 3000 + ./src/wikigdrived.sh --service_account /service_account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com --workdir /data server 3000 docker rm -f wikigdrive @@ -139,7 +139,7 @@ ZIPKIN_URL=http://localhost:9411` ## Debugging ``` -./src/cli.sh --inspect --workdir ~/wikigdrive --service_account ~/workspaces/mieweb/wikigdrive-with-service-account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com server 3000 +./src/wikigdrived.sh --inspect --workdir ~/wikigdrive --service_account ~/workspaces/mieweb/wikigdrive-with-service-account.json --share_email mie-docs-wikigdrive@wikigdrive.iam.gserviceaccount.com server 3000 ``` Chrome diff --git a/doc/internals.md b/doc/internals.md index b1e231c1..c8aa891b 100644 --- a/doc/internals.md +++ b/doc/internals.md @@ -66,7 +66,6 @@ ``` { "drive": "https://drive.google.com/drive/folders/FOLDER_ID", - "drive_id": "", "dest": "/home/user/mieweb/wikigdrive-test", "link_mode": "mdURLs", "service_account": "wikigdrive.json" diff --git a/package-lock.json b/package-lock.json index a29fef2f..0e2113fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,9 +71,9 @@ "xmldoc": "^1.1.2" }, "bin": { - "wgd": "src/cli.sh", - "wikigdrive": "src/cli.sh", - "wikigdrive-ts": "src/cli.sh" + "wgd": "src/wikigdrive.sh", + "wikigdrive": "src/wikigdrive.sh", + "wikigdrivectl": "src/wikigdrivectl.sh" }, "devDependencies": { "@swc/core": "1.3.24", diff --git a/package.json b/package.json index cd824954..dea64e6a 100644 --- a/package.json +++ b/package.json @@ -15,15 +15,15 @@ "npm": ">= 8.1.2" }, "bin": { - "wgd": "./src/cli.sh", - "wikigdrive": "./src/cli.sh", - "wikigdrive-ts": "./src/cli.sh" + "wgd": "src/wikigdrive.sh", + "wikigdrive": "src/wikigdrive.sh", + "wikigdrivectl": "src/wikigdrivectl.sh" }, - "main": "src/main.ts", + "main": "src/cli/wikigdrive.ts", "scripts": { "test": "mocha --require ts-node/register --ui bdd test/*.ts test/**/*.ts", "lint": "eslint ./apps/**/*.ts ./apps/**/*.vue", - "start": "wikigdrive-ts watch --server 3000", + "start": "wikigdrive server 3000", "build": "" }, "repository": { diff --git a/src/ContainerEngine.ts b/src/ContainerEngine.ts index 763670b5..3f7ae99b 100644 --- a/src/ContainerEngine.ts +++ b/src/ContainerEngine.ts @@ -109,6 +109,10 @@ export class ContainerEngine { return this.containers[name]; } + hasContainer(name: string) { + return !!this.containers[name]; + } + async flushData() { for (const container of Object.values(this.containers)) { await container.flushData(); diff --git a/src/MainService.ts b/src/MainService.ts deleted file mode 100644 index a5ca2085..00000000 --- a/src/MainService.ts +++ /dev/null @@ -1,317 +0,0 @@ -'use strict'; - -import {EventEmitter} from 'events'; -import winston from 'winston'; - -import {createLogger} from './utils/logger/logger'; -import {ContainerEngine} from './ContainerEngine'; -import {GoogleFolderContainer} from './containers/google_folder/GoogleFolderContainer'; -import {GoogleApiContainer} from './containers/google_api/GoogleApiContainer'; -import {FileContentService} from './utils/FileContentService'; -import {default as envPaths, Paths} from 'env-paths'; -import path from 'path'; -import {urlToFolderId} from './utils/idParsers'; -import {TransformContainer} from './containers/transform/TransformContainer'; -import {CliParams} from './model/CliParams'; -import {AuthConfig} from './model/AccountJson'; -import {loadRunningInstance} from './containers/server/loadRunningInstance'; -import {FolderRegistryContainer} from './containers/folder_registry/FolderRegistryContainer'; -import {JobManagerContainer} from './containers/job/JobManagerContainer'; -import {WatchChangesContainer} from './containers/changes/WatchChangesContainer'; -import {UserConfigService} from './containers/google_folder/UserConfigService'; -import {ServerContainer} from './containers/server/ServerContainer'; - -export class MainService { - private readonly eventBus: EventEmitter; - private readonly command: string; - private readonly logger: winston.Logger; - private containerEngine: ContainerEngine; - private paths: Paths; - private mainFileService: FileContentService; - private authConfig: AuthConfig; - - constructor(private params: CliParams) { - this.command = this.params.command; - this.eventBus = new EventEmitter(); - this.eventBus.setMaxListeners(0); - if (params.debug.indexOf('main') > -1) { - this.attachDebug(); - } - - this.paths = envPaths('wikigdrive', {suffix: null}); - this.logger = createLogger(this.eventBus, this.params.workdir); - } - - attachDebug() { - const eventNames = {}; - this.eventBus.on('newListener', (event) => { - if (eventNames[event]) return; - eventNames[event] = event; - - this.eventBus.on(event, () => { - this.logger.debug('OnEvent', event); - }); - }); - } - - async init() { - this.mainFileService = new FileContentService(this.params.workdir || process.cwd()); - await this.mainFileService.mkdir('/'); - - const requireAuth = ['config', 'server', 'drives', 'pull', 'register']; - - if (requireAuth.indexOf(this.command) > -1) { - if (this.params.service_account) { - const rootFileService = new FileContentService('/'); - this.authConfig = { - service_account: await rootFileService.readJson(path.resolve(this.params.service_account)) - }; - } else - if (this.params.client_id && this.params.client_secret) { - this.authConfig = { - user_account: { - type: 'user_account', - client_id: this.params.client_id, - client_secret: this.params.client_secret - } - }; - } else { - this.authConfig = await this.mainFileService.readJson('auth_config.json'); - } - - if (!this.authConfig) { - throw new Error('No authentication credentials provided'); - } - - if (this.params.share_email) { - this.authConfig.share_email = this.params.share_email; - } - } - - this.containerEngine = new ContainerEngine(this.logger, this.mainFileService); - - this.eventBus.on('panic:invalid_grant', () => { - // if (configService) { - // await configService.saveGoogleAuth(null); - // await configService.flushData(); - // } - process.exit(1); - }); - this.eventBus.on('panic', (error) => { - throw error; - /* - this.logger.error(error.stack ? error.stack : error.message); - console.error(error.message); - process.exit(1); - */ - }); - } - - async cmdDrives() { - const apiContainer: GoogleApiContainer = this.containerEngine.getContainer('google_api'); - const drives = await apiContainer.listDrives(); - console.log('Available drives:'); - console.table(drives); - } - - async cmdTransform() { - const folderId = urlToFolderId(this.params.args[0]); - if (!folderId) { - throw new Error('No folderId'); - } - - this.logger.info('Transforming'); - - const filesIds = this.params.args.slice(1); - const transformContainer = new TransformContainer({ - name: folderId - }, { filesIds }); - const googleFileSystem = await this.mainFileService.getSubFileService(folderId, '/'); - await transformContainer.mount2( - googleFileSystem, - await this.mainFileService.getSubFileService(folderId + '_transform', '/') - ); - - const userConfigService = new UserConfigService(googleFileSystem); - await userConfigService.load(); - - await this.containerEngine.registerContainer(transformContainer); - - await transformContainer.run(folderId); - - await this.containerEngine.unregisterContainer(transformContainer.params.name); - } - - async cmdPull() { - const folderId = urlToFolderId(this.params.args[0]); - if (!folderId) { - throw new Error('No folderId'); - } - - this.logger.info('Downloading'); - - const filesIds = this.params.args.slice(1); - const downloadContainer = new GoogleFolderContainer({ - cmd: 'pull', - name: folderId, - folderId: folderId, - apiContainer: 'google_api' - }, { filesIds }); - await downloadContainer.mount(await this.mainFileService.getSubFileService(folderId, '/')); - await this.containerEngine.registerContainer(downloadContainer); - await downloadContainer.run(); - await this.containerEngine.unregisterContainer(downloadContainer.params.name); - - await this.cmdTransform(); - } - - async cmdPrune() { - const folderId = urlToFolderId(this.params.args[0]); - if (!folderId) { - throw new Error('No folderId'); - } - - const folderRegistryContainer = this.containerEngine.getContainer('folder_registry'); - await folderRegistryContainer.pruneFolder(folderId); - } - - async cmdRegister() { - const folderId = urlToFolderId(this.params.args[0]); - if (!folderId) { - throw new Error('No folderId'); - } - - const folderRegistryContainer = this.containerEngine.getContainer('folder_registry'); - const folder = await folderRegistryContainer.registerFolder(folderId); - if (folder.new) { - await this.cmdPull(); - } - } - - async cmdUnregister() { - const folderId = urlToFolderId(this.params.args[0]); - if (!folderId) { - throw new Error('No folderId'); - } - - const folderRegistryContainer = this.containerEngine.getContainer('folder_registry'); - await folderRegistryContainer.unregisterFolder(folderId); - } - - async cmdServer() { - const instance = await loadRunningInstance(); - if (instance) { - this.logger.error('WikiGDrive server already running, PID: ' + instance.pid); - process.exit(1); - } - - const changesContainer = new WatchChangesContainer({ name: 'watch_changes', share_email: this.params.share_email }); - await changesContainer.mount(await this.mainFileService); - await this.containerEngine.registerContainer(changesContainer); - await changesContainer.run(); - - const port = parseInt(this.params.args[0]) || 3000; - const serverContainer = new ServerContainer({ name: 'server', share_email: this.params.share_email }, port); - await serverContainer.mount(await this.mainFileService); - await this.containerEngine.registerContainer(serverContainer); - await serverContainer.run(); - - const containerEnginePromise = this.containerEngine.run(); - // eslint-disable-next-line @typescript-eslint/no-empty-function - containerEnginePromise.then(() => { - }); - - await new Promise(resolve => { - this.eventBus.on('end', resolve); - }); - } - - async cmdInspect() { - const folderId = urlToFolderId(this.params.args[0]); - if (!folderId) { - throw new Error('No folderId'); - } - - const instance = await loadRunningInstance(); - if (!instance) { - this.logger.error('WikiGDrive server is not running'); - process.exit(1); - } - - const response = await fetch(`http://localhost:${instance.port}/api/drive/${folderId}/inspect`); - const json = await response.json(); - - console.log(json); - } - - async cmdPs() { - const instance = await loadRunningInstance(); - if (!instance) { - this.logger.error('WikiGDrive server is not running'); - process.exit(1); - } - - const response = await fetch(`http://localhost:${instance.port}/api/ps`); - const json = await response.json(); - - console.table(json); - } - - async start() { - if (this.authConfig) { - const apiContainer = new GoogleApiContainer({ name: 'google_api' }, this.authConfig); - await apiContainer.mount(await this.mainFileService); - await this.containerEngine.registerContainer(apiContainer); - await apiContainer.run(); - } - - const folderRegistryContainer = new FolderRegistryContainer({ name: 'folder_registry' }); - await folderRegistryContainer.mount(await this.mainFileService); - await this.containerEngine.registerContainer(folderRegistryContainer); - await folderRegistryContainer.run(); - - - const jobManagerContainer = new JobManagerContainer({ name: 'job_manager' }); - await jobManagerContainer.mount(await this.mainFileService); - await this.containerEngine.registerContainer(jobManagerContainer); - await jobManagerContainer.run(); - - switch (this.command) { - case 'config': - if (this.authConfig) { - await this.mainFileService.writeJson('auth_config.json', this.authConfig); - } - break; - case 'drives': - await this.cmdDrives(); - break; - case 'server': - await this.cmdServer(); - break; - case 'pull': - await this.cmdPull(); - break; - case 'transform': - await this.cmdTransform(); - break; - case 'prune': - await this.cmdPrune(); - break; - case 'register': - await this.cmdRegister(); - break; - case 'unregister': - await this.cmdUnregister(); - break; - case 'ps': - await this.cmdPs(); - break; - case 'inspect': - await this.cmdInspect(); - break; - } - - await this.containerEngine.flushData(); - } - -} diff --git a/src/cli.sh b/src/cli.sh deleted file mode 100755 index 04e5cfcb..00000000 --- a/src/cli.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -FULL_PATH="$(readlink -f ${BASH_SOURCE[0]})" -MAIN_DIR=$(dirname "$FULL_PATH")/.. -NODE_MODULES=$MAIN_DIR/node_modules - -#export NODE_PATH=$NODE_MODULES -cd $MAIN_DIR - -if test "$1" = "--inspect"; then - /usr/bin/env node --inspect --no-warnings --experimental-specifier-resolution=node --loader $NODE_MODULES/ts-node/esm $MAIN_DIR/src/main.ts "$@" -else - /usr/bin/env node --no-warnings --experimental-specifier-resolution=node --loader $NODE_MODULES/ts-node/esm $MAIN_DIR/src/main.ts "$@" -fi diff --git a/src/cli/getAuthConfig.ts b/src/cli/getAuthConfig.ts new file mode 100644 index 00000000..18102fb7 --- /dev/null +++ b/src/cli/getAuthConfig.ts @@ -0,0 +1,32 @@ +import {FileContentService} from '../utils/FileContentService'; +import {AuthConfig} from '../model/AccountJson'; + +interface Params { + client_id?: string; + client_secret?: string; + service_account?: string; +} + +export async function getAuthConfig(params: Params, mainFileService: FileContentService): Promise { + if (params.service_account) { + const rootFileService = new FileContentService('/'); + return { + service_account: await rootFileService.readJson(params.service_account) + }; + } else + if (params.client_id && params.client_secret) { + return { + user_account: { + type: 'user_account', + client_id: params.client_id, + client_secret: params.client_secret + } + }; + } else { + const authConfig = await mainFileService.readJson('auth_config.json'); + if (!authConfig) { + throw new Error('No authentication credentials provided'); + } + return authConfig; + } +} diff --git a/src/cli/usage.ts b/src/cli/usage.ts new file mode 100644 index 00000000..bc227cc8 --- /dev/null +++ b/src/cli/usage.ts @@ -0,0 +1,49 @@ +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +export async function usage(filename: string) { + const pkg = JSON.parse(fs.readFileSync(path.resolve(__dirname, '..', '..', 'package.json')).toString()); + + console.log( + `version: ${pkg.version}, ${process.env.GIT_SHA}${` +Usage: + $ wikigdrive [args] [] + +Main commands: + + wikigdrive config + --client_id + --client_secret + --service_account=./private_key.json + + wikigdrive service + + wikigdrive add [folder_id_or_url] + --drive [shared drive url] + --workdir (current working folder) + --link_mode [mdURLs|dirURLs|uglyURLs] + + wikigdrive pull [URL to specific file] + + wikigdrive watch (keep scanning for changes, ie: daemon) + +Other commands: + + wikigdrive status [ID of document] - Show status of the document or stats of the entire path. + wikigdrive drives + wikigdrive sync + wikigdrive download + wikigdrive transform + +Options: + --workdir (current working folder) + +Examples: + $ wikigdrive init + $ wikigdrive add https://google.drive... + `}`); +} diff --git a/src/cli/wikigdrive-config.ts b/src/cli/wikigdrive-config.ts new file mode 100644 index 00000000..b6ab34a2 --- /dev/null +++ b/src/cli/wikigdrive-config.ts @@ -0,0 +1,55 @@ +'use strict'; + +import path from 'path'; +import fs from 'fs'; +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; + +import {addTelemetry} from '../telemetry'; +import {FileContentService} from '../utils/FileContentService'; +import {getAuthConfig} from './getAuthConfig'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + + const mainFileService = new FileContentService(workdir); + await mainFileService.mkdir('/'); + + const params = { + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + service_account: argv['service_account'] || null, + }; + + const authConfig = await getAuthConfig(params, mainFileService); + await mainFileService.writeJson('auth_config.json', authConfig); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-drives.ts b/src/cli/wikigdrive-drives.ts new file mode 100644 index 00000000..85e8d289 --- /dev/null +++ b/src/cli/wikigdrive-drives.ts @@ -0,0 +1,62 @@ +'use strict'; + +import path from 'path'; +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; + +import {addTelemetry} from '../telemetry'; +import {GoogleApiContainer} from '../containers/google_api/GoogleApiContainer'; +import {FileContentService} from '../utils/FileContentService'; +import {getAuthConfig} from './getAuthConfig'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + + const mainFileService = new FileContentService(workdir); + await mainFileService.mkdir('/'); + + const params = { + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + service_account: argv['service_account'] || null, + }; + + const authConfig = await getAuthConfig(params, mainFileService); + + const apiContainer = new GoogleApiContainer({ name: 'google_api' }, authConfig); + await apiContainer.mount(await mainFileService); + await apiContainer.run(); + + const drives = await apiContainer.listDrives(); + console.log('Available drives:'); + console.table(drives); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-prune.ts b/src/cli/wikigdrive-prune.ts new file mode 100644 index 00000000..f7508a7f --- /dev/null +++ b/src/cli/wikigdrive-prune.ts @@ -0,0 +1,90 @@ +'use strict'; + +import path from 'path'; +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; +import {EventEmitter} from 'events'; + +import {addTelemetry} from '../telemetry'; +import {GoogleApiContainer} from '../containers/google_api/GoogleApiContainer'; +import {FileContentService} from '../utils/FileContentService'; +import {getAuthConfig} from './getAuthConfig'; +import {urlToFolderId} from '../utils/idParsers'; +import {FolderRegistryContainer} from '../containers/folder_registry/FolderRegistryContainer'; +import {ContainerEngine} from '../ContainerEngine'; +import {createLogger} from '../utils/logger/logger'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + const args = argv._.slice(1); + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + + const mainFileService = new FileContentService(workdir); + await mainFileService.mkdir('/'); + + const params = { + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + service_account: argv['service_account'] || null, + }; + + const authConfig = await getAuthConfig(params, mainFileService); + + const apiContainer = new GoogleApiContainer({ name: 'google_api' }, authConfig); + await apiContainer.mount(await mainFileService); + await apiContainer.run(); + + const folderId = urlToFolderId(args[0]); + if (!folderId) { + throw new Error('No folderId'); + } + + const eventBus = new EventEmitter(); + eventBus.setMaxListeners(0); + eventBus.on('panic:invalid_grant', () => { + process.exit(1); + }); + eventBus.on('panic', (error) => { + throw error; + }); + + const logger = createLogger(workdir, eventBus); + const containerEngine = new ContainerEngine(logger, mainFileService); + + const folderRegistryContainer = new FolderRegistryContainer({ name: 'folder_registry' }); + await folderRegistryContainer.mount(await mainFileService); + await this.containerEngine.registerContainer(folderRegistryContainer); + await folderRegistryContainer.run(); + await folderRegistryContainer.pruneFolder(folderId); + + await containerEngine.flushData(); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-pull.ts b/src/cli/wikigdrive-pull.ts new file mode 100644 index 00000000..301435e7 --- /dev/null +++ b/src/cli/wikigdrive-pull.ts @@ -0,0 +1,118 @@ +'use strict'; + +import path from 'path'; +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; +import {EventEmitter} from 'events'; + +import {addTelemetry} from '../telemetry'; +import {GoogleApiContainer} from '../containers/google_api/GoogleApiContainer'; +import {FileContentService} from '../utils/FileContentService'; +import {getAuthConfig} from './getAuthConfig'; +import {urlToFolderId} from '../utils/idParsers'; +import {GoogleFolderContainer} from '../containers/google_folder/GoogleFolderContainer'; +import {TransformContainer} from '../containers/transform/TransformContainer'; +import {ContainerEngine} from '../ContainerEngine'; +import {createLogger} from '../utils/logger/logger'; +import {FolderRegistryContainer} from '../containers/folder_registry/FolderRegistryContainer'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + const args = argv._.slice(1); + + const folderId = urlToFolderId(args[0]); + if (!folderId) { + throw new Error('No folderId'); + } + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + + const mainFileService = new FileContentService(workdir); + await mainFileService.mkdir('/'); + + const params = { + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + service_account: argv['service_account'] || null, + }; + const authConfig = await getAuthConfig(params, mainFileService); + + const apiContainer = new GoogleApiContainer({ name: 'google_api' }, authConfig); + await apiContainer.mount(await mainFileService); + await apiContainer.run(); + + const folderRegistryContainer = new FolderRegistryContainer({ name: 'folder_registry' }); + await folderRegistryContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(folderRegistryContainer); + await folderRegistryContainer.run(); + + const eventBus = new EventEmitter(); + eventBus.setMaxListeners(0); + eventBus.on('panic:invalid_grant', () => { + process.exit(1); + }); + eventBus.on('panic', (error) => { + throw error; + }); + + const logger = createLogger(workdir, eventBus); + + + const containerEngine = new ContainerEngine(logger, mainFileService); + + const filesIds = args.slice(1); + + const googleFileSystem = await mainFileService.getSubFileService(folderId, '/'); + const transformFileSystem = await mainFileService.getSubFileService(folderId + '_transform', '/'); + + logger.info('Downloading'); + const downloadContainer = new GoogleFolderContainer({ + cmd: 'pull', + name: folderId, + folderId: folderId, + apiContainer: 'google_api' + }, { filesIds }); + + await downloadContainer.mount(googleFileSystem); + await containerEngine.registerContainer(downloadContainer); + await downloadContainer.run(); + await containerEngine.unregisterContainer(downloadContainer.params.name); + + logger.info('Transforming'); + const transformContainer = new TransformContainer({ name: folderId }, { filesIds }); + await transformContainer.mount2(googleFileSystem, transformFileSystem); + + await containerEngine.registerContainer(transformContainer); + await transformContainer.run(folderId); + await containerEngine.unregisterContainer(transformContainer.params.name); + + await containerEngine.flushData(); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-register.ts b/src/cli/wikigdrive-register.ts new file mode 100644 index 00000000..7affdaa7 --- /dev/null +++ b/src/cli/wikigdrive-register.ts @@ -0,0 +1,93 @@ +'use strict'; + +import path from 'path'; +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; + +import {addTelemetry} from '../telemetry'; +import {GoogleApiContainer} from '../containers/google_api/GoogleApiContainer'; +import {FileContentService} from '../utils/FileContentService'; +import {getAuthConfig} from './getAuthConfig'; +import {urlToFolderId} from '../utils/idParsers'; +import {FolderRegistryContainer} from '../containers/folder_registry/FolderRegistryContainer'; +import {ContainerEngine} from '../ContainerEngine'; +import {createLogger} from '../utils/logger/logger'; +import {EventEmitter} from 'events'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + const args = argv._.slice(1); + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + + const mainFileService = new FileContentService(workdir); + await mainFileService.mkdir('/'); + + const params = { + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + service_account: argv['service_account'] || null, + }; + + const authConfig = await getAuthConfig(params, mainFileService); + + const apiContainer = new GoogleApiContainer({ name: 'google_api' }, authConfig); + await apiContainer.mount(await mainFileService); + await apiContainer.run(); + + const folderId = urlToFolderId(args[0]); + if (!folderId) { + throw new Error('No folderId'); + } + + const eventBus = new EventEmitter(); + eventBus.setMaxListeners(0); + eventBus.on('panic:invalid_grant', () => { + process.exit(1); + }); + eventBus.on('panic', (error) => { + throw error; + }); + + const logger = createLogger(workdir, eventBus); + const containerEngine = new ContainerEngine(logger, mainFileService); + + const folderRegistryContainer = new FolderRegistryContainer({ name: 'folder_registry' }); + await folderRegistryContainer.mount(await mainFileService); + await this.containerEngine.registerContainer(folderRegistryContainer); + await folderRegistryContainer.run(); + const folder = await folderRegistryContainer.registerFolder(folderId); + if (folder.new) { + logger.info('New folder registered. Run: wikigdrive pull'); + } + + await containerEngine.flushData(); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-server.ts b/src/cli/wikigdrive-server.ts new file mode 100644 index 00000000..143bbca4 --- /dev/null +++ b/src/cli/wikigdrive-server.ts @@ -0,0 +1,195 @@ +'use strict'; + +import path from 'path'; +import minimist from 'minimist'; +import winston from 'winston'; +import {fileURLToPath} from 'url'; +import {EventEmitter} from 'events'; +import dotenv from 'dotenv'; +import {default as envPaths, Paths} from 'env-paths'; + +import {addTelemetry} from '../telemetry'; +import {CliParams} from '../model/CliParams'; + +import {createLogger} from '../utils/logger/logger'; +import {ContainerEngine} from '../ContainerEngine'; +import {GoogleApiContainer} from '../containers/google_api/GoogleApiContainer'; +import {FileContentService} from '../utils/FileContentService'; +import {AuthConfig} from '../model/AccountJson'; +import {loadRunningInstance} from '../containers/server/loadRunningInstance'; +import {FolderRegistryContainer} from '../containers/folder_registry/FolderRegistryContainer'; +import {JobManagerContainer} from '../containers/job/JobManagerContainer'; +import {WatchChangesContainer} from '../containers/changes/WatchChangesContainer'; +import {ServerContainer} from '../containers/server/ServerContainer'; +import {getAuthConfig} from './getAuthConfig'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +export class MainService { + private readonly eventBus: EventEmitter; + private readonly logger: winston.Logger; + private containerEngine: ContainerEngine; + private paths: Paths; + private mainFileService: FileContentService; + private authConfig: AuthConfig; + + constructor(private params: CliParams) { + this.eventBus = new EventEmitter(); + this.eventBus.setMaxListeners(0); + if (params.debug.indexOf('main') > -1) { + this.attachDebug(); + } + + this.paths = envPaths('wikigdrive', {suffix: null}); + this.logger = createLogger(this.params.workdir, this.eventBus); + } + + attachDebug() { + const eventNames = {}; + this.eventBus.on('newListener', (event) => { + if (eventNames[event]) return; + eventNames[event] = event; + + this.eventBus.on(event, () => { + this.logger.debug('OnEvent', event); + }); + }); + } + + async init() { + this.mainFileService = new FileContentService(this.params.workdir || process.cwd()); + await this.mainFileService.mkdir('/'); + + this.authConfig = await getAuthConfig(this.params, this.mainFileService); + + if (this.params.share_email) { + this.authConfig.share_email = this.params.share_email; + } + + this.containerEngine = new ContainerEngine(this.logger, this.mainFileService); + + this.eventBus.on('panic:invalid_grant', () => { + // if (configService) { + // await configService.saveGoogleAuth(null); + // await configService.flushData(); + // } + process.exit(1); + }); + this.eventBus.on('panic', (error) => { + throw error; + /* + this.logger.error(error.stack ? error.stack : error.message); + console.error(error.message); + process.exit(1); + */ + }); + } + + async cmdServer() { + const instance = await loadRunningInstance(); + if (instance) { + this.logger.error('WikiGDrive server already running, PID: ' + instance.pid); + process.exit(1); + } + + const changesContainer = new WatchChangesContainer({ name: 'watch_changes', share_email: this.params.share_email }); + await changesContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(changesContainer); + await changesContainer.run(); + + const port = parseInt(this.params.args[1]) || 3000; + const serverContainer = new ServerContainer({ name: 'server', share_email: this.params.share_email }, port); + await serverContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(serverContainer); + await serverContainer.run(); + + const containerEnginePromise = this.containerEngine.run(); + // eslint-disable-next-line @typescript-eslint/no-empty-function + containerEnginePromise.then(() => { + }); + + await new Promise(resolve => { + this.eventBus.on('end', resolve); + }); + } + + async start() { + const apiContainer = new GoogleApiContainer({ name: 'google_api' }, this.authConfig); + await apiContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(apiContainer); + await apiContainer.run(); + + const folderRegistryContainer = new FolderRegistryContainer({ name: 'folder_registry' }); + await folderRegistryContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(folderRegistryContainer); + await folderRegistryContainer.run(); + + const jobManagerContainer = new JobManagerContainer({ name: 'job_manager' }); + await jobManagerContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(jobManagerContainer); + await jobManagerContainer.run(); + + await this.cmdServer(); + + await this.containerEngine.flushData(); + } + +} + +async function main() { + const argv = minimist(process.argv.slice(2)); + + console.log(argv._); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const params: CliParams = { + args: argv._.slice(1), + drive: argv['drive'], + workdir: argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data', + + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + + link_mode: argv['link_mode'] || 'mdURLs', + + debug: (argv['debug'] || '').split(',').map(str => str.toLocaleString().trim()), + + service_account: argv['service_account'] || null, + share_email: argv['share_email'] || process.env.SHARE_EMAIL || null, + server_port: +argv['server'] + }; + + const mainService = new MainService(params); + + try { + await mainService.init(); + } catch (err) { + await usage(__filename); + console.error(err); + process.exit(1); + } + return await mainService.start(); +} + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-transform.ts b/src/cli/wikigdrive-transform.ts new file mode 100644 index 00000000..bd4842dc --- /dev/null +++ b/src/cli/wikigdrive-transform.ts @@ -0,0 +1,102 @@ +'use strict'; + +import path from 'path'; +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; +import {EventEmitter} from 'events'; + +import {addTelemetry} from '../telemetry'; +import {GoogleApiContainer} from '../containers/google_api/GoogleApiContainer'; +import {FileContentService} from '../utils/FileContentService'; +import {getAuthConfig} from './getAuthConfig'; +import {urlToFolderId} from '../utils/idParsers'; +import {TransformContainer} from '../containers/transform/TransformContainer'; +import {ContainerEngine} from '../ContainerEngine'; +import {createLogger} from '../utils/logger/logger'; +import {FolderRegistryContainer} from '../containers/folder_registry/FolderRegistryContainer'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + const args = argv._.slice(1); + + const folderId = urlToFolderId(args[0]); + if (!folderId) { + throw new Error('No folderId'); + } + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const filesIds = args.slice(1); + const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + + const mainFileService = new FileContentService(workdir); + await mainFileService.mkdir('/'); + + const params = { + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + service_account: argv['service_account'] || null, + }; + const authConfig = await getAuthConfig(params, mainFileService); + + const apiContainer = new GoogleApiContainer({ name: 'google_api' }, authConfig); + await apiContainer.mount(await mainFileService); + await apiContainer.run(); + + const folderRegistryContainer = new FolderRegistryContainer({ name: 'folder_registry' }); + await folderRegistryContainer.mount(await this.mainFileService); + await this.containerEngine.registerContainer(folderRegistryContainer); + await folderRegistryContainer.run(); + + const eventBus = new EventEmitter(); + eventBus.setMaxListeners(0); + eventBus.on('panic:invalid_grant', () => { + process.exit(1); + }); + eventBus.on('panic', (error) => { + throw error; + }); + + const logger = createLogger(workdir, eventBus); + + const containerEngine = new ContainerEngine(logger, mainFileService); + + const googleFileSystem = await mainFileService.getSubFileService(folderId, '/'); + const transformFileSystem = await mainFileService.getSubFileService(folderId + '_transform', '/'); + + logger.info('Transforming'); + const transformContainer = new TransformContainer({ name: folderId }, { filesIds }); + await transformContainer.mount2(googleFileSystem, transformFileSystem); + + await containerEngine.registerContainer(transformContainer); + await transformContainer.run(folderId); + await containerEngine.unregisterContainer(transformContainer.params.name); + + await containerEngine.flushData(); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-unregister.ts b/src/cli/wikigdrive-unregister.ts new file mode 100644 index 00000000..8ac67cec --- /dev/null +++ b/src/cli/wikigdrive-unregister.ts @@ -0,0 +1,90 @@ +'use strict'; + +import path from 'path'; +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; +import {EventEmitter} from 'events'; + +import {addTelemetry} from '../telemetry'; +import {GoogleApiContainer} from '../containers/google_api/GoogleApiContainer'; +import {FileContentService} from '../utils/FileContentService'; +import {getAuthConfig} from './getAuthConfig'; +import {urlToFolderId} from '../utils/idParsers'; +import {FolderRegistryContainer} from '../containers/folder_registry/FolderRegistryContainer'; +import {ContainerEngine} from '../ContainerEngine'; +import {createLogger} from '../utils/logger/logger'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname); + +async function main() { + const argv = minimist(process.argv.slice(2)); + + if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); + } + + const args = argv._.slice(1); + + // PWD is null on Windows, so we can set it here + process.env.PWD = process.cwd(); + + const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + + const mainFileService = new FileContentService(workdir); + await mainFileService.mkdir('/'); + + const params = { + client_id: argv['client_id'] || process.env.CLIENT_ID, + client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, + service_account: argv['service_account'] || null, + }; + + const authConfig = await getAuthConfig(params, mainFileService); + + const apiContainer = new GoogleApiContainer({ name: 'google_api' }, authConfig); + await apiContainer.mount(await mainFileService); + await apiContainer.run(); + + const folderId = urlToFolderId(args[0]); + if (!folderId) { + throw new Error('No folderId'); + } + + const eventBus = new EventEmitter(); + eventBus.setMaxListeners(0); + eventBus.on('panic:invalid_grant', () => { + process.exit(1); + }); + eventBus.on('panic', (error) => { + throw error; + }); + + const logger = createLogger(workdir, eventBus); + const containerEngine = new ContainerEngine(logger, mainFileService); + + const folderRegistryContainer = new FolderRegistryContainer({ name: 'folder_registry' }); + await folderRegistryContainer.mount(await mainFileService); + await this.containerEngine.registerContainer(folderRegistryContainer); + await folderRegistryContainer.run(); + await folderRegistryContainer.unregisterFolder(folderId); + + await containerEngine.flushData(); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/src/cli/wikigdrive-usage.ts b/src/cli/wikigdrive-usage.ts new file mode 100644 index 00000000..125f49d3 --- /dev/null +++ b/src/cli/wikigdrive-usage.ts @@ -0,0 +1,7 @@ +import {usage} from './usage'; +import {fileURLToPath} from 'url'; + +const __filename = fileURLToPath(import.meta.url); + +await usage(__filename); +process.exit(1); diff --git a/src/cli/wikigdrivectl-inspect.ts b/src/cli/wikigdrivectl-inspect.ts new file mode 100644 index 00000000..cd473444 --- /dev/null +++ b/src/cli/wikigdrivectl-inspect.ts @@ -0,0 +1,46 @@ +'use strict'; + +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; + +import {createLogger} from '../utils/logger/logger'; +import {loadRunningInstance} from '../containers/server/loadRunningInstance'; +import {urlToFolderId} from '../utils/idParsers'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +const argv = minimist(process.argv.slice(2)); + +if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); +} + +// PWD is null on Windows, so we can set it here +process.env.PWD = process.cwd(); + +const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; +const args = argv._.slice(1); + +const logger = createLogger(workdir); + +const folderId = urlToFolderId(args[0]); +if (!folderId) { + throw new Error('No folderId'); +} + +const instance = await loadRunningInstance(); +if (!instance) { + logger.error('WikiGDrive server is not running'); + process.exit(1); +} + +const response = await fetch(`http://localhost:${instance.port}/api/drive/${folderId}/inspect`); +const json = await response.json(); + +console.log(json); diff --git a/src/cli/wikigdrivectl-ps.ts b/src/cli/wikigdrivectl-ps.ts new file mode 100644 index 00000000..44b899bf --- /dev/null +++ b/src/cli/wikigdrivectl-ps.ts @@ -0,0 +1,39 @@ +'use strict'; + +import minimist from 'minimist'; +import dotenv from 'dotenv'; +import {fileURLToPath} from 'url'; + +import {createLogger} from '../utils/logger/logger'; +import {loadRunningInstance} from '../containers/server/loadRunningInstance'; +import {usage} from './usage'; + +const __filename = fileURLToPath(import.meta.url); + +process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; + +dotenv.config(); +const argv = minimist(process.argv.slice(2)); + +if (argv._.length < 1 || argv.h || argv.help) { + await usage(__filename); + process.exit(0); +} + +// PWD is null on Windows, so we can set it here +process.env.PWD = process.cwd(); + +const workdir = argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data'; + +const logger = createLogger(workdir); + +const instance = await loadRunningInstance(); +if (!instance) { + logger.error('WikiGDrive server is not running'); + process.exit(1); +} + +const response = await fetch(`http://localhost:${instance.port}/api/ps`); +const json = await response.json(); + +console.table(json); diff --git a/src/cli/wikigdrivectl-usage.ts b/src/cli/wikigdrivectl-usage.ts new file mode 100644 index 00000000..125f49d3 --- /dev/null +++ b/src/cli/wikigdrivectl-usage.ts @@ -0,0 +1,7 @@ +import {usage} from './usage'; +import {fileURLToPath} from 'url'; + +const __filename = fileURLToPath(import.meta.url); + +await usage(__filename); +process.exit(1); diff --git a/src/containers/folder_registry/FolderRegistryContainer.ts b/src/containers/folder_registry/FolderRegistryContainer.ts index 3e6c5b58..89c7c81e 100644 --- a/src/containers/folder_registry/FolderRegistryContainer.ts +++ b/src/containers/folder_registry/FolderRegistryContainer.ts @@ -53,6 +53,11 @@ export class FolderRegistryContainer extends Container { } async refreshDrives() { + if (!this.engine.hasContainer('google_api')) { + this.logger.warn('Not authenticated to Google API. Skipping drives refresh.'); + return; + } + const oldDrives = Object.values(await this.getFolders()); const apiContainer: GoogleApiContainer = this.engine.getContainer('google_api'); diff --git a/src/containers/server/loadRunningInstance.ts b/src/containers/server/loadRunningInstance.ts index 88023607..9c95faa7 100644 --- a/src/containers/server/loadRunningInstance.ts +++ b/src/containers/server/loadRunningInstance.ts @@ -11,6 +11,7 @@ export async function loadRunningInstance() { const json = JSON.parse(content); if (json.pid > 0) { try { + // sending the signal 0 to a given PID just checks if any process with the given PID is running, and you have the permission to send a signal to it. process.kill(json.pid, 0); } catch (err) { return null; diff --git a/src/containers/server/routes/ConfigController.ts b/src/containers/server/routes/ConfigController.ts index 86aa6f97..2be06815 100644 --- a/src/containers/server/routes/ConfigController.ts +++ b/src/containers/server/routes/ConfigController.ts @@ -3,7 +3,7 @@ import {FileContentService} from '../../../utils/FileContentService'; import {GitScanner} from '../../../git/GitScanner'; import {UserConfigService} from '../../google_folder/UserConfigService'; import {FolderRegistryContainer} from '../../folder_registry/FolderRegistryContainer'; -import {ContainerEngine} from "../../../ContainerEngine"; +import {ContainerEngine} from '../../../ContainerEngine'; export interface ConfigBody { config: { diff --git a/src/main.ts b/src/main.ts index 03e3f762..e69de29b 100755 --- a/src/main.ts +++ b/src/main.ts @@ -1,116 +0,0 @@ -'use strict'; - -import path from 'path'; -import fs from 'fs'; -import minimist from 'minimist'; -import {addTelemetry} from './telemetry'; -import dotenv from 'dotenv'; -import {CliParams} from './model/CliParams'; -import {fileURLToPath} from 'url'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -process.env.GIT_SHA = process.env.GIT_SHA || 'dev'; - -function usage() { - const pkg = JSON.parse(fs.readFileSync(path.resolve(__dirname, '..', 'package.json')).toString()); - - console.log( - `version: ${pkg.version}, ${process.env.GIT_SHA}${` -Usage: - $ wikigdrive [args] [] - -Main commands: - - wikigdrive config - --client_id - --client_secret - --service_account=./private_key.json - - wikigdrive service - - wikigdrive add [folder_id_or_url] - --drive_id - --drive [shared drive url] - --workdir (current working folder) - --link_mode [mdURLs|dirURLs|uglyURLs] - --without-folder-structure - - wikigdrive pull [URL to specific file] - - wikigdrive watch (keep scanning for changes, ie: daemon) - -Other commands: - - wikigdrive status [ID of document] - Show status of the document or stats of the entire path. - wikigdrive drives - wikigdrive sync - wikigdrive download - wikigdrive transform - -Options: - --workdir (current working folder) - -Examples: - $ wikigdrive init - $ wikigdrive add https://google.drive... - `}`); -} - -async function main() { - const argv = minimist(process.argv.slice(2)); - - if (argv._.length < 1 || argv.h || argv.help) { - usage(); - process.exit(0); - } - - // PWD is null on Windows, so we can set it here - process.env.PWD = process.cwd(); - - const params: CliParams = { - command: argv._[0], - args: argv._.slice(1), - drive: argv['drive'], - workdir: argv['workdir'] || process.env.WIKIGDRIVE_WORKDIR || '/data', - - client_id: argv['client_id'] || process.env.CLIENT_ID, - client_secret: argv['client_secret'] || process.env.CLIENT_SECRET, - - link_mode: argv['link_mode'] || 'mdURLs', - - debug: (argv['debug'] || '').split(',').map(str => str.toLocaleString().trim()), - - drive_id: argv['drive_id'] || '', - service_account: argv['service_account'] || null, - share_email: argv['share_email'] || process.env.SHARE_EMAIL || null, - git_update_delay: argv['git_update_delay'] || 60, - force: !!argv['force'], - server_port: +argv['server'] - }; - - const {MainService} = await import('./MainService'); - const mainService = new MainService(params); - - try { - await mainService.init(); - } catch (err) { - usage(); - console.error(err); - process.exit(1); - } - return await mainService.start(); -} - -dotenv.config(); -await addTelemetry(process.env.ZIPKIN_SERVICE || 'wikigdrive', __dirname) - -main() - .then(() => { - process.exit(0); - }) - .catch((err) => { - console.error('Error', err); - process.exit(1); - }); diff --git a/src/model/CliParams.ts b/src/model/CliParams.ts index 63f2228d..76d1b74a 100644 --- a/src/model/CliParams.ts +++ b/src/model/CliParams.ts @@ -3,17 +3,13 @@ import {LinkMode} from './model'; export interface CliParams { link_mode: LinkMode; workdir: string; - drive_id: string; drive: string; - command: string; args: string[]; debug: string[]; - force: boolean; client_id?: string; client_secret?: string; service_account?: string; - git_update_delay: number; server_port?: number; share_email?: string; } diff --git a/src/utils/logger/logger.ts b/src/utils/logger/logger.ts index bcd72366..a08d444e 100644 --- a/src/utils/logger/logger.ts +++ b/src/utils/logger/logger.ts @@ -110,7 +110,7 @@ export function instrumentLogger(logger, childOpts = {}) { }; } -export function createLogger(eventBus: EventEmitter, workdir: string) { +export function createLogger(workdir: string, eventBus?: EventEmitter) { const logger = winston.createLogger({ level: 'info', format: winston.format.combine( @@ -171,7 +171,7 @@ export function createLogger(eventBus: EventEmitter, workdir: string) { console.error('unhandledRejection', reason); logger.error('unhandledRejection: ' + reason.stack ? reason.stack : reason.message, reason); - if (reason?.response?.data?.error === 'invalid_grant') { + if (eventBus && reason?.response?.data?.error === 'invalid_grant') { eventBus.emit('panic:invalid_grant'); return; } diff --git a/src/wikigdrive.sh b/src/wikigdrive.sh new file mode 100755 index 00000000..8ea98290 --- /dev/null +++ b/src/wikigdrive.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +FULL_PATH="$(readlink -f ${BASH_SOURCE[0]})" +MAIN_DIR=$(dirname "$FULL_PATH")/.. +NODE_MODULES=$MAIN_DIR/node_modules + +#export NODE_PATH=$NODE_MODULES +cd $MAIN_DIR + +POSITIONAL_ARGS=() +CMD="" +INSPECT="" + +ORIG_ARGS=$@ + +while [[ $# -gt 0 ]]; do + case $1 in + --inspect) + INSPECT="$1" + shift # past argument + ;; + --link_mode | --workdir | --drive | --debug | --client_id | --client_secret | --service_account | --share_email) + POSITIONAL_ARGS+=("$1") # save positional arg1 + POSITIONAL_ARGS+=("$2") # save positional arg2 + shift # past argument + shift # past value + ;; + *) + if [[ -z "$CMD" ]]; then + CMD=$1 + fi + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +if [[ -z "$CMD" ]]; then + CMD="usage" +fi + +if [[ ! -f "$MAIN_DIR/src/cli/wikigdrive-$CMD.ts" ]]; then + CMD="usage" +fi + +if test "$INSPECT" = "--inspect"; then + /usr/bin/env node --inspect --no-warnings --experimental-specifier-resolution=node --loader $NODE_MODULES/ts-node/esm $MAIN_DIR/src/cli/wikigdrive-$CMD.ts $ORIG_ARGS +else + /usr/bin/env node --no-warnings --experimental-specifier-resolution=node --loader $NODE_MODULES/ts-node/esm $MAIN_DIR/src/cli/wikigdrive-$CMD.ts $ORIG_ARGS +fi diff --git a/src/wikigdrivectl.sh b/src/wikigdrivectl.sh new file mode 100755 index 00000000..480b686b --- /dev/null +++ b/src/wikigdrivectl.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +FULL_PATH="$(readlink -f ${BASH_SOURCE[0]})" +MAIN_DIR=$(dirname "$FULL_PATH")/.. +NODE_MODULES=$MAIN_DIR/node_modules + +#export NODE_PATH=$NODE_MODULES +cd $MAIN_DIR + +POSITIONAL_ARGS=() +CMD="" +INSPECT="" + +while [[ $# -gt 0 ]]; do + case $1 in + --inspect) + INSPECT="$1" + shift # past argument + ;; + --link_mode | --workdir | --drive | --debug | --client_id | --client_secret | --service_account | --share_email) + POSITIONAL_ARGS+=("$1") # save positional arg1 + POSITIONAL_ARGS+=("$2") # save positional arg2 + shift # past argument + shift # past value + ;; + *) + if [[ -z "$CMD" ]]; then + CMD=$1 + fi + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +if [[ -z "$CMD" ]]; then + echo "No command" + exit 1 +fi + +if [[ ! -f "$MAIN_DIR/src/cli/wikigdrivectl-$CMD.ts" ]]; then + echo "Invalid command: $CMD" + exit 2 +fi + +if test "$INSPECT" = "--inspect"; then + /usr/bin/env node --inspect --no-warnings --experimental-specifier-resolution=node --loader $NODE_MODULES/ts-node/esm $MAIN_DIR/src/cli/wikigdrivectl-$CMD.ts $ORIG_ARGS +else + /usr/bin/env node --no-warnings --experimental-specifier-resolution=node --loader $NODE_MODULES/ts-node/esm $MAIN_DIR/src/cli/wikigdrivectl-$CMD.ts $ORIG_ARGS +fi