From 233a83cf3d15056733f6dfb9901311fc4c57c59b Mon Sep 17 00:00:00 2001 From: Max Wang Date: Sun, 17 Mar 2024 14:23:13 -0700 Subject: [PATCH] Revert "added local minio s3 and backup:mongodb script (#649)" This reverts commit d6959a2070ac05f1c65512576b3f46c6cc9c21a0. --- .env.template | 6 - backend/Dockerfile | 6 - backend/package-lock.json | 329 +----------------------- backend/package.json | 5 +- backend/src/config.ts | 14 - backend/src/scripts/backup-mongodb.ts | 114 -------- backend/src/scripts/download-mongodb.ts | 58 ----- docker-compose.yml | 18 -- 8 files changed, 6 insertions(+), 544 deletions(-) delete mode 100644 backend/src/scripts/backup-mongodb.ts delete mode 100644 backend/src/scripts/download-mongodb.ts diff --git a/.env.template b/.env.template index 52970c0cd..7b0b0d830 100644 --- a/.env.template +++ b/.env.template @@ -14,9 +14,3 @@ SIS_COURSE_APP_KEY=_ GOOGLE_CLIENT_ID=_ GOOGLE_CLIENT_SECRET=_ SESSION_SECRET=_ - -S3_ENDPOINT=minio -S3_PORT=9000 -S3_ACCESS_KEY_ID=root -S3_SECRET_ACCESS_KEY=password -S3_MONGO_BACKUP_BUCKET=mongo-backup diff --git a/backend/Dockerfile b/backend/Dockerfile index 6acae3dd6..36e6ef2ec 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,12 +1,6 @@ FROM node:16.14.2-slim AS builder RUN mkdir /backend WORKDIR /backend -RUN apt-get update \ - && apt-get install -y wget \ - && rm -rf /var/lib/apt/lists/* -RUN apt-get update && wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu1804-arm64-100.9.0.deb && \ - apt-get -y install ./mongodb-database-tools-*100.9.0.deb && \ - rm -f mongodb-database-tools-*.deb COPY package.json . RUN npm install --prefer-offline --no-audit COPY . . diff --git a/backend/package-lock.json b/backend/package-lock.json index bca48164e..4eee5a4f8 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -22,7 +22,6 @@ "graphql-type-json": "^0.3.2", "helmet": "^3.22.0", "lodash": "^4.17.21", - "minio": "^7.1.3", "mongodb": "^6.2.0", "mongoose": "^7.6.3", "passport": "^0.6.0", @@ -41,7 +40,7 @@ "@types/helmet": "0.0.46", "@types/jest": "^25.2.1", "@types/lodash": "^4.14.186", - "@types/node": "^13.13.52", + "@types/node": "^13.13.5", "@types/passport-google-oauth20": "^2.0.11", "@typescript-eslint/eslint-plugin": "^5.40.1", "@typescript-eslint/parser": "^5.40.1", @@ -4397,12 +4396,6 @@ "tslib": "^2.3.1" } }, - "node_modules/@zxing/text-encoding": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", - "integrity": "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==", - "optional": true - }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -4605,11 +4598,6 @@ "node": ">=8" } }, - "node_modules/async": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", - "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" - }, "node_modules/async-retry": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", @@ -4635,20 +4623,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/available-typed-arrays": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", - "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", - "dependencies": { - "possible-typed-array-names": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/axios": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", @@ -4864,14 +4838,6 @@ "readable-stream": "^3.4.0" } }, - "node_modules/block-stream2": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz", - "integrity": "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==", - "dependencies": { - "readable-stream": "^3.4.0" - } - }, "node_modules/body-parser": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", @@ -4935,11 +4901,6 @@ "node": ">=8" } }, - "node_modules/browser-or-node": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", - "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" - }, "node_modules/browserslist": { "version": "4.22.1", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.22.1.tgz", @@ -5025,14 +4986,6 @@ "ieee754": "^1.1.13" } }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -5618,14 +5571,6 @@ "node": ">=0.10.0" } }, - "node_modules/decode-uri-component": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", - "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", - "engines": { - "node": ">=0.10" - } - }, "node_modules/dedent": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.1.tgz", @@ -6444,27 +6389,6 @@ "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", "dev": true }, - "node_modules/fast-xml-parser": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.5.tgz", - "integrity": "sha512-sWvP1Pl8H03B8oFJpFR3HE31HUfwtX7Rlf9BNsvdpujD4n7WMhfmu8h9wOV2u+c1k0ZilTADhPqypzx2J690ZQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -6560,14 +6484,6 @@ "node": ">=8" } }, - "node_modules/filter-obj": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", - "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/finalhandler": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", @@ -6653,14 +6569,6 @@ } } }, - "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dependencies": { - "is-callable": "^1.1.3" - } - }, "node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -7134,20 +7042,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/hasown": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", @@ -7491,21 +7385,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -7524,17 +7403,6 @@ "node": ">=8" } }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -7574,20 +7442,6 @@ "node": ">=6" } }, - "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -7660,20 +7514,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-typed-array": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", - "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", - "dependencies": { - "which-typed-array": "^1.1.14" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-unc-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz", @@ -8819,11 +8659,6 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, - "node_modules/json-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-stream/-/json-stream-1.0.0.tgz", - "integrity": "sha512-H/ZGY0nIAg3QcOwE1QN/rK/Fa7gJn7Ii5obwp6zyPO4xiPNwpIMjqy2gwjBEGqzkF/vSWEIBQCBuN19hYiL6Qg==" - }, "node_modules/json-to-pretty-yaml": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.2.tgz", @@ -9302,38 +9137,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/minio": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minio/-/minio-7.1.3.tgz", - "integrity": "sha512-xPrLjWkTT5E7H7VnzOjF//xBp9I40jYB4aWhb2xTFopXXfw+Wo82DDWngdUju7Doy3Wk7R8C4LAgwhLHHnf0wA==", - "dependencies": { - "async": "^3.2.4", - "block-stream2": "^2.1.0", - "browser-or-node": "^2.1.1", - "buffer-crc32": "^0.2.13", - "fast-xml-parser": "^4.2.2", - "ipaddr.js": "^2.0.1", - "json-stream": "^1.0.0", - "lodash": "^4.17.21", - "mime-types": "^2.1.35", - "query-string": "^7.1.3", - "through2": "^4.0.2", - "web-encoding": "^1.1.5", - "xml": "^1.0.1", - "xml2js": "^0.5.0" - }, - "engines": { - "node": "^16 || ^18 || >=20" - } - }, - "node_modules/minio/node_modules/ipaddr.js": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", - "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==", - "engines": { - "node": ">= 10" - } - }, "node_modules/mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", @@ -10111,14 +9914,6 @@ "node": ">=8" } }, - "node_modules/possible-typed-array-names": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", - "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -10285,23 +10080,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/query-string": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", - "integrity": "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==", - "dependencies": { - "decode-uri-component": "^0.2.2", - "filter-obj": "^1.1.0", - "split-on-first": "^1.0.0", - "strict-uri-encode": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -10371,6 +10149,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -10622,11 +10401,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "node_modules/sax": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", - "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" - }, "node_modules/scuid": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/scuid/-/scuid-1.1.0.tgz", @@ -10899,14 +10673,6 @@ "memory-pager": "^1.0.2" } }, - "node_modules/split-on-first": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", - "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==", - "engines": { - "node": ">=6" - } - }, "node_modules/sponge-case": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/sponge-case/-/sponge-case-1.0.1.tgz", @@ -10960,18 +10726,11 @@ "node": ">=10.0.0" } }, - "node_modules/strict-uri-encode": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", - "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==", - "engines": { - "node": ">=4" - } - }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -11051,11 +10810,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -11115,14 +10869,6 @@ "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "dev": true }, - "node_modules/through2": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", - "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", - "dependencies": { - "readable-stream": "3" - } - }, "node_modules/title-case": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/title-case/-/title-case-3.0.3.tgz", @@ -11675,22 +11421,11 @@ "integrity": "sha512-Qp95D4TPJl1kC9SKigDcqgyM2VDVO4RiJc2d4qe5GrYm+zbIQCWWKAFaJNQ4BhdFeDGwBmAxqJBwWSJDb9T3BQ==", "dev": true }, - "node_modules/util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "dependencies": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true }, "node_modules/utils-merge": { "version": "1.0.1", @@ -11766,17 +11501,6 @@ "defaults": "^1.0.3" } }, - "node_modules/web-encoding": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", - "integrity": "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==", - "dependencies": { - "util": "^0.12.3" - }, - "optionalDependencies": { - "@zxing/text-encoding": "0.9.0" - } - }, "node_modules/web-streams-polyfill": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", @@ -11848,24 +11572,6 @@ "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==", "dev": true }, - "node_modules/which-typed-array": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.14.tgz", - "integrity": "sha512-VnXFiIW8yNn9kIHN88xvZ4yOWchftKDsRJ8fEPacX/wl1lOvBrhsJ/OeJCXq7B0AaijRuqgzSKalJoPk+D8MPg==", - "dependencies": { - "available-typed-arrays": "^1.0.6", - "call-bind": "^1.0.5", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", @@ -11928,31 +11634,6 @@ "node": ">=4.0.0" } }, - "node_modules/xml": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", - "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==" - }, - "node_modules/xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "engines": { - "node": ">=4.0" - } - }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/backend/package.json b/backend/package.json index c696e8230..ba915ce72 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,8 +11,6 @@ "coverage": "jest --coverage", "generate": "graphql-codegen --config codegen.ts", "lint": "tsc --noEmit && eslint --ext .ts,.tsx .", - "backup:mongodb": "node ./build/scripts/backup-mongodb.js", - "download:mongodb": "node ./build/scripts/download-mongodb.js", "update:catalog": "node ./build/scripts/update-catalog.js" }, "repository": { @@ -38,7 +36,7 @@ "@types/helmet": "0.0.46", "@types/jest": "^25.2.1", "@types/lodash": "^4.14.186", - "@types/node": "^13.13.52", + "@types/node": "^13.13.5", "@types/passport-google-oauth20": "^2.0.11", "@typescript-eslint/eslint-plugin": "^5.40.1", "@typescript-eslint/parser": "^5.40.1", @@ -64,7 +62,6 @@ "graphql-type-json": "^0.3.2", "helmet": "^3.22.0", "lodash": "^4.17.21", - "minio": "^7.1.3", "mongodb": "^6.2.0", "mongoose": "^7.6.3", "passport": "^0.6.0", diff --git a/backend/src/config.ts b/backend/src/config.ts index 362749ee6..1d0d9d569 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -30,13 +30,6 @@ export interface Config { SESSION_SECRET: string; GOOGLE_CLIENT_ID: string; GOOGLE_CLIENT_SECRET: string; - s3: { - endpoint: string, - port: number, - access_key_id: string, - secret_access_key: string, - mongo_backup_bucket: string - } } // All your secrets, keys go here @@ -58,11 +51,4 @@ export const config: Config = { SESSION_SECRET: env("SESSION_SECRET"), GOOGLE_CLIENT_ID: env("GOOGLE_CLIENT_ID"), GOOGLE_CLIENT_SECRET: env("GOOGLE_CLIENT_SECRET"), - s3: { - endpoint: env("S3_ENDPOINT"), - port: parseInt(env("S3_PORT")), - access_key_id: env("S3_ACCESS_KEY_ID"), - secret_access_key: env("S3_SECRET_ACCESS_KEY"), - mongo_backup_bucket: env("S3_MONGO_BACKUP_BUCKET") - } }; diff --git a/backend/src/scripts/backup-mongodb.ts b/backend/src/scripts/backup-mongodb.ts deleted file mode 100644 index 173fb1e6a..000000000 --- a/backend/src/scripts/backup-mongodb.ts +++ /dev/null @@ -1,114 +0,0 @@ -import * as Minio from 'minio'; -import * as child from 'child_process'; -import * as Fs from "fs"; -import { config } from "../config"; - -const ensureBucketExists = async (client : Minio.Client) => { - const exists = await client.bucketExists(config.s3.mongo_backup_bucket) - if (!exists) { - const res = await client.makeBucket(config.s3.mongo_backup_bucket) - console.log(`Bucket ${config.s3.mongo_backup_bucket} created`) - } - const bucketLifeCycleExists = await client.getBucketLifecycle(config.s3.mongo_backup_bucket); - if (bucketLifeCycleExists) return; - - const lifecycleConfig = { - Rule: [ - { - ID: 'Transition and Expiration Rule', - Status: 'Enabled', - Filter: { - Prefix: '', - }, - Expiration: { - Days: '7', - }, - }, - ], - } - - const err = await client.setBucketLifecycle(config.s3.mongo_backup_bucket, lifecycleConfig) - if (err != null) { - console.error("Could not set bucket lifecycle") - } -} - -const error = async (client : Minio.Client, toDelete : string[]) => { - client.removeObjects(config.s3.mongo_backup_bucket, toDelete, (e) => { - if (e) { - console.error("Unable to delete incorrect objects. Please do so manually.") - } - }); -} - -(async () => { - - const client = new Minio.Client({ - endPoint: config.s3.endpoint, - port: config.s3.port, - useSSL: config.s3.endpoint != "minio", - accessKey: config.s3.access_key_id, - secretKey: config.s3.secret_access_key, - }) - - await ensureBucketExists(client) - - const dump = child.spawn("mongodump", ["--uri", config.mongoDB.uri]) - - console.log("RUNNING MONGODUMP\n---------------------\n") - - const dumpedFiles:string[] = []; - - const fileFinder = new RegExp(/(?:writing )(.*)(?: to )(.*)\n/) - dump.stderr.on('data', (data) => { - // mongodump output goes by default to stderr - const matches = data.toString().match(fileFinder) - if (matches != null) { - dumpedFiles.push(matches[2]) - dumpedFiles.push(matches[2].substring(0, matches[2].length - 5) + ".metadata.json") - } - console.log(data.toString()) - }); - - dump.on('close', (code) => { - if (code != 0) { - console.error('mongodump failed') - process.exit(1); - } - const complete:string[] = [] - let puts = 0 - const folderName = (new Date()).getTime().toString() - console.log(`--------------------------\nSAVING BACKUP TO: ${folderName} \n\n`) - let errored = false - dumpedFiles.forEach((file:string) => { - const fileStream = Fs.createReadStream(file) - Fs.stat(file, (err, stats) => { - if (err) { - errored = true - return - } - const name = `${folderName}/${file}` - client.putObject(config.s3.mongo_backup_bucket, name, fileStream, stats.size, (err:any, objInfo:any) => { - puts++ - if (err) { - console.error(err) // err should be null - errored = true - } else { - console.log('Object Created:', objInfo) - complete.push(name) - } - if (puts == dumpedFiles.length) { - if (errored) { - error(client, complete) - console.log("\n--------------------\nBACKUP FAILED") - } else { - console.log("\n--------------------\nBACKUP SUCCESSFUL") - } - } - }) - }) - }) - - }); - -})(); \ No newline at end of file diff --git a/backend/src/scripts/download-mongodb.ts b/backend/src/scripts/download-mongodb.ts deleted file mode 100644 index d18b011db..000000000 --- a/backend/src/scripts/download-mongodb.ts +++ /dev/null @@ -1,58 +0,0 @@ -import * as Minio from 'minio'; -import { config } from "../config"; -import * as Fs from "fs"; - -const getBucketExists = async (client : Minio.Client) => { - return await client.bucketExists(config.s3.mongo_backup_bucket) -} - -(async () => { - - const client = new Minio.Client({ - endPoint: config.s3.endpoint, - port: config.s3.port, - useSSL: config.s3.endpoint != "minio", - accessKey: config.s3.access_key_id, - secretKey: config.s3.secret_access_key, - }) - - const exists = await getBucketExists(client) - - if (!exists) { - console.error("Bucket does not exist") - return - } - - if (process.argv.length < 3) { - console.error("Please give timestamp argument") - return - } - - const timeStamp = parseInt(process.argv[2]) - - const objectStream = client.listObjectsV2(config.s3.mongo_backup_bucket, timeStamp + "/dump/bt/") - - const objects:string[] = [] - - objectStream.on("data", (chunk : any) => { - objects.push(chunk.name) - }) - - objectStream.on("end", () => { - Fs.mkdir(`/backend/dump/bt`, { recursive: true }, (err) => { console.error(err) }) - objects.forEach(async (objectName) => { - const dataStream = await client.getObject(config.s3.mongo_backup_bucket, objectName) - const fileStream = Fs.createWriteStream("/backend/" + objectName.substring(timeStamp.toString().length + 1)) - fileStream.on("open", () => { - dataStream.on("data", (chunk : any) => { - fileStream.write(chunk) - }) - dataStream.on("end", () => { - fileStream.close() - console.log(`${"/backend/" + objectName.substring(timeStamp.toString().length + 1)} done writing`) - }) - }) - }) - }) - -})(); \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4da0b71b7..ea6ae8221 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,23 +55,5 @@ services: ports: - 6379:6379 restart: always - networks: - - bt - # LOCAL S3 STORAGE - FOR TESTING MONGODB BACKUP SCRIPT - minio: - image: 'minio/minio:latest' - ports: - - '9000:9000' - - '9090:9090' - environment: - MINIO_ROOT_USER: 'root' - MINIO_ROOT_PASSWORD: 'password' networks: - bt - volumes: - - 'minio:/data/minio' - command: minio server /data/minio --console-address ":9090" -volumes: - minio: - driver: local -