feat: revet download function

packages/api/app/src/index.ts

@@ -55,7 +55,7 @@ export class AppAPI {
   }
 
   public async checkEnvironment(nodeVersion?: string) {
-    if (semver.lt(process.versions.node, nodeVersion || '16.20.2')) {
+    if (semver.lt(process.versions.node, nodeVersion || '14.18.3')) {
       this.generatorCore.logger.warn(
         `🟡 ${i18n.t(localeKeys.environment.node_version)}`,
       );

packages/core/package.json

@@ -32,7 +32,7 @@
     "@swc/helpers": "0.5.1",
     "axios": "^1.6.0",
     "debug": "4.3.7",
-    "pacote": "^18.0.6",
+    "tar": "^6.1.13",
     "@modern-js/codesmith-utils": "workspace:*"
   },
   "devDependencies": {
@@ -42,7 +42,6 @@
     "@types/node": "^14.18.42",
     "@types/tar": "^4.0.5",
     "@types/debug": "^4.1.12",
-    "@types/pacote": "^11.1.8",
     "typescript": "^4.9.5"
   },
   "sideEffects": false

packages/core/src/generator/index.ts

@@ -281,8 +281,19 @@ check path: ${chalk.blue.underline(
     this.setbasePath(preBasePath);
     this.logger?.timing?.(`🕒 RunSubGenerator ${subGenerator}`, true);
   }
-
   public async prepareGenerators(generators: string[]) {
+    if ((global as any).CODESMITH_PREPARE_GLOBAL) {
+      return;
+    }
     await this.materialsManager.prepareGenerators(generators);
+    (global as any).CODESMITH_PREPARE_GLOBAL = true;
+  }
+
+  public async prepareGlobal() {
+    if ((global as any).CODESMITH_PREPARE_GLOBAL) {
+      return;
+    }
+    await this.materialsManager.prepareGlobal();
+    (global as any).CODESMITH_PREPARE_GLOBAL = true;
   }
 }

packages/core/src/utils/downloadPackage.ts

@@ -4,45 +4,13 @@ import type { Logger } from '@/logger';
 import type { ILogger } from '@/logger/constants';
 import { fs } from '@modern-js/codesmith-utils/fs-extra';
 import { semver } from '@modern-js/codesmith-utils/semver';
-import pacote from 'pacote';
+import axios from 'axios';
+import tar from 'tar';
+import { getNpmTarballUrl } from './getNpmTarballUrl';
 import { fsExists } from './fsExists';
 import { getNpmVersion } from './getNpmVersion';
 import { runInstall } from './packageManager';
 
-async function isValidCache(cacheDir: string) {
-  /* generator cache can use
-   * 1. .codesmith.completed exist
-   * 2. cache time is within the validity period
-   */
-  if (await fsExists(`${cacheDir}/.codesmith.completed`)) {
-    const preCacheTimeStr = await fs.readFile(
-      `${cacheDir}/.codesmith.completed`,
-      {
-        encoding: 'utf-8',
-      },
-    );
-    const preCacheTime = preCacheTimeStr
-      ? new Date(preCacheTimeStr)
-      : new Date(0);
-    if (Number(new Date()) - Number(preCacheTime) < CATCHE_VALIDITY_PREIOD) {
-      return true;
-    }
-    return false;
-  }
-  return false;
-}
-
-async function downloadAndDecompressTargz(
-  packageName: string,
-  version: string,
-  targetDir: string,
-  registryUrl?: string,
-): Promise<void> {
-  await pacote.extract(`${packageName}@${version}`, targetDir, {
-    registry: registryUrl,
-  });
-}
-
 const GeneratorVersionMap = new Map<string, string>();
 
 export async function getGeneratorVersion(
@@ -89,6 +57,76 @@ export async function getGeneratorVersion(
   return version;
 }
 
+async function isValidCache(cacheDir: string) {
+  /* generator cache can use
+   * 1. .codesmith.completed exist
+   * 2. cache time is within the validity period
+   */
+  if (await fsExists(`${cacheDir}/.codesmith.completed`)) {
+    const preCacheTimeStr = await fs.readFile(
+      `${cacheDir}/.codesmith.completed`,
+      {
+        encoding: 'utf-8',
+      },
+    );
+    const preCacheTime = preCacheTimeStr
+      ? new Date(preCacheTimeStr)
+      : new Date(0);
+    if (Number(new Date()) - Number(preCacheTime) < CATCHE_VALIDITY_PREIOD) {
+      return true;
+    }
+    return false;
+  }
+  return false;
+}
+
+async function downloadAndDecompressTargz(
+  tarballPkg: string,
+  targetDir: string,
+) {
+  const response = await axios({
+    method: 'get',
+    url: tarballPkg,
+    responseType: 'stream',
+    adapter: 'http',
+  });
+  if (response.status !== 200) {
+    throw new Error(
+      `download tar package get bad status code: ${response.status}`,
+    );
+  }
+  // create tmp file
+  const randomId = Math.floor(Math.random() * 10000);
+  const tempTgzFilePath = `${os.tmpdir()}/temp-${randomId}.tgz`;
+
+  const dest = fs.createWriteStream(tempTgzFilePath);
+
+  await new Promise<void>((resolve, reject) => {
+    response.data.pipe(dest);
+    response.data.on('error', (err: any) => {
+      reject(err);
+    });
+    dest.on('finish', () => {
+      resolve();
+    });
+  });
+  await new Promise<void>((resolve, reject) => {
+    fs.createReadStream(tempTgzFilePath)
+      .pipe(
+        tar.x({
+          strip: 1,
+          C: `${targetDir}`,
+        }),
+      )
+      .on('finish', () => {
+        resolve();
+      })
+      .on('error', (err: any) => {
+        reject(err);
+      });
+  });
+}
+
 /**
  * download npm package
  * @param {string} pkgName
@@ -106,10 +144,21 @@ export async function downloadPackage(
   } = {},
 ) {
   const { registryUrl, install, logger } = options;
-  const version = await getGeneratorVersion(pkgName, pkgVersion, {
-    registryUrl,
-    logger,
-  });
+  let version: string | undefined;
+  if (!semver.valid(pkgVersion)) {
+    // get pkgName version
+    logger?.timing(`🕒 get ${pkgName} version`);
+    version = await getNpmVersion(pkgName, {
+      registryUrl,
+      version: pkgVersion,
+    });
+    logger?.timing(`🕒 get ${pkgName} version`, true);
+    if (version === undefined) {
+      throw new Error(`package ${pkgName}@${pkgVersion} not found in registry`);
+    }
+  } else {
+    version = pkgVersion;
+  }
   const targetDir = `${os.tmpdir()}/csmith-generator/${pkgName}@${version}`;
   logger?.debug?.(
     `💡 [Download Generator Package]: ${pkgName}@${version} to ${targetDir}`,
@@ -120,9 +169,16 @@ export async function downloadPackage(
   await fs.remove(targetDir);
   await fs.mkdirp(targetDir);
 
+  logger?.timing(`🕒 get ${pkgName}@${version} tarball url`);
+  // get package tarball
+  const tarballPkg = await getNpmTarballUrl(pkgName, version, {
+    registryUrl,
+  });
+  logger?.timing(`🕒 get ${pkgName}@${version} tarball url`, true);
+
   logger?.timing(`🕒 download ${pkgName}@${version} tarball`);
   // download tarball and compress it to target directory
-  await downloadAndDecompressTargz(pkgName, version, targetDir, registryUrl);
+  await downloadAndDecompressTargz(tarballPkg, targetDir);
   logger?.timing(`🕒 download ${pkgName}@${version} tarball`, true);
 
   if (install) {

packages/core/src/utils/getNpmTarballUrl.ts

@@ -0,0 +1,37 @@
+import { NPM_API_TIMEOUT } from '@/constants';
+import { execa } from '@modern-js/codesmith-utils/execa';
+import { timeoutPromise } from './timeoutPromise';
+
+interface Options {
+  registryUrl?: string;
+}
+
+export async function getNpmTarballUrl(
+  pkgName: string,
+  pkgVersion: string,
+  options?: Options,
+): Promise<string> {
+  const { registryUrl } = options || {};
+  const params = ['view', `${pkgName}@${pkgVersion}`, 'dist', '--json'];
+
+  if (registryUrl) {
+    params.push('--registry');
+    params.push(registryUrl);
+  }
+
+  const getPkgInfoPromise = execa('npm', params);
+  const { stdout } = await timeoutPromise(
+    getPkgInfoPromise,
+    NPM_API_TIMEOUT,
+    `Get npm tarball of '${pkgName}'`,
+  );
+
+  try {
+    const pkgDistInfo = JSON.parse(stdout);
+    return pkgDistInfo.tarball;
+  } catch (e) {
+    throw new Error(
+      `Version \`${pkgVersion}\` for package \`${pkgName}\` could not be found`,
+    );
+  }
+}

packages/global/modern.config.ts

@@ -2,10 +2,10 @@ import { defineConfig, moduleTools } from '@modern-js/module-tools';
 
 export default defineConfig({
   buildConfig: {
+    target: 'es2020',
     dts: false,
     format: 'umd',
     autoExternal: false,
-    externals: ['bluebird', '@sigstore/core', '@sigstore/verify'],
   },
   plugins: [moduleTools()],
 });

packages/global/package.json

@@ -13,84 +13,7 @@
   ],
   "version": "2.5.1",
   "jsnext:source": "./src/index.ts",
-  "types": "./dist/types/index.d.ts",
-  "main": "./dist/cjs/index.js",
-  "module": "./dist/esm-node/index.js",
-  "exports": {
-    ".": {
-      "jsnext:source": "./src/index.ts",
-      "types": "./dist/types/index.d.ts",
-      "default": "./dist/cjs/index.js"
-    },
-    "./chalk": {
-      "jsnext:source": "./src/chalk.ts",
-      "import": "./dist/esm/chalk.js",
-      "default": "./dist/cjs/chalk.js"
-    },
-    "./execa": {
-      "jsnext:source": "./src/execa.ts",
-      "import": "./dist/esm/execa.js",
-      "default": "./dist/cjs/execa.js"
-    },
-    "./fs-extra": {
-      "jsnext:source": "./src/fs-extra.ts",
-      "import": "./dist/esm/fs-extra.js",
-      "default": "./dist/cjs/fs-extra.js"
-    },
-    "./glob": {
-      "jsnext:source": "./src/glob.ts",
-      "import": "./dist/esm/glob.js",
-      "default": "./dist/cjs/glob.js"
-    },
-    "./lodash": {
-      "jsnext:source": "./src/lodash.ts",
-      "import": "./dist/esm/lodash.js",
-      "default": "./dist/cjs/lodash.js"
-    },
-    "./ora": {
-      "jsnext:source": "./src/ora.ts",
-      "import": "./dist/esm/ora.js",
-      "default": "./dist/cjs/ora.js"
-    },
-    "./semver": {
-      "jsnext:source": "./src/semver.ts",
-      "import": "./dist/esm/semver.js",
-      "default": "./dist/cjs/semver.js"
-    },
-    "./npm": {
-      "jsnext:source": "./src/npm.ts",
-      "import": "./dist/esm/npm.js",
-      "default": "./dist/cjs/npm.js"
-    }
-  },
-  "typesVersions": {
-    "*": {
-      "chalk": [
-        "./dist/types/chalk.d.ts"
-      ],
-      "execa": [
-        "./dist/types/execa.d.ts"
-      ],
-      "fs-extra": [
-        "./dist/types/fs-extra.d.ts"
-      ],
-      "glob": [
-        "./dist/types/glob.d.ts"
-      ],
-      "lodash": [
-        "./dist/types/lodash.d.ts"
-      ],
-      "ora": [
-        "./dist/types/ora.d.ts"
-      ],
-      "semver": [
-        "./dist/types/semver.d.ts"
-      ],
-      "npm": [
-        "./dist/types/npm.d.ts"
-      ]
-    }
-  },
+  "main": "./dist/index.js",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public",