From 36b903f0fc41559ad516ab1a7d091b332cb714de Mon Sep 17 00:00:00 2001 From: Tycho Bokdam Date: Thu, 27 Jul 2023 23:19:28 +0200 Subject: [PATCH 1/3] feat(vercel): Improve `buildTarget` logic BREAKING CHANGE: The logic update could break your build / deploy, add `buildTarget` option now with the project name and optional config --- .../vercel/src/executors/build/build.impl.ts | 54 ++++++++--------- .../src/executors/deploy/deploy.impl.ts | 60 +++++++++++-------- .../get-output-directory-from-build-target.ts | 14 +++++ packages/vercel/src/utils/is-github-ci.ts | 1 - 4 files changed, 74 insertions(+), 55 deletions(-) create mode 100644 packages/vercel/src/utils/get-output-directory-from-build-target.ts delete mode 100644 packages/vercel/src/utils/is-github-ci.ts diff --git a/packages/vercel/src/executors/build/build.impl.ts b/packages/vercel/src/executors/build/build.impl.ts index bafe1e4a..cfb5cf48 100644 --- a/packages/vercel/src/executors/build/build.impl.ts +++ b/packages/vercel/src/executors/build/build.impl.ts @@ -1,5 +1,7 @@ -import { readJsonFile, writeJsonFile } from '@nx/devkit' -import { buildCommand, copyFile, execCommand } from '@nx-extend/core' +import { parseTargetString, readJsonFile, writeJsonFile } from '@nx/devkit' +import { targetToTargetString } from '@nx/devkit/src/executors/parse-target-string' +import { readCachedProjectGraph } from '@nx/workspace/src/core/project-graph' +import { buildCommand, copyFile, execCommand, USE_VERBOSE_LOGGING } from '@nx-extend/core' import { existsSync, rmSync } from 'fs' import { join } from 'path' @@ -8,17 +10,17 @@ import type { ExecutorContext } from '@nx/devkit' import { addEnvVariablesToFile } from '../../utils/add-env-variables-to-file' import { enrichVercelEnvFile } from '../../utils/enrich-vercel-env-file' import { getEnvVars } from '../../utils/get-env-vars' +import { getOutputDirectoryFromBuildTarget } from '../../utils/get-output-directory-from-build-target' import { vercelToken } from '../../utils/vercel-token' import { getOutputDirectory } from './utils/get-output-directory' export interface BuildOptions { projectId: string orgId: string - debug?: boolean envVars?: Record buildTarget?: string - buildConfig?: string framework?: string + outputPath?: string nodeVersion?: '16.x' } @@ -26,9 +28,14 @@ export function buildExecutor( options: BuildOptions, context: ExecutorContext ): Promise<{ success: boolean }> { - const { targets } = context.workspace.projects[context.projectName] const framework = options.framework || 'nextjs' - const buildTarget = options.buildTarget || (framework === 'nextjs' ? 'build-next' : 'build') + let buildTarget = options.buildTarget || (framework === 'nextjs' ? 'build-next' : 'build') + + if (!buildTarget.includes(':')) { + buildTarget = `${context.projectName}:${buildTarget}` + } + + const targetString = parseTargetString(buildTarget, readCachedProjectGraph()) if (!options.orgId) { throw new Error(`"orgId" option is required!`) @@ -38,22 +45,15 @@ export function buildExecutor( throw new Error(`"projectId" option is required!`) } - if (!targets[buildTarget]) { - throw new Error( - `"${context.projectName}" is missing the "${buildTarget}" target!` - ) + if (!targetString) { + throw new Error(`Invalid build target "${buildTarget}"!`) } - if (!targets[buildTarget]?.options?.outputPath) { + const outputDirectory = options.outputPath || getOutputDirectoryFromBuildTarget(context, buildTarget) + if (!outputDirectory) { throw new Error(`"${buildTarget}" target has no "outputPath" configured!`) } - if (options.buildConfig && !targets[buildTarget]?.configurations[options.buildConfig]) { - throw new Error( - `"${buildTarget}" target has no configuration "${options.buildConfig}"!` - ) - } - const vercelDirectory = '.vercel' const vercelDirectoryLocation = join(context.root, vercelDirectory) @@ -70,15 +70,15 @@ export function buildExecutor( settings: {} }) - const vercelEnironment = context.configurationName === 'production' ? 'production' : 'preview' + const vercelEnvironment = context.configurationName === 'production' ? 'production' : 'preview' // Pull latest const { success: pullSuccess } = execCommand(buildCommand([ 'npx vercel pull --yes', - `--environment=${vercelEnironment}`, + `--environment=${vercelEnvironment}`, vercelToken && `--token=${vercelToken}`, - options.debug && '--debug' + USE_VERBOSE_LOGGING && '--debug' ])) if (!pullSuccess) { @@ -86,9 +86,7 @@ export function buildExecutor( } const vercelProjectJson = `./${vercelDirectory}/project.json` - const outputDirectory = targets[buildTarget]?.options?.outputPath - - const vercelEnvFile = `.env.${vercelEnironment}.local` + const vercelEnvFile = `.env.${vercelEnvironment}.local` const vercelEnvFileLocation = join(context.root, vercelDirectory) const envVars = getEnvVars(options.envVars, true) @@ -106,10 +104,8 @@ export function buildExecutor( createdAt: new Date().getTime(), framework, devCommand: null, - installCommand: "echo ''", - buildCommand: `nx run ${context.projectName}:${buildTarget}:${ - options.buildConfig || context.configurationName - }`, + installCommand: 'echo ""', + buildCommand: `nx run ${targetToTargetString(targetString)}`, outputDirectory: getOutputDirectory(framework, outputDirectory), rootDirectory: null, directoryListing: false, @@ -119,11 +115,11 @@ export function buildExecutor( const { success } = execCommand(buildCommand([ 'npx vercel build', - `--output ${targets[buildTarget].options.outputPath}/.vercel/output`, + `--output ${outputDirectory}/.vercel/output`, context.configurationName === 'production' && '--prod', vercelToken && `--token=${vercelToken}`, - options.debug && '--debug' + USE_VERBOSE_LOGGING && '--debug' ])) if (success) { diff --git a/packages/vercel/src/executors/deploy/deploy.impl.ts b/packages/vercel/src/executors/deploy/deploy.impl.ts index f3c567e5..64bd4d3b 100644 --- a/packages/vercel/src/executors/deploy/deploy.impl.ts +++ b/packages/vercel/src/executors/deploy/deploy.impl.ts @@ -1,16 +1,15 @@ import * as githubCore from '@actions/core' -import { setOutput } from '@actions/core' -import { buildCommand, execCommand } from '@nx-extend/core' +import { buildCommand, execCommand, isCI, USE_VERBOSE_LOGGING } from '@nx-extend/core' import { existsSync } from 'fs' import { join } from 'path' import type { ExecutorContext } from '@nx/devkit' -import { isGithubCi } from '../../utils/is-github-ci' +import { getOutputDirectoryFromBuildTarget } from '../../utils/get-output-directory-from-build-target' import { vercelToken } from '../../utils/vercel-token' export interface DeployOptions { - debug?: boolean + buildTarget?: string regions?: string } @@ -20,35 +19,46 @@ export async function deployExecutor( ): Promise<{ success: boolean }> { const { targets } = context.workspace.projects[context.projectName] - const vercelBuildTarget = Object.keys(targets).find( - (target) => targets[target].executor === '@nx-extend/vercel:build' - ) - const buildTarget = targets[vercelBuildTarget]?.options?.buildTarget || 'build-next' + let outputDirectory = '' - if (!targets[buildTarget]?.options?.outputPath) { - throw new Error(`"${buildTarget}" target has no "outputPath" configured!`) + if (options.buildTarget) { + outputDirectory = getOutputDirectoryFromBuildTarget(context, options.buildTarget) + } else { + const projectVercelBuildTarget = Object.keys(targets).find((target) => ( + targets[target].executor === '@nx-extend/vercel:build' + )) + + if (projectVercelBuildTarget) { + let projectBuildTarget = targets[projectVercelBuildTarget]?.options?.buildTarget || 'build-next' + if (!projectBuildTarget.includes(':')) { + projectBuildTarget = `${context.projectName}:${projectBuildTarget}` + } + + outputDirectory = getOutputDirectoryFromBuildTarget(context, projectBuildTarget) + } } - if (!existsSync(join(targets[buildTarget].options.outputPath, '.vercel/project.json'))) { + if (!outputDirectory) { + throw new Error(`Could not find the builds output path!`) + } + + if (!existsSync(join(outputDirectory, '.vercel/project.json'))) { throw new Error('No ".vercel/project.json" found in dist folder! ') } - const { success, output } = execCommand( - buildCommand([ - 'npx vercel deploy --prebuilt', - context.configurationName === 'production' && '--prod', - vercelToken && `--token=${vercelToken}`, - options.regions && `--regions=${options.regions}`, - - options.debug && '--debug' - ]), - { - cwd: targets[buildTarget].options.outputPath - } - ) + const { success, output } = execCommand(buildCommand([ + 'npx vercel deploy --prebuilt', + context.configurationName === 'production' && '--prod', + vercelToken && `--token=${vercelToken}`, + options.regions && `--regions=${options.regions}`, + + USE_VERBOSE_LOGGING && '--debug' + ]), { + cwd: outputDirectory + }) // When running in GitHub CI add the URL of the deployment as summary - if (isGithubCi) { + if (isCI()) { // Add comment instead of summary (Look at https://github.com/mshick/add-pr-comment) const parts = output.split('\n') diff --git a/packages/vercel/src/utils/get-output-directory-from-build-target.ts b/packages/vercel/src/utils/get-output-directory-from-build-target.ts new file mode 100644 index 00000000..c22eccb7 --- /dev/null +++ b/packages/vercel/src/utils/get-output-directory-from-build-target.ts @@ -0,0 +1,14 @@ +import { ExecutorContext, parseTargetString, readTargetOptions } from '@nx/devkit' +import { readCachedProjectGraph } from '@nx/workspace/src/core/project-graph' + +export function getOutputDirectoryFromBuildTarget(context: ExecutorContext, buildTarget: string): string | undefined { + const targetString = parseTargetString(buildTarget, readCachedProjectGraph()) + const targetOptions = readTargetOptions(targetString, context) + const outputDirectory = targetOptions?.outputPath + + if (!outputDirectory && targetOptions?.buildTarget) { + return getOutputDirectoryFromBuildTarget(context, targetOptions?.buildTarget) + } + + return outputDirectory +} diff --git a/packages/vercel/src/utils/is-github-ci.ts b/packages/vercel/src/utils/is-github-ci.ts deleted file mode 100644 index 1aacfe79..00000000 --- a/packages/vercel/src/utils/is-github-ci.ts +++ /dev/null @@ -1 +0,0 @@ -export const isGithubCi = process.env.CI && process.env.GITHUB_ACTIONS From 9b999c55b365132e2e0abd66f90ff6c04251fc9a Mon Sep 17 00:00:00 2001 From: Tycho Bokdam Date: Fri, 28 Jul 2023 00:34:30 +0200 Subject: [PATCH 2/3] perf(gcp-task-runner): Tar files before uploading --- package.json | 2 + packages/gcp-task-runner/src/gcp-cache.ts | 124 ++++++++++++---------- packages/gcp-task-runner/src/logger.ts | 14 ++- packages/gcp-task-runner/src/runner.ts | 6 ++ yarn.lock | 14 ++- 5 files changed, 101 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index 6f1db9ab..0fc37f15 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,13 @@ "@actions/core": "^1.10.0", "@nx/devkit": "16.3.2", "@nx/workspace": "16.3.2", + "@types/tar": "^6.1.5", "axios": "^1.4.0", "crypto-js": "^4.1.1", "deepmerge": "^4.3.1", "rxjs-for-await": "^1.0.0", "shelljs": "^0.8.5", + "tar": "^6.1.15", "tslib": "^2.5.3", "yargs": "^17.7.2" }, diff --git a/packages/gcp-task-runner/src/gcp-cache.ts b/packages/gcp-task-runner/src/gcp-cache.ts index 76ccfdbf..11131fe5 100644 --- a/packages/gcp-task-runner/src/gcp-cache.ts +++ b/packages/gcp-task-runner/src/gcp-cache.ts @@ -1,31 +1,25 @@ import { File, Storage } from '@google-cloud/storage' -import { mkdirSync, promises } from 'fs' -import { dirname, join, relative } from 'path' +import { join } from 'path' +import { create, extract } from 'tar' import type { MessageReporter } from './message-reporter' -import type { Bucket, UploadResponse } from '@google-cloud/storage' +import type { Bucket } from '@google-cloud/storage' import type { RemoteCache } from '@nx/workspace/src/tasks-runner/default-tasks-runner' import { Logger } from './logger' export class GcpCache implements RemoteCache { - private readonly bucket: Bucket + private readonly bucket: Bucket private readonly logger = new Logger() - private readonly messages: MessageReporter - private uploadQueue: Array> = [] - public constructor(bucket: string, messages: MessageReporter) { this.bucket = new Storage().bucket(bucket) this.messages = messages } - public async retrieve( - hash: string, - cacheDirectory: string - ): Promise { + public async retrieve(hash: string, cacheDirectory: string): Promise { if (this.messages.error) { return false } @@ -33,23 +27,19 @@ export class GcpCache implements RemoteCache { try { this.logger.debug(`Downloading ${hash}`) - const commitFile = this.bucket.file(`${hash}.commit`) - + const commitFile = this.bucket.file(this.getCommitFileName(hash)) if (!(await commitFile.exists())[0]) { this.logger.debug(`Cache miss ${hash}`) return false } - // Get all the files for this hash - const [files] = await this.bucket.getFiles({ prefix: `${hash}/` }) + const tarFile = this.bucket.file(this.getTarFileName(hash)) + await this.downloadFile(cacheDirectory, tarFile) + await this.extractFile(cacheDirectory, tarFile) - // Download all the files - await Promise.all( - files.map((file) => this.downloadFile(cacheDirectory, file)) - ) - - await this.downloadFile(cacheDirectory, commitFile) // commit file after we're sure all content is downloaded + // commit file after we're sure all content is downloaded + await this.downloadFile(cacheDirectory, commitFile) this.logger.success(`Cache hit ${hash}`) @@ -63,47 +53,33 @@ export class GcpCache implements RemoteCache { } } - public async store(hash: string, cacheDirectory: string): Promise { + public store(hash: string, cacheDirectory: string): Promise { if (this.messages.error) { return Promise.resolve(false) } - try { - await this.createAndUploadFiles(hash, cacheDirectory) - - return true - } catch (err) { - this.logger.warn(`Failed to upload cache`, err) - - return false - } + return this.createAndUploadFile(hash, cacheDirectory) } private async downloadFile(cacheDirectory: string, file: File) { - const destination = join(cacheDirectory, file.name) - mkdirSync(dirname(destination), { - recursive: true - }) - - await file.download({ destination }) + await file.download({ destination: join(cacheDirectory, file.name) }) } - private async createAndUploadFiles( - hash: string, - cacheDirectory: string - ): Promise { + private async createAndUploadFile(hash: string, cacheDirectory: string): Promise { try { this.logger.debug(`Storage Cache: Uploading ${hash}`) - // Add all the files to the upload queue - await this.uploadDirectory(cacheDirectory, join(cacheDirectory, hash)) - // Upload all the files - await Promise.all(this.uploadQueue) + const tarFilePath = this.getTarFilePath(hash, cacheDirectory) + await this.createTarFile(tarFilePath, hash, cacheDirectory) + await this.uploadFile(cacheDirectory, this.getTarFileName(hash)) // commit file once we're sure all content is uploaded - await this.bucket.upload(join(cacheDirectory, `${hash}.commit`)) + await this.uploadFile(cacheDirectory, this.getCommitFileName(hash)) this.logger.debug(`Storage Cache: Stored ${hash}`) + + return true + } catch (err) { this.messages.error = err @@ -113,17 +89,53 @@ export class GcpCache implements RemoteCache { } } - private async uploadDirectory(cacheDirectory: string, dir: string) { - for (const entry of await promises.readdir(dir)) { - const full = join(dir, entry) - const stats = await promises.stat(full) + private async uploadFile(cacheDirectory: string, file: string): Promise { + const destination = join(cacheDirectory, file) - if (stats.isDirectory()) { - await this.uploadDirectory(cacheDirectory, full) - } else if (stats.isFile()) { - const destination = relative(cacheDirectory, full) - this.uploadQueue.push(this.bucket.upload(full, { destination })) - } + try { + await this.bucket.upload(destination, { destination: file }) + } catch (err) { + throw new Error(`Storage Cache: Upload error - ${err}`) } } + + private async createTarFile(tgzFilePath: string, hash: string, cacheDirectory: string): Promise { + try { + await create({ + gzip: true, + file: tgzFilePath, + cwd: cacheDirectory + }, [hash]) + } catch (err) { + this.logger.error(`Error creating tar file for has "${hash}"`, err) + + throw new Error(`Error creating tar file - ${err}`) + } + } + + private async extractFile(cacheDirectory: string, file: File): Promise { + try { + await extract({ + file: join(cacheDirectory, file.name), + cwd: cacheDirectory + }) + } catch (err) { + this.logger.error(`Error extracting tar file "${file.name}"`, err) + + throw new Error(`\`Error extracting tar file "${file.name}" - ${err}`) + } + } + + private getTarFileName(hash: string): string { + return `${hash}.tar.gz` + } + + private getTarFilePath(hash: string, cacheDirectory: string): string { + return join(cacheDirectory, this.getTarFileName(hash)) + } + + private getCommitFileName(hash: string): string { + return `${hash}.commit` + } + } diff --git a/packages/gcp-task-runner/src/logger.ts b/packages/gcp-task-runner/src/logger.ts index fea03174..ca7f3ddd 100644 --- a/packages/gcp-task-runner/src/logger.ts +++ b/packages/gcp-task-runner/src/logger.ts @@ -7,7 +7,7 @@ export class Logger { } output.log({ - title: 'GCP-NX', + title: 'GCP Task runner', bodyLines: message.split('\n') ?? [] }) } @@ -28,7 +28,17 @@ export class Logger { public success(message: string): void { output.success({ - title: message + title: 'GCP Task runner', + bodyLines: message.split('\n') }) } + + public note(message: string): void { + output.addNewline() + output.note({ + title: 'GCP Task runner', + bodyLines: message.split('\n') + }) + output.addNewline() + } } diff --git a/packages/gcp-task-runner/src/runner.ts b/packages/gcp-task-runner/src/runner.ts index dc9b0c70..c70f7853 100644 --- a/packages/gcp-task-runner/src/runner.ts +++ b/packages/gcp-task-runner/src/runner.ts @@ -12,6 +12,12 @@ export const tasksRunner = ( // Create the logger const logger = new Logger() + if (options.skipNxCache) { + logger.note('Using local cache') + + return defaultTaskRunner(tasks, options, context) + } + try { const messages = new MessageReporter(logger) const remoteCache = new GcpCache(options.bucket, messages) diff --git a/yarn.lock b/yarn.lock index c845894c..806588e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6116,6 +6116,16 @@ __metadata: languageName: node linkType: hard +"@types/tar@npm:^6.1.5": + version: 6.1.5 + resolution: "@types/tar@npm:6.1.5" + dependencies: + "@types/node": "*" + minipass: ^4.0.0 + checksum: 1efa71c8d72f2b02d16ffecb65c4a96be9dacce000c4207532a686477386524dd99b6e26904319354be3ac9ac4d41296a0cb9cf87d05f8f21ae31c68fe446348 + languageName: node + linkType: hard + "@types/through@npm:*": version: 0.0.30 resolution: "@types/through@npm:0.0.30" @@ -16820,6 +16830,7 @@ __metadata: "@types/jest": 29.5.2 "@types/node": 18.15.11 "@types/shelljs": ^0.8.12 + "@types/tar": ^6.1.5 "@types/yargs": ^17.0.24 "@typescript-eslint/eslint-plugin": 5.60.0 "@typescript-eslint/parser": 5.60.0 @@ -16839,6 +16850,7 @@ __metadata: prettier: 2.8.8 rxjs-for-await: ^1.0.0 shelljs: ^0.8.5 + tar: ^6.1.15 ts-jest: 29.1.0 ts-node: 10.9.1 tslib: ^2.5.3 @@ -21073,7 +21085,7 @@ __metadata: languageName: node linkType: hard -"tar@npm:^6.1.11, tar@npm:^6.1.2": +"tar@npm:^6.1.11, tar@npm:^6.1.15, tar@npm:^6.1.2": version: 6.1.15 resolution: "tar@npm:6.1.15" dependencies: From af1d5be76d874f356282e371c0e469034aa3cbd6 Mon Sep 17 00:00:00 2001 From: Tycho Bokdam Date: Fri, 28 Jul 2023 00:40:53 +0200 Subject: [PATCH 3/3] refactor(gcp-task-runner): Log that we are downloading before download starts --- packages/gcp-task-runner/src/gcp-cache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gcp-task-runner/src/gcp-cache.ts b/packages/gcp-task-runner/src/gcp-cache.ts index 11131fe5..45ffb572 100644 --- a/packages/gcp-task-runner/src/gcp-cache.ts +++ b/packages/gcp-task-runner/src/gcp-cache.ts @@ -25,8 +25,6 @@ export class GcpCache implements RemoteCache { } try { - this.logger.debug(`Downloading ${hash}`) - const commitFile = this.bucket.file(this.getCommitFileName(hash)) if (!(await commitFile.exists())[0]) { this.logger.debug(`Cache miss ${hash}`) @@ -34,6 +32,8 @@ export class GcpCache implements RemoteCache { return false } + this.logger.debug(`Downloading ${hash}`) + const tarFile = this.bucket.file(this.getTarFileName(hash)) await this.downloadFile(cacheDirectory, tarFile) await this.extractFile(cacheDirectory, tarFile)