Merge pull request #137 from TriPSs/improvements

feat(vercel): Improve `buildTarget` logic

package.json

@@ -35,11 +35,13 @@
     "@actions/core": "^1.10.0",
     "@nx/devkit": "16.3.2",
     "@nx/workspace": "16.3.2",
+    "@types/tar": "^6.1.5",
     "axios": "^1.4.0",
     "crypto-js": "^4.1.1",
     "deepmerge": "^4.3.1",
     "rxjs-for-await": "^1.0.0",
     "shelljs": "^0.8.5",
+    "tar": "^6.1.15",
     "tslib": "^2.5.3",
     "yargs": "^17.7.2"
   },

packages/gcp-task-runner/src/gcp-cache.ts

@@ -1,55 +1,45 @@
 import { File, Storage } from '@google-cloud/storage'
-import { mkdirSync, promises } from 'fs'
-import { dirname, join, relative } from 'path'
+import { join } from 'path'
+import { create, extract } from 'tar'
 
 import type { MessageReporter } from './message-reporter'
-import type { Bucket, UploadResponse } from '@google-cloud/storage'
+import type { Bucket } from '@google-cloud/storage'
 import type { RemoteCache } from '@nx/workspace/src/tasks-runner/default-tasks-runner'
 
 import { Logger } from './logger'
 
 export class GcpCache implements RemoteCache {
-  private readonly bucket: Bucket
 
+  private readonly bucket: Bucket
   private readonly logger = new Logger()
-
   private readonly messages: MessageReporter
 
-  private uploadQueue: Array<Promise<UploadResponse>> = []
-
   public constructor(bucket: string, messages: MessageReporter) {
     this.bucket = new Storage().bucket(bucket)
     this.messages = messages
   }
 
-  public async retrieve(
-    hash: string,
-    cacheDirectory: string
-  ): Promise<boolean> {
+  public async retrieve(hash: string, cacheDirectory: string): Promise<boolean> {
     if (this.messages.error) {
       return false
     }
 
     try {
-      this.logger.debug(`Downloading ${hash}`)
-
-      const commitFile = this.bucket.file(`${hash}.commit`)
-
+      const commitFile = this.bucket.file(this.getCommitFileName(hash))
       if (!(await commitFile.exists())[0]) {
         this.logger.debug(`Cache miss ${hash}`)
 
         return false
       }
 
-      // Get all the files for this hash
-      const [files] = await this.bucket.getFiles({ prefix: `${hash}/` })
+      this.logger.debug(`Downloading ${hash}`)
 
-      // Download all the files
-      await Promise.all(
-        files.map((file) => this.downloadFile(cacheDirectory, file))
-      )
+      const tarFile = this.bucket.file(this.getTarFileName(hash))
+      await this.downloadFile(cacheDirectory, tarFile)
+      await this.extractFile(cacheDirectory, tarFile)
 
-      await this.downloadFile(cacheDirectory, commitFile) // commit file after we're sure all content is downloaded
+      // commit file after we're sure all content is downloaded
+      await this.downloadFile(cacheDirectory, commitFile)
 
       this.logger.success(`Cache hit ${hash}`)
 
@@ -63,47 +53,33 @@ export class GcpCache implements RemoteCache {
     }
   }
 
-  public async store(hash: string, cacheDirectory: string): Promise<boolean> {
+  public store(hash: string, cacheDirectory: string): Promise<boolean> {
     if (this.messages.error) {
       return Promise.resolve(false)
     }
 
-    try {
-      await this.createAndUploadFiles(hash, cacheDirectory)
-
-      return true
-    } catch (err) {
-      this.logger.warn(`Failed to upload cache`, err)
-
-      return false
-    }
+    return this.createAndUploadFile(hash, cacheDirectory)
   }
 
   private async downloadFile(cacheDirectory: string, file: File) {
-    const destination = join(cacheDirectory, file.name)
-    mkdirSync(dirname(destination), {
-      recursive: true
-    })
-
-    await file.download({ destination })
+    await file.download({ destination: join(cacheDirectory, file.name) })
   }
 
-  private async createAndUploadFiles(
-    hash: string,
-    cacheDirectory: string
-  ): Promise<boolean> {
+  private async createAndUploadFile(hash: string, cacheDirectory: string): Promise<boolean> {
     try {
       this.logger.debug(`Storage Cache: Uploading ${hash}`)
 
-      // Add all the files to the upload queue
-      await this.uploadDirectory(cacheDirectory, join(cacheDirectory, hash))
-      // Upload all the files
-      await Promise.all(this.uploadQueue)
+      const tarFilePath = this.getTarFilePath(hash, cacheDirectory)
+      await this.createTarFile(tarFilePath, hash, cacheDirectory)
 
+      await this.uploadFile(cacheDirectory, this.getTarFileName(hash))
       // commit file once we're sure all content is uploaded
-      await this.bucket.upload(join(cacheDirectory, `${hash}.commit`))
+      await this.uploadFile(cacheDirectory, this.getCommitFileName(hash))
 
       this.logger.debug(`Storage Cache: Stored ${hash}`)
+
+      return true
+
     } catch (err) {
       this.messages.error = err
 
@@ -113,17 +89,53 @@ export class GcpCache implements RemoteCache {
     }
   }
 
-  private async uploadDirectory(cacheDirectory: string, dir: string) {
-    for (const entry of await promises.readdir(dir)) {
-      const full = join(dir, entry)
-      const stats = await promises.stat(full)
+  private async uploadFile(cacheDirectory: string, file: string): Promise<void> {
+    const destination = join(cacheDirectory, file)
 
-      if (stats.isDirectory()) {
-        await this.uploadDirectory(cacheDirectory, full)
-      } else if (stats.isFile()) {
-        const destination = relative(cacheDirectory, full)
-        this.uploadQueue.push(this.bucket.upload(full, { destination }))
-      }
+    try {
+      await this.bucket.upload(destination, { destination: file })
+    } catch (err) {
+      throw new Error(`Storage Cache: Upload error - ${err}`)
+    }
+  }
+
+  private async createTarFile(tgzFilePath: string, hash: string, cacheDirectory: string): Promise<void> {
+    try {
+      await create({
+        gzip: true,
+        file: tgzFilePath,
+        cwd: cacheDirectory
+      }, [hash])
+    } catch (err) {
+      this.logger.error(`Error creating tar file for has "${hash}"`, err)
+
+      throw new Error(`Error creating tar file - ${err}`)
+    }
+  }
+
+  private async extractFile(cacheDirectory: string, file: File): Promise<void> {
+    try {
+      await extract({
+        file: join(cacheDirectory, file.name),
+        cwd: cacheDirectory
+      })
+    } catch (err) {
+      this.logger.error(`Error extracting tar file "${file.name}"`, err)
+
+      throw new Error(`\`Error extracting tar file "${file.name}" - ${err}`)
     }
   }
+
+  private getTarFileName(hash: string): string {
+    return `${hash}.tar.gz`
+  }
+
+  private getTarFilePath(hash: string, cacheDirectory: string): string {
+    return join(cacheDirectory, this.getTarFileName(hash))
+  }
+
+  private getCommitFileName(hash: string): string {
+    return `${hash}.commit`
+  }
+
 }

packages/gcp-task-runner/src/logger.ts

@@ -7,7 +7,7 @@ export class Logger {
     }
 
     output.log({
-      title: 'GCP-NX',
+      title: 'GCP Task runner',
       bodyLines: message.split('\n') ?? []
     })
   }
@@ -28,7 +28,17 @@ export class Logger {
 
   public success(message: string): void {
     output.success({
-      title: message
+      title: 'GCP Task runner',
+      bodyLines: message.split('\n')
     })
   }
+
+  public note(message: string): void {
+    output.addNewline()
+    output.note({
+      title: 'GCP Task runner',
+      bodyLines: message.split('\n')
+    })
+    output.addNewline()
+  }
 }

packages/gcp-task-runner/src/runner.ts

@@ -12,6 +12,12 @@ export const tasksRunner = (
   // Create the logger
   const logger = new Logger()
 
+  if (options.skipNxCache) {
+    logger.note('Using local cache')
+
+    return defaultTaskRunner(tasks, options, context)
+  }
+
   try {
     const messages = new MessageReporter(logger)
     const remoteCache = new GcpCache(options.bucket, messages)

packages/vercel/src/executors/build/build.impl.ts

@@ -1,5 +1,7 @@
-import { readJsonFile, writeJsonFile } from '@nx/devkit'
-import { buildCommand, copyFile, execCommand } from '@nx-extend/core'
+import { parseTargetString, readJsonFile, writeJsonFile } from '@nx/devkit'
+import { targetToTargetString } from '@nx/devkit/src/executors/parse-target-string'
+import { readCachedProjectGraph } from '@nx/workspace/src/core/project-graph'
+import { buildCommand, copyFile, execCommand, USE_VERBOSE_LOGGING } from '@nx-extend/core'
 import { existsSync, rmSync } from 'fs'
 import { join } from 'path'
 
@@ -8,27 +10,32 @@ import type { ExecutorContext } from '@nx/devkit'
 import { addEnvVariablesToFile } from '../../utils/add-env-variables-to-file'
 import { enrichVercelEnvFile } from '../../utils/enrich-vercel-env-file'
 import { getEnvVars } from '../../utils/get-env-vars'
+import { getOutputDirectoryFromBuildTarget } from '../../utils/get-output-directory-from-build-target'
 import { vercelToken } from '../../utils/vercel-token'
 import { getOutputDirectory } from './utils/get-output-directory'
 
 export interface BuildOptions {
   projectId: string
   orgId: string
-  debug?: boolean
   envVars?: Record<string, string>
   buildTarget?: string
-  buildConfig?: string
   framework?: string
+  outputPath?: string
   nodeVersion?: '16.x'
 }
 
 export function buildExecutor(
   options: BuildOptions,
   context: ExecutorContext
 ): Promise<{ success: boolean }> {
-  const { targets } = context.workspace.projects[context.projectName]
   const framework = options.framework || 'nextjs'
-  const buildTarget = options.buildTarget || (framework === 'nextjs' ? 'build-next' : 'build')
+  let buildTarget = options.buildTarget || (framework === 'nextjs' ? 'build-next' : 'build')
+
+  if (!buildTarget.includes(':')) {
+    buildTarget = `${context.projectName}:${buildTarget}`
+  }
+
+  const targetString = parseTargetString(buildTarget, readCachedProjectGraph())
 
   if (!options.orgId) {
     throw new Error(`"orgId" option is required!`)
@@ -38,22 +45,15 @@ export function buildExecutor(
     throw new Error(`"projectId" option is required!`)
   }
 
-  if (!targets[buildTarget]) {
-    throw new Error(
-      `"${context.projectName}" is missing the "${buildTarget}" target!`
-    )
+  if (!targetString) {
+    throw new Error(`Invalid build target "${buildTarget}"!`)
   }
 
-  if (!targets[buildTarget]?.options?.outputPath) {
+  const outputDirectory = options.outputPath || getOutputDirectoryFromBuildTarget(context, buildTarget)
+  if (!outputDirectory) {
     throw new Error(`"${buildTarget}" target has no "outputPath" configured!`)
   }
 
-  if (options.buildConfig && !targets[buildTarget]?.configurations[options.buildConfig]) {
-    throw new Error(
-      `"${buildTarget}" target has no configuration "${options.buildConfig}"!`
-    )
-  }
-
   const vercelDirectory = '.vercel'
   const vercelDirectoryLocation = join(context.root, vercelDirectory)
 
@@ -70,25 +70,23 @@ export function buildExecutor(
     settings: {}
   })
 
-  const vercelEnironment = context.configurationName === 'production' ? 'production' : 'preview'
+  const vercelEnvironment = context.configurationName === 'production' ? 'production' : 'preview'
 
   // Pull latest
   const { success: pullSuccess } = execCommand(buildCommand([
     'npx vercel pull --yes',
-    `--environment=${vercelEnironment}`,
+    `--environment=${vercelEnvironment}`,
     vercelToken && `--token=${vercelToken}`,
 
-    options.debug && '--debug'
+    USE_VERBOSE_LOGGING && '--debug'
   ]))
 
   if (!pullSuccess) {
     throw new Error(`Was unable to pull!`)
   }
 
   const vercelProjectJson = `./${vercelDirectory}/project.json`
-  const outputDirectory = targets[buildTarget]?.options?.outputPath
-
-  const vercelEnvFile = `.env.${vercelEnironment}.local`
+  const vercelEnvFile = `.env.${vercelEnvironment}.local`
   const vercelEnvFileLocation = join(context.root, vercelDirectory)
 
   const envVars = getEnvVars(options.envVars, true)
@@ -106,10 +104,8 @@ export function buildExecutor(
       createdAt: new Date().getTime(),
       framework,
       devCommand: null,
-      installCommand: "echo ''",
-      buildCommand: `nx run ${context.projectName}:${buildTarget}:${
-        options.buildConfig || context.configurationName
-      }`,
+      installCommand: 'echo ""',
+      buildCommand: `nx run ${targetToTargetString(targetString)}`,
       outputDirectory: getOutputDirectory(framework, outputDirectory),
       rootDirectory: null,
       directoryListing: false,
@@ -119,11 +115,11 @@ export function buildExecutor(
 
   const { success } = execCommand(buildCommand([
     'npx vercel build',
-    `--output ${targets[buildTarget].options.outputPath}/.vercel/output`,
+    `--output ${outputDirectory}/.vercel/output`,
     context.configurationName === 'production' && '--prod',
     vercelToken && `--token=${vercelToken}`,
 
-    options.debug && '--debug'
+    USE_VERBOSE_LOGGING && '--debug'
   ]))
 
   if (success) {