From cf0fa5b46980f12cc029869f11a46168fa09cdab Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Sun, 18 Aug 2024 19:23:27 -0700 Subject: [PATCH] fix(runfiles): support bzlmod repo mappings --- packages/runfiles/index.ts | 8 +- packages/runfiles/paths.ts | 4 + packages/runfiles/repository.ts | 42 +++++ packages/runfiles/runfiles.ts | 172 +++++++++++++----- .../runfiles/test/runfile_resolution.spec.js | 120 +++++++----- 5 files changed, 255 insertions(+), 91 deletions(-) create mode 100644 packages/runfiles/repository.ts diff --git a/packages/runfiles/index.ts b/packages/runfiles/index.ts index 8b154cc061..04de980fc6 100644 --- a/packages/runfiles/index.ts +++ b/packages/runfiles/index.ts @@ -1,12 +1,8 @@ -import {BAZEL_OUT_REGEX} from './paths'; -import {Runfiles} from './runfiles'; +import { Runfiles } from "./runfiles"; // Re-export the `Runfiles` class. This class if the runfile helpers need to be // mocked for testing purposes. This is used by the linker but also publicly exposed. -export {Runfiles}; -// Re-export a RegExp for matching `bazel-out` paths. This is used by the linker -// but not intended for public use. -export {BAZEL_OUT_REGEX as _BAZEL_OUT_REGEX}; +export { Runfiles }; /** Instance of the runfile helpers. */ export const runfiles = new Runfiles(process.env); diff --git a/packages/runfiles/paths.ts b/packages/runfiles/paths.ts index e43ec09cf2..1cae1e112a 100755 --- a/packages/runfiles/paths.ts +++ b/packages/runfiles/paths.ts @@ -1,3 +1,7 @@ // NB: on windows thanks to legacy 8-character path segments it might be like // c:/b/ojvxx6nx/execroot/build_~1/bazel-~1/x64_wi~1/bin/internal/npm_in~1/test export const BAZEL_OUT_REGEX = /(\/bazel-out\/|\/bazel-~1\/x64_wi~1\/)/; + +// The runfiles root symlink under which the repository mapping can be found. +// https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java;l=424 +export const REPO_MAPPING_RLOCATION = "_repo_mapping"; diff --git a/packages/runfiles/repository.ts b/packages/runfiles/repository.ts new file mode 100644 index 0000000000..fa63186a32 --- /dev/null +++ b/packages/runfiles/repository.ts @@ -0,0 +1,42 @@ +/** + * Utils for managing repository mappings. + * + * The majority of this code is ported from [rules_go](https://github.com/bazelbuild/rules_go/pull/3347). + */ + +export interface RepoMappings { + [repo: string]: { + [mapped: string]: string; + }; +} + +const legacyExternalGeneratedFile = + /^bazel-out[\/][^\/]+\/bin\/external\/([^\/]+)\//; +const legacyExternalFile = /^external\/([^\/]+)\//; + +// CurrentRepository returns the canonical name of the Bazel repository that +// contains the source file of the caller of CurrentRepository. +export function currentRepository(): string { + return callerRepositoryFromStack(1); +} + +// CallerRepository returns the canonical name of the Bazel repository that +// contains the source file of the caller of the function that itself calls +// CallerRepository. +export function callerRepository(): string { + return callerRepositoryFromStack(2); +} + +export function callerRepositoryFromStack(skip: number): string { + const file = new Error().stack.split("\n")[skip + 1]; + const match = + file.match(legacyExternalGeneratedFile) || file.match(legacyExternalFile); + + if (match) { + return match[0]; + } + + // If a file is not in an external repository, it is in the main repository, + // which has the empty string as its canonical name. + return ""; +} diff --git a/packages/runfiles/runfiles.ts b/packages/runfiles/runfiles.ts index 9b47b762a8..71e8cf1cfd 100644 --- a/packages/runfiles/runfiles.ts +++ b/packages/runfiles/runfiles.ts @@ -1,45 +1,59 @@ -import * as fs from 'fs'; -import * as path from 'path'; +import * as fs from "fs"; +import * as path from "path"; -import {BAZEL_OUT_REGEX} from './paths'; +import { BAZEL_OUT_REGEX, REPO_MAPPING_RLOCATION } from "./paths"; +import { RepoMappings, callerRepository } from "./repository"; /** * Class that provides methods for resolving Bazel runfiles. */ export class Runfiles { - manifest: Map|undefined; - runfilesDir: string|undefined; + manifest: Map | undefined; + runfilesDir: string | undefined; /** * If the environment gives us enough hints, we can know the workspace name */ - workspace: string|undefined; + workspace: string | undefined; /** * If the environment gives us enough hints, we can know the package path */ - package: string|undefined; + package: string | undefined; + /** + * If the environment has repo mappings, we can use them to resolve repo relative paths. + */ + repoMappings: RepoMappings | undefined; - constructor(private _env: typeof process.env) { + constructor(private _env = process.env) { // If Bazel sets a variable pointing to a runfiles manifest, // we'll always use it. // Note that this has a slight performance implication on Mac/Linux // where we could use the runfiles tree already laid out on disk // but this just costs one file read for the external npm/node_modules // and one for each first-party module, not one per file. - if (!!_env['RUNFILES_MANIFEST_FILE']) { - this.manifest = this.loadRunfilesManifest(_env['RUNFILES_MANIFEST_FILE']!); - } else if (!!_env['RUNFILES_DIR']) { - this.runfilesDir = path.resolve(_env['RUNFILES_DIR']!); - } else if (!!_env['RUNFILES']) { - this.runfilesDir = path.resolve(_env['RUNFILES']!); + if (!!_env["RUNFILES_MANIFEST_FILE"]) { + this.manifest = this.loadRunfilesManifest( + _env["RUNFILES_MANIFEST_FILE"]!, + ); + } else if (!!_env["RUNFILES_DIR"]) { + this.runfilesDir = path.resolve(_env["RUNFILES_DIR"]!); + this.repoMappings = this.parseRepoMapping(this.runfilesDir); + } else if (!!_env["RUNFILES"]) { + this.runfilesDir = path.resolve(_env["RUNFILES"]!); + this.repoMappings = this.parseRepoMapping(this.runfilesDir); } else { throw new Error( - 'Every node program run under Bazel must have a $RUNFILES_DIR, $RUNFILES or $RUNFILES_MANIFEST_FILE environment variable'); + "Every node program run under Bazel must have a $RUNFILES_DIR, $RUNFILES or $RUNFILES_MANIFEST_FILE environment variable", + ); } + // Under --noenable_runfiles (in particular on Windows) // Bazel sets RUNFILES_MANIFEST_ONLY=1. // When this happens, we need to read the manifest file to locate // inputs - if (_env['RUNFILES_MANIFEST_ONLY'] === '1' && !_env['RUNFILES_MANIFEST_FILE']) { + if ( + _env["RUNFILES_MANIFEST_ONLY"] === "1" && + !_env["RUNFILES_MANIFEST_FILE"] + ) { console.warn(`Workaround https://github.com/bazelbuild/bazel/issues/7994 RUNFILES_MANIFEST_FILE should have been set but wasn't. falling back to using runfiles symlinks. @@ -47,22 +61,23 @@ export class Runfiles { --spawn_strategy=standalone to the command line.`); } // Bazel starts actions with pwd=execroot/my_wksp or pwd=runfiles/my_wksp - this.workspace = _env['BAZEL_WORKSPACE'] || _env['JS_BINARY__WORKSPACE'] || undefined; + this.workspace = + _env["BAZEL_WORKSPACE"] || _env["JS_BINARY__WORKSPACE"] || undefined; // If target is from an external workspace such as @npm//rollup/bin:rollup // resolvePackageRelative is not supported since package is in an external // workspace. - let target = _env['BAZEL_TARGET'] || _env['JS_BINARY__TARGET']; - if (!!target && !target.startsWith('@')) { + let target = _env["BAZEL_TARGET"] || _env["JS_BINARY__TARGET"]; + if (!!target && !target.startsWith("@")) { // //path/to:target -> path/to - this.package = target.split(':')[0].replace(/^\/\//, ''); + this.package = target.split(":")[0].replace(/^\/\//, ""); } } /** Resolves the given path from the runfile manifest. */ - private _resolveFromManifest(searchPath: string): string|undefined { + private _resolveFromManifest(searchPath: string): string | undefined { if (!this.manifest) return undefined; - let result: string|undefined; + let result: string | undefined; for (const [k, v] of this.manifest) { // Account for Bazel --legacy_external_runfiles // which pollutes the workspace with 'my_wksp/external/...' @@ -93,7 +108,6 @@ export class Runfiles { return result; } - /** * The runfiles manifest maps from short_path * https://docs.bazel.build/versions/main/skylark/lib/File.html#short_path @@ -105,31 +119,77 @@ export class Runfiles { */ loadRunfilesManifest(manifestPath: string) { const runfilesEntries = new Map(); - const input = fs.readFileSync(manifestPath, {encoding: 'utf-8'}); + const input = fs.readFileSync(manifestPath, { encoding: "utf-8" }); - for (const line of input.split('\n')) { + for (const line of input.split("\n")) { if (!line) continue; - const [runfilesPath, realPath] = line.split(' '); + const [runfilesPath, realPath] = line.split(" "); runfilesEntries.set(runfilesPath, realPath); } return runfilesEntries; } + parseRepoMapping(runfilesDir: string): RepoMappings | undefined { + const repoMappingPath = path.join(runfilesDir, REPO_MAPPING_RLOCATION); + + if (!fs.existsSync(repoMappingPath)) { + // The repo mapping manifest only exists with Bzlmod, so it's not an + // error if it's missing. Since any repository name not contained in the + // mapping is assumed to be already canonical, an empty map is + // equivalent to not applying any mapping. + return Object.create(null); + } + + const repoMappings: RepoMappings = Object.create(null); + const mappings = fs.readFileSync(repoMappingPath, { encoding: "utf-8" }); + + // Each line of the repository mapping manifest has the form: + // canonical name of source repo,apparent name of target repo,target repo runfiles directory + // https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java;l=117 + for (const line of mappings.split("\n")) { + if (!line) continue; + + const [from, repoName, repoPath] = line.split(","); + + (repoMappings[from] ??= Object.create(null))[repoName] = repoPath; + } + return repoMappings; + } + /** Resolves the given module path. */ - resolve(modulePath: string) { + resolve(modulePath: string, sourceRepo?: string): string { // Normalize path by converting to forward slashes and removing all trailing // forward slashes - modulePath = modulePath.replace(/\\/g, '/').replace(/\/+$/g, '') + modulePath = modulePath.replace(/\\/g, "/").replace(/\/+$/g, ""); if (path.isAbsolute(modulePath)) { return modulePath; } - const result = this._resolve(modulePath, undefined); + + if (this.repoMappings) { + // Determine the repository which runfiles is being invoked from by default. + if (sourceRepo === undefined) { + sourceRepo = callerRepository(); + } + + // If the repository mappings were loaded ensure the source repository is valid. + if (!(sourceRepo in this.repoMappings)) { + throw new Error( + `source repository ${sourceRepo} not found in repo mappings: ${JSON.stringify( + this.repoMappings, + null, + 2, + )}`, + ); + } + } + + const result = this._resolve(sourceRepo, modulePath, undefined); if (result) { return result; } const e = new Error(`could not resolve module ${modulePath}`); - (e as any).code = 'MODULE_NOT_FOUND'; + (e as any).code = "MODULE_NOT_FOUND"; throw e; } @@ -137,10 +197,11 @@ export class Runfiles { resolveWorkspaceRelative(modulePath: string) { // Normalize path by converting to forward slashes and removing all trailing // forward slashes - modulePath = modulePath.replace(/\\/g, '/').replace(/\/+$/g, '') + modulePath = modulePath.replace(/\\/g, "/").replace(/\/+$/g, ""); if (!this.workspace) { throw new Error( - 'workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set'); + "workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set", + ); } return this.resolve(path.posix.join(this.workspace, modulePath)); } @@ -149,17 +210,21 @@ export class Runfiles { resolvePackageRelative(modulePath: string) { // Normalize path by converting to forward slashes and removing all trailing // forward slashes - modulePath = modulePath.replace(/\\/g, '/').replace(/\/+$/g, '') + modulePath = modulePath.replace(/\\/g, "/").replace(/\/+$/g, ""); if (!this.workspace) { throw new Error( - 'workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set'); + "workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set", + ); } // NB: this.package may be '' if at the root of the workspace if (this.package === undefined) { throw new Error( - 'package could not be determined from the environment; make sure BAZEL_TARGET is set'); + "package could not be determined from the environment; make sure BAZEL_TARGET is set", + ); } - return this.resolve(path.posix.join(this.workspace, this.package, modulePath)); + return this.resolve( + path.posix.join(this.workspace, this.package, modulePath), + ); } /** @@ -167,20 +232,26 @@ export class Runfiles { * @deprecated Use the runfile helpers directly instead. **/ patchRequire() { - const requirePatch = this._env['BAZEL_NODE_PATCH_REQUIRE']; + const requirePatch = this._env["BAZEL_NODE_PATCH_REQUIRE"]; if (!requirePatch) { - throw new Error('require patch location could not be determined from the environment'); + throw new Error( + "require patch location could not be determined from the environment", + ); } require(requirePatch); } /** Helper for resolving a given module recursively in the runfiles. */ - private _resolve(moduleBase: string, moduleTail: string|undefined): string|undefined { + private _resolve( + sourceRepo: string, + moduleBase: string, + moduleTail: string | undefined, + ): string | undefined { if (this.manifest) { const result = this._resolveFromManifest(moduleBase); if (result) { if (moduleTail) { - const maybe = path.join(result, moduleTail || ''); + const maybe = path.join(result, moduleTail || ""); if (fs.existsSync(maybe)) { return maybe; } @@ -189,17 +260,32 @@ export class Runfiles { } } } + + // Apply repo mappings to the moduleBase if it is a known repo. + if (this.repoMappings && moduleBase in this.repoMappings[sourceRepo]) { + const mappedRepo = this.repoMappings[sourceRepo][moduleBase]; + if (mappedRepo !== moduleBase) { + const maybe = this._resolve(sourceRepo, mappedRepo, moduleTail); + if (maybe !== undefined) { + return maybe; + } + } + } if (this.runfilesDir) { - const maybe = path.join(this.runfilesDir, moduleBase, moduleTail || ''); + const maybe = path.join(this.runfilesDir, moduleBase, moduleTail || ""); if (fs.existsSync(maybe)) { return maybe; } } const dirname = path.dirname(moduleBase); - if (dirname == '.') { + if (dirname == ".") { // no match return undefined; } - return this._resolve(dirname, path.join(path.basename(moduleBase), moduleTail || '')); + return this._resolve( + sourceRepo, + dirname, + path.join(path.basename(moduleBase), moduleTail || ""), + ); } } diff --git a/packages/runfiles/test/runfile_resolution.spec.js b/packages/runfiles/test/runfile_resolution.spec.js index a8292c2080..dfc66929c5 100644 --- a/packages/runfiles/test/runfile_resolution.spec.js +++ b/packages/runfiles/test/runfile_resolution.spec.js @@ -1,55 +1,91 @@ -const {join, dirname} = require('path') -const {runfiles} = require('@bazel/runfiles'); +const { join, dirname } = require("path"); +const { runfiles } = require("@bazel/runfiles"); -describe('runfile resolution', () => { +describe("runfile resolution", () => { it('should properly resolve the "test_fixture.md" file', () => { - const testFixturePath = - runfiles.resolve('rules_nodejs/packages/runfiles/test/test_fixture.md'); - const expectedPath = join(__dirname, 'test_fixture.md'); - - expect(normalizePath(testFixturePath)) - .toEqual( - normalizePath(expectedPath), - 'Expected the test fixture to be resolved next to the spec source file.'); + const testFixturePath = runfiles.resolve( + "rules_nodejs/packages/runfiles/test/test_fixture.md", + ); + const expectedPath = join(__dirname, "test_fixture.md"); + + expect(normalizePath(testFixturePath)).toEqual( + normalizePath(expectedPath), + "Expected the test fixture to be resolved next to the spec source file.", + ); + }); + + it("should properly resolve with forward slashes", () => { + const testFixturePath = runfiles.resolve( + "rules_nodejs\\packages\\runfiles\\test\\test_fixture.md", + ); + const expectedPath = join(__dirname, "test_fixture.md"); + + expect(normalizePath(testFixturePath)).toEqual( + normalizePath(expectedPath), + "Expected the test fixture to be resolved next to the spec source file.", + ); }); - it('should properly resolve with forward slashes', () => { - const testFixturePath = - runfiles.resolve('rules_nodejs\\packages\\runfiles\\test\\test_fixture.md'); - const expectedPath = join(__dirname, 'test_fixture.md'); + it('should properly resolve the "test_fixture.md" file with the __main__ repo', () => { + const testFixturePath = runfiles.resolve( + "__main__/packages/runfiles/test/test_fixture.md", + ); + const expectedPath = join(__dirname, "test_fixture.md"); - expect(normalizePath(testFixturePath)) - .toEqual( - normalizePath(expectedPath), - 'Expected the test fixture to be resolved next to the spec source file.'); + expect(normalizePath(testFixturePath)).toEqual( + normalizePath(expectedPath), + "Expected the test fixture to be resolved next to the spec source file.", + ); }); - it('should properly resolve a subdirectory of a runfile', () => { - const packagePath = runfiles.resolve('rules_nodejs/packages'); + it("should properly resolve a subdirectory of a runfile", () => { + const packagePath = runfiles.resolve("rules_nodejs/packages"); // Alternate with trailing slash - const packagePath2 = runfiles.resolve('rules_nodejs/packages/'); - const expectedPath = dirname(dirname(dirname(runfiles.resolve( - 'rules_nodejs/packages/runfiles/test/test_fixture.md.generated_file_suffix')))); - - expect(normalizePath(packagePath)) - .toEqual(normalizePath(expectedPath), 'Expected to resolve a subdirectory of a runfile.'); - expect(normalizePath(packagePath2)) - .toEqual(normalizePath(expectedPath), 'Expected to resolve a subdirectory of a runfile.'); + const packagePath2 = runfiles.resolve("rules_nodejs/packages/"); + const expectedPath = dirname( + dirname( + dirname( + runfiles.resolve( + "rules_nodejs/packages/runfiles/test/test_fixture.md.generated_file_suffix", + ), + ), + ), + ); + + expect(normalizePath(packagePath)).toEqual( + normalizePath(expectedPath), + "Expected to resolve a subdirectory of a runfile.", + ); + expect(normalizePath(packagePath2)).toEqual( + normalizePath(expectedPath), + "Expected to resolve a subdirectory of a runfile.", + ); }); - it('should properly resolve the workspace root of a runfile', () => { - const packagePath = runfiles.resolve('rules_nodejs'); + it("should properly resolve the workspace root of a runfile", () => { + const packagePath = runfiles.resolve("rules_nodejs"); // Alternate with trailing slash - const packagePath2 = runfiles.resolve('rules_nodejs/'); - const expectedPath = dirname(dirname(dirname(dirname(runfiles.resolve( - 'rules_nodejs/packages/runfiles/test/test_fixture.md.generated_file_suffix'))))); - - expect(normalizePath(packagePath)) - .toEqual( - normalizePath(expectedPath), 'Expected to resolve the workspace root of a runfile.'); - expect(normalizePath(packagePath2)) - .toEqual( - normalizePath(expectedPath), 'Expected to resolve the workspace root of a runfile.'); + const packagePath2 = runfiles.resolve("rules_nodejs/"); + const expectedPath = dirname( + dirname( + dirname( + dirname( + runfiles.resolve( + "rules_nodejs/packages/runfiles/test/test_fixture.md.generated_file_suffix", + ), + ), + ), + ), + ); + + expect(normalizePath(packagePath)).toEqual( + normalizePath(expectedPath), + "Expected to resolve the workspace root of a runfile.", + ); + expect(normalizePath(packagePath2)).toEqual( + normalizePath(expectedPath), + "Expected to resolve the workspace root of a runfile.", + ); }); }); @@ -58,5 +94,5 @@ describe('runfile resolution', () => { * where paths might be computed using different path delimiters. */ function normalizePath(value) { - return value.replace(/\\/g, '/'); + return value.replace(/\\/g, "/"); }