Skip to content

Commit

Permalink
fix(runfiles): support bzlmod repo mappings
Browse files Browse the repository at this point in the history
  • Loading branch information
@jbedard
jbedard committed Sep 12, 2024
1 parent e910541 commit cf0fa5b
Show file tree
Hide file tree
Showing 5 changed files with 255 additions and 91 deletions.
8 changes: 2 additions & 6 deletions packages/runfiles/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
import {BAZEL_OUT_REGEX} from './paths';
import {Runfiles} from './runfiles';
import { Runfiles } from "./runfiles";

// Re-export the `Runfiles` class. This class if the runfile helpers need to be
// mocked for testing purposes. This is used by the linker but also publicly exposed.
export {Runfiles};
// Re-export a RegExp for matching `bazel-out` paths. This is used by the linker
// but not intended for public use.
export {BAZEL_OUT_REGEX as _BAZEL_OUT_REGEX};
export { Runfiles };

/** Instance of the runfile helpers. */
export const runfiles = new Runfiles(process.env);
4 changes: 4 additions & 0 deletions packages/runfiles/paths.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
// NB: on windows thanks to legacy 8-character path segments it might be like
// c:/b/ojvxx6nx/execroot/build_~1/bazel-~1/x64_wi~1/bin/internal/npm_in~1/test
export const BAZEL_OUT_REGEX = /(\/bazel-out\/|\/bazel-~1\/x64_wi~1\/)/;

// The runfiles root symlink under which the repository mapping can be found.
// https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java;l=424
export const REPO_MAPPING_RLOCATION = "_repo_mapping";
42 changes: 42 additions & 0 deletions packages/runfiles/repository.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/**
* Utils for managing repository mappings.
*
* The majority of this code is ported from [rules_go](https://github.com/bazelbuild/rules_go/pull/3347).
*/

export interface RepoMappings {
[repo: string]: {
[mapped: string]: string;
};
}

const legacyExternalGeneratedFile =
/^bazel-out[\/][^\/]+\/bin\/external\/([^\/]+)\//;
const legacyExternalFile = /^external\/([^\/]+)\//;

// CurrentRepository returns the canonical name of the Bazel repository that
// contains the source file of the caller of CurrentRepository.
export function currentRepository(): string {
return callerRepositoryFromStack(1);
}

// CallerRepository returns the canonical name of the Bazel repository that
// contains the source file of the caller of the function that itself calls
// CallerRepository.
export function callerRepository(): string {
return callerRepositoryFromStack(2);
}

export function callerRepositoryFromStack(skip: number): string {
const file = new Error().stack.split("\n")[skip + 1];
const match =
file.match(legacyExternalGeneratedFile) || file.match(legacyExternalFile);

if (match) {
return match[0];
}

// If a file is not in an external repository, it is in the main repository,
// which has the empty string as its canonical name.
return "";
}
172 changes: 129 additions & 43 deletions packages/runfiles/runfiles.ts
View file
Original file line number Diff line number Diff line change
@@ -1,68 +1,83 @@
import * as fs from 'fs';
import * as path from 'path';
import * as fs from "fs";
import * as path from "path";

import {BAZEL_OUT_REGEX} from './paths';
import { BAZEL_OUT_REGEX, REPO_MAPPING_RLOCATION } from "./paths";
import { RepoMappings, callerRepository } from "./repository";

/**
* Class that provides methods for resolving Bazel runfiles.
*/
export class Runfiles {
manifest: Map<string, string>|undefined;
runfilesDir: string|undefined;
manifest: Map<string, string> | undefined;
runfilesDir: string | undefined;
/**
* If the environment gives us enough hints, we can know the workspace name
*/
workspace: string|undefined;
workspace: string | undefined;
/**
* If the environment gives us enough hints, we can know the package path
*/
package: string|undefined;
package: string | undefined;
/**
* If the environment has repo mappings, we can use them to resolve repo relative paths.
*/
repoMappings: RepoMappings | undefined;

constructor(private _env: typeof process.env) {
constructor(private _env = process.env) {
// If Bazel sets a variable pointing to a runfiles manifest,
// we'll always use it.
// Note that this has a slight performance implication on Mac/Linux
// where we could use the runfiles tree already laid out on disk
// but this just costs one file read for the external npm/node_modules
// and one for each first-party module, not one per file.
if (!!_env['RUNFILES_MANIFEST_FILE']) {
this.manifest = this.loadRunfilesManifest(_env['RUNFILES_MANIFEST_FILE']!);
} else if (!!_env['RUNFILES_DIR']) {
this.runfilesDir = path.resolve(_env['RUNFILES_DIR']!);
} else if (!!_env['RUNFILES']) {
this.runfilesDir = path.resolve(_env['RUNFILES']!);
if (!!_env["RUNFILES_MANIFEST_FILE"]) {
this.manifest = this.loadRunfilesManifest(
_env["RUNFILES_MANIFEST_FILE"]!,
);
} else if (!!_env["RUNFILES_DIR"]) {
this.runfilesDir = path.resolve(_env["RUNFILES_DIR"]!);
this.repoMappings = this.parseRepoMapping(this.runfilesDir);
} else if (!!_env["RUNFILES"]) {
this.runfilesDir = path.resolve(_env["RUNFILES"]!);
this.repoMappings = this.parseRepoMapping(this.runfilesDir);
} else {
throw new Error(
'Every node program run under Bazel must have a $RUNFILES_DIR, $RUNFILES or $RUNFILES_MANIFEST_FILE environment variable');
"Every node program run under Bazel must have a $RUNFILES_DIR, $RUNFILES or $RUNFILES_MANIFEST_FILE environment variable",
);
}

// Under --noenable_runfiles (in particular on Windows)
// Bazel sets RUNFILES_MANIFEST_ONLY=1.
// When this happens, we need to read the manifest file to locate
// inputs
if (_env['RUNFILES_MANIFEST_ONLY'] === '1' && !_env['RUNFILES_MANIFEST_FILE']) {
if (
_env["RUNFILES_MANIFEST_ONLY"] === "1" &&
!_env["RUNFILES_MANIFEST_FILE"]
) {
console.warn(`Workaround https://github.com/bazelbuild/bazel/issues/7994
RUNFILES_MANIFEST_FILE should have been set but wasn't.
falling back to using runfiles symlinks.
If you want to test runfiles manifest behavior, add
--spawn_strategy=standalone to the command line.`);
}
// Bazel starts actions with pwd=execroot/my_wksp or pwd=runfiles/my_wksp
this.workspace = _env['BAZEL_WORKSPACE'] || _env['JS_BINARY__WORKSPACE'] || undefined;
this.workspace =
_env["BAZEL_WORKSPACE"] || _env["JS_BINARY__WORKSPACE"] || undefined;
// If target is from an external workspace such as @npm//rollup/bin:rollup
// resolvePackageRelative is not supported since package is in an external
// workspace.
let target = _env['BAZEL_TARGET'] || _env['JS_BINARY__TARGET'];
if (!!target && !target.startsWith('@')) {
let target = _env["BAZEL_TARGET"] || _env["JS_BINARY__TARGET"];
if (!!target && !target.startsWith("@")) {
// //path/to:target -> path/to
this.package = target.split(':')[0].replace(/^\/\//, '');
this.package = target.split(":")[0].replace(/^\/\//, "");
}
}

/** Resolves the given path from the runfile manifest. */
private _resolveFromManifest(searchPath: string): string|undefined {
private _resolveFromManifest(searchPath: string): string | undefined {
if (!this.manifest) return undefined;

let result: string|undefined;
let result: string | undefined;
for (const [k, v] of this.manifest) {
// Account for Bazel --legacy_external_runfiles
// which pollutes the workspace with 'my_wksp/external/...'
Expand Down Expand Up @@ -93,7 +108,6 @@ export class Runfiles {
return result;
}


/**
* The runfiles manifest maps from short_path
* https://docs.bazel.build/versions/main/skylark/lib/File.html#short_path
Expand All @@ -105,42 +119,89 @@ export class Runfiles {
*/
loadRunfilesManifest(manifestPath: string) {
const runfilesEntries = new Map();
const input = fs.readFileSync(manifestPath, {encoding: 'utf-8'});
const input = fs.readFileSync(manifestPath, { encoding: "utf-8" });

for (const line of input.split('\n')) {
for (const line of input.split("\n")) {
if (!line) continue;
const [runfilesPath, realPath] = line.split(' ');
const [runfilesPath, realPath] = line.split(" ");
runfilesEntries.set(runfilesPath, realPath);
}

return runfilesEntries;
}

parseRepoMapping(runfilesDir: string): RepoMappings | undefined {
const repoMappingPath = path.join(runfilesDir, REPO_MAPPING_RLOCATION);

if (!fs.existsSync(repoMappingPath)) {
// The repo mapping manifest only exists with Bzlmod, so it's not an
// error if it's missing. Since any repository name not contained in the
// mapping is assumed to be already canonical, an empty map is
// equivalent to not applying any mapping.
return Object.create(null);
}

const repoMappings: RepoMappings = Object.create(null);
const mappings = fs.readFileSync(repoMappingPath, { encoding: "utf-8" });

// Each line of the repository mapping manifest has the form:
// canonical name of source repo,apparent name of target repo,target repo runfiles directory
// https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java;l=117
for (const line of mappings.split("\n")) {
if (!line) continue;

const [from, repoName, repoPath] = line.split(",");

(repoMappings[from] ??= Object.create(null))[repoName] = repoPath;
}
return repoMappings;
}

/** Resolves the given module path. */
resolve(modulePath: string) {
resolve(modulePath: string, sourceRepo?: string): string {
// Normalize path by converting to forward slashes and removing all trailing
// forward slashes
modulePath = modulePath.replace(/\\/g, '/').replace(/\/+$/g, '')
modulePath = modulePath.replace(/\\/g, "/").replace(/\/+$/g, "");
if (path.isAbsolute(modulePath)) {
return modulePath;
}
const result = this._resolve(modulePath, undefined);

if (this.repoMappings) {
// Determine the repository which runfiles is being invoked from by default.
if (sourceRepo === undefined) {
sourceRepo = callerRepository();
}

// If the repository mappings were loaded ensure the source repository is valid.
if (!(sourceRepo in this.repoMappings)) {
throw new Error(
`source repository ${sourceRepo} not found in repo mappings: ${JSON.stringify(
this.repoMappings,
null,
2,
)}`,
);
}
}

const result = this._resolve(sourceRepo, modulePath, undefined);
if (result) {
return result;
}
const e = new Error(`could not resolve module ${modulePath}`);
(e as any).code = 'MODULE_NOT_FOUND';
(e as any).code = "MODULE_NOT_FOUND";
throw e;
}

/** Resolves the given path relative to the current Bazel workspace. */
resolveWorkspaceRelative(modulePath: string) {
// Normalize path by converting to forward slashes and removing all trailing
// forward slashes
modulePath = modulePath.replace(/\\/g, '/').replace(/\/+$/g, '')
modulePath = modulePath.replace(/\\/g, "/").replace(/\/+$/g, "");
if (!this.workspace) {
throw new Error(
'workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set');
"workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set",
);
}
return this.resolve(path.posix.join(this.workspace, modulePath));
}
Expand All @@ -149,38 +210,48 @@ export class Runfiles {
resolvePackageRelative(modulePath: string) {
// Normalize path by converting to forward slashes and removing all trailing
// forward slashes
modulePath = modulePath.replace(/\\/g, '/').replace(/\/+$/g, '')
modulePath = modulePath.replace(/\\/g, "/").replace(/\/+$/g, "");
if (!this.workspace) {
throw new Error(
'workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set');
"workspace could not be determined from the environment; make sure BAZEL_WORKSPACE is set",
);
}
// NB: this.package may be '' if at the root of the workspace
if (this.package === undefined) {
throw new Error(
'package could not be determined from the environment; make sure BAZEL_TARGET is set');
"package could not be determined from the environment; make sure BAZEL_TARGET is set",
);
}
return this.resolve(path.posix.join(this.workspace, this.package, modulePath));
return this.resolve(
path.posix.join(this.workspace, this.package, modulePath),
);
}

/**
* Patches the default Node.js resolution to support runfile resolution.
* @deprecated Use the runfile helpers directly instead.
**/
patchRequire() {
const requirePatch = this._env['BAZEL_NODE_PATCH_REQUIRE'];
const requirePatch = this._env["BAZEL_NODE_PATCH_REQUIRE"];
if (!requirePatch) {
throw new Error('require patch location could not be determined from the environment');
throw new Error(
"require patch location could not be determined from the environment",
);
}
require(requirePatch);
}

/** Helper for resolving a given module recursively in the runfiles. */
private _resolve(moduleBase: string, moduleTail: string|undefined): string|undefined {
private _resolve(
sourceRepo: string,
moduleBase: string,
moduleTail: string | undefined,
): string | undefined {
if (this.manifest) {
const result = this._resolveFromManifest(moduleBase);
if (result) {
if (moduleTail) {
const maybe = path.join(result, moduleTail || '');
const maybe = path.join(result, moduleTail || "");
if (fs.existsSync(maybe)) {
return maybe;
}
Expand All @@ -189,17 +260,32 @@ export class Runfiles {
}
}
}

// Apply repo mappings to the moduleBase if it is a known repo.
if (this.repoMappings && moduleBase in this.repoMappings[sourceRepo]) {
const mappedRepo = this.repoMappings[sourceRepo][moduleBase];
if (mappedRepo !== moduleBase) {
const maybe = this._resolve(sourceRepo, mappedRepo, moduleTail);
if (maybe !== undefined) {
return maybe;
}
}
}
if (this.runfilesDir) {
const maybe = path.join(this.runfilesDir, moduleBase, moduleTail || '');
const maybe = path.join(this.runfilesDir, moduleBase, moduleTail || "");
if (fs.existsSync(maybe)) {
return maybe;
}
}
const dirname = path.dirname(moduleBase);
if (dirname == '.') {
if (dirname == ".") {
// no match
return undefined;
}
return this._resolve(dirname, path.join(path.basename(moduleBase), moduleTail || ''));
return this._resolve(
sourceRepo,
dirname,
path.join(path.basename(moduleBase), moduleTail || ""),
);
}
}
Loading

0 comments on commit cf0fa5b

Please sign in to comment.