Skip to content

Commit

Permalink
Fallback from Node 20 to Node 16
Browse files Browse the repository at this point in the history
  • Loading branch information
@DenisRumyantsev
DenisRumyantsev committed Sep 19, 2024
1 parent 363ba84 commit 2c3cdd9
Show file tree
Hide file tree
Showing 6 changed files with 130 additions and 73 deletions.
1 change: 1 addition & 0 deletions src/Agent.Sdk/ContainerInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContain
public string ContainerName { get; set; }
public string ContainerCommand { get; set; }
public string CustomNodePath { get; set; }
public string ResultNodePath { get; set; }
public Guid ContainerRegistryEndpoint { get; private set; }
public string ContainerCreateOptions { get; set; }
public bool SkipContainerImagePull { get; private set; }
Expand Down
2 changes: 2 additions & 0 deletions src/Agent.Sdk/ExecutionTargetInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ public interface ExecutionTargetInfo
{
PlatformUtil.OS ExecutionOS { get; }
string CustomNodePath { get; set; }
string ResultNodePath { get; set; }

string TranslateContainerPathForImageOS(PlatformUtil.OS runningOs, string path);
string TranslateToContainerPath(string path);
Expand All @@ -18,6 +19,7 @@ public class HostInfo : ExecutionTargetInfo
{
public PlatformUtil.OS ExecutionOS => PlatformUtil.HostOS;
public string CustomNodePath { get; set; }
public string ResultNodePath { get; set; }

public string TranslateToContainerPath(string path)
{
Expand Down
9 changes: 9 additions & 0 deletions src/Agent.Worker/Container/DockerCommandManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public interface IDockerCommandManager : IAgentService
Task<string> DockerCreate(IExecutionContext context, ContainerInfo container);
Task<int> DockerStart(IExecutionContext context, string containerId);
Task<int> DockerLogs(IExecutionContext context, string containerId);
Task<List<string>> GetDockerLogs(IExecutionContext context, string containerId);
Task<List<string>> DockerPS(IExecutionContext context, string options);
Task<int> DockerRemove(IExecutionContext context, string containerId);
Task<int> DockerNetworkCreate(IExecutionContext context, string network);
Expand Down Expand Up @@ -229,6 +230,14 @@ public async Task<int> DockerLogs(IExecutionContext context, string containerId)
return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}", context.CancellationToken);
}

public async Task<List<string>> GetDockerLogs(IExecutionContext context, string containerId)
{
ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(containerId, nameof(containerId));

return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}");
}

public async Task<List<string>> DockerPS(IExecutionContext context, string options)
{
ArgUtil.NotNull(context, nameof(context));
Expand Down
100 changes: 82 additions & 18 deletions src/Agent.Worker/ContainerOperationProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,37 +525,51 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
}

bool useNode20ToStartContainer = AgentKnobs.UseNode20ToStartContainer.GetValue(executionContext).AsBoolean();

string labelContainerStartupUsingNode20 = "container-startup-using-node-20";
string labelContainerStartupUsingNode16 = "container-startup-using-node-16";
string labelContainerStartupFailed = "container-startup-failed";

string containerNodePath(string nodeFolder)
{
return container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeFolder, "bin", $"node{IOUtil.ExeExtension}"));
}

string nodeContainerPath = containerNodePath(NodeHandler.NodeFolder);
string node16ContainerPath = containerNodePath(NodeHandler.Node16Folder);
string node20ContainerPath = containerNodePath(NodeHandler.Node20_1Folder);

if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.CustomNodePath = await _dockerManger.DockerInspect(context: executionContext,
dockerObject: container.ContainerImage,
options: $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");

string node;
string nodeSetInterval(string node)
{
return $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
}

if (!string.IsNullOrEmpty(container.CustomNodePath))
{
node = container.CustomNodePath;
container.ContainerCommand = nodeSetInterval(container.CustomNodePath);
container.ResultNodePath = container.CustomNodePath;
}
else if (PlatformUtil.RunningOnMacOS || (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux))
{
// require container to have node if running on macOS, or if running on Windows and attempting to run Linux container
container.CustomNodePath = "node";
container.ContainerCommand = nodeSetInterval(container.CustomNodePath);
container.ResultNodePath = container.CustomNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), AgentKnobs.UseNode20ToStartContainer.GetValue(executionContext).AsBoolean() ? NodeHandler.Node20_1Folder : NodeHandler.NodeFolder, "bin", $"node{IOUtil.ExeExtension}"));

// if on Mac OS X, require container to have node
if (PlatformUtil.RunningOnMacOS)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
// if running on Windows, and attempting to run linux container, require container to have node
else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
string sleepCommand = useNode20ToStartContainer ? $"\"{node20ContainerPath}\" --version && echo \"{labelContainerStartupUsingNode20}\" && {nodeSetInterval(node20ContainerPath)} || \"{node16ContainerPath}\" --version && echo \"{labelContainerStartupUsingNode16}\" && {nodeSetInterval(node16ContainerPath)} || echo \"{labelContainerStartupFailed}\"" : nodeSetInterval(nodeContainerPath);
container.ContainerCommand = PlatformUtil.RunningOnWindows ? $"cmd.exe /c call {sleepCommand}" : $"bash -c \"{sleepCommand.Replace('"', '\'')}\"";
container.ResultNodePath = nodeContainerPath;
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}

container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
Expand Down Expand Up @@ -588,6 +602,56 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta

executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
else if (useNode20ToStartContainer)
{
bool containerStartupCompleted = false;
int containerStartupTimeoutInMilliseconds = 10000;
int delayInMilliseconds = 100;
int checksCount = 0;

while (true)
{
List<string> containerLogs = await _dockerManger.GetDockerLogs(executionContext, container.ContainerId);

foreach (string logLine in containerLogs)
{
if (logLine.Contains(labelContainerStartupUsingNode20))
{
executionContext.Debug("Using Node 20 for container startup.");
containerStartupCompleted = true;
container.ResultNodePath = node20ContainerPath;
break;
}
else if (logLine.Contains(labelContainerStartupUsingNode16))
{
executionContext.Warning("Can not run Node 20 in container. Falling back to Node 16 for container startup.");
containerStartupCompleted = true;
container.ResultNodePath = node16ContainerPath;
break;
}
else if (logLine.Contains(labelContainerStartupFailed))
{
executionContext.Error("Can not run both Node 20 and Node 16 in container. Container startup failed.");
containerStartupCompleted = true;
break;
}
}

if (containerStartupCompleted)
{
break;
}

checksCount++;
if (checksCount * delayInMilliseconds > containerStartupTimeoutInMilliseconds)
{
executionContext.Warning("Can not get startup status from container.");
break;
}

await Task.Delay(delayInMilliseconds);
}
}
}
catch (Exception ex)
{
Expand Down
12 changes: 1 addition & 11 deletions src/Agent.Worker/Handlers/StepHost.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,16 +182,6 @@ public async Task<int> ExecuteAsync(string workingDirectory,
HostContext.GetTrace(nameof(ContainerStepHost)).Info($"Copying containerHandlerInvoker.js to {tempDir}");
File.Copy(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "containerHandlerInvoker.js.template"), targetEntryScript, true);

string node;
if (!string.IsNullOrEmpty(Container.CustomNodePath))
{
node = Container.CustomNodePath;
}
else
{
node = Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
}

string entryScript = Container.TranslateContainerPathForImageOS(PlatformUtil.HostOS, Container.TranslateToContainerPath(targetEntryScript));

string userArgs = "";
Expand All @@ -209,7 +199,7 @@ public async Task<int> ExecuteAsync(string workingDirectory,
}
}

string containerExecutionArgs = $"exec -i {userArgs} {workingDirectoryParam} {Container.ContainerId} {node} {entryScript}";
string containerExecutionArgs = $"exec -i {userArgs} {workingDirectoryParam} {Container.ContainerId} {Container.ResultNodePath} {entryScript}";

using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
{
Expand Down
79 changes: 35 additions & 44 deletions src/Misc/layoutbin/containerHandlerInvoker.js.template
View file
Original file line number Diff line number Diff line change
@@ -1,62 +1,53 @@
const { spawn } = require('child_process');
var stdinString = "";
process.stdin.on('data', function (chunk) {
stdinString += chunk;
});

process.stdin.on('end', function () {
var stdinData = JSON.parse(stdinString);
var handler = stdinData.handler;
var handlerArg = stdinData.args;
var handlerWorkDir = stdinData.workDir;
var prependPath = stdinData.prependPath;
const debug = log => console.log(`##vso[task.debug]${log}`);

console.log("##vso[task.debug]Handler: " + handler);
console.log("##vso[task.debug]HandlerArg: " + handlerArg);
console.log("##vso[task.debug]HandlerWorkDir: " + handlerWorkDir);
Object.keys(stdinData.environment).forEach(function (key) {
console.log("##vso[task.debug]Set env: " + key + "=" + stdinData.environment[key].toString().replace(/\r/g, '%0D').replace(/\n/g, '%0A'));
process.env[key] = stdinData.environment[key];
});
let stdinString = '';
process.stdin.on('data', chunk => stdinString += chunk);

process.stdin.on('end', () => {
const { handler, args: handlerArg, workDir: handlerWorkDir, prependPath, environment } = JSON.parse(stdinString);

var currentPath = process.env['PATH'];
var options = {
debug(`Handler: ${handler}`);
debug(`HandlerArg: ${handlerArg}`);
debug(`HandlerWorkDir: ${handlerWorkDir}`);

for (const key in environment) {
const value = environment[key].toString().replace(/\r/g, '%0D').replace(/\n/g, '%0A');
debug(`Set env: ${key}=${value}`);
process.env[key] = environment[key];
}

const options = {
stdio: 'inherit',
cwd: handlerWorkDir
};
if (process.platform == 'win32') {

const isWindows = process.platform == 'win32';

if (isWindows) {
options.argv0 = `"${handler}"`;
options.windowsVerbatimArguments = true;

if (prependPath && prependPath.length > 0) {
if (currentPath && currentPath.length > 0) {
process.env['PATH'] = prependPath + ';' + currentPath;
}
else {
process.env['PATH'] = prependPath;
}
}
}
else {
if (prependPath && prependPath.length > 0) {
if (currentPath && currentPath.length > 0) {
process.env['PATH'] = prependPath + ':' + currentPath;
}
else {
process.env['PATH'] = prependPath;
}
}
}

if (prependPath && prependPath.length > 0) {
console.log("##vso[task.debug]Prepend Path: " + process.env['PATH']);
const currentPath = process.env['PATH'];
process.env['PATH'] = prependPath;

if (currentPath && currentPath.length > 0) {
process.env['PATH'] += `${isWindows ? ';' : ':'}${currentPath}`;
}

debug(`Prepend Path: ${process.env['PATH']}`);
}

process.env['TF_BUILD'] = 'True';
console.log("##vso[task.debug]Handler Setup Complete");
var launch = spawn(handler, [handlerArg], options);
launch.on('exit', function (code) {
console.log("##vso[task.debug]Handler exit code: " + code);
debug(`Handler Setup Complete`);
const launch = spawn(handler, [handlerArg], options);

launch.on('exit', code => {
debug(`Handler exit code: ${code}`);

if (code != 0) {
process.exit(code);
}
Expand Down

0 comments on commit 2c3cdd9

Please sign in to comment.