Fallback from Node 20 to Node 16 on container startup

src/Agent.Sdk/ContainerInfo.cs

@@ -83,6 +83,7 @@ public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContain
         public string ContainerName { get; set; }
         public string ContainerCommand { get; set; }
         public string CustomNodePath { get; set; }
+        public string ResultNodePath { get; set; }
         public Guid ContainerRegistryEndpoint { get; private set; }
         public string ContainerCreateOptions { get; set; }
         public bool SkipContainerImagePull { get; private set; }

src/Agent.Sdk/ExecutionTargetInfo.cs

@@ -8,6 +8,7 @@ public interface ExecutionTargetInfo
     {
         PlatformUtil.OS ExecutionOS { get; }
         string CustomNodePath { get; set; }
+        string ResultNodePath { get; set; }
 
         string TranslateContainerPathForImageOS(PlatformUtil.OS runningOs, string path);
         string TranslateToContainerPath(string path);
@@ -18,6 +19,7 @@ public class HostInfo : ExecutionTargetInfo
     {
         public PlatformUtil.OS ExecutionOS => PlatformUtil.HostOS;
         public string CustomNodePath { get; set; }
+        public string ResultNodePath { get; set; }
 
         public string TranslateToContainerPath(string path)
         {

src/Agent.Worker/Container/DockerCommandManager.cs

@@ -26,6 +26,7 @@ public interface IDockerCommandManager : IAgentService
         Task<string> DockerCreate(IExecutionContext context, ContainerInfo container);
         Task<int> DockerStart(IExecutionContext context, string containerId);
         Task<int> DockerLogs(IExecutionContext context, string containerId);
+        Task<List<string>> GetDockerLogs(IExecutionContext context, string containerId);
         Task<List<string>> DockerPS(IExecutionContext context, string options);
         Task<int> DockerRemove(IExecutionContext context, string containerId);
         Task<int> DockerNetworkCreate(IExecutionContext context, string network);
@@ -229,6 +230,14 @@ public async Task<int> DockerLogs(IExecutionContext context, string containerId)
             return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}", context.CancellationToken);
         }
 
+        public async Task<List<string>> GetDockerLogs(IExecutionContext context, string containerId)
+        {
+            ArgUtil.NotNull(context, nameof(context));
+            ArgUtil.NotNull(containerId, nameof(containerId));
+
+            return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}");
+        }
+
         public async Task<List<string>> DockerPS(IExecutionContext context, string options)
         {
             ArgUtil.NotNull(context, nameof(context));

src/Agent.Worker/ContainerOperationProvider.cs

@@ -525,37 +525,58 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta
                 container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
             }
 
+            bool useNode20ToStartContainer = AgentKnobs.UseNode20ToStartContainer.GetValue(executionContext).AsBoolean();
+            bool useAgentNode = false;
+
+            string labelContainerStartupUsingNode20 = "container-startup-using-node-20";
+            string labelContainerStartupUsingNode16 = "container-startup-using-node-16";
+            string labelContainerStartupFailed = "container-startup-failed";
+
+            string containerNodePath(string nodeFolder)
+            {
+                return container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeFolder, "bin", $"node{IOUtil.ExeExtension}"));
+            }
+
+            string nodeContainerPath = containerNodePath(NodeHandler.NodeFolder);
+            string node16ContainerPath = containerNodePath(NodeHandler.Node16Folder);
+            string node20ContainerPath = containerNodePath(NodeHandler.Node20_1Folder);
+
             if (container.IsJobContainer)
             {
                 // See if this container brings its own Node.js
                 container.CustomNodePath = await _dockerManger.DockerInspect(context: executionContext,
                                                                     dockerObject: container.ContainerImage,
                                                                     options: $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
 
-                string node;
+                string nodeSetInterval(string node)
+                {
+                    return $"'{node}' -e 'setInterval(function(){{}}, 24 * 60 * 60 * 1000);'";
+                }
+
+                string useDoubleQuotes(string value)
+                {
+                    return value.Replace('\'', '"');
+                }
+
                 if (!string.IsNullOrEmpty(container.CustomNodePath))
                 {
-                    node = container.CustomNodePath;
+                    container.ContainerCommand = useDoubleQuotes(nodeSetInterval(container.CustomNodePath));
+                    container.ResultNodePath = container.CustomNodePath;
+                }
+                else if (PlatformUtil.RunningOnMacOS || (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux))
+                {
+                    // require container to have node if running on macOS, or if running on Windows and attempting to run Linux container
+                    container.CustomNodePath = "node";
+                    container.ContainerCommand = useDoubleQuotes(nodeSetInterval(container.CustomNodePath));
+                    container.ResultNodePath = container.CustomNodePath;
                 }
                 else
                 {
-                    node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), AgentKnobs.UseNode20ToStartContainer.GetValue(executionContext).AsBoolean() ? NodeHandler.Node20_1Folder : NodeHandler.NodeFolder, "bin", $"node{IOUtil.ExeExtension}"));
-
-                    // if on Mac OS X, require container to have node
-                    if (PlatformUtil.RunningOnMacOS)
-                    {
-                        container.CustomNodePath = "node";
-                        node = container.CustomNodePath;
-                    }
-                    // if running on Windows, and attempting to run linux container, require container to have node
-                    else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux)
-                    {
-                        container.CustomNodePath = "node";
-                        node = container.CustomNodePath;
-                    }
+                    useAgentNode = true;
+                    string sleepCommand = useNode20ToStartContainer ? $"'{node20ContainerPath}' --version && echo '{labelContainerStartupUsingNode20}' && {nodeSetInterval(node20ContainerPath)} || '{node16ContainerPath}' --version && echo '{labelContainerStartupUsingNode16}' && {nodeSetInterval(node16ContainerPath)} || echo '{labelContainerStartupFailed}'" : nodeSetInterval(nodeContainerPath);
+                    container.ContainerCommand = PlatformUtil.RunningOnWindows ? $"cmd.exe /c call {useDoubleQuotes(sleepCommand)}" : $"bash -c \"{sleepCommand}\"";
+                    container.ResultNodePath = nodeContainerPath;
                 }
-                string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
-                container.ContainerCommand = sleepCommand;
             }
 
             container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
@@ -588,6 +609,56 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta
 
                     executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
                 }
+                else if (useAgentNode && useNode20ToStartContainer)
+                {
+                    bool containerStartupCompleted = false;
+                    int containerStartupTimeoutInMilliseconds = 10000;
+                    int delayInMilliseconds = 100;
+                    int checksCount = 0;
+
+                    while (true)
+                    {
+                        List<string> containerLogs = await _dockerManger.GetDockerLogs(executionContext, container.ContainerId);
+
+                        foreach (string logLine in containerLogs)
+                        {
+                            if (logLine.Contains(labelContainerStartupUsingNode20))
+                            {
+                                executionContext.Debug("Using Node 20 for container startup.");
+                                containerStartupCompleted = true;
+                                container.ResultNodePath = node20ContainerPath;
+                                break;
+                            }
+                            else if (logLine.Contains(labelContainerStartupUsingNode16))
+                            {
+                                executionContext.Warning("Can not run Node 20 in container. Falling back to Node 16 for container startup.");
+                                containerStartupCompleted = true;
+                                container.ResultNodePath = node16ContainerPath;
+                                break;
+                            }
+                            else if (logLine.Contains(labelContainerStartupFailed))
+                            {
+                                executionContext.Error("Can not run both Node 20 and Node 16 in container. Container startup failed.");
+                                containerStartupCompleted = true;
+                                break;
+                            }
+                        }
+
+                        if (containerStartupCompleted)
+                        {
+                            break;
+                        }
+
+                        checksCount++;
+                        if (checksCount * delayInMilliseconds > containerStartupTimeoutInMilliseconds)
+                        {
+                            executionContext.Warning("Can not get startup status from container.");
+                            break;
+                        }
+
+                        await Task.Delay(delayInMilliseconds);
+                    }
+                }
             }
             catch (Exception ex)
             {

src/Agent.Worker/Handlers/StepHost.cs

@@ -182,16 +182,6 @@ public async Task<int> ExecuteAsync(string workingDirectory,
             HostContext.GetTrace(nameof(ContainerStepHost)).Info($"Copying containerHandlerInvoker.js to {tempDir}");
             File.Copy(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "containerHandlerInvoker.js.template"), targetEntryScript, true);
 
-            string node;
-            if (!string.IsNullOrEmpty(Container.CustomNodePath))
-            {
-                node = Container.CustomNodePath;
-            }
-            else
-            {
-                node = Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
-            }
-
             string entryScript = Container.TranslateContainerPathForImageOS(PlatformUtil.HostOS, Container.TranslateToContainerPath(targetEntryScript));
 
             string userArgs = "";
@@ -209,7 +199,7 @@ public async Task<int> ExecuteAsync(string workingDirectory,
                 }
             }
 
-            string containerExecutionArgs = $"exec -i {userArgs} {workingDirectoryParam} {Container.ContainerId} {node} {entryScript}";
+            string containerExecutionArgs = $"exec -i {userArgs} {workingDirectoryParam} {Container.ContainerId} {Container.ResultNodePath} {entryScript}";
 
             using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
             {

src/Misc/layoutbin/containerHandlerInvoker.js.template

@@ -1,62 +1,53 @@
 const { spawn } = require('child_process');
-var stdinString = "";
-process.stdin.on('data', function (chunk) {
-    stdinString += chunk;
-});
 
-process.stdin.on('end', function () {
-    var stdinData = JSON.parse(stdinString);
-    var handler = stdinData.handler;
-    var handlerArg = stdinData.args;
-    var handlerWorkDir = stdinData.workDir;
-    var prependPath = stdinData.prependPath;
+const debug = log => console.log(`##vso[task.debug]${log}`);
 
-    console.log("##vso[task.debug]Handler: " + handler);
-    console.log("##vso[task.debug]HandlerArg: " + handlerArg);
-    console.log("##vso[task.debug]HandlerWorkDir: " + handlerWorkDir);
-    Object.keys(stdinData.environment).forEach(function (key) {
-        console.log("##vso[task.debug]Set env: " + key + "=" + stdinData.environment[key].toString().replace(/\r/g, '%0D').replace(/\n/g, '%0A'));
-        process.env[key] = stdinData.environment[key];
-    });
+let stdinString = '';
+process.stdin.on('data', chunk => stdinString += chunk);
+
+process.stdin.on('end', () => {
+    const { handler, args: handlerArg, workDir: handlerWorkDir, prependPath, environment } = JSON.parse(stdinString);
 
-    var currentPath = process.env['PATH'];
-    var options = {
+    debug(`Handler: ${handler}`);
+    debug(`HandlerArg: ${handlerArg}`);
+    debug(`HandlerWorkDir: ${handlerWorkDir}`);
+
+    for (const key in environment) {
+        const value = environment[key].toString().replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+        debug(`Set env: ${key}=${value}`);
+        process.env[key] = environment[key];
+    }
+
+    const options = {
         stdio: 'inherit',
         cwd: handlerWorkDir
     };
-    if (process.platform == 'win32') {
+
+    const isWindows = process.platform == 'win32';
+
+    if (isWindows) {
         options.argv0 = `"${handler}"`;
         options.windowsVerbatimArguments = true;
-
-        if (prependPath && prependPath.length > 0) {
-            if (currentPath && currentPath.length > 0) {
-                process.env['PATH'] = prependPath + ';' + currentPath;
-            }
-            else {
-                process.env['PATH'] = prependPath;
-            }
-        }
-    }
-    else {
-        if (prependPath && prependPath.length > 0) {
-            if (currentPath && currentPath.length > 0) {
-                process.env['PATH'] = prependPath + ':' + currentPath;
-            }
-            else {
-                process.env['PATH'] = prependPath;
-            }
-        }
     }
 
     if (prependPath && prependPath.length > 0) {
-        console.log("##vso[task.debug]Prepend Path: " + process.env['PATH']);
+        const currentPath = process.env['PATH'];
+        process.env['PATH'] = prependPath;
+
+        if (currentPath && currentPath.length > 0) {
+            process.env['PATH'] += `${isWindows ? ';' : ':'}${currentPath}`;
+        }
+
+        debug(`Prepend Path: ${process.env['PATH']}`);
     }
 
     process.env['TF_BUILD'] = 'True';
-    console.log("##vso[task.debug]Handler Setup Complete");
-    var launch = spawn(handler, [handlerArg], options);
-    launch.on('exit', function (code) {
-        console.log("##vso[task.debug]Handler exit code: " + code);
+    debug(`Handler Setup Complete`);
+    const launch = spawn(handler, [handlerArg], options);
+
+    launch.on('exit', code => {
+        debug(`Handler exit code: ${code}`);
+
         if (code != 0) {
             process.exit(code);
         }