From 7529cc68a4aa89fe326cfc594b5b5df7c113b7fa Mon Sep 17 00:00:00 2001 From: Konstantin Tyukalov Date: Sat, 13 Apr 2024 16:40:20 +0400 Subject: [PATCH 1/4] Move context to separate folder --- src/Agent.Worker/{ => ExecutionContext}/ExecutionContext.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Agent.Worker/{ => ExecutionContext}/ExecutionContext.cs (100%) diff --git a/src/Agent.Worker/ExecutionContext.cs b/src/Agent.Worker/ExecutionContext/ExecutionContext.cs similarity index 100% rename from src/Agent.Worker/ExecutionContext.cs rename to src/Agent.Worker/ExecutionContext/ExecutionContext.cs From ed50bbc365aaac2baa78d500770fbd1f748332e6 Mon Sep 17 00:00:00 2001 From: Konstantin Tyukalov Date: Sat, 13 Apr 2024 16:52:03 +0400 Subject: [PATCH 2/4] Add inheritance from exec context --- .../ExecutionContext/ExecutionContext.cs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/Agent.Worker/ExecutionContext/ExecutionContext.cs b/src/Agent.Worker/ExecutionContext/ExecutionContext.cs index ffcb6d7ccf..845fb1f8cd 100644 --- a/src/Agent.Worker/ExecutionContext/ExecutionContext.cs +++ b/src/Agent.Worker/ExecutionContext/ExecutionContext.cs @@ -91,7 +91,7 @@ public interface IExecutionContext : IAgentService, IKnobValueContext void EmitHostNode20FallbackTelemetry(bool node20ResultsInGlibCErrorHost); } - public sealed class ExecutionContext : AgentService, IExecutionContext, IDisposable + public class ExecutionContext : AgentService, IExecutionContext, IDisposable { private const int _maxIssueCount = 10; @@ -937,13 +937,23 @@ private void PublishTelemetry( public void Dispose() { - _cancellationTokenSource?.Dispose(); - _forceCompleteCancellationTokenSource?.Dispose(); + Dispose(true); + + GC.SuppressFinalize(this); + } - _buildLogsWriter?.Dispose(); - _buildLogsWriter = null; - _buildLogsData?.Dispose(); - _buildLogsData = null; + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _cancellationTokenSource?.Dispose(); + _forceCompleteCancellationTokenSource?.Dispose(); + + _buildLogsWriter?.Dispose(); + _buildLogsWriter = null; + _buildLogsData?.Dispose(); + _buildLogsData = null; + } } } From f401c04bceed51d225b879a7c0b50e0893cbf38d Mon Sep 17 00:00:00 2001 From: Konstantin Tyukalov Date: Sat, 13 Apr 2024 17:05:12 +0400 Subject: [PATCH 3/4] Init task and job contexts --- .../ExecutionContext/JobExecutionContext.cs | 33 +++++++++++++++++++ .../ExecutionContext/TaskExecutionContext.cs | 14 ++++++++ 2 files changed, 47 insertions(+) create mode 100644 src/Agent.Worker/ExecutionContext/JobExecutionContext.cs create mode 100644 src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs diff --git a/src/Agent.Worker/ExecutionContext/JobExecutionContext.cs b/src/Agent.Worker/ExecutionContext/JobExecutionContext.cs new file mode 100644 index 0000000000..39068f6316 --- /dev/null +++ b/src/Agent.Worker/ExecutionContext/JobExecutionContext.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Microsoft.TeamFoundation.DistributedTask.WebApi; + +namespace Microsoft.VisualStudio.Services.Agent.Worker; + +public interface IJobExecutionContext : IExecutionContext +{ + public IExecutionContext CreateTaskExecutionContext( + Guid recordId, + string displayName, + string refName, + Variables taskVariables = null, + bool outputForward = false, + List taskRestrictions = null); +} + +public sealed class JobExecutionContext : ExecutionContext, IJobExecutionContext +{ + public IExecutionContext CreateTaskExecutionContext( + Guid recordId, + string displayName, + string refName, + Variables taskVariables = null, + bool outputForward = false, + List taskRestrictions = null) + { + return CreateChild(recordId, displayName, refName, taskVariables, outputForward, taskRestrictions); + } +} diff --git a/src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs b/src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs new file mode 100644 index 0000000000..7c4b013881 --- /dev/null +++ b/src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.VisualStudio.Services.Agent.Worker; + +public interface ITaskExecutionContext : IExecutionContext +{ + public IJobExecutionContext JobContext { get; } +} + +public sealed class TaskExecutionContext : ExecutionContext, ITaskExecutionContext +{ + public IJobExecutionContext JobContext { get; private set; } +} From 44b6e1707192ab4d6bd184dfb4357004e1154533 Mon Sep 17 00:00:00 2001 From: Konstantin Tyukalov Date: Sat, 13 Apr 2024 19:14:53 +0400 Subject: [PATCH 4/4] Use job context in job extension --- .../ExecutionContext/ExecutionContext.cs | 43 +++++------ .../ExecutionContext/JobExecutionContext.cs | 38 +++++++++- .../ExecutionContext/TaskExecutionContext.cs | 73 +++++++++++++++++++ src/Agent.Worker/JobExtension.cs | 18 ++--- src/Agent.Worker/JobRunner.cs | 4 +- 5 files changed, 141 insertions(+), 35 deletions(-) diff --git a/src/Agent.Worker/ExecutionContext/ExecutionContext.cs b/src/Agent.Worker/ExecutionContext/ExecutionContext.cs index 845fb1f8cd..e0b90c05ca 100644 --- a/src/Agent.Worker/ExecutionContext/ExecutionContext.cs +++ b/src/Agent.Worker/ExecutionContext/ExecutionContext.cs @@ -95,7 +95,16 @@ public class ExecutionContext : AgentService, IExecutionContext, IDisposable { private const int _maxIssueCount = 10; - private readonly TimelineRecord _record = new TimelineRecord(); + protected readonly TimelineRecord _record = new(); + protected ExecutionTargetInfo _defaultStepTarget; + protected ExecutionTargetInfo _currentStepTarget; + protected IPagingLogger _logger; + protected bool _outputForward = false; + protected Guid _mainTimelineId; + protected int _childTimelineRecordOrder = 0; + protected CancellationTokenSource _cancellationTokenSource; + protected IExecutionContext _parentExecutionContext; + private readonly Dictionary _detailRecords = new Dictionary(); private readonly object _loggerLock = new object(); private readonly List _asyncCommands = new List(); @@ -103,19 +112,11 @@ public class ExecutionContext : AgentService, IExecutionContext, IDisposable private readonly List _restrictions = new List(); private readonly string _buildLogsFolderName = "buildlogs"; private IAgentLogPlugin _logPlugin; - private IPagingLogger _logger; private IJobServerQueue _jobServerQueue; - private IExecutionContext _parentExecutionContext; - private bool _outputForward = false; - private Guid _mainTimelineId; private Guid _detailTimelineId; - private int _childTimelineRecordOrder = 0; - private CancellationTokenSource _cancellationTokenSource; private CancellationTokenSource _forceCompleteCancellationTokenSource = new CancellationTokenSource(); private TaskCompletionSource _forceCompleted = new TaskCompletionSource(); private bool _throttlingReported = false; - private ExecutionTargetInfo _defaultStepTarget; - private ExecutionTargetInfo _currentStepTarget; private bool _disableLogUploads; private string _buildLogsFolderPath; private string _buildLogsFile; @@ -130,17 +131,17 @@ public class ExecutionContext : AgentService, IExecutionContext, IDisposable public Task ForceCompleted => _forceCompleted.Task; public CancellationToken CancellationToken => _cancellationTokenSource.Token; public CancellationToken ForceCompleteCancellationToken => _forceCompleteCancellationTokenSource.Token; - public List Endpoints { get; private set; } - public List SecureFiles { get; private set; } - public List Repositories { get; private set; } - public Dictionary JobSettings { get; private set; } - public Variables Variables { get; private set; } - public Variables TaskVariables { get; private set; } + public List Endpoints { get; protected set; } + public List SecureFiles { get; protected set; } + public List Repositories { get; protected set; } + public Dictionary JobSettings { get; protected set; } + public Variables Variables { get; protected set; } + public Variables TaskVariables { get; protected set; } public HashSet OutputVariables => _outputvariables; - public bool WriteDebug { get; private set; } - public List PrependPath { get; private set; } - public List Containers { get; private set; } - public List SidecarContainers { get; private set; } + public bool WriteDebug { get; protected set; } + public List PrependPath { get; protected set; } + public List Containers { get; protected set; } + public List SidecarContainers { get; protected set; } public List Restrictions => _restrictions; public List AsyncCommands => _asyncCommands; @@ -172,7 +173,7 @@ public string ResultCode } } - public PlanFeatures Features { get; private set; } + public PlanFeatures Features { get; protected set; } public override void Initialize(IHostContext hostContext) { @@ -761,7 +762,7 @@ public ITraceWriter GetTraceWriter() return Trace; } - private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order) + protected internal void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order) { _mainTimelineId = timelineId; _record.Id = timelineRecordId; diff --git a/src/Agent.Worker/ExecutionContext/JobExecutionContext.cs b/src/Agent.Worker/ExecutionContext/JobExecutionContext.cs index 39068f6316..d0fb6069c5 100644 --- a/src/Agent.Worker/ExecutionContext/JobExecutionContext.cs +++ b/src/Agent.Worker/ExecutionContext/JobExecutionContext.cs @@ -3,13 +3,14 @@ using System; using System.Collections.Generic; +using System.Threading; using Microsoft.TeamFoundation.DistributedTask.WebApi; namespace Microsoft.VisualStudio.Services.Agent.Worker; public interface IJobExecutionContext : IExecutionContext { - public IExecutionContext CreateTaskExecutionContext( + public ITaskExecutionContext CreateTaskExecutionContext( Guid recordId, string displayName, string refName, @@ -20,7 +21,7 @@ public IExecutionContext CreateTaskExecutionContext( public sealed class JobExecutionContext : ExecutionContext, IJobExecutionContext { - public IExecutionContext CreateTaskExecutionContext( + public ITaskExecutionContext CreateTaskExecutionContext( Guid recordId, string displayName, string refName, @@ -28,6 +29,37 @@ public IExecutionContext CreateTaskExecutionContext( bool outputForward = false, List taskRestrictions = null) { - return CreateChild(recordId, displayName, refName, taskVariables, outputForward, taskRestrictions); + Trace.Entering(); + + var taskContext = new TaskExecutionContext(); + taskContext.Initialize(HostContext); + taskContext.InitContextProperties( + jobContext: this, + features: Features, + variables: Variables, + endpoints: Endpoints, + repositories: Repositories, + jobSettings: JobSettings, + secureFiles: SecureFiles, + taskVariables: taskVariables, + writeDebug: WriteDebug, + prependPath: PrependPath, + containers: Containers, + sidecarContainers: SidecarContainers, + outputForward: outputForward, + defaultStepTarget: _defaultStepTarget, + currentStepTarget: _currentStepTarget, + cancellationTokenSource: new CancellationTokenSource()); + + if (taskRestrictions != null) + { + taskContext.Restrictions.AddRange(taskRestrictions); + } + + taskContext.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder); + + taskContext.InitLogger(_mainTimelineId, recordId); + + return taskContext; } } diff --git a/src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs b/src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs index 7c4b013881..91e5173fb1 100644 --- a/src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs +++ b/src/Agent.Worker/ExecutionContext/TaskExecutionContext.cs @@ -1,14 +1,87 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System; +using System.Threading; +using System.Collections.Generic; +using Microsoft.TeamFoundation.DistributedTask.WebApi; +using Microsoft.VisualStudio.Services.Agent.Util; +using Microsoft.TeamFoundation.DistributedTask.Pipelines; +using Agent.Sdk; + namespace Microsoft.VisualStudio.Services.Agent.Worker; public interface ITaskExecutionContext : IExecutionContext { public IJobExecutionContext JobContext { get; } + + public void InitLogger(Guid mainTimelineId, Guid recordId); + + public void InitContextProperties( + IJobExecutionContext jobContext, + PlanFeatures features, + Variables variables, + List endpoints, + List repositories, + Dictionary jobSettings, + List secureFiles, + Variables taskVariables, + bool writeDebug, + List prependPath, + List containers, + List sidecarContainers, + bool outputForward, + ExecutionTargetInfo defaultStepTarget, + ExecutionTargetInfo currentStepTarget, + CancellationTokenSource cancellationTokenSource); } public sealed class TaskExecutionContext : ExecutionContext, ITaskExecutionContext { public IJobExecutionContext JobContext { get; private set; } + + public TaskExecutionContext() { } + + public void InitContextProperties( + IJobExecutionContext jobContext, + PlanFeatures features, + Variables variables, + List endpoints, + List repositories, + Dictionary jobSettings, + List secureFiles, + Variables taskVariables, + bool writeDebug, + List prependPath, + List containers, + List sidecarContainers, + bool outputForward, + ExecutionTargetInfo defaultStepTarget, + ExecutionTargetInfo currentStepTarget, + CancellationTokenSource cancellationTokenSource) + { + JobContext = jobContext; + _parentExecutionContext = jobContext; + Features = features; + Variables = variables; + Endpoints = endpoints; + Repositories = repositories; + JobSettings = jobSettings; + SecureFiles = secureFiles; + TaskVariables = taskVariables; + _cancellationTokenSource = cancellationTokenSource; + WriteDebug = writeDebug; + PrependPath = prependPath; + Containers = containers; + SidecarContainers = sidecarContainers; + _outputForward = outputForward; + _defaultStepTarget = defaultStepTarget; + _currentStepTarget = currentStepTarget; + } + + public void InitLogger(Guid mainTimelineId, Guid recordId) + { + _logger = HostContext.CreateService(); + _logger.Setup(mainTimelineId, recordId); + } } diff --git a/src/Agent.Worker/JobExtension.cs b/src/Agent.Worker/JobExtension.cs index 390f1f37e1..7018206407 100644 --- a/src/Agent.Worker/JobExtension.cs +++ b/src/Agent.Worker/JobExtension.cs @@ -21,8 +21,8 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker public interface IJobExtension : IExtension { HostTypes HostType { get; } - Task> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message); - Task FinalizeJob(IExecutionContext jobContext); + Task> InitializeJob(IJobExecutionContext jobContext, Pipelines.AgentJobRequestMessage message); + Task FinalizeJob(IJobExecutionContext jobContext); string GetRootedPath(IExecutionContext context, string path); void ConvertLocalPath(IExecutionContext context, string localPath, out string repoName, out string sourcePath); } @@ -55,14 +55,14 @@ public abstract class JobExtension : AgentService, IJobExtension // download all required tasks. // make sure all task's condition inputs are valid. // build up three list of steps for jobrunner. (pre-job, job, post-job) - public async Task> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message) + public async Task> InitializeJob(IJobExecutionContext jobContext, Pipelines.AgentJobRequestMessage message) { Trace.Entering(); ArgUtil.NotNull(jobContext, nameof(jobContext)); ArgUtil.NotNull(message, nameof(message)); // create a new timeline record node for 'Initialize job' - IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), $"{nameof(JobExtension)}_Init"); + ITaskExecutionContext context = jobContext.CreateTaskExecutionContext(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), $"{nameof(JobExtension)}_Init"); List preJobSteps = new List(); List jobSteps = new List(); @@ -373,7 +373,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel { ITaskRunner taskStep = step as ITaskRunner; ArgUtil.NotNull(taskStep, step.DisplayName); - taskStep.ExecutionContext = jobContext.CreateChild( + taskStep.ExecutionContext = jobContext.CreateTaskExecutionContext( Guid.NewGuid(), StringUtil.Loc("PreJob", taskStep.DisplayName), taskStep.Task.Name, @@ -388,7 +388,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel { ITaskRunner taskStep = step as ITaskRunner; ArgUtil.NotNull(taskStep, step.DisplayName); - taskStep.ExecutionContext = jobContext.CreateChild( + taskStep.ExecutionContext = jobContext.CreateTaskExecutionContext( taskStep.Task.Id, taskStep.DisplayName, taskStep.Task.Name, @@ -418,7 +418,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel { ITaskRunner taskStep = step as ITaskRunner; ArgUtil.NotNull(taskStep, step.DisplayName); - taskStep.ExecutionContext = jobContext.CreateChild( + taskStep.ExecutionContext = jobContext.CreateTaskExecutionContext( Guid.NewGuid(), StringUtil.Loc("PostJob", taskStep.DisplayName), taskStep.Task.Name, @@ -515,13 +515,13 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } - public async Task FinalizeJob(IExecutionContext jobContext) + public async Task FinalizeJob(IJobExecutionContext jobContext) { Trace.Entering(); ArgUtil.NotNull(jobContext, nameof(jobContext)); // create a new timeline record node for 'Finalize job' - IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("FinalizeJob"), $"{nameof(JobExtension)}_Final"); + ITaskExecutionContext context = jobContext.CreateTaskExecutionContext(Guid.NewGuid(), StringUtil.Loc("FinalizeJob"), $"{nameof(JobExtension)}_Final"); using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); })) { try diff --git a/src/Agent.Worker/JobRunner.cs b/src/Agent.Worker/JobRunner.cs index c24e441dc2..2910349fed 100644 --- a/src/Agent.Worker/JobRunner.cs +++ b/src/Agent.Worker/JobRunner.cs @@ -96,7 +96,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, _jobServerQueue.Start(message); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); - IExecutionContext jobContext = null; + IJobExecutionContext jobContext = null; CancellationTokenRegistration? agentShutdownRegistration = null; VssConnection taskConnection = null; VssConnection legacyTaskConnection = null; @@ -104,7 +104,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, try { // Create the job execution context. - jobContext = HostContext.CreateService(); + jobContext = HostContext.CreateService(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context.");