diff --git a/Directory.Packages.props b/Directory.Packages.props index 8beeef79..adc0944e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -21,9 +21,12 @@ + + + diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj index f5008404..8f873607 100644 --- a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj +++ b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj @@ -9,7 +9,10 @@ + + + diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetMSBuildBinaryLogComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetMSBuildBinaryLogComponentDetector.cs new file mode 100644 index 00000000..d62d6979 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetMSBuildBinaryLogComponentDetector.cs @@ -0,0 +1,212 @@ +namespace Microsoft.ComponentDetection.Detectors.NuGet; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Microsoft.Build.Locator; +using Microsoft.Build.Logging.StructuredLogger; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +using Task = System.Threading.Tasks.Task; + +public class NuGetMSBuildBinaryLogComponentDetector : FileComponentDetector +{ + private static readonly HashSet TopLevelPackageItemNames = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "PackageReference", + }; + + // the items listed below represent collection names that NuGet will resolve a package into, along with the metadata value names to get the package name and version + private static readonly Dictionary ResolvedPackageItemNames = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["NativeCopyLocalItems"] = ("NuGetPackageId", "NuGetPackageVersion"), + ["ResourceCopyLocalItems"] = ("NuGetPackageId", "NuGetPackageVersion"), + ["RuntimeCopyLocalItems"] = ("NuGetPackageId", "NuGetPackageVersion"), + ["ResolvedAnalyzers"] = ("NuGetPackageId", "NuGetPackageVersion"), + ["_PackageDependenciesDesignTime"] = ("Name", "Version"), + }; + + private static bool isMSBuildRegistered; + + public NuGetMSBuildBinaryLogComponentDetector( + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id { get; } = "NuGetMSBuildBinaryLog"; + + public override IEnumerable Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.NuGet) }; + + public override IList SearchPatterns { get; } = new List { "*.binlog" }; + + public override IEnumerable SupportedComponentTypes { get; } = new[] { ComponentType.NuGet }; + + public override int Version { get; } = 1; + + private static void ProcessResolvedPackageReference(Dictionary> topLevelDependencies, Dictionary> projectResolvedDependencies, NamedNode node) + { + var doRemoveOperation = node is RemoveItem; + var doAddOperation = node is AddItem; + if (TopLevelPackageItemNames.Contains(node.Name)) + { + var projectEvaluation = node.GetNearestParent(); + if (projectEvaluation is not null) + { + foreach (var child in node.Children.OfType()) + { + var packageName = child.Name; + if (!topLevelDependencies.TryGetValue(projectEvaluation.ProjectFile, out var topLevel)) + { + topLevel = new(StringComparer.OrdinalIgnoreCase); + topLevelDependencies[projectEvaluation.ProjectFile] = topLevel; + } + + if (doRemoveOperation) + { + topLevel.Remove(packageName); + } + + if (doAddOperation) + { + topLevel.Add(packageName); + } + } + } + } + else if (ResolvedPackageItemNames.TryGetValue(node.Name, out var metadataNames)) + { + var nameMetadata = metadataNames.NameMetadata; + var versionMetadata = metadataNames.VersionMetadata; + var originalProject = node.GetNearestParent(); + if (originalProject is not null) + { + foreach (var child in node.Children.OfType()) + { + var packageName = GetChildMetadataValue(child, nameMetadata); + var packageVersion = GetChildMetadataValue(child, versionMetadata); + if (packageName is not null && packageVersion is not null) + { + var project = originalProject; + while (project is not null) + { + if (!projectResolvedDependencies.TryGetValue(project.ProjectFile, out var projectDependencies)) + { + projectDependencies = new(StringComparer.OrdinalIgnoreCase); + projectResolvedDependencies[project.ProjectFile] = projectDependencies; + } + + if (doRemoveOperation) + { + projectDependencies.Remove(packageName); + } + + if (doAddOperation) + { + projectDependencies[packageName] = packageVersion; + } + + project = project.GetNearestParent(); + } + } + } + } + } + } + + private static string GetChildMetadataValue(TreeNode node, string metadataItemName) + { + var metadata = node.Children.OfType(); + var metadataValue = metadata.FirstOrDefault(m => m.Name.Equals(metadataItemName, StringComparison.OrdinalIgnoreCase))?.Value; + return metadataValue; + } + + protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + try + { + if (!isMSBuildRegistered) + { + // this must happen once per process, and never again + var defaultInstance = MSBuildLocator.QueryVisualStudioInstances().First(); + MSBuildLocator.RegisterInstance(defaultInstance); + isMSBuildRegistered = true; + } + + var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(processRequest.ComponentStream.Location); + var buildRoot = BinaryLog.ReadBuild(processRequest.ComponentStream.Stream); + this.RecordLockfileVersion(buildRoot.FileFormatVersion); + this.ProcessBinLog(buildRoot, singleFileComponentRecorder); + } + catch (Exception e) + { + // If something went wrong, just ignore the package + this.Logger.LogError(e, "Failed to process MSBuild binary log {BinLogFile}", processRequest.ComponentStream.Location); + } + + return Task.CompletedTask; + } + + protected override Task OnDetectionFinishedAsync() + { + return Task.CompletedTask; + } + + private void ProcessBinLog(Build buildRoot, ISingleFileComponentRecorder componentRecorder) + { + // maps a project path to a set of resolved dependencies + var projectTopLevelDependencies = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var projectResolvedDependencies = new Dictionary>(StringComparer.OrdinalIgnoreCase); + buildRoot.VisitAllChildren(node => + { + switch (node) + { + case NamedNode namedNode when namedNode is AddItem or RemoveItem: + ProcessResolvedPackageReference(projectTopLevelDependencies, projectResolvedDependencies, namedNode); + break; + default: + break; + } + }); + + // dependencies were resolved per project, we need to re-arrange them to be per package/version + var projectsPerPackage = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var projectPath in projectResolvedDependencies.Keys) + { + var projectDependencies = projectResolvedDependencies[projectPath]; + foreach (var (packageName, packageVersion) in projectDependencies) + { + var key = $"{packageName}/{packageVersion}"; + if (!projectsPerPackage.TryGetValue(key, out var projectPaths)) + { + projectPaths = new HashSet(StringComparer.OrdinalIgnoreCase); + projectsPerPackage[key] = projectPaths; + } + + projectPaths.Add(projectPath); + } + } + + // report it all + foreach (var (packageNameAndVersion, projectPaths) in projectsPerPackage) + { + var parts = packageNameAndVersion.Split('/', 2); + var packageName = parts[0]; + var packageVersion = parts[1]; + var component = new NuGetComponent(packageName, packageVersion); + var libraryComponent = new DetectedComponent(component); + foreach (var projectPath in projectPaths) + { + libraryComponent.FilePaths.Add(projectPath); + } + + componentRecorder.RegisterUsage(libraryComponent); + } + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 41d16ccc..4da100fe 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -105,6 +105,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // PIP services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/MSBuildTestUtilities.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/MSBuildTestUtilities.cs new file mode 100644 index 00000000..5bdc0161 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/MSBuildTestUtilities.cs @@ -0,0 +1,337 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Xml.XPath; +using FluentAssertions; + +public static class MSBuildTestUtilities +{ + public const int TestTargetFrameworkVersion = 6; + public static readonly string TestTargetFramework = $"net{TestTargetFrameworkVersion}.0"; + + // we need to find the file `Microsoft.NETCoreSdk.BundledVersions.props` in the SDK directory + private static readonly Lazy BundledVersionsPropsPath = new(static () => + { + // get the sdk version + using var tempDir = new TemporaryProjectDirectory(); + var projectContents = """ + + + + + + """; + var projectPath = Path.Combine(tempDir.DirectoryPath, "project.csproj"); + File.WriteAllText(projectPath, projectContents); + var (exitCode, stdout, stderr) = RunProcessAsync("dotnet", $"msbuild {projectPath} /t:_ReportCurrentSdkVersion").Result; + if (exitCode != 0) + { + throw new NotSupportedException($"Failed to report the current SDK version:\n{stdout}\n{stderr}"); + } + + var matches = Regex.Matches(stdout, "_CurrentSdkVersion=(?.*)$", RegexOptions.Multiline); + if (matches.Count == 0) + { + throw new NotSupportedException($"Failed to find the current SDK version in the output:\n{stdout}"); + } + + var sdkVersionString = matches.First().Groups["SdkVersion"].Value.Trim(); + + // find the actual SDK directory + var privateCoreLibPath = typeof(object).Assembly.Location; // e.g., C:\Program Files\dotnet\shared\Microsoft.NETCore.App\8.0.4\System.Private.CoreLib.dll + var sdkDirectory = Path.Combine(Path.GetDirectoryName(privateCoreLibPath), "..", "..", "..", "sdk", sdkVersionString); // e.g., C:\Program Files\dotnet\sdk\8.0.204 + var bundledVersionsPropsPath = Path.Combine(sdkDirectory, "Microsoft.NETCoreSdk.BundledVersions.props"); + var normalizedPath = new FileInfo(bundledVersionsPropsPath); + return normalizedPath.FullName; + }); + + public static async Task GetBinLogStreamFromFileContentsAsync( + string projectContents, + (string FileName, string Contents)[] additionalFiles = null, + (string Name, string Version, string TargetFramework, string AdditionalMetadataXml)[] mockedPackages = null) + { + // write all files + using var tempDir = new TemporaryProjectDirectory(); + var fullProjectPath = Path.Combine(tempDir.DirectoryPath, "project.csproj"); + await File.WriteAllTextAsync(fullProjectPath, projectContents); + foreach (var (fileName, contents) in additionalFiles ?? []) + { + var fullFilePath = Path.Combine(tempDir.DirectoryPath, fileName); + var fullFileDirectory = Path.GetDirectoryName(fullFilePath); + Directory.CreateDirectory(fullFileDirectory); + await File.WriteAllTextAsync(fullFilePath, contents); + } + + await MockNuGetPackagesInDirectoryAsync(tempDir, mockedPackages); + + // generate the binlog + var (exitCode, stdOut, stdErr) = await RunProcessAsync("dotnet", $"build \"{fullProjectPath}\" /t:GenerateBuildDependencyFile /bl:msbuild.binlog", workingDirectory: tempDir.DirectoryPath); + exitCode.Should().Be(0, $"STDOUT:\n{stdOut}\n\nSTDERR:\n{stdErr}"); + + // copy it to memory so the temporary directory can be cleaned up + var fullBinLogPath = Path.Combine(tempDir.DirectoryPath, "msbuild.binlog"); + using var binLogStream = File.OpenRead(fullBinLogPath); + var inMemoryStream = new MemoryStream(); + await binLogStream.CopyToAsync(inMemoryStream); + inMemoryStream.Position = 0; + return inMemoryStream; + } + + private static async Task MockNuGetPackagesInDirectoryAsync( + TemporaryProjectDirectory tempDir, + (string Name, string Version, string TargetFramework, string AdditionalMetadataXml)[] mockedPackages) + { + if (mockedPackages is not null) + { + var nugetConfig = """ + + + + + + + """; + await File.WriteAllTextAsync(Path.Combine(tempDir.DirectoryPath, "NuGet.Config"), nugetConfig); + var packagesPath = Path.Combine(tempDir.DirectoryPath, "local-packages"); + Directory.CreateDirectory(packagesPath); + + var mockedPackagesWithFiles = mockedPackages.Select(p => + { + return ( + p.Name, + p.Version, + p.TargetFramework, + p.AdditionalMetadataXml, + Files: new[] { ($"lib/{p.TargetFramework}/{p.Name}.dll", Array.Empty()) }); + }); + + var allPackages = mockedPackagesWithFiles.Concat(GetCommonPackages()); + + using var sha512 = SHA512.Create(); // this is used to compute the hash of each package below + foreach (var package in allPackages) + { + var nuspec = NugetTestUtilities.GetValidNuspec(package.Name, package.Version, []); + if (package.AdditionalMetadataXml is not null) + { + // augment the nuspec + var doc = XDocument.Parse(nuspec); + var additionalMetadata = XElement.Parse(package.AdditionalMetadataXml); + additionalMetadata = WithNamespace(additionalMetadata, doc.Root.Name.Namespace); + + var metadataElement = doc.Root.Descendants().First(e => e.Name.LocalName == "metadata"); + metadataElement.Add(additionalMetadata); + nuspec = doc.ToString(); + } + + var nupkg = await NugetTestUtilities.ZipNupkgComponentAsync(package.Name, nuspec, additionalFiles: package.Files); + + // to create a local nuget package source, we need a directory structure like this: + // local-packages/// + var packagePath = Path.Combine(packagesPath, package.Name.ToLower(), package.Version.ToLower()); + Directory.CreateDirectory(packagePath); + + // and we need the following files: + // 1. the package + var nupkgPath = Path.Combine(packagePath, $"{package.Name}.{package.Version}.nupkg".ToLower()); + using (var nupkgFileStream = File.OpenWrite(nupkgPath)) + { + await nupkg.CopyToAsync(nupkgFileStream); + } + + // 2. the nuspec + var nuspecPath = Path.Combine(packagePath, $"{package.Name}.nuspec".ToLower()); + await File.WriteAllTextAsync(nuspecPath, nuspec); + + // 3. SHA512 hash of the package + var hash = sha512.ComputeHash(File.ReadAllBytes(nupkgPath)); + var hashString = Convert.ToBase64String(hash); + var hashPath = $"{nupkgPath}.sha512"; + await File.WriteAllTextAsync(hashPath, hashString); + + // 4. a JSON metadata file + var metadata = $$""" + { + "version": 2, + "contentHash": "{{hashString}}", + "source": null + } + """; + var metadataPath = Path.Combine(packagePath, ".nupkg.metadata"); + await File.WriteAllTextAsync(metadataPath, metadata); + } + } + } + + private static XElement WithNamespace(XElement element, XNamespace ns) + { + return new XElement( + ns + element.Name.LocalName, + element.Attributes(), + element.Elements().Select(e => WithNamespace(e, ns)), + element.Value); + } + + private static IEnumerable<(string Name, string Version, string TargetFramework, string AdditionalMetadataXml, (string Path, byte[] Content)[] Files)> GetCommonPackages() + { + // to allow the tests to not require the network, we need to mock some common packages + yield return MakeWellKnownReferencePackage("Microsoft.AspNetCore.App", []); + yield return MakeWellKnownReferencePackage("Microsoft.WindowsDesktop.App", []); + + var frameworksXml = $""" + + + """; + yield return MakeWellKnownReferencePackage("Microsoft.NETCore.App", [("data/FrameworkList.xml", Encoding.UTF8.GetBytes(frameworksXml))]); + } + + private static (string Name, string Version, string TargetFramework, string AdditionalMetadataXml, (string Path, byte[] Content)[] Files) MakeWellKnownReferencePackage(string packageName, (string Path, byte[] Content)[] files) + { + var propsDocument = XDocument.Load(BundledVersionsPropsPath.Value); + var xpathQuery = $""" + /Project/ItemGroup/KnownFrameworkReference + [ + @Include='{packageName}' and + @TargetingPackName='{packageName}.Ref' and + @TargetFramework='{TestTargetFramework}' + ] + """; + var matchingFrameworkElement = propsDocument.XPathSelectElement(xpathQuery); + if (matchingFrameworkElement is null) + { + throw new NotSupportedException($"Unable to find {packageName}.Ref"); + } + + var expectedVersion = matchingFrameworkElement.Attribute("TargetingPackVersion").Value; + return ( + $"{packageName}.Ref", + expectedVersion, + TestTargetFramework, + "", + files); + } + + public static Task<(int ExitCode, string Output, string Error)> RunProcessAsync(string fileName, string arguments = "", string workingDirectory = null) + { + var tcs = new TaskCompletionSource<(int, string, string)>(); + + var redirectInitiated = new ManualResetEventSlim(); + var process = new Process + { + StartInfo = + { + FileName = fileName, + Arguments = arguments, + UseShellExecute = false, // required to redirect output + RedirectStandardOutput = true, + RedirectStandardError = true, + }, + EnableRaisingEvents = true, + }; + + if (workingDirectory is not null) + { + process.StartInfo.WorkingDirectory = workingDirectory; + } + + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + + process.OutputDataReceived += (_, e) => stdout.AppendLine(e.Data); + process.ErrorDataReceived += (_, e) => stderr.AppendLine(e.Data); + + process.Exited += (sender, args) => + { + // It is necessary to wait until we have invoked 'BeginXReadLine' for our redirected IO. Then, + // we must call WaitForExit to make sure we've received all OutputDataReceived/ErrorDataReceived calls + // or else we'll be returning a list we're still modifying. For paranoia, we'll start a task here rather + // than enter right back into the Process type and start a wait which isn't guaranteed to be safe. + var unused = Task.Run(() => + { + redirectInitiated.Wait(); + redirectInitiated.Dispose(); + redirectInitiated = null; + + process.WaitForExit(); + + tcs.TrySetResult((process.ExitCode, stdout.ToString(), stderr.ToString())); + process.Dispose(); + }); + }; + + if (!process.Start()) + { + throw new InvalidOperationException("Process failed to start"); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + redirectInitiated.Set(); + + return tcs.Task; + } + + private class TemporaryProjectDirectory : IDisposable + { + private const string DirectoryBuildPropsContents = """ + + + false + + + """; + + private readonly Dictionary originalEnvironment = new(); + + public TemporaryProjectDirectory() + { + var testDataPath = Path.Combine(Path.GetDirectoryName(this.GetType().Assembly.Location), "test-data"); + + // ensure tests don't crawl the directory tree + File.WriteAllText(Path.Combine(testDataPath, "Directory.Build.props"), DirectoryBuildPropsContents); + File.WriteAllText(Path.Combine(testDataPath, "Directory.Build.targets"), ""); + File.WriteAllText(Path.Combine(testDataPath, "Directory.Packages.props"), ""); + + // create temporary project directory + this.DirectoryPath = Path.Combine(testDataPath, Guid.NewGuid().ToString("d")); + Directory.CreateDirectory(this.DirectoryPath); + + // ensure each project gets a fresh package cache + foreach (var envName in new[] { "NUGET_PACKAGES", "NUGET_HTTP_CACHE_PATH", "NUGET_SCRATCH", "NUGET_PLUGINS_CACHE_PATH" }) + { + this.originalEnvironment[envName] = Environment.GetEnvironmentVariable(envName); + var dir = Path.Join(this.DirectoryPath, envName); + Directory.CreateDirectory(dir); + Environment.SetEnvironmentVariable(envName, dir); + } + } + + public string DirectoryPath { get; } + + public void Dispose() + { + foreach (var (key, value) in this.originalEnvironment) + { + Environment.SetEnvironmentVariable(key, value); + } + + try + { + Directory.Delete(this.DirectoryPath, recursive: true); + } + catch + { + } + } + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetMSBuildBinaryLogComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetMSBuildBinaryLogComponentDetectorTests.cs new file mode 100644 index 00000000..4e2de13d --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetMSBuildBinaryLogComponentDetectorTests.cs @@ -0,0 +1,120 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.NuGet; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class NuGetMSBuildBinaryLogComponentDetectorTests : BaseDetectorTest +{ + [TestMethod] + public async Task DependenciesAreReportedForEachProjectFile() + { + // the contents of `projectContents` are the root entrypoint to the detector, but MSBuild will crawl to the other project file + var (scanResult, componentRecorder) = await this.ExecuteDetectorAndGetBinLogAsync( + projectContents: $""" + + + {MSBuildTestUtilities.TestTargetFramework} + + + + + + """, + additionalFiles: [ + ("other-project/other-project.csproj", $""" + + + {MSBuildTestUtilities.TestTargetFramework} + + + + + + """) + ], + mockedPackages: [ + ("Some.Package", "1.2.3", MSBuildTestUtilities.TestTargetFramework, ""), + ("Transitive.Dependency", "4.5.6", MSBuildTestUtilities.TestTargetFramework, null), + ]); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + // components are reported for each project file + var originalFileComponents = detectedComponents + .Where(d => d.FilePaths.Any(p => p.Replace("\\", "/").EndsWith("/project.csproj"))) + .Select(d => d.Component) + .Cast() + .OrderBy(c => c.Name) + .Select(c => $"{c.Name}/{c.Version}"); + originalFileComponents.Should().Equal("Some.Package/1.2.3", "Transitive.Dependency/4.5.6"); + + var referencedFileComponents = detectedComponents + .Where(d => d.FilePaths.Any(p => p.Replace("\\", "/").EndsWith("/other-project/other-project.csproj"))) + .Select(d => d.Component) + .Cast() + .OrderBy(c => c.Name) + .Select(c => $"{c.Name}/{c.Version}"); + referencedFileComponents.Should().Equal("Some.Package/1.2.3", "Transitive.Dependency/4.5.6"); + } + + [TestMethod] + public async Task RemovedPackagesAreNotReported() + { + // This is a very specific scenario that should be tested, but we don't want any changes to this repo's SDK to + // change the outcome of the test, so we're doing it manually. The scenario is the SDK knowingly replacing an + // assembly from an outdated transitive package. One example common in the wild is the package + // `Microsoft.Extensions.Configuration.Json/6.0.0` which contains a transitive dependency on + // `System.Text.Json/6.0.0`, but the SDK version 6.0.424 or later pulls this reference out of the dependency + // set and replaces the .dll with a local updated copy. The end result is the `package.assets.json` file + // reports that `System.Text.Json/6.0.0` is referenced by a project, but after build and at runtime, this isn't + // the case and can lead to false positives when reporting dependencies. The way the SDK accomplishes this is + // by removing `System.Text.Json.dll` from the group `@(RuntimeCopyLocalItems)`. To accomplish this in the + // test, we're inserting a custom target that does this same action. + var (scanResult, componentRecorder) = await this.ExecuteDetectorAndGetBinLogAsync( + projectContents: $""" + + + {MSBuildTestUtilities.TestTargetFramework} + + + + + + + + + + + + """, + mockedPackages: [ + ("Some.Package", "1.2.3", MSBuildTestUtilities.TestTargetFramework, ""), + ("Transitive.Dependency", "4.5.6", MSBuildTestUtilities.TestTargetFramework, null), + ]); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + var packages = detectedComponents.Select(d => d.Component).Cast().OrderBy(c => c.Name).Select(c => $"{c.Name}/{c.Version}"); + packages.Should().Equal("Some.Package/1.2.3"); + } + + private async Task<(Contracts.IndividualDetectorScanResult ScanResult, Contracts.IComponentRecorder ComponentRecorder)> ExecuteDetectorAndGetBinLogAsync( + string projectContents, + (string FileName, string Content)[] additionalFiles = null, + (string Name, string Version, string TargetFramework, string DependenciesXml)[] mockedPackages = null) + { + using var binLogStream = await MSBuildTestUtilities.GetBinLogStreamFromFileContentsAsync(projectContents, additionalFiles, mockedPackages); + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("msbuild.binlog", binLogStream) + .ExecuteDetectorAsync(); + return (scanResult, componentRecorder); + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NugetTestUtilities.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NugetTestUtilities.cs index 37fa45ce..4c05971a 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NugetTestUtilities.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NugetTestUtilities.cs @@ -3,6 +3,7 @@ using System.IO; using System.IO.Compression; using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.TestsUtilities; @@ -11,6 +12,8 @@ public static class NugetTestUtilities { + private const string NuSpecNamespace = "http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd"; + public static string GetRandomValidNuSpecComponent() { var componentName = GetRandomString(); @@ -60,7 +63,7 @@ public static string GetValidNuspec(string componentName, string version, string return GetTemplatedNuspec(componentName, version, authors); } - public static async Task ZipNupkgComponentAsync(string filename, string content) + public static async Task ZipNupkgComponentAsync(string filename, string content, (string Path, byte[] Contents)[] additionalFiles = null) { var stream = new MemoryStream(); @@ -68,10 +71,18 @@ public static async Task ZipNupkgComponentAsync(string filename, string { var entry = archive.CreateEntry($"{filename}.nuspec"); - using var entryStream = entry.Open(); - - var templateBytes = Encoding.UTF8.GetBytes(content); - await entryStream.WriteAsync(templateBytes); + using (var entryStream = entry.Open()) + { + var templateBytes = Encoding.UTF8.GetBytes(content); + await entryStream.WriteAsync(templateBytes); + } + + foreach (var file in additionalFiles ?? []) + { + var additionalEntry = archive.CreateEntry(file.Path); + using var additionalEntryStream = additionalEntry.Open(); + await additionalEntryStream.WriteAsync(file.Contents, CancellationToken.None); + } } stream.Seek(0, SeekOrigin.Begin);