[ASM] Attacker fingerprint (#5982)

## Summary of changes

Fingerprinting is a technique used to identify and track users through
the use of available data which, when combined through a certain set of
algorithms, can provide a unique fingerprint for said user.
Fingerprinting can be performed on many contexts with different data
sets, such as the browser, which can provide the algorithm with specific
data about the user’s software and hardware stack, or the server, which
typically provides data at the different levels of the network stack.

This PR contains the implementation of the attacker fingerprint feature
described in [this
RFC](https://docs.google.com/document/d/1DivOa9XsCggmZVzMI57vyxH2_EBJ0-qqIkRHm_sEvSs/edit?pli=1).

## Reason for change

## Implementation details

There are two small issues detected that seem related to the WAF:
If we don't send the request body, no endpoint fingerprint
(_dd.appsec.fp.http.endpoint) is generated.
The agent header fingerprint is not generated if we send a value in a
dictionary instead of a regular string.

These issues will be discussed with the libdwaf team.

## Test coverage

Some unit tests have been added. 

Since this feature will be enabled by default and, in order to cover
different situations while not impacting the CI performance, the ASM
integration tests have been modified to include the fingerprint in the
snapshots.

## Other details
<!-- Fixes #{issue} -->

<!-- ⚠️ Note: where possible, please obtain 2 approvals prior to
merging. Unless CODEOWNERS specifies otherwise, for external teams it is
typically best to have one review from a team member, and one review
from apm-dotnet. Trivial changes do not require 2 reviews. -->

tracer/src/Datadog.Trace/AppSec/AddressesConstants.cs

@@ -32,6 +32,7 @@ internal static class AddressesConstants
         public const string ResponseHeaderNoCookies = "server.response.headers.no_cookies";
 
         public const string UserId = "usr.id";
+        public const string UserSessionId = "usr.session_id";
         public const string WafContextProcessor = "waf.context.processor";
     }
 }

tracer/src/Datadog.Trace/AppSec/AttackerFingerprint/AttackerFingerprintHelper.cs

@@ -0,0 +1,63 @@
+// <copyright file="AttackerFingerprintHelper.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System.Collections.Generic;
+using Datadog.Trace.AppSec.Coordinator;
+using Datadog.Trace.AppSec.Waf;
+using Datadog.Trace.Logging;
+
+#nullable enable
+
+namespace Datadog.Trace.AppSec.AttackerFingerprint;
+
+internal static class AttackerFingerprintHelper
+{
+    private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AttackerFingerprintHelper));
+    private static readonly Dictionary<string, object> _fingerprintRequest = new() { { AddressesConstants.WafContextProcessor, new Dictionary<string, object> { { "fingerprint", true } } } };
+    private static bool _warningLogged = false;
+
+    public static void AddSpanTags(Span span)
+    {
+        if (span.IsFinished || span.Type != SpanTypes.Web)
+        {
+            return;
+        }
+
+        var securityCoordinator = new SecurityCoordinator(Security.Instance, span);
+
+        // We need a context
+        if (!securityCoordinator.HasContext() || securityCoordinator.IsAdditiveContextDisposed())
+        {
+            return;
+        }
+
+        var result = securityCoordinator.RunWaf(_fingerprintRequest);
+        AddSpanTags(result?.FingerprintDerivatives, span);
+    }
+
+    private static void AddSpanTags(Dictionary<string, object?>? fingerPrintDerivatives, ISpan span)
+    {
+        if (fingerPrintDerivatives is not null)
+        {
+            foreach (var derivative in fingerPrintDerivatives)
+            {
+                var value = derivative.Value?.ToString();
+                if (!string.IsNullOrEmpty(value))
+                {
+                    span.SetTag(derivative.Key, value);
+                }
+                else
+                {
+                    if (!_warningLogged)
+                    {
+                        // This should not happen
+                        Log.Warning("Fingerprint derivative {DerivativeKey} has no value", derivative.Key);
+                        _warningLogged = true;
+                    }
+                }
+            }
+        }
+    }
+}

tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Reporter.cs

@@ -9,6 +9,7 @@
 using System.Collections.Specialized;
 using System.IO;
 using System.IO.Compression;
+using Datadog.Trace.AppSec.AttackerFingerprint;
 using Datadog.Trace.AppSec.Waf;
 using Datadog.Trace.ExtensionMethods;
 using Datadog.Trace.Headers;
@@ -131,6 +132,8 @@ internal void TryReport(IResult result, bool blocked, int? status = null)
                 traceContext.AddWafSecurityEvents(result.Data);
             }
 
+            AttackerFingerprintHelper.AddSpanTags(_localRootSpan);
+
             var clientIp = _localRootSpan.GetTag(Tags.HttpClientIp);
             if (!string.IsNullOrEmpty(clientIp))
             {
@@ -157,9 +160,9 @@ internal void TryReport(IResult result, bool blocked, int? status = null)
 
         AddRaspSpanMetrics(result, _localRootSpan);
 
-        if (result.ShouldReportSchema)
+        if (result.ExtractSchemaDerivatives?.Count > 0)
         {
-            foreach (var derivative in result.Derivatives)
+            foreach (var derivative in result.ExtractSchemaDerivatives)
             {
                 var serializeObject = JsonConvert.SerializeObject(derivative.Value);
                 var bytes = System.Text.Encoding.UTF8.GetBytes(serializeObject);

tracer/src/Datadog.Trace/AppSec/Security.cs

@@ -11,6 +11,7 @@
 using System.Numerics;
 using System.Threading;
 using Datadog.Trace.Agent.DiscoveryService;
+using Datadog.Trace.AppSec.AttackerFingerprint;
 using Datadog.Trace.AppSec.Rcm;
 using Datadog.Trace.AppSec.Rcm.Models.AsmDd;
 using Datadog.Trace.AppSec.Waf;
@@ -503,6 +504,10 @@ private void SetRemoteConfigCapabilites()
             rcm.SetCapability(RcmCapabilitiesIndices.AsmRaspShi, _settings.RaspEnabled && _noLocalRules && WafSupportsCapability(RcmCapabilitiesIndices.AsmRaspShi));
             rcm.SetCapability(RcmCapabilitiesIndices.AsmRaspSqli, _settings.RaspEnabled && _noLocalRules && WafSupportsCapability(RcmCapabilitiesIndices.AsmRaspSqli));
             rcm.SetCapability(RcmCapabilitiesIndices.AsmExclusionData, _noLocalRules && WafSupportsCapability(RcmCapabilitiesIndices.AsmExclusionData));
+            rcm.SetCapability(RcmCapabilitiesIndices.AsmEnpointFingerprint, _noLocalRules && WafSupportsCapability(RcmCapabilitiesIndices.AsmEnpointFingerprint));
+            rcm.SetCapability(RcmCapabilitiesIndices.AsmHeaderFingerprint, _noLocalRules && WafSupportsCapability(RcmCapabilitiesIndices.AsmHeaderFingerprint));
+            rcm.SetCapability(RcmCapabilitiesIndices.AsmNetworkFingerprint, _noLocalRules && WafSupportsCapability(RcmCapabilitiesIndices.AsmNetworkFingerprint));
+            rcm.SetCapability(RcmCapabilitiesIndices.AsmSessionFingerprint, _noLocalRules && WafSupportsCapability(RcmCapabilitiesIndices.AsmSessionFingerprint));
             // follows a different pattern to rest of ASM remote config, if available it's the RC value
             // that takes precedence. This follows what other products do.
             rcm.SetCapability(RcmCapabilitiesIndices.AsmAutoUserInstrumentationMode, true);

tracer/src/Datadog.Trace/AppSec/Waf/IResult.cs

@@ -40,9 +40,9 @@ internal interface IResult
 
         bool Timeout { get; }
 
-        Dictionary<string, object?> Derivatives { get; }
+        Dictionary<string, object?>? ExtractSchemaDerivatives { get; }
 
-        bool ShouldReportSchema { get; }
+        Dictionary<string, object?>? FingerprintDerivatives { get; }
 
         bool ShouldReportSecurityResult { get; }
     }

tracer/src/Datadog.Trace/AppSec/Waf/RCMCapabilitiesHelper.cs

@@ -17,8 +17,12 @@ internal static class RCMCapabilitiesHelper
 {
     private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(RCMCapabilitiesHelper));
 
-    private static readonly Dictionary<BigInteger, Version> _CapabilitiesVersion = new Dictionary<BigInteger, Version>()
+    private static readonly Dictionary<BigInteger, Version> _capabilitiesByWafVersion = new Dictionary<BigInteger, Version>()
     {
+        { RcmCapabilitiesIndices.AsmEnpointFingerprint, new Version(1, 19) },
+        { RcmCapabilitiesIndices.AsmHeaderFingerprint, new Version(1, 19) },
+        { RcmCapabilitiesIndices.AsmNetworkFingerprint, new Version(1, 19) },
+        { RcmCapabilitiesIndices.AsmSessionFingerprint, new Version(1, 19) },
         { RcmCapabilitiesIndices.AsmRaspSqli, new Version(1, 18) },
         { RcmCapabilitiesIndices.AsmRaspLfi, new Version(1, 17) },
         { RcmCapabilitiesIndices.AsmRaspSsrf, new Version(1, 17) },
@@ -35,7 +39,7 @@ internal static bool WafSupportsCapability(BigInteger capability, string? wafVer
             return false;
         }
 
-        if (_CapabilitiesVersion.TryGetValue(capability, out var version))
+        if (_capabilitiesByWafVersion.TryGetValue(capability, out var version))
         {
             try
             {

tracer/src/Datadog.Trace/AppSec/Waf/Result.cs

@@ -4,6 +4,7 @@
 // </copyright>
 
 #nullable enable
+using System;
 using System.Collections.Generic;
 using Datadog.Trace.AppSec.Waf.NativeBindings;
 
@@ -16,8 +17,8 @@ public Result(DdwafResultStruct returnStruct, WafReturnCode returnCode, ulong ag
             ReturnCode = returnCode;
             Actions = returnStruct.Actions.DecodeMap();
             ShouldReportSecurityResult = returnCode >= WafReturnCode.Match;
-            Derivatives = returnStruct.Derivatives.DecodeMap();
-            ShouldReportSchema = Derivatives is { Count: > 0 };
+            var derivatives = returnStruct.Derivatives.DecodeMap();
+            BuildDerivatives(derivatives);
             if (ShouldReportSecurityResult)
             {
                 Data = returnStruct.Events.DecodeObjectArray();
@@ -59,13 +60,13 @@ public Result(DdwafResultStruct returnStruct, WafReturnCode returnCode, ulong ag
 
         public WafReturnCode ReturnCode { get; }
 
-        public bool ShouldReportSchema { get; }
-
         public IReadOnlyCollection<object>? Data { get; }
 
         public Dictionary<string, object?>? Actions { get; }
 
-        public Dictionary<string, object?> Derivatives { get; }
+        public Dictionary<string, object?>? ExtractSchemaDerivatives { get; private set; }
+
+        public Dictionary<string, object?>? FingerprintDerivatives { get; private set; }
 
         /// <summary>
         /// Gets the total runtime in nanoseconds
@@ -103,5 +104,30 @@ public Result(DdwafResultStruct returnStruct, WafReturnCode returnCode, ulong ag
         public bool ShouldReportSecurityResult { get; }
 
         public bool Timeout { get; }
+
+        private void BuildDerivatives(Dictionary<string, object?> derivatives)
+        {
+            foreach (var derivative in derivatives)
+            {
+                if ((derivative.Key == Tags.AppSecFpEndpoint) || (derivative.Key == Tags.AppSecFpHeader) || (derivative.Key == Tags.AppSecFpHttpNetwork) || (derivative.Key == Tags.AppSecFpSession))
+                {
+                    if (FingerprintDerivatives is null)
+                    {
+                        FingerprintDerivatives = new Dictionary<string, object?>();
+                    }
+
+                    FingerprintDerivatives.Add(derivative.Key, derivative.Value);
+                }
+                else
+                {
+                    if (ExtractSchemaDerivatives is null)
+                    {
+                        ExtractSchemaDerivatives = new Dictionary<string, object?>();
+                    }
+
+                    ExtractSchemaDerivatives.Add(derivative.Key, derivative.Value);
+                }
+            }
+        }
     }
 }

tracer/src/Datadog.Trace/RemoteConfigurationManagement/RcmCapabilitiesIndices.cs

@@ -73,6 +73,14 @@ internal static class RcmCapabilitiesIndices
 
         public static readonly BigInteger AsmAutoUserInstrumentationMode = Create(31);
 
+        public static readonly BigInteger AsmEnpointFingerprint = Create(32);
+
+        public static readonly BigInteger AsmSessionFingerprint = Create(33);
+
+        public static readonly BigInteger AsmNetworkFingerprint = Create(34);
+
+        public static readonly BigInteger AsmHeaderFingerprint = Create(35);
+
         private static BigInteger Create(int index) => new(1UL << index);
     }
 }

tracer/src/Datadog.Trace/Tags.cs

@@ -601,6 +601,26 @@ public static partial class Tags
         /// </summary>
         internal const string AppSecWafInitRuleErrors = "_dd.appsec.event_rules.errors";
 
+        /// <summary>
+        /// Indicates the http endpoint fingerprint
+        /// </summary>
+        internal const string AppSecFpEndpoint = "_dd.appsec.fp.http.endpoint";
+
+        /// <summary>
+        /// Indicates the http header fingerprint
+        /// </summary>
+        internal const string AppSecFpHeader = "_dd.appsec.fp.http.header";
+
+        /// <summary>
+        /// Indicates the http network fingerprint
+        /// </summary>
+        internal const string AppSecFpHttpNetwork = "_dd.appsec.fp.http.network";
+
+        /// <summary>
+        /// Indicates the session fingerprint
+        /// </summary>
+        internal const string AppSecFpSession = "_dd.appsec.fp.session";
+
         /// <summary>
         /// Should contain the public IP of the host initiating the request.
         /// </summary>

tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetBase.cs

@@ -48,6 +48,7 @@ public class AspNetBase : TestHelper
         private static readonly Regex AppSecErrorCount = new(@"\s*_dd.appsec.event_rules.error_count: 0.0,?", RegexOptions.IgnoreCase | RegexOptions.Compiled);
         private static readonly Regex AppSecRaspWafDuration = new(@"_dd.appsec.rasp.duration: \d+\.0", RegexOptions.IgnoreCase | RegexOptions.Compiled);
         private static readonly Regex AppSecRaspWafDurationWithBindings = new(@"_dd.appsec.rasp.duration_ext: \d+\.0", RegexOptions.IgnoreCase | RegexOptions.Compiled);
+        private static readonly Regex AppSecFingerPrintHeaders = new(@"_dd.appsec.fp.http.header: hdr-\d+-\S*-\d+-\S*", RegexOptions.IgnoreCase | RegexOptions.Compiled);
         private static readonly Regex AppSecSpanIdRegex = (new Regex("\"span_id\":\\d+"));
         private static readonly Type MetaStructHelperType = Type.GetType("Datadog.Trace.AppSec.Rasp.MetaStructHelper, Datadog.Trace");
         private static readonly MethodInfo MetaStructByteArrayToObject = MetaStructHelperType.GetMethod("ByteArrayToObject", BindingFlags.Public | BindingFlags.Static);
@@ -114,6 +115,11 @@ public async Task TestAppSecRequestWithVerifyAsync(MockTracerAgent agent, string
             await VerifySpans(spans, settings, testInit, methodNameOverride, fileNameOverride: fileNameOverride);
         }
 
+        public void ScrubFingerprintHeaders(VerifySettings settings)
+        {
+            settings.AddRegexScrubber(AppSecFingerPrintHeaders, "_dd.appsec.fp.http.header: <HeaderPrint>");
+        }
+
         public async Task VerifySpans(IImmutableList<MockSpan> spans, VerifySettings settings, bool testInit = false, string methodNameOverride = null, string testName = null, bool forceMetaStruct = false, string fileNameOverride = null)
         {
             settings.ModifySerialization(

tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetCoreWithExternalRulesFileBase.cs

@@ -41,6 +41,7 @@ public async Task TestBlockedHeader(string test, HttpStatusCode expectedStatusCo
             var agent = Fixture.Agent;
             var sanitisedUrl = VerifyHelper.SanitisePathsForVerify(url);
             var settings = VerifyHelper.GetSpanVerifierSettings(test, (int)expectedStatusCode, sanitisedUrl);
+            ScrubFingerprintHeaders(settings);
             await TestAppSecRequestWithVerifyAsync(agent, url, null, 5, 1, settings);
         }
     }
@@ -86,6 +87,7 @@ public async Task TestGet(string test, HttpStatusCode expectedStatusCode, string
 
             var sanitisedUrl = VerifyHelper.SanitisePathsForVerify(url);
             var settings = VerifyHelper.GetSpanVerifierSettings(test, (int)expectedStatusCode, sanitisedUrl);
+            ScrubFingerprintHeaders(settings);
             await TestAppSecRequestWithVerifyAsync(agent, url, null, 5, 1, settings);
         }
     }

tracer/test/Datadog.Trace.Security.IntegrationTests/RASP/AspNetCore2Rasp.cs

@@ -82,6 +82,13 @@ public async Task TryStartApp()
     [Trait("RunOnWindows", "True")]
     public async Task TestRaspRequest(string url, string exploit)
     {
+        AddHeaders(new()
+        {
+            { "Accept-Language", "en_UK" },
+            { "X-Custom-Header", "42" },
+            { "AnotherHeader", "Value" },
+        });
+
         var testName = IastEnabled ? "RaspIast.AspNetCore2" : "Rasp.AspNetCore2";
         IncludeAllHttpSpans = true;
         await TryStartApp();

tracer/test/Datadog.Trace.Security.IntegrationTests/RASP/AspNetCore5Rasp.cs

@@ -12,12 +12,10 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Datadog.Trace.AppSec.Rcm.Models.Asm;
-using Datadog.Trace.AppSec.Rcm.Models.AsmFeatures;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Iast.Telemetry;
 using Datadog.Trace.Security.IntegrationTests.IAST;
 using Datadog.Trace.TestHelpers;
-using VerifyTests;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -108,6 +106,12 @@ public async Task TryStartApp()
     [Trait("RunOnWindows", "True")]
     public async Task TestRaspRequest(string url, string exploit)
     {
+        AddHeaders(new()
+        {
+            { "Accept-Language", "en_UK" },
+            { "X-Custom-Header", "42" },
+            { "AnotherHeader", "Value" },
+        });
         var testName = IastEnabled ? "RaspIast.AspNetCore5" : "Rasp.AspNetCore5";
         IncludeAllHttpSpans = true;
         await TryStartApp();

tracer/test/Datadog.Trace.Security.IntegrationTests/RASP/AspNetMvc5Rasp.cs

@@ -95,6 +95,13 @@ public AspNetMvc5RaspTests(IisFixture iisFixture, ITestOutputHelper output, bool
     [InlineData("/Iast/ExecuteCommand?file=ls&argumentLine=;evilCommand&fromShell=true", "CmdI")]
     public async Task TestRaspRequest(string url, string exploit)
     {
+        AddHeaders(new()
+        {
+            { "Accept-Language", "en_UK" },
+            { "X-Custom-Header", "42" },
+            { "AnotherHeader", "Value" },
+        });
+
         var agent = _iisFixture.Agent;
         var settings = VerifyHelper.GetSpanVerifierSettings();
         settings.UseParameters(url, exploit);