Merge branch 'master' into conti/system-tests-aws-access

DataDog · Sep 26, 2024 · c5ea23c · c5ea23c
2 parents 3cd013b + 8ee4a5d
commit c5ea23c
Show file tree

Hide file tree

Showing 520 changed files with 15,319 additions and 1,862 deletions.
diff --git a/dd-java-agent/agent-bootstrap/gradle.lockfile b/dd-java-agent/agent-bootstrap/gradle.lockfile
@@ -39,7 +39,7 @@ commons-fileupload:commons-fileupload:1.5=jmhRuntimeClasspath,testCompileClasspa
 commons-io:commons-io:2.11.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
 de.thetaphi:forbiddenapis:3.1=compileClasspath,jmhCompileClasspath
 info.picocli:picocli:4.6.3=jmhRuntimeClasspath,testRuntimeClasspath
-io.sqreen:libsqreen:11.0.0=jmhRuntimeClasspath,testRuntimeClasspath
+io.sqreen:libsqreen:11.0.1=jmhRuntimeClasspath,testRuntimeClasspath
 javax.servlet:javax.servlet-api:3.1.0=jmhRuntimeClasspath,testCompileClasspath,testRuntimeClasspath
 jaxen:jaxen:1.2.0=spotbugs
 jline:jline:2.14.6=jmhRuntimeClasspath,testRuntimeClasspath

diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java
@@ -103,6 +103,8 @@ private enum AgentFeature {
     USM(propertyNameToSystemPropertyName(UsmConfig.USM_ENABLED), false),
     TELEMETRY(propertyNameToSystemPropertyName(GeneralConfig.TELEMETRY_ENABLED), true),
     DEBUGGER(propertyNameToSystemPropertyName(DebuggerConfig.DEBUGGER_ENABLED), false),
+    EXCEPTION_DEBUGGING(
+        propertyNameToSystemPropertyName(DebuggerConfig.EXCEPTION_REPLAY_ENABLED), false),
     DATA_JOBS(propertyNameToSystemPropertyName(GeneralConfig.DATA_JOBS_ENABLED), false),
     AGENTLESS_LOG_SUBMISSION(
         propertyNameToSystemPropertyName(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED), false);
@@ -149,6 +151,7 @@ public boolean isEnabledByDefault() {
   private static boolean usmEnabled = false;
   private static boolean telemetryEnabled = true;
   private static boolean debuggerEnabled = false;
+  private static boolean exceptionDebuggingEnabled = false;
   private static boolean agentlessLogSubmissionEnabled = false;
 
   /**
@@ -259,6 +262,7 @@ public static void start(
     cwsEnabled = isFeatureEnabled(AgentFeature.CWS);
     telemetryEnabled = isFeatureEnabled(AgentFeature.TELEMETRY);
     debuggerEnabled = isFeatureEnabled(AgentFeature.DEBUGGER);
+    exceptionDebuggingEnabled = isFeatureEnabled(AgentFeature.EXCEPTION_DEBUGGING);
     agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION);
 
     if (profilingEnabled) {
@@ -1069,7 +1073,7 @@ private static void shutdownProfilingAgent(final boolean sync) {
   }
 
   private static void maybeStartDebugger(Instrumentation inst, Class<?> scoClass, Object sco) {
-    if (!debuggerEnabled) {
+    if (!debuggerEnabled && !exceptionDebuggingEnabled) {
       return;
     }
     if (!remoteConfigEnabled) {

diff --git a/.../src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java b/.../src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java
@@ -95,6 +95,10 @@ public abstract class HttpServerDecorator<REQUEST, CONNECTION, RESPONSE, REQUEST
 
   protected abstract int status(RESPONSE response);
 
+  protected String requestedSessionId(REQUEST request) {
+    return null;
+  }
+
   public CharSequence operationName() {
     return SpanNaming.instance()
         .namingSchema()
@@ -166,6 +170,11 @@ public AgentSpan onRequest(
           requestContext.setBlockResponseFunction(brf);
         }
       }
+      Flow<Void> flow = callIGCallbackRequestSessionId(span, request);
+      Flow.Action action = flow.getAction();
+      if (action instanceof Flow.Action.RequestBlockingAction) {
+        span.setRequestBlockingAction((Flow.Action.RequestBlockingAction) flow.getAction());
+      }
     }
 
     if (context != null) {
@@ -432,6 +441,25 @@ private Flow<Void> callIGCallbackRequestHeaders(AgentSpan span, REQUEST_CARRIER
     return Flow.ResultFlow.empty();
   }
 
+  @SuppressWarnings("UnusedReturnValue")
+  private Flow<Void> callIGCallbackRequestSessionId(final AgentSpan span, final REQUEST request) {
+    final String sessionId = requestedSessionId(request);
+    if (sessionId == null) {
+      return Flow.ResultFlow.empty();
+    }
+    final CallbackProvider cbp = tracer().getCallbackProvider(RequestContextSlot.APPSEC);
+    final RequestContext requestContext = span.getRequestContext();
+    if (cbp == null || requestContext == null) {
+      return Flow.ResultFlow.empty();
+    }
+    final BiFunction<RequestContext, String, Flow<Void>> addrCallback =
+        cbp.getCallback(EVENTS.requestSession());
+    if (addrCallback == null) {
+      return Flow.ResultFlow.empty();
+    }
+    return addrCallback.apply(requestContext, sessionId);
+  }
+
   private Flow<Void> callIGCallbackResponseAndHeaders(
       AgentSpan span, RESPONSE carrier, int status) {
     return callIGCallbackResponseAndHeaders(span, carrier, status, responseGetter());

diff --git a/dd-java-agent/agent-builder/gradle.lockfile b/dd-java-agent/agent-builder/gradle.lockfile
@@ -39,7 +39,7 @@ commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClassp
 commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath
 de.thetaphi:forbiddenapis:3.1=compileClasspath
 info.picocli:picocli:4.6.3=testRuntimeClasspath
-io.sqreen:libsqreen:11.0.0=testRuntimeClasspath
+io.sqreen:libsqreen:11.0.1=testRuntimeClasspath
 javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath
 jaxen:jaxen:1.2.0=spotbugs
 jline:jline:2.14.6=testRuntimeClasspath

diff --git a/dd-java-agent/agent-ci-visibility/gradle.lockfile b/dd-java-agent/agent-ci-visibility/gradle.lockfile
@@ -54,7 +54,7 @@ commons-io:commons-io:2.11.0=testCompileClasspath,testFixturesApiDependenciesMet
 de.thetaphi:forbiddenapis:3.1=compileClasspath,compileOnlyDependenciesMetadata
 info.picocli:picocli:4.6.3=testFixturesRuntimeClasspath,testRuntimeClasspath
 io.github.java-diff-utils:java-diff-utils:4.12=zinc
-io.sqreen:libsqreen:11.0.0=testFixturesRuntimeClasspath,testRuntimeClasspath
+io.sqreen:libsqreen:11.0.1=testFixturesRuntimeClasspath,testRuntimeClasspath
 javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testFixturesApiDependenciesMetadata,testFixturesCompileClasspath,testFixturesImplementationDependenciesMetadata,testFixturesRuntimeClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 jaxen:jaxen:1.2.0=spotbugs
 jline:jline:2.14.6=testFixturesRuntimeClasspath,testRuntimeClasspath

diff --git a/...t-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java b/...t-ci-visibility/src/main/java/datadog/trace/civisibility/decorator/TestDecoratorImpl.java
@@ -15,6 +15,7 @@ public class TestDecoratorImpl implements TestDecorator {
   private final String component;
   private final String sessionName;
   private final Map<String, String> ciTags;
+  private final int cpuCount;
 
   public TestDecoratorImpl(
       String component, String sessionName, String testCommand, Map<String, String> ciTags) {
@@ -27,6 +28,7 @@ public TestDecoratorImpl(
       this.sessionName =
           Strings.isNotBlank(ciJobName) ? ciJobName + "-" + testCommand : testCommand;
     }
+    cpuCount = Runtime.getRuntime().availableProcessors();
   }
 
   protected String testType() {
@@ -46,6 +48,7 @@ public CharSequence component() {
   public AgentSpan afterStart(final AgentSpan span) {
     span.setSamplingPriority(PrioritySampling.SAMPLER_KEEP);
     span.setTag(DDTags.ORIGIN_KEY, CIAPP_TEST_ORIGIN);
+    span.setTag(DDTags.HOST_VCPU_COUNT, cpuCount);
     span.setTag(Tags.TEST_TYPE, testType());
     span.setTag(Tags.COMPONENT, component());
     span.setTag(Tags.TEST_SESSION_NAME, sessionName);

diff --git a/...ibility/src/test/groovy/datadog/trace/civisibility/decorator/TestDecoratorImplTest.groovy b/...ibility/src/test/groovy/datadog/trace/civisibility/decorator/TestDecoratorImplTest.groovy
@@ -23,6 +23,7 @@ class TestDecoratorImplTest extends Specification {
     1 * span.setTag(Tags.TEST_TYPE, decorator.testType())
     1 * span.setSamplingPriority(PrioritySampling.SAMPLER_KEEP)
     1 * span.setTag(DDTags.ORIGIN_KEY, decorator.origin())
+    1 * span.setTag(DDTags.HOST_VCPU_COUNT, Runtime.runtime.availableProcessors())
     1 * span.setTag("ci-tag-1", "value")
     1 * span.setTag("ci-tag-2", "another value")
 

diff --git a/...ava-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashLogParser.java b/...ava-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashLogParser.java
@@ -0,0 +1,10 @@
+package com.datadog.crashtracking;
+
+import com.datadog.crashtracking.dto.CrashLog;
+import com.datadog.crashtracking.parsers.HotspotCrashLogParser;
+
+public final class CrashLogParser {
+  public static CrashLog fromHotspotCrashLog(String logText) {
+    return new HotspotCrashLogParser().parse(logText);
+  }
+}
diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java b/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java
@@ -7,6 +7,7 @@
 import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT;
 import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT;
 
+import com.datadog.crashtracking.dto.CrashLog;
 import com.squareup.moshi.JsonWriter;
 import datadog.common.container.ContainerInfo;
 import datadog.common.version.VersionInfo;
@@ -16,13 +17,12 @@
 import datadog.trace.bootstrap.config.provider.ConfigProvider;
 import datadog.trace.util.PidHelper;
 import de.thetaphi.forbiddenapis.SuppressForbidden;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
+import java.io.*;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -54,7 +54,7 @@ public final class CrashUploader {
   static final String JAVA_TRACING_LIBRARY = "dd-trace-java";
   static final String HEADER_DD_EVP_ORIGIN_VERSION = "DD-EVP-ORIGIN-VERSION";
   static final String HEADER_DD_TELEMETRY_API_VERSION = "DD-Telemetry-API-Version";
-  static final String TELEMETRY_API_VERSION = "v1";
+  static final String TELEMETRY_API_VERSION = "v2";
   static final String HEADER_DD_TELEMETRY_REQUEST_TYPE = "DD-Telemetry-Request-Type";
   static final String TELEMETRY_REQUEST_TYPE = "logs";
 
@@ -114,43 +114,44 @@ private String tagsToString(final Map<String, String> tags) {
         .collect(Collectors.joining(","));
   }
 
-  public void upload(@Nonnull List<InputStream> files) throws IOException {
-    List<String> filesContent = new ArrayList<>(files.size());
-    for (InputStream file : files) {
-      filesContent.add(readContent(file));
+  public void upload(@Nonnull List<Path> files) throws IOException {
+    for (Path file : files) {
+      uploadToLogs(file);
+      uploadToTelemetry(file);
     }
-    uploadToLogs(filesContent);
-    uploadToTelemetry(filesContent);
   }
 
-  void uploadToLogs(@Nonnull List<String> filesContent) throws IOException {
-    uploadToLogs(filesContent, System.out);
+  boolean uploadToLogs(@Nonnull Path file) {
+    try {
+      uploadToLogs(new String(Files.readAllBytes(file), StandardCharsets.UTF_8), System.out);
+    } catch (IOException e) {
+      log.error("Failed to upload crash file: {}", file, e);
+      return false;
+    }
+    return true;
   }
 
-  void uploadToLogs(@Nonnull List<String> filesContent, @Nonnull PrintStream out)
-      throws IOException {
+  void uploadToLogs(@Nonnull String message, @Nonnull PrintStream out) throws IOException {
     // print on the output, and the application/container/host log will pick it up
-    for (String message : filesContent) {
-      try (Buffer buf = new Buffer()) {
-        try (JsonWriter writer = JsonWriter.of(buf)) {
-          writer.beginObject();
-          writer.name("ddsource").value("crashtracker");
-          writer.name("ddtags").value(tags);
-          writer.name("hostname").value(config.getHostName());
-          writer.name("service").value(config.getServiceName());
-          writer.name("message").value(message);
-          writer.name("level").value("ERROR");
-          writer.name("error");
-          writer.beginObject();
-          writer.name("kind").value(extractErrorKind(message));
-          writer.name("message").value(extractErrorMessage(message));
-          writer.name("stack").value(extractErrorStackTrace(message, false));
-          writer.endObject();
-          writer.endObject();
-        }
-
-        out.println(buf.readByteString().utf8());
+    try (Buffer buf = new Buffer()) {
+      try (JsonWriter writer = JsonWriter.of(buf)) {
+        writer.beginObject();
+        writer.name("ddsource").value("crashtracker");
+        writer.name("ddtags").value(tags);
+        writer.name("hostname").value(config.getHostName());
+        writer.name("service").value(config.getServiceName());
+        writer.name("message").value(message);
+        writer.name("level").value("ERROR");
+        writer.name("error");
+        writer.beginObject();
+        writer.name("kind").value(extractErrorKind(message));
+        writer.name("message").value(extractErrorMessage(message));
+        writer.name("stack").value(extractErrorStackTrace(message, false));
+        writer.endObject();
+        writer.endObject();
       }
+
+      out.println(buf.readByteString().utf8());
     }
   }
 
@@ -235,16 +236,26 @@ private String extractErrorStackTrace(String fileContent) {
     return extractErrorStackTrace(fileContent, true);
   }
 
-  void uploadToTelemetry(@Nonnull List<String> filesContent) throws IOException {
-    handleCall(makeTelemetryRequest(filesContent));
+  boolean uploadToTelemetry(@Nonnull Path file) {
+    try {
+      String content = new String(Files.readAllBytes(file), Charset.defaultCharset());
+      handleCall(makeTelemetryRequest(content));
+    } catch (IOException e) {
+      log.error("Failed to upload crash file: {}", file, e);
+      return false;
+    }
+    return true;
   }
 
-  private Call makeTelemetryRequest(@Nonnull List<String> filesContent) throws IOException {
-    final RequestBody requestBody = makeTelemetryRequestBody(filesContent);
+  private Call makeTelemetryRequest(@Nonnull String content) throws IOException {
+    final RequestBody requestBody = makeTelemetryRequestBody(content);
 
     final Map<String, String> headers = new HashMap<>();
     // Set chunked transfer
-    headers.put("Content-Type", requestBody.contentType().toString());
+    MediaType contentType = requestBody.contentType();
+    if (contentType != null) {
+      headers.put("Content-Type", contentType.toString());
+    }
     headers.put("Content-Length", Long.toString(requestBody.contentLength()));
     headers.put("Transfer-Encoding", "chunked");
     headers.put(HEADER_DD_EVP_ORIGIN, JAVA_TRACING_LIBRARY);
@@ -258,8 +269,11 @@ private Call makeTelemetryRequest(@Nonnull List<String> filesContent) throws IOE
             .build());
   }
 
-  private RequestBody makeTelemetryRequestBody(@Nonnull List<String> filesContent)
-      throws IOException {
+  private RequestBody makeTelemetryRequestBody(@Nonnull String content) throws IOException {
+    CrashLog crashLog = CrashLogParser.fromHotspotCrashLog(content);
+    if (crashLog == null) {
+      throw new IOException("Failed to parse crash log");
+    }
     try (Buffer buf = new Buffer()) {
       try (JsonWriter writer = JsonWriter.of(buf)) {
         writer.beginObject();
@@ -275,13 +289,11 @@ private RequestBody makeTelemetryRequestBody(@Nonnull List<String> filesContent)
         writer.name("debug").value(true);
         writer.name("payload");
         writer.beginArray();
-        for (String message : filesContent) {
-          writer.beginObject();
-          writer.name("message").value(extractErrorStackTrace(message));
-          writer.name("level").value("ERROR");
-          writer.name("tags").value("severity:crash");
-          writer.endObject();
-        }
+        writer.beginObject();
+        writer.name("message").value(crashLog.toJson());
+        writer.name("level").value("ERROR");
+        writer.name("tags").value("severity:crash");
+        writer.endObject();
         writer.endArray();
         writer.name("application");
         writer.beginObject();