Skip to content

Commit

Permalink
Add support for new telemetry crash data format
Browse files Browse the repository at this point in the history
  • Loading branch information
jbachorik committed Sep 24, 2024
1 parent b4718bd commit 8dd0cb5
Show file tree
Hide file tree
Showing 20 changed files with 11,554 additions and 142 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.datadog.crashtracking;

import com.datadog.crashtracking.dto.CrashLog;
import com.datadog.crashtracking.parsers.HotspotCrashLogParser;

public final class CrashLogParser {
public static CrashLog fromHotspotCrashLog(String logText) {
return new HotspotCrashLogParser().parse(logText);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT;
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT;

import com.datadog.crashtracking.dto.CrashLog;
import com.squareup.moshi.JsonWriter;
import datadog.common.container.ContainerInfo;
import datadog.common.version.VersionInfo;
Expand All @@ -16,13 +17,11 @@
import datadog.trace.bootstrap.config.provider.ConfigProvider;
import datadog.trace.util.PidHelper;
import de.thetaphi.forbiddenapis.SuppressForbidden;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -54,7 +53,7 @@ public final class CrashUploader {
static final String JAVA_TRACING_LIBRARY = "dd-trace-java";
static final String HEADER_DD_EVP_ORIGIN_VERSION = "DD-EVP-ORIGIN-VERSION";
static final String HEADER_DD_TELEMETRY_API_VERSION = "DD-Telemetry-API-Version";
static final String TELEMETRY_API_VERSION = "v1";
static final String TELEMETRY_API_VERSION = "v2";
static final String HEADER_DD_TELEMETRY_REQUEST_TYPE = "DD-Telemetry-Request-Type";
static final String TELEMETRY_REQUEST_TYPE = "logs";

Expand Down Expand Up @@ -114,43 +113,44 @@ private String tagsToString(final Map<String, String> tags) {
.collect(Collectors.joining(","));
}

public void upload(@Nonnull List<InputStream> files) throws IOException {
List<String> filesContent = new ArrayList<>(files.size());
for (InputStream file : files) {
filesContent.add(readContent(file));
public void upload(@Nonnull List<Path> files) throws IOException {
for (Path file : files) {
uploadToLogs(file);
uploadToTelemetry(file);
}
uploadToLogs(filesContent);
uploadToTelemetry(filesContent);
}

void uploadToLogs(@Nonnull List<String> filesContent) throws IOException {
uploadToLogs(filesContent, System.out);
boolean uploadToLogs(@Nonnull Path file) {
try {
uploadToLogs(new String(Files.readAllBytes(file), StandardCharsets.UTF_8), System.out);
} catch (IOException e) {
log.error("Failed to upload crash file: {}", file, e);
return false;
}
return true;
}

void uploadToLogs(@Nonnull List<String> filesContent, @Nonnull PrintStream out)
throws IOException {
void uploadToLogs(@Nonnull String message, @Nonnull PrintStream out) throws IOException {
// print on the output, and the application/container/host log will pick it up
for (String message : filesContent) {
try (Buffer buf = new Buffer()) {
try (JsonWriter writer = JsonWriter.of(buf)) {
writer.beginObject();
writer.name("ddsource").value("crashtracker");
writer.name("ddtags").value(tags);
writer.name("hostname").value(config.getHostName());
writer.name("service").value(config.getServiceName());
writer.name("message").value(message);
writer.name("level").value("ERROR");
writer.name("error");
writer.beginObject();
writer.name("kind").value(extractErrorKind(message));
writer.name("message").value(extractErrorMessage(message));
writer.name("stack").value(extractErrorStackTrace(message, false));
writer.endObject();
writer.endObject();
}

out.println(buf.readByteString().utf8());
try (Buffer buf = new Buffer()) {
try (JsonWriter writer = JsonWriter.of(buf)) {
writer.beginObject();
writer.name("ddsource").value("crashtracker");
writer.name("ddtags").value(tags);
writer.name("hostname").value(config.getHostName());
writer.name("service").value(config.getServiceName());
writer.name("message").value(message);
writer.name("level").value("ERROR");
writer.name("error");
writer.beginObject();
writer.name("kind").value(extractErrorKind(message));
writer.name("message").value(extractErrorMessage(message));
writer.name("stack").value(extractErrorStackTrace(message, false));
writer.endObject();
writer.endObject();
}

out.println(buf.readByteString().utf8());
}
}

Expand Down Expand Up @@ -235,16 +235,26 @@ private String extractErrorStackTrace(String fileContent) {
return extractErrorStackTrace(fileContent, true);
}

void uploadToTelemetry(@Nonnull List<String> filesContent) throws IOException {
handleCall(makeTelemetryRequest(filesContent));
boolean uploadToTelemetry(@Nonnull Path file) {
try {
String content = new String(Files.readAllBytes(file), StandardCharsets.UTF_8);
handleCall(makeTelemetryRequest(content));
} catch (IOException e) {
log.error("Failed to upload crash file: {}", file, e);
return false;
}
return true;
}

private Call makeTelemetryRequest(@Nonnull List<String> filesContent) throws IOException {
final RequestBody requestBody = makeTelemetryRequestBody(filesContent);
private Call makeTelemetryRequest(@Nonnull String content) throws IOException {
final RequestBody requestBody = makeTelemetryRequestBody(content);

final Map<String, String> headers = new HashMap<>();
// Set chunked transfer
headers.put("Content-Type", requestBody.contentType().toString());
MediaType contentType = requestBody.contentType();
if (contentType != null) {
headers.put("Content-Type", contentType.toString());
}
headers.put("Content-Length", Long.toString(requestBody.contentLength()));
headers.put("Transfer-Encoding", "chunked");
headers.put(HEADER_DD_EVP_ORIGIN, JAVA_TRACING_LIBRARY);
Expand All @@ -258,8 +268,11 @@ private Call makeTelemetryRequest(@Nonnull List<String> filesContent) throws IOE
.build());
}

private RequestBody makeTelemetryRequestBody(@Nonnull List<String> filesContent)
throws IOException {
private RequestBody makeTelemetryRequestBody(@Nonnull String content) throws IOException {
CrashLog crashLog = CrashLogParser.fromHotspotCrashLog(content);
if (crashLog == null) {
throw new IOException("Failed to parse crash log");
}
try (Buffer buf = new Buffer()) {
try (JsonWriter writer = JsonWriter.of(buf)) {
writer.beginObject();
Expand All @@ -275,13 +288,11 @@ private RequestBody makeTelemetryRequestBody(@Nonnull List<String> filesContent)
writer.name("debug").value(true);
writer.name("payload");
writer.beginArray();
for (String message : filesContent) {
writer.beginObject();
writer.name("message").value(extractErrorStackTrace(message));
writer.name("level").value("ERROR");
writer.name("tags").value("severity:crash");
writer.endObject();
}
writer.beginObject();
writer.name("message").value(crashLog.toJson());
writer.name("level").value("ERROR");
writer.name("tags").value("severity:crash");
writer.endObject();
writer.endArray();
writer.name("application");
writer.beginObject();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package com.datadog.crashtracking.dto;

import com.squareup.moshi.Json;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
import java.util.UUID;

public final class CrashLog {
private static final int VERSION = 0;

private static final JsonAdapter<CrashLog> ADAPTER;

static {
Moshi moshi = new Moshi.Builder().add(new SemanticVersion.SemanticVersionAdapter()).build();
ADAPTER = moshi.adapter(CrashLog.class);
}

public final String uuid = UUID.randomUUID().toString();
public final String timestamp;
public final boolean incomplete;
public final ErrorData error;
public final Metadata metadata;

@Json(name = "os_info")
public final OSInfo osInfo;

@Json(name = "proc_info")
public final ProcInfo procInfo;

@Json(name = "version_id")
public final int version = VERSION;

public CrashLog(
boolean incomplete,
String timestamp,
ErrorData error,
Metadata metadata,
OSInfo osInfo,
ProcInfo procInfo) {
this.incomplete = incomplete;
this.timestamp = timestamp;
this.error = error;
this.metadata = metadata;
this.osInfo = osInfo;
this.procInfo = procInfo;
}

public String toJson() {
return ADAPTER.toJson(this);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package com.datadog.crashtracking.dto;

import com.squareup.moshi.Json;

public final class ErrorData {
@Json(name = "is_crash")
public final boolean isCrash = true;

public final String kind;
public final String message;

@Json(name = "source_type")
public final String sourceType = "crashtracking";

public final StackTrace stack;

public ErrorData(String kind, String message, StackTrace stack) {
this.kind = kind;
this.message = message;
this.stack = stack;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package com.datadog.crashtracking.dto;

import com.squareup.moshi.Json;
import java.util.Collections;
import java.util.Map;

public final class Metadata {
@Json(name = "library_name")
public final String libraryName;

@Json(name = "library_version")
public final String libraryVersion;

public final String family;
public final Map<String, String> tags;

public Metadata(
String libraryName, String libraryVersion, String family, Map<String, String> tags) {
this.libraryName = libraryName;
this.libraryVersion = libraryVersion;
this.family = family;
this.tags = tags != null ? Collections.unmodifiableMap(tags) : Collections.emptyMap();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package com.datadog.crashtracking.dto;

import com.squareup.moshi.Json;

public final class OSInfo {
public final String architecture;
public final String bitness;

@Json(name = "os_type")
public final String osType;

public final SemanticVersion version;

public OSInfo(String architecture, String bitness, String osType, SemanticVersion version) {
this.architecture = architecture;
this.bitness = bitness;
this.osType = osType;
this.version = version;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package com.datadog.crashtracking.dto;

public final class ProcInfo {
public final String pid;

public ProcInfo(String pid) {
this.pid = pid;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package com.datadog.crashtracking.dto;

import com.squareup.moshi.FromJson;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import com.squareup.moshi.ToJson;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public final class SemanticVersion {
public static final class SemanticVersionAdapter {

@ToJson
public void toJson(JsonWriter writer, SemanticVersion version) throws IOException {
writer.beginObject();
writer.name("Semantic");
writer.beginArray();
writer.value(version.major);
writer.value(version.minor);
writer.value(version.patch);
writer.endArray();
writer.endObject();
}

@FromJson
public SemanticVersion fromJson(JsonReader reader) throws IOException {
reader.beginObject();
String name = reader.nextName();
if (!"Semantic".equals(name)) {
throw new IOException("Expected 'Semantic' key");
}
reader.beginArray();
int major = reader.nextInt();
int minor = reader.nextInt();
int patch = reader.nextInt();
reader.endArray();
reader.endObject();
return new SemanticVersion(major, minor, patch);
}
}

public final int major;
public final int minor;
public final int patch;

public SemanticVersion(int major, int minor, int patch) {
this.major = major;
this.minor = minor;
this.patch = patch;
}

public static SemanticVersion of(String version) {
String[] parts = version.split("\\.");
if (parts.length == 3) {
return new SemanticVersion(
safeParseInteger(parts[0]), safeParseInteger(parts[1]), safeParseInteger(parts[2]));
} else if (parts.length == 2) {
return new SemanticVersion(safeParseInteger(parts[0]), safeParseInteger(parts[1]), 0);
} else if (parts.length == 1) {
return new SemanticVersion(safeParseInteger(parts[0]), 0, 0);
} else {
throw new IllegalArgumentException("Invalid version string: " + version);
}
}

private static final Pattern INTEGER_PATTERN = Pattern.compile("(\\d+).*");

private static int safeParseInteger(String value) {
try {
return Integer.parseInt(value);
} catch (NumberFormatException e) {
Matcher matcher = INTEGER_PATTERN.matcher(value);
if (matcher.matches()) {
// this is guaranteed to be an integer
return Integer.parseInt(matcher.group(1));
}
return 0;
}
}
}
Loading

0 comments on commit 8dd0cb5

Please sign in to comment.