Replace Json Tag Extraction implementation. Migrate from jayway.jsonp…

…ath to jsurfer-core

dd-trace-core/build.gradle

@@ -70,12 +70,7 @@ dependencies {
   implementation libs.slf4j
   implementation libs.moshi
   implementation libs.jctools
-  implementation (libs.jsonPath) {
-    // exclude json-smart to reduce the size of the resulting jar (this is possible by using a custom MoshiJsonProvider)
-    exclude group: 'net.minidev', module: 'json-smart'
-    // exclude slf4j-api to avoid conflicts with the slf4j-api dependency in the agent
-    exclude group: 'org.slf4j', module: 'slf4j-api'
-  }
+  implementation libs.jsurfer
 
   implementation group: 'com.datadoghq', name: 'sketches-java', version: '0.8.3'
 

dd-trace-core/src/main/java/datadog/trace/payloadtags/JsonPosition.java

@@ -0,0 +1,58 @@
+package datadog.trace.payloadtags;
+
+import org.jsfr.json.path.ArrayIndex;
+import org.jsfr.json.path.ChildNode;
+import org.jsfr.json.path.JsonPath;
+import org.jsfr.json.path.PathOperator;
+
+class JsonPosition extends JsonPath {
+
+  static JsonPosition start() {
+    return new JsonPosition();
+  }
+
+  void stepIntoObject() {
+    if (operators.length > size) {
+      PathOperator next = operators[size];
+      if (next instanceof ChildNode) {
+        size++;
+        ((ChildNode) next).setKey(null);
+        return;
+      }
+    }
+    push(new ChildNode(null));
+  }
+
+  void updateObjectEntry(String key) {
+    ((ChildNode) peek()).setKey(key);
+  }
+
+  void stepOutObject() {
+    pop();
+  }
+
+  void stepIntoArray() {
+    if (operators.length > size) {
+      PathOperator next = operators[size];
+      if (next instanceof ArrayIndex) {
+        size++;
+        ((ArrayIndex) next).reset();
+        return;
+      }
+    }
+    push(new ArrayIndex());
+  }
+
+  boolean accumulateArrayIndex() {
+    PathOperator top = this.peek();
+    if (top.getType() == PathOperator.Type.ARRAY) {
+      ((ArrayIndex) top).increaseArrayIndex();
+      return true;
+    }
+    return false;
+  }
+
+  void stepOutArray() {
+    pop();
+  }
+}

dd-trace-core/src/main/java/datadog/trace/payloadtags/JsonToTags.java

@@ -1,17 +1,18 @@
 package datadog.trace.payloadtags;
 
-import com.jayway.jsonpath.Configuration;
-import com.jayway.jsonpath.DocumentContext;
-import com.jayway.jsonpath.InvalidJsonException;
-import com.jayway.jsonpath.JsonPath;
-import com.jayway.jsonpath.ParseContext;
-import com.jayway.jsonpath.PathNotFoundException;
+import com.squareup.moshi.JsonAdapter;
+import com.squareup.moshi.Moshi;
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import okio.BufferedSource;
+import okio.Okio;
+import org.jsfr.json.compiler.JsonPathCompiler;
+import org.jsfr.json.path.JsonPath;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,6 +22,9 @@ public final class JsonToTags {
 
   private static final String DD_PAYLOAD_TAGS_INCOMPLETE = "_dd.payload_tags_incomplete";
 
+  private final Moshi moshi = new Moshi.Builder().build();
+  private final JsonAdapter<Object> jsonAdapter = moshi.adapter(Object.class).lenient();
+
   public static final class Builder {
     private List<JsonPath> expansionRules = Collections.emptyList();
     private List<JsonPath> redactionRules = Collections.emptyList();
@@ -44,10 +48,10 @@ private static List<JsonPath> parseRules(List<String> rules) {
       List<JsonPath> result = new ArrayList<>(rules.size());
       for (String rule : rules) {
         try {
-          JsonPath jp = JsonPath.compile(rule);
+          JsonPath jp = JsonPathCompiler.compile(rule);
           result.add(jp);
         } catch (Exception ex) {
-          log.debug("Skipping failed to parse JSON path rule: '{}'", rule);
+          log.debug("Skipping failed to parse JSON path rule: '{}'", rule, ex);
         }
       }
       return result;
@@ -75,8 +79,6 @@ public JsonToTags build() {
   private final int limitTags;
   private final int limitDeepness;
 
-  private final ParseContext parseContext;
-
   private JsonToTags(
       List<JsonPath> expansionRules,
       List<JsonPath> redactionRules,
@@ -86,57 +88,26 @@ private JsonToTags(
     this.redactionRules = redactionRules;
     this.limitTags = limitTags;
     this.limitDeepness = limitDeepness;
-
-    Configuration configuration =
-        Configuration.builder()
-            .jsonProvider(new MoshiJsonProvider())
-            .mappingProvider(new MoshiMappingProvider())
-            .build();
-
-    parseContext = JsonPath.using(configuration);
   }
 
   public Map<String, Object> process(InputStream is, String tagPrefix) {
-    DocumentContext dc;
+    Object json;
 
     try {
-      dc = parseContext.parse(is);
-    } catch (Exception ex) {
-      log.debug("Failed to parse JSON body for tag extraction", ex);
+      json = parse(is);
+    } catch (IOException ex) {
+      log.debug("Failed to parse JSON body for tag extraction: {}", ex.getMessage());
       return Collections.emptyMap();
     }
 
-    if (!(dc.json() instanceof Map)) {
+    if (!(json instanceof Map)) {
       log.debug("Failed to parse JSON body for tag extraction. Expected JSON object at the root.");
       return Collections.emptyMap();
     }
 
-    for (JsonPath jp : expansionRules) {
-      try {
-        dc.map(jp, (obj, conf) -> expandInnerJson(jp, obj));
-      } catch (PathNotFoundException ex) {
-        // ignore
-      }
-    }
-
-    for (JsonPath jp : redactionRules) {
-      try {
-        dc.set(jp, REDACTED);
-      } catch (PathNotFoundException ex) {
-        // ignore
-      }
-    }
-
     LinkedHashMap<String, Object> tags = new LinkedHashMap<>();
     boolean visitedAll =
-        traverse(
-            dc.json(),
-            new StringBuilder(tagPrefix),
-            (path, value) -> {
-              tags.put(path.toString(), value);
-              return tags.size() < limitTags;
-            },
-            0);
+        traverse(json, new StringBuilder(tagPrefix), JsonPosition.start(), 0, tags);
 
     if (!visitedAll) {
       tags.put(DD_PAYLOAD_TAGS_INCOMPLETE, true);
@@ -145,62 +116,100 @@ public Map<String, Object> process(InputStream is, String tagPrefix) {
     return tags;
   }
 
-  private Object expandInnerJson(JsonPath jp, Object obj) {
+  private Object parse(InputStream is) throws IOException {
+    try (BufferedSource source = Okio.buffer(Okio.source(is))) {
+      return jsonAdapter.fromJson(source);
+    }
+  }
+
+  public Object parse(String s) throws IOException {
+    return jsonAdapter.fromJson(s);
+  }
+
+  private Object expandInnerJson(CharSequence path, Object obj) {
     if (obj instanceof String) {
       String str = (String) obj;
       if (!str.startsWith("{") && !str.startsWith("[")) {
         log.debug(
             "Couldn't expand inner JSON {} for path: {} because it neither start with { or [",
             str,
-            jp.getPath());
-        return str;
+            path);
+        return null;
       }
       try {
-        return parseContext.parse((String) obj).json();
-      } catch (InvalidJsonException ex) {
-        log.debug("Failed to parse inner JSON for path: {}", jp.getPath(), ex);
+        return parse((String) obj);
+      } catch (IOException ex) {
+        log.debug("Failed to parse inner JSON for path: {}", path, ex);
       }
     }
-    return obj;
-  }
-
-  private interface JsonVisitor {
-    /* return true to continue or false to stop visiting */
-    boolean visit(CharSequence path, Object value);
+    return null;
   }
 
-  private boolean traverse(Object json, StringBuilder pathBuf, JsonVisitor visitor, int depth) {
+  private boolean traverse(
+      Object jsonValue,
+      StringBuilder pathBuf,
+      JsonPosition jp,
+      int depth,
+      LinkedHashMap<String, Object> tagsAcc) {
     if (depth > limitDeepness) {
       return true;
     }
-    if (json instanceof Map) {
-      Map<String, Object> map = (Map<String, Object>) json;
+    if (jsonValue instanceof Map) {
+      Map<String, Object> map = (Map<String, Object>) jsonValue;
+      jp.stepIntoObject();
       for (Map.Entry<String, Object> entry : map.entrySet()) {
         int i = pathBuf.length();
         pathBuf.append('.');
         pathBuf.append(entry.getKey().replace(".", "\\."));
-        boolean visitedAll = traverse(entry.getValue(), pathBuf, visitor, depth + 1);
+        jp.updateObjectEntry(entry.getKey());
+        boolean visitedAll = traverse(entry.getValue(), pathBuf, jp, depth + 1, tagsAcc);
         pathBuf.delete(i, pathBuf.length());
         if (!visitedAll) {
           return false;
         }
       }
-    } else if (json instanceof Iterable) {
-      Iterable<Object> iterable = (Iterable<Object>) json;
+      jp.stepOutObject();
+    } else if (jsonValue instanceof Iterable) {
+      Iterable<Object> iterable = (Iterable<Object>) jsonValue;
       int index = 0;
+      jp.stepIntoArray();
       for (Object item : iterable) {
         int i = pathBuf.length();
         pathBuf.append('.');
         pathBuf.append(index);
-        boolean visitedAll = traverse(item, pathBuf, visitor, depth + 1);
+        jp.accumulateArrayIndex();
+        boolean visitedAll = traverse(item, pathBuf, jp, depth + 1, tagsAcc);
         pathBuf.delete(i, pathBuf.length());
         if (!visitedAll) {
           return false;
         }
         index += 1;
       }
+      jp.stepOutArray();
     } else {
-      return visitor.visit(pathBuf, json);
+      // 1. for each matching expansion rule, run traverse for inner object
+      for (JsonPath er : expansionRules) {
+        if (er.matchWithDeepScan(jp)) {
+          Object innerJson = expandInnerJson(pathBuf, jsonValue);
+          if (innerJson == null) {
+            // matched but failed to expand
+            // skip and continue
+            continue;
+          }
+          return traverse(innerJson, pathBuf, jp, 0, tagsAcc);
+        }
+      }
+
+      // 2. for each matching redaction rules change value to REDACTED
+      for (JsonPath rr : redactionRules) {
+        if (rr.matchWithDeepScan(jp)) {
+          tagsAcc.put(pathBuf.toString(), REDACTED);
+          return tagsAcc.size() < limitTags;
+        }
+      }
+
+      tagsAcc.put(pathBuf.toString(), jsonValue);
+      return tagsAcc.size() < limitTags;
     }
     return true;
   }

dd-trace-core/src/test/groovy/datadog/trace/payloadtags/JsonToTagsTest.groovy

@@ -232,7 +232,7 @@ class JsonToTagsTest extends Specification {
   }
 
   def "skip invalid rules"() {
-    def invalidRuleWithLeadingSpace = ' $.Message'
+    def invalidRuleWithLeadingSpace = '$$.Message'
     JsonToTags jsonToTags = new JsonToTags.Builder()
       .parseExpansionRules([invalidRuleWithLeadingSpace])
       .parseRedactionRules([invalidRuleWithLeadingSpace])