From 92d12dddaaece512ae5ddf32aa7abc40209aad8a Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Fri, 29 Sep 2023 10:48:14 -0400 Subject: [PATCH] Use a Map instead of Map for subplans to facilitate serialization and comparison --- .../querymetric/BaseQueryMetric.java | 11 ++-- .../BaseQueryMetricSubplanResponse.java | 5 +- .../microservice/querymetric/QueryMetric.java | 14 ++--- .../StringIntegerListMapAdapter.java | 54 +++++++++++++++++++ .../ContentQueryMetricsIngestHelper.java | 8 +-- .../handler/ShardTableQueryMetricHandler.java | 15 +++--- .../QueryMetricConsistencyTest.java | 16 +++--- 7 files changed, 88 insertions(+), 35 deletions(-) create mode 100644 api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index 3e6f8839..d4b11c66 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -1,7 +1,6 @@ package datawave.microservice.querymetric; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.protobuf.Internal; import datawave.marking.MarkingFunctions; import datawave.webservice.query.Query; import datawave.webservice.query.QueryImpl.Parameter; @@ -684,8 +683,8 @@ public int getFieldNumber(String name) { protected Set predictions = new HashSet<>(); @XmlElement(name = "subplans") - @XmlJavaTypeAdapter(StringMapAdapter.class) - protected Map subPlans = new HashMap<>(); + @XmlJavaTypeAdapter(StringIntegerListMapAdapter.class) + protected Map> subPlans = new HashMap<>(); public static final String DATAWAVE = "DATAWAVE"; protected static final Map discoveredVersionMap = BaseQueryMetric.getVersionsFromClasspath(); @@ -696,15 +695,15 @@ public enum Lifecycle { NONE, DEFINED, INITIALIZED, RESULTS, CLOSED, CANCELLED, MAXRESULTS, NEXTTIMEOUT, TIMEOUT, SHUTDOWN, MAXWORK } - public void addSubPlan(String plan, int[] rangeCounts) { + public void addSubPlan(String plan, List rangeCounts) { subPlans.put(plan, rangeCounts); } - public Map getSubPlans() { + public Map> getSubPlans() { return subPlans; } - public void setSubPlans(Map subPlans) { + public void setSubPlans(Map> subPlans) { this.subPlans = subPlans; } diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java index db1e8652..a9f1ff77 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java @@ -9,6 +9,7 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -133,7 +134,7 @@ public String getMainContent() { builder.append("RangeSub Plan"); if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) { int s = 0; - for (Map.Entry e : metric.getSubPlans().entrySet()) { + for (Map.Entry> e : metric.getSubPlans().entrySet()) { // highlight alternating rows if (s % 2 == 0) { builder.append(""); @@ -141,7 +142,7 @@ public String getMainContent() { builder.append(""); } builder.append("").append(e.getKey()).append(""); - builder.append("").append(Arrays.toString(e.getValue())).append(""); + builder.append("").append(e.getValue()).append(""); builder.append("\n\n"); s++; } diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java index fefbd90e..926f158f 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java @@ -26,6 +26,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -410,8 +411,8 @@ public void writeTo(Output output, QueryMetric message) throws IOException { } if (message.subPlans != null) { - for (Map.Entry entry : message.subPlans.entrySet()) { - output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true); + for (Map.Entry> entry : message.subPlans.entrySet()) { + output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), StringUtils.join(entry.getValue(), ",")), "\0"), true); } } @@ -570,11 +571,10 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException { String encodedPlans = input.readString(); String[] splitPlans = StringUtils.split(encodedPlans, "\0"); if (splitPlans.length == 2) { - int[] rangeCounts = new int[2]; - int index = 0; - for (String count : splitPlans[1].substring(1, splitPlans[1].length() - 1).split(", ")) { - rangeCounts[index] = Integer.parseInt(count); - index++; + List rangeCounts = new ArrayList<>(); + String[] rangeCountSplit = StringUtils.split(splitPlans[1], ","); + for (String count : rangeCountSplit) { + rangeCounts.add(Integer.parseInt(count)); } message.subPlans.put(splitPlans[0], rangeCounts); } diff --git a/api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java b/api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java new file mode 100644 index 00000000..1dcfddde --- /dev/null +++ b/api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java @@ -0,0 +1,54 @@ +package datawave.microservice.querymetric; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlValue; +import javax.xml.bind.annotation.adapters.XmlAdapter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Provides JAX-B marshalling/unmarshalling of {@link Map} of String to List of Integer. This allows the marshalled type to be in our own namespace rather than + * in the "default" one, which when triggered will then assume the "" prefix and push all of our own elements into the "ns2" prefix. + */ +public class StringIntegerListMapAdapter extends XmlAdapter>> { + + @Override + public Map> unmarshal(StringIntegerListMap v) throws Exception { + HashMap> map = new HashMap<>(); + for (StringIntegerListMapEntry entry : v.entries) { + map.put(entry.key, entry.value); + } + return map; + } + + @Override + public StringIntegerListMap marshal(Map> v) throws Exception { + StringIntegerListMap map = new StringIntegerListMap(); + for (Map.Entry> entry : v.entrySet()) { + map.entries.add(new StringIntegerListMapEntry(entry.getKey(), entry.getValue())); + } + return map; + } + + public static class StringIntegerListMap { + @XmlElement(name = "entry") + private List entries = new ArrayList<>(); + } + + public static class StringIntegerListMapEntry { + @XmlAttribute(name = "name") + private String key; + @XmlValue + private List value; + + public StringIntegerListMapEntry() {} + + public StringIntegerListMapEntry(String key, List value) { + this.key = key; + this.value = value; + } + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index b859969c..09b7614c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -297,12 +297,12 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored == null ? -1 : stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(updated.getSourceCount())); } - Map updatedSubPlans = updated.getSubPlans(); + Map> updatedSubPlans = updated.getSubPlans(); if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { - Map storedSubPlans = stored == null ? null : stored.getSubPlans(); - for (Map.Entry entry : updatedSubPlans.entrySet()) { + Map> storedSubPlans = stored == null ? null : stored.getSubPlans(); + for (Map.Entry> entry : updatedSubPlans.entrySet()) { if (storedSubPlans == null || !storedSubPlans.containsKey(entry.getKey())) { - fields.put("SUBPLAN", entry.getKey() + " : " + Arrays.toString(entry.getValue())); + fields.put("SUBPLAN", entry.getKey() + " : " + StringUtils.join(entry.getValue(), ",")); } } } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 213f23d3..4155d5c9 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -509,7 +509,7 @@ public T toMetric(EventBase event) { List field = event.getFields(); m.setMarkings(event.getMarkings()); TreeMap pageMetrics = Maps.newTreeMap(); - Map subplans = new HashMap<>(); + Map> subplans = new HashMap<>(); boolean createDateSet = false; for (FieldBase f : field) { @@ -635,13 +635,12 @@ public T toMetric(EventBase event) { } else if (fieldName.equals("SUBPLAN")) { if (fieldValue != null) { String[] arr = fieldValue.split(" : ", 2); - int[] rangeCounts = new int[2]; - int index = 0; - for (String count : arr[1].substring(1, arr[1].length() - 1).split(", ")) { - rangeCounts[index] = Integer.parseInt(count); - index++; - } - if (arr.length >= 2) { + if (arr.length == 2) { + List rangeCounts = new ArrayList<>(); + String[] rangeCountSplit = StringUtils.split(arr[1], ","); + for (String count : rangeCountSplit) { + rangeCounts.add(Integer.parseInt(count)); + } subplans.put(arr[0], rangeCounts); } } diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 7c3c2d3d..261bfda1 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -30,6 +30,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -88,8 +89,7 @@ public void PageMetricTest() throws Exception { assertNoDuplicateFields(queryId); } - public void updateRangeCounts(Map shardCounts, Map documentCounts, - String subplan, Range range) { + public void updateRangeCounts(Map shardCounts, Map documentCounts, String subplan, Range range) { Key key = range.getStartKey(); String cf = key.getColumnFamily().toString(); if (cf.length() > 0) { @@ -106,13 +106,13 @@ public void updateRangeCounts(Map shardCounts, Map shardCounts = new HashMap<>(); - Map documentCounts = new HashMap<>(); + Map shardCounts = new HashMap<>(); + Map documentCounts = new HashMap<>(); List plans = new ArrayList<>(); plans.add("F1 == 'value1' || F2 == 'value2'"); plans.add("F3 == 'value3' || F4 == 'value4'"); @@ -120,7 +120,7 @@ public void SubPlanTest() throws Exception { plans.add("F1 == 'value1' || F6 == 'value6'"); plans.add("F1 == 'value1' || F5 == 'value5'"); Random r = new Random(System.currentTimeMillis()); - + Date now = new Date(); SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); String date = sdf.format(now); @@ -144,9 +144,9 @@ public void SubPlanTest() throws Exception { for (String p : plans) { Integer shardCount = shardCounts.getOrDefault(p, 0); Integer documentCount = documentCounts.getOrDefault(p, 0); - m.addSubPlan(p, new int[] {shardCount, documentCount}); + m.addSubPlan(p, Arrays.asList(shardCount, documentCount)); } - + // @formatter:off client.submit(new QueryMetricClient.Request.Builder() .withMetric(m)