WIP: DW-issue-1229: Save shard-specific query subplans to metrics

api/pom.xml

@@ -23,7 +23,7 @@
         <version.base-rest-responses>3.0.0</version.base-rest-responses>
         <version.commons-lang3>3.12.0</version.commons-lang3>
         <version.commons-text>1.9</version.commons-text>
-        <version.datawave>6.0.0</version.datawave>
+        <version.datawave>6.4.0-SNAPSHOT</version.datawave>
         <version.glassfish>2.3.6</version.glassfish>
         <version.guava>31.1-jre</version.guava>
         <version.jackson>2.12.5</version.jackson>

api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java

@@ -684,6 +684,10 @@ public int getFieldNumber(String name) {
     @XmlElement(name = "prediction")
     protected Set<Prediction> predictions = new HashSet<>();
 
+    @XmlElement(name = "subplans")
+    @XmlJavaTypeAdapter(StringIntegerListMapAdapter.class)
+    protected Map<String,RangeCounts> subPlans = new HashMap<>();
+
     public static final String DATAWAVE = "DATAWAVE";
     protected static final Map<String,String> discoveredVersionMap = BaseQueryMetric.getVersionsFromClasspath();
     protected long numUpdates = 0;
@@ -693,6 +697,28 @@ public enum Lifecycle {
         NONE, DEFINED, INITIALIZED, RESULTS, CLOSED, CANCELLED, MAXRESULTS, NEXTTIMEOUT, TIMEOUT, SHUTDOWN, MAXWORK
     }
 
+    public void addSubPlan(String plan, RangeCounts rangeCounts) {
+        synchronized (this.subPlans) {
+            if (subPlans.containsKey(plan)) {
+                RangeCounts combinedCounts = new RangeCounts();
+                RangeCounts currentCounts = subPlans.get(plan);
+                combinedCounts.setDocumentRangeCount(currentCounts.getDocumentRangeCount() + rangeCounts.getDocumentRangeCount());
+                combinedCounts.setShardRangeCount(currentCounts.getShardRangeCount() + rangeCounts.getShardRangeCount());
+                subPlans.put(plan, combinedCounts);
+            } else {
+                subPlans.put(plan, rangeCounts);
+            }
+        }
+    }
+
+    public Map<String,RangeCounts> getSubPlans() {
+        return subPlans;
+    }
+
+    public void setSubPlans(Map<String,RangeCounts> subPlans) {
+        this.subPlans = subPlans;
+    }
+
     public String getQueryType() {
         return queryType;
     }

api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java

@@ -0,0 +1,118 @@
+package datawave.microservice.querymetric;
+
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import javax.xml.bind.annotation.XmlTransient;
+
+import org.apache.commons.text.StringEscapeUtils;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+import datawave.webservice.query.QueryImpl;
+
+public abstract class BaseQueryMetricSubplanResponse<T extends BaseQueryMetric> extends BaseQueryMetricListResponse<T> {
+
+    private static final long serialVersionUID = 1L;
+    private static final String TITLE = "Query Metrics / Subplans";
+
+    @Override
+    public String getTitle() {
+        return TITLE;
+    }
+
+    @JsonIgnore
+    @XmlTransient
+    @Override
+    public String getPageHeader() {
+        return getTitle();
+    }
+
+    @JsonIgnore
+    @XmlTransient
+
+    @Override
+    public String getMainContent() {
+        TreeMap<Date,T> metricMap = new TreeMap<>(Collections.reverseOrder());
+        for (T metric : this.getResult()) {
+            metricMap.put(metric.getCreateDate(), metric);
+        }
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss");
+        StringBuilder builder = new StringBuilder();
+        int x = 0;
+        for (T metric : metricMap.values()) {
+            builder.append("<table>\n");
+            builder.append("<tr>");
+            builder.append("<th>Visibility</th>");
+            builder.append("<th>Query Date</th>");
+            builder.append("<th>User</th>");
+            builder.append("<th>UserDN</th>");
+            builder.append("<th>Query ID</th>");
+            builder.append("<th>Query Logic</th>");
+            builder.append("<th id=\"query-header\">Query</th>");
+            builder.append("<th>Query Plan</th>");
+            builder.append("</tr>");
+
+            Set<QueryImpl.Parameter> parameters = metric.getParameters();
+            // highlight alternating rows
+            if (x % 2 == 0) {
+                builder.append("<tr class=\"highlight\">\n");
+            } else {
+                builder.append("<tr>\n");
+            }
+            x++;
+
+            builder.append("<td>").append(metric.getColumnVisibility()).append("</td>");
+            builder.append("<td style=\"min-width:125px !important;\">").append(sdf.format(metric.getCreateDate())).append("</td>");
+            builder.append("<td>").append(metric.getUser()).append("</td>");
+            String userDN = metric.getUserDN();
+            builder.append("<td style=\"min-width:500px !important;\">").append(userDN == null ? "" : userDN).append("</td>");
+            if (this.isAdministratorMode()) {
+                builder.append("<td><a href=\"" + BASE_URL + "/user/").append(metric.getUser()).append("/").append(metric.getQueryId()).append("/")
+                                .append("\">").append(metric.getQueryId()).append("</a></td>");
+            } else {
+                builder.append("<td><a href=\"" + BASE_URL + "/id/").append(metric.getQueryId()).append("/").append("\">").append(metric.getQueryId())
+                                .append("</a></td>");
+            }
+            builder.append("<td>").append(metric.getQueryLogic()).append("</td>");
+            builder.append(isJexlQuery(parameters) ? "<td style=\"white-space: pre; word-wrap: break-word;\">" : "<td style=\"word-wrap: break-word;\">")
+                            .append(StringEscapeUtils.escapeHtml4(metric.getQuery())).append("</td>");
+            builder.append("<td style=\"white-space: pre; word-wrap: break-word;\">").append(StringEscapeUtils.escapeHtml4(metric.getPlan())).append("</td>");
+            builder.append("</tr>\n");
+            builder.append("</table>\n");
+
+            builder.append("<br/>");
+            builder.append("<table>\n");
+            builder.append("<tr><th>Range</th><th>Sub Plan</th></tr>");
+            if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) {
+                int s = 0;
+                for (Map.Entry<String,RangeCounts> e : metric.getSubPlans().entrySet()) {
+                    // highlight alternating rows
+                    if (s % 2 == 0) {
+                        builder.append("<tr class=\"highlight\">");
+                    } else {
+                        builder.append("<tr>");
+                    }
+                    builder.append("<td>").append(e.getKey()).append("</td>");
+                    builder.append("<td>").append("[" + e.getValue().getDocumentRangeCount() + "," + e.getValue().getShardRangeCount() + "]").append("</td>");
+                    builder.append("\n</tr>\n");
+                    s++;
+                }
+            } else {
+                builder.append("<tr><td colspan=\"2\">NONE<td/></tr>");
+            }
+            builder.append("</td></tr>\n");
+            builder.append("</table>\n");
+            builder.append("\n<br/><br/>\n");
+        }
+        return builder.toString();
+    }
+
+    private static boolean isJexlQuery(Set<QueryImpl.Parameter> params) {
+        return params.stream().anyMatch(p -> p.getParameterName().equals("query.syntax") && p.getParameterValue().equals("JEXL"));
+    }
+}

api/src/main/java/datawave/microservice/querymetric/QueryMetric.java

@@ -110,6 +110,12 @@ public QueryMetric(QueryMetric other) {
                 this.predictions.add(p.duplicate());
             }
         }
+
+        if (other.subPlans != null) {
+            this.subPlans = new HashMap<>();
+            this.subPlans.putAll(other.subPlans);
+        }
+
     }
 
     @Override
@@ -141,7 +147,8 @@ public int hashCode() {
                         .append(this.getErrorMessage()).append(this.getCreateCallTime()).append(this.getErrorCode()).append(this.getQueryName())
                         .append(this.getParameters()).append(this.getSourceCount()).append(this.getNextCount()).append(this.getSeekCount())
                         .append(this.getYieldCount()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime())
-                        .append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode();
+                        .append(this.getPredictions()).append(this.getSubPlans()).append(this.getMarkings()).append(this.getNumUpdates())
+                        .append(this.getVersionMap()).toHashCode();
     }
 
     @Override
@@ -169,8 +176,8 @@ public boolean equals(Object o) {
                             .append(this.getYieldCount(), other.getYieldCount()).append(this.getDocRanges(), other.getDocRanges())
                             .append(this.getFiRanges(), other.getFiRanges()).append(this.getPlan(), other.getPlan())
                             .append(this.getLoginTime(), other.getLoginTime()).append(this.getPredictions(), other.getPredictions())
-                            .append(this.getMarkings(), other.getMarkings()).append(this.getNumUpdates(), other.getNumUpdates())
-                            .append(this.getVersionMap(), other.getVersionMap()).isEquals();
+                            .append(this.getSubPlans(), other.getSubPlans()).append(this.getMarkings(), other.getMarkings())
+                            .append(this.getNumUpdates(), other.getNumUpdates()).append(this.getVersionMap(), other.getVersionMap()).isEquals();
         } else {
             return false;
         }
@@ -212,6 +219,7 @@ public String toString() {
         buf.append(" FI Ranges: ").append(this.getFiRanges());
         buf.append(" Login Time: ").append(this.getLoginTime());
         buf.append(" Predictions: ").append(this.getPredictions());
+        buf.append(" Subplans: ").append(this.getSubPlans());
         buf.append(" Markings: ").append(this.getMarkings());
         buf.append(" NumUpdates: ").append(this.getNumUpdates());
         buf.append(" VersionMap: ").append(this.getVersionMap());
@@ -403,6 +411,13 @@ public void writeTo(Output output, QueryMetric message) throws IOException {
                     output.writeString(38, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true);
                 }
             }
+
+            if (message.subPlans != null) {
+                for (Map.Entry<String,RangeCounts> entry : message.subPlans.entrySet()) {
+                    output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), StringUtils.join(entry.getValue(), ",")), "\0"), true);
+                }
+            }
+
         }
 
         public void mergeFrom(Input input, QueryMetric message) throws IOException {
@@ -551,6 +566,26 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException {
                             message.versionMap.put(split[0], split[1]);
                         }
                         break;
+                    case 39:
+                        if (message.subPlans == null) {
+                            message.subPlans = new TreeMap<>();
+                        }
+                        String encodedPlans = input.readString();
+                        String[] splitPlans = StringUtils.split(encodedPlans, "\0");
+                        if (splitPlans.length == 2) {
+                            RangeCounts rangeCounts = new RangeCounts();
+                            int index = 0;
+                            for (String count : StringUtils.split(splitPlans[1], ",")) {
+                                if (index == 0) {
+                                    rangeCounts.setDocumentRangeCount(Integer.parseInt(count));
+                                } else if (index == 1) {
+                                    rangeCounts.setShardRangeCount(Integer.parseInt(count));
+                                }
+                                index++;
+                            }
+                            message.subPlans.put(splitPlans[0], rangeCounts);
+                        }
+                        break;
                     default:
                         input.handleUnknownField(number, this);
                         break;
@@ -637,6 +672,8 @@ public String getFieldName(int number) {
                     return "version";
                 case 38:
                     return "versionMap";
+                case 39:
+                    return "subPlans";
                 default:
                     return null;
             }
@@ -688,6 +725,7 @@ public int getFieldNumber(String name) {
             fieldMap.put("predictions", 36);
             fieldMap.put("version", 37);
             fieldMap.put("versionMap", 38);
+            fieldMap.put("subPlans", 39);
         }
     };
 

api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java

@@ -48,7 +48,7 @@ public String getMainContent() {
         builder.append("<table>\n");
         builder.append("<tr>");
         builder.append("<th>Visibility</th><th>Query Date</th><th>User</th><th>UserDN</th><th>Proxy Server(s)</th><th>Query ID</th><th>Query Type</th>");
-        builder.append("<th>Query Logic</th><th id=\"query-header\">Query</th><th>Query Plan</th><th>Query Name</th><th>Begin Date</th><th>End Date</th><th>Parameters</th><th>Query Auths</th>");
+        builder.append("<th>Query Logic</th><th id=\"query-header\">Query</th><th>Query Plan</th><th>Query Subplan(s)</th><th>Query Name</th><th>Begin Date</th><th>End Date</th><th>Parameters</th><th>Query Auths</th>");
         builder.append("<th>Server</th>");
         builder.append("<th>Predictions</th>");
         builder.append("<th>Login Time (ms)</th>");
@@ -106,6 +106,8 @@ public String getMainContent() {
             builder.append(isJexlQuery(parameters) ? "<td id='query" + x + "'" + " style=\"white-space: pre; word-wrap: break-word;\">"
                             : "<td id='query" + x + "'" + " style=\"word-wrap: break-word;\">").append("</td>");
             builder.append("<td id='query-plan" + x + "'" + " style=\"white-space: pre; word-wrap: break-word;\">").append("</td>");
+            builder.append("<td style=\"white-space: pre; word-wrap: break-word;\">").append("<a href=\"/DataWave/Query/Metrics/id/")
+                            .append(metric.getQueryId()).append("/subplans\">Subplan(s)</a>").append("</td>");
             builder.append("<td>").append(metric.getQueryName()).append("</td>");
 
             String beginDate = metric.getBeginDate() == null ? "" : sdf.format(metric.getBeginDate());

api/src/main/java/datawave/microservice/querymetric/QueryMetricsSubplanResponse.java

@@ -0,0 +1,15 @@
+package datawave.microservice.querymetric;
+
+import javax.xml.bind.annotation.XmlAccessOrder;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorOrder;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "QueryMetricListResponse")
+@XmlAccessorType(XmlAccessType.NONE)
+@XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL)
+public class QueryMetricsSubplanResponse extends BaseQueryMetricSubplanResponse<QueryMetric> {
+
+    private static final long serialVersionUID = 1L;
+}

api/src/main/java/datawave/microservice/querymetric/RangeCounts.java

@@ -0,0 +1,48 @@
+package datawave.microservice.querymetric;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+
+public class RangeCounts implements Serializable {
+
+    private long documentRangeCount;
+    private long shardRangeCount;
+
+    public long getDocumentRangeCount() {
+        return documentRangeCount;
+    }
+
+    public long getShardRangeCount() {
+        return shardRangeCount;
+    }
+
+    public void setDocumentRangeCount(long newDocumentRangeCount) {
+        this.documentRangeCount = newDocumentRangeCount;
+    }
+
+    public void setShardRangeCount(long newShardRangeCount) {
+        this.shardRangeCount = newShardRangeCount;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        RangeCounts that = (RangeCounts) o;
+
+        return new EqualsBuilder().append(documentRangeCount, that.documentRangeCount).append(shardRangeCount, that.shardRangeCount).isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(documentRangeCount, shardRangeCount);
+    }
+}