WIP : DW-issue-2096: Add metrics showing the number of events evaluated

api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java

@@ -668,6 +668,10 @@ public int getFieldNumber(String name) {
     @XmlElement
     protected long seekCount = 0;
     @XmlElement
+    protected long evaluatedCount = 0;
+    @XmlElement
+    protected long rejectedCount = 0;
+    @XmlElement
     protected long yieldCount = 0L;
     @XmlElement
     @XmlJavaTypeAdapter(StringMapAdapter.class)
@@ -891,6 +895,22 @@ public void setSeekCount(long seekCount) {
         this.seekCount = seekCount;
     }
 
+    public long getEvaluatedCount() {
+        return this.evaluatedCount;
+    }
+
+    public void setEvaluatedCount(long evaluatedCount) {
+        this.evaluatedCount = evaluatedCount;
+    }
+
+    public long getRejectedCount() {
+        return this.rejectedCount;
+    }
+
+    public void setRejectedCount(long rejectedCount) {
+        this.rejectedCount = rejectedCount;
+    }
+
     public long getYieldCount() {
         return this.yieldCount;
     }

api/src/main/java/datawave/microservice/querymetric/QueryMetric.java

@@ -97,6 +97,8 @@ public QueryMetric(QueryMetric other) {
         this.sourceCount = other.sourceCount;
         this.nextCount = other.nextCount;
         this.seekCount = other.seekCount;
+        this.evaluatedCount = other.evaluatedCount;
+        this.rejectedCount = other.rejectedCount;
         this.yieldCount = other.yieldCount;
         this.versionMap = other.versionMap;
         this.docRanges = other.docRanges;
@@ -140,8 +142,9 @@ public int hashCode() {
                         .append(this.getHost()).append(this.getPageTimes()).append(this.getProxyServers()).append(this.getLifecycle())
                         .append(this.getErrorMessage()).append(this.getCreateCallTime()).append(this.getErrorCode()).append(this.getQueryName())
                         .append(this.getParameters()).append(this.getSourceCount()).append(this.getNextCount()).append(this.getSeekCount())
-                        .append(this.getYieldCount()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime())
-                        .append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode();
+                        .append(this.getEvaluatedCount()).append(this.getRejectedCount()).append(this.getYieldCount()).append(this.getDocRanges())
+                        .append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime()).append(this.getPredictions()).append(this.getMarkings())
+                        .append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode();
     }
 
     @Override
@@ -166,6 +169,7 @@ public boolean equals(Object o) {
                             .append(this.getLifecycle(), other.getLifecycle()).append(this.getErrorMessage(), other.getErrorMessage())
                             .append(this.getErrorCode(), other.getErrorCode()).append(this.getSourceCount(), other.getSourceCount())
                             .append(this.getNextCount(), other.getNextCount()).append(this.getSeekCount(), other.getSeekCount())
+                            .append(this.getEvaluatedCount(), other.getEvaluatedCount()).append(this.getRejectedCount(), other.getRejectedCount())
                             .append(this.getYieldCount(), other.getYieldCount()).append(this.getDocRanges(), other.getDocRanges())
                             .append(this.getFiRanges(), other.getFiRanges()).append(this.getPlan(), other.getPlan())
                             .append(this.getLoginTime(), other.getLoginTime()).append(this.getPredictions(), other.getPredictions())
@@ -207,6 +211,8 @@ public String toString() {
         buf.append(" Source Count: ").append(this.getSourceCount());
         buf.append(" NextCount: ").append(this.getNextCount());
         buf.append(" Seek Count: ").append(this.getSeekCount());
+        buf.append(" Evaluated Count: ").append(this.getEvaluatedCount());
+        buf.append(" Rejected Count: ").append(this.getRejectedCount());
         buf.append(" Yield Count: ").append(this.getYieldCount());
         buf.append(" Doc Ranges: ").append(this.getDocRanges());
         buf.append(" FI Ranges: ").append(this.getFiRanges());
@@ -378,6 +384,8 @@ public void writeTo(Output output, QueryMetric message) throws IOException {
             output.writeInt64(28, message.sourceCount, false);
             output.writeInt64(29, message.nextCount, false);
             output.writeInt64(30, message.seekCount, false);
+            output.writeInt64(39, message.evaluatedCount, false);
+            output.writeInt64(40, message.rejectedCount, false);
             output.writeInt64(31, message.yieldCount, false);
             output.writeInt64(32, message.docRanges, false);
             output.writeInt64(33, message.fiRanges, false);
@@ -551,6 +559,12 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException {
                             message.versionMap.put(split[0], split[1]);
                         }
                         break;
+                    case 39:
+                        message.evaluatedCount = input.readInt64();
+                        break;
+                    case 40:
+                        message.rejectedCount = input.readInt64();
+                        break;
                     default:
                         input.handleUnknownField(number, this);
                         break;
@@ -637,6 +651,10 @@ public String getFieldName(int number) {
                     return "version";
                 case 38:
                     return "versionMap";
+                case 39:
+                    return "evaluatedCount";
+                case 40:
+                    return "rejectedCount";
                 default:
                     return null;
             }
@@ -688,6 +706,8 @@ public int getFieldNumber(String name) {
             fieldMap.put("predictions", 36);
             fieldMap.put("version", 37);
             fieldMap.put("versionMap", 38);
+            fieldMap.put("evaluatedCount", 39);
+            fieldMap.put("rejectedCount", 40);
         }
     };
 

api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java

@@ -50,7 +50,7 @@ public String getMainContent() {
         builder.append("<th>Query Setup Time (ms)</th><th>Query Setup Call Time (ms)</th><th>Number Pages</th><th>Number Results</th>");
 
         builder.append("<th>Doc Ranges</th><th>FI Ranges</th>");
-        builder.append("<th>Sources</th><th>Next Calls</th><th>Seek Calls</th><th>Yield Count</th><th>Versions</th>");
+        builder.append("<th>Sources</th><th>Next Calls</th><th>Seek Calls</th><th>Evaluated Events</th><th>Rejected Events</th><th>Yield Count</th><th>Versions</th>");
 
         builder.append("<th>Total Page Time (ms)</th><th>Total Page Call Time (ms)</th><th>Total Page Serialization Time (ms)</th>");
         builder.append("<th>Total Page Bytes Sent (uncompressed)</th><th>Lifecycle</th><th>Elapsed Time</th><th>Error Code</th><th>Error Message</th>");
@@ -131,6 +131,8 @@ public String getMainContent() {
             builder.append("<td>").append(metric.getSourceCount()).append("</td>");
             builder.append("<td>").append(metric.getNextCount()).append("</td>");
             builder.append("<td>").append(metric.getSeekCount()).append("</td>");
+            builder.append("<td>").append(metric.getEvaluatedCount()).append("</td>");
+            builder.append("<td>").append(metric.getRejectedCount()).append("</td>");
             builder.append("<td>").append(metric.getYieldCount()).append("</td>");
             String versions = metric.getVersionMap().entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("<br/>"));
             builder.append("<td style=\"min-width:250px !important;\">").append(versions).append("</td>");

service/pom.xml

@@ -28,7 +28,7 @@
         <version.in-memory-accumulo>3.0.1</version.in-memory-accumulo>
         <version.microservice.accumulo-api>3.0.0</version.microservice.accumulo-api>
         <version.microservice.hazelcast>3.0.0</version.microservice.hazelcast>
-        <version.microservice.query-metric-api>3.0.0</version.microservice.query-metric-api>
+        <version.microservice.query-metric-api>3.0.1-SNAPSHOT</version.microservice.query-metric-api>
         <version.microservice.starter>3.0.0</version.microservice.starter>
         <version.microservice.starter-datawave-query-metric>2.0.0</version.microservice.starter-datawave-query-metric>
         <version.microservice.starter-metadata>2.0.0</version.microservice.starter-metadata>

service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java

@@ -40,6 +40,8 @@ public Long process(Map.Entry<String,QueryMetricUpdateHolder> entry) {
             updatedHolder.addValue("sourceCount", updatedMetric.getSourceCount());
             updatedHolder.addValue("nextCount", updatedMetric.getNextCount());
             updatedHolder.addValue("seekCount", updatedMetric.getSeekCount());
+            updatedHolder.addValue("evaluatedCount", updatedMetric.getEvaluatedCount());
+            updatedHolder.addValue("rejectedCount", updatedMetric.getRejectedCount());
             updatedHolder.addValue("yieldCount", updatedMetric.getYieldCount());
             updatedHolder.addValue("docRanges", updatedMetric.getDocRanges());
             updatedHolder.addValue("fiRanges", updatedMetric.getFiRanges());

service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java

@@ -113,6 +113,8 @@ public class QueryMetricHandlerProperties {
             "QUERY_NAME",
             "QUERY_TYPE",
             "SEEK_COUNT",
+            "EVALUATED_COUNT",
+            "REJECTED_COUNT",
             "SETUP_TIME",
             "SOURCE_COUNT",
             "USER",
@@ -146,6 +148,8 @@ public class QueryMetricHandlerProperties {
             "LOGIN_TIME",
             "NEXT_COUNT",
             "SEEK_COUNT",
+            "EVALUATED_COUNT",
+            "REJECTED_COUNT",
             "SETUP_TIME",
             "SOURCE_COUNT",
             "NUM_PAGES",

service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java

@@ -296,6 +296,12 @@ public Multimap<String,String> getEventFieldsToWrite(T updated, T stored) {
             if (isChanged(updated.getSeekCount(), stored == null ? -1 : stored.getSeekCount())) {
                 fields.put("SEEK_COUNT", Long.toString(updated.getSeekCount()));
             }
+            if (isChanged(updated.getEvaluatedCount(), stored == null ? -1 : stored.getEvaluatedCount())) {
+                fields.put("EVALUATED_COUNT", Long.toString(updated.getEvaluatedCount()));
+            }
+            if (isChanged(updated.getRejectedCount(), stored == null ? -1 : stored.getRejectedCount())) {
+                fields.put("REJECTED_COUNT", Long.toString(updated.getRejectedCount()));
+            }
             if (isChanged(updated.getSourceCount(), stored == null ? -1 : stored.getSourceCount())) {
                 fields.put("SOURCE_COUNT", Long.toString(updated.getSourceCount()));
             }
@@ -406,6 +412,12 @@ public Multimap<String,String> getEventFieldsToDelete(T updated, T stored) {
                 if (isChanged(updated.getSeekCount(), stored.getSeekCount())) {
                     fields.put("SEEK_COUNT", Long.toString(stored.getSeekCount()));
                 }
+                if (isChanged(updated.getEvaluatedCount(), stored.getEvaluatedCount())) {
+                    fields.put("EVALUATED_COUNT", Long.toString(stored.getEvaluatedCount()));
+                }
+                if (isChanged(updated.getRejectedCount(), stored.getRejectedCount())) {
+                    fields.put("REJECTED_COUNT", Long.toString(stored.getRejectedCount()));
+                }
                 if (isChanged(updated.getSetupTime(), stored.getSetupTime())) {
                     fields.put("SETUP_TIME", Long.toString(stored.getSetupTime()));
                 }

service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java

@@ -173,13 +173,17 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy
                 combinedMetric.setSourceCount(combinedMetric.getSourceCount() + updatedQueryMetric.getSourceCount());
                 combinedMetric.setNextCount(combinedMetric.getNextCount() + updatedQueryMetric.getNextCount());
                 combinedMetric.setSeekCount(combinedMetric.getSeekCount() + updatedQueryMetric.getSeekCount());
+                combinedMetric.setEvaluatedCount(combinedMetric.getEvaluatedCount() + updatedQueryMetric.getEvaluatedCount());
+                combinedMetric.setRejectedCount(combinedMetric.getRejectedCount() + updatedQueryMetric.getRejectedCount());
                 combinedMetric.setYieldCount(combinedMetric.getYieldCount() + updatedQueryMetric.getYieldCount());
                 combinedMetric.setDocRanges(combinedMetric.getDocRanges() + updatedQueryMetric.getDocRanges());
                 combinedMetric.setFiRanges(combinedMetric.getFiRanges() + updatedQueryMetric.getFiRanges());
             } else {
                 combinedMetric.setSourceCount(updatedQueryMetric.getSourceCount());
                 combinedMetric.setNextCount(updatedQueryMetric.getNextCount());
                 combinedMetric.setSeekCount(updatedQueryMetric.getSeekCount());
+                combinedMetric.setEvaluatedCount(updatedQueryMetric.getEvaluatedCount());
+                combinedMetric.setRejectedCount(updatedQueryMetric.getRejectedCount());
                 combinedMetric.setYieldCount(updatedQueryMetric.getYieldCount());
                 combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges());
                 combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges());

service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java

@@ -605,6 +605,24 @@ public T toMetric(EventBase event) {
                         } catch (Exception e) {
                             log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
                         }
+                    } else if (fieldName.equals("EVALUATED_COUNT")) {
+                        try {
+                            long l = Long.parseLong(fieldValue);
+                            if (l > m.getEvaluatedCount()) {
+                                m.setEvaluatedCount(l);
+                            }
+                        } catch (Exception e) {
+                            log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
+                        }
+                    } else if (fieldName.equals("REJECTED_COUNT")) {
+                        try {
+                            long l = Long.parseLong(fieldValue);
+                            if (l > m.getRejectedCount()) {
+                                m.setRejectedCount(l);
+                            }
+                        } catch (Exception e) {
+                            log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
+                        }
                     } else if (fieldName.equals("SETUP_TIME")) {
                         m.setSetupTime(Long.parseLong(fieldValue));
                     } else if (fieldName.equals("SOURCE_COUNT")) {

service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java

@@ -128,6 +128,8 @@ public void store(QueryMetricUpdateHolder<T> queryMetricUpdate) throws Exception
                     updatedMetric.setSourceCount(queryMetricUpdate.getValue("sourceCount"));
                     updatedMetric.setNextCount(queryMetricUpdate.getValue("nextCount"));
                     updatedMetric.setSeekCount(queryMetricUpdate.getValue("seekCount"));
+                    updatedMetric.setEvaluatedCount(queryMetricUpdate.getValue("evaluatedCount"));
+                    updatedMetric.setRejectedCount(queryMetricUpdate.getValue("rejectedCount"));
                     updatedMetric.setYieldCount(queryMetricUpdate.getValue("yieldCount"));
                     updatedMetric.setDocRanges(queryMetricUpdate.getValue("docRanges"));
                     updatedMetric.setFiRanges(queryMetricUpdate.getValue("fiRanges"));
@@ -173,6 +175,8 @@ public void store(QueryMetricUpdateHolder<T> queryMetricUpdate) throws Exception
                     queryMetricUpdate.getMetric().setSourceCount(updatedMetric.getSourceCount());
                     queryMetricUpdate.getMetric().setNextCount(updatedMetric.getNextCount());
                     queryMetricUpdate.getMetric().setSeekCount(updatedMetric.getSeekCount());
+                    queryMetricUpdate.getMetric().setEvaluatedCount(updatedMetric.getEvaluatedCount());
+                    queryMetricUpdate.getMetric().setRejectedCount(updatedMetric.getRejectedCount());
                     queryMetricUpdate.getMetric().setYieldCount(updatedMetric.getYieldCount());
                     queryMetricUpdate.getMetric().setDocRanges(updatedMetric.getDocRanges());
                     queryMetricUpdate.getMetric().setFiRanges(updatedMetric.getFiRanges());