From 0f5fd77c62b0f5b8412904a76f7bb1b318340a32 Mon Sep 17 00:00:00 2001 From: foster33 Date: Tue, 30 Aug 2022 11:55:56 -0400 Subject: [PATCH 1/7] Save each unique suplan and a count of shardRanges and documentRanges to query metrics --- .../querymetric/BaseQueryMetric.java | 17 ++ .../BaseQueryMetricSubplanResponse.java | 160 ++++++++++++++++++ .../microservice/querymetric/QueryMetric.java | 34 +++- .../QueryMetricsDetailListResponse.java | 5 +- .../QueryMetricsSubplanResponse.java | 15 ++ .../querymetric/QueryMetricOperations.java | 80 ++++++--- .../QueryMetricHandlerConfiguration.java | 7 + ...BaseQueryMetricSubplanResponseFactory.java | 8 + .../QueryMetricSubplanResponseFactory.java | 12 ++ .../ContentQueryMetricsIngestHelper.java | 10 +- .../handler/QueryMetricCombiner.java | 8 +- .../handler/ShardTableQueryMetricHandler.java | 22 ++- .../QueryMetricConsistencyTest.java | 54 ++++++ .../querymetric/QueryMetricTestBase.java | 2 + 14 files changed, 405 insertions(+), 29 deletions(-) create mode 100644 api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java create mode 100644 api/src/main/java/datawave/microservice/querymetric/QueryMetricsSubplanResponse.java create mode 100644 service/src/main/java/datawave/microservice/querymetric/factory/BaseQueryMetricSubplanResponseFactory.java create mode 100644 service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricSubplanResponseFactory.java diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index 3a1168a0..adb26ce0 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -1,6 +1,7 @@ package datawave.microservice.querymetric; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.protobuf.Internal; import datawave.marking.MarkingFunctions; import datawave.webservice.query.Query; import datawave.webservice.query.QueryImpl.Parameter; @@ -682,6 +683,10 @@ public int getFieldNumber(String name) { @XmlElement(name = "prediction") protected Set predictions = new HashSet<>(); + @XmlElement(name = "subplans") + @XmlJavaTypeAdapter(StringMapAdapter.class) + protected Map subPlans = new HashMap<>(); + public static final String DATAWAVE = "DATAWAVE"; protected static final Map discoveredVersionMap = BaseQueryMetric.getVersionsFromClasspath(); protected long numUpdates = 0; @@ -691,6 +696,18 @@ public enum Lifecycle { NONE, DEFINED, INITIALIZED, RESULTS, CLOSED, CANCELLED, MAXRESULTS, NEXTTIMEOUT, TIMEOUT, SHUTDOWN, MAXWORK } + public void addSubPlan(String range, String plan) { + subPlans.put(range, plan); + } + + public Map getSubPlans() { + return subPlans; + } + + public void setSubPlans(Map subPlans) { + this.subPlans = subPlans; + } + public String getQueryType() { return queryType; } diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java new file mode 100644 index 00000000..be2df79b --- /dev/null +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java @@ -0,0 +1,160 @@ +package datawave.microservice.querymetric; + +import datawave.webservice.HtmlProvider; +import datawave.webservice.query.QueryImpl; +import datawave.webservice.result.BaseResponse; +import org.apache.commons.text.StringEscapeUtils; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlTransient; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +public abstract class BaseQueryMetricSubplanResponse extends BaseResponse implements HtmlProvider { + + private static final long serialVersionUID = 1L; + private static final String TITLE = "Query Metrics / Subplans"; + private static final String EMPTY = ""; + @XmlElementWrapper(name = "querySubplans") + @XmlElement(name = "querySubplan") + protected List result = null; + @XmlElement + protected int numResults = 0; + @XmlTransient + private boolean isGeoQuery = false; + + public List getResult() { + return result; + } + + public int getNumResults() { + return numResults; + } + + public void setResult(List result) { + this.result = result; + this.numResults = this.result.size(); + } + + public void setNumResults(int numResults) { + this.numResults = numResults; + } + + public boolean isGeoQuery() { + return isGeoQuery; + } + + public void setGeoQuery(boolean geoQuery) { + isGeoQuery = geoQuery; + } + + @Override + public String getTitle() { + return TITLE; + } + + @Override + public String getPageHeader() { + return getTitle(); + } + + @Override + public String getHeadContent() { + if (isGeoQuery) { + // @formatter:off + return "" + + ""; + // @formatter: on + } else { + return EMPTY; + } + } + + @Override + public String getMainContent() { + TreeMap metricMap = new TreeMap<>(Collections.reverseOrder()); + for (T metric : this.getResult()) { + metricMap.put(metric.getCreateDate(), metric); + } + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss"); + StringBuilder builder = new StringBuilder(); + int x = 0; + for (T metric : metricMap.values()) { + builder.append("\n"); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(""); + + Set parameters = metric.getParameters(); + // highlight alternating rows + if (x % 2 == 0) { + builder.append("\n"); + } else { + builder.append("\n"); + } + x++; + + builder.append(""); + builder.append(""); + builder.append(""); + String userDN = metric.getUserDN(); + builder.append(""); + builder.append(""); + builder.append(""); + builder.append(isJexlQuery(parameters) ? ""); + builder.append(""); + builder.append("\n"); + builder.append("
VisibilityQuery DateUserUserDNQuery IDQuery LogicQueryQuery Plan
").append(metric.getColumnVisibility()).append("").append(sdf.format(metric.getCreateDate())).append("").append(metric.getUser()).append("").append(userDN == null ? "" : userDN).append("").append(metric.getQueryId()).append("").append(metric.getQueryLogic()).append("" : "") + .append(StringEscapeUtils.escapeHtml4(metric.getQuery())).append("").append(StringEscapeUtils.escapeHtml4(metric.getPlan())).append("
\n"); + + builder.append("
"); + builder.append("\n"); + builder.append(""); + if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) { + int s = 0; + for (Map.Entry e : metric.getSubPlans().entrySet()) { + // highlight alternating rows + if (s % 2 == 0) { + builder.append(""); + } else { + builder.append(""); + } + builder.append(""); + builder.append(""); + builder.append("\n\n"); + s++; + } + } else { + builder.append(""); + } + builder.append("\n"); + builder.append("
RangeSub Plan
").append(e.getKey()).append("").append(e.getValue()).append("
NONE
\n"); + builder.append("\n

\n"); + } + return builder.toString(); + } + + private static boolean isJexlQuery(Set params) { + return params.stream().anyMatch(p -> p.getParameterName().equals("query.syntax") && p.getParameterValue().equals("JEXL")); + } +} \ No newline at end of file diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java index e52f7835..361d857a 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java @@ -107,6 +107,12 @@ public QueryMetric(QueryMetric other) { this.predictions.add(p.duplicate()); } } + + if (other.subPlans != null) { + this.subPlans = new HashMap<>(); + this.subPlans.putAll(other.subPlans); + } + } @Override @@ -138,7 +144,8 @@ public int hashCode() { .append(this.getErrorMessage()).append(this.getCreateCallTime()).append(this.getErrorCode()).append(this.getQueryName()) .append(this.getParameters()).append(this.getSourceCount()).append(this.getNextCount()).append(this.getSeekCount()) .append(this.getYieldCount()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime()) - .append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode(); + .append(this.getPredictions()).append(this.getSubPlans()).append(this.getMarkings()).append(this.getNumUpdates()) + .append(this.getVersionMap()).toHashCode(); } @Override @@ -166,8 +173,8 @@ public boolean equals(Object o) { .append(this.getYieldCount(), other.getYieldCount()).append(this.getDocRanges(), other.getDocRanges()) .append(this.getFiRanges(), other.getFiRanges()).append(this.getPlan(), other.getPlan()) .append(this.getLoginTime(), other.getLoginTime()).append(this.getPredictions(), other.getPredictions()) - .append(this.getMarkings(), other.getMarkings()).append(this.getNumUpdates(), other.getNumUpdates()) - .append(this.getVersionMap(), other.getVersionMap()).isEquals(); + .append(this.getSubPlans(), other.getSubPlans()).append(this.getMarkings(), other.getMarkings()) + .append(this.getNumUpdates(), other.getNumUpdates()).append(this.getVersionMap(), other.getVersionMap()).isEquals(); } else { return false; } @@ -209,6 +216,7 @@ public String toString() { buf.append(" FI Ranges: ").append(this.getFiRanges()); buf.append(" Login Time: ").append(this.getLoginTime()); buf.append(" Predictions: ").append(this.getPredictions()); + buf.append(" Subplans: ").append(this.getSubPlans()); buf.append(" Markings: ").append(this.getMarkings()); buf.append(" NumUpdates: ").append(this.getNumUpdates()); buf.append(" VersionMap: ").append(this.getVersionMap()); @@ -400,6 +408,13 @@ public void writeTo(Output output, QueryMetric message) throws IOException { output.writeString(38, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true); } } + + if (message.subPlans != null) { + for (Map.Entry entry : message.subPlans.entrySet()) { + output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true); + } + } + } public void mergeFrom(Input input, QueryMetric message) throws IOException { @@ -548,6 +563,16 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException { message.versionMap.put(split[0], split[1]); } break; + case 39: + if (message.subPlans == null) { + message.subPlans = new TreeMap<>(); + } + String encodedPlans = input.readString(); + String[] splitPlans = StringUtils.split(encodedPlans, "\0"); + if (splitPlans.length == 2) { + message.subPlans.put(splitPlans[0], splitPlans[1]); + } + break; default: input.handleUnknownField(number, this); break; @@ -634,6 +659,8 @@ public String getFieldName(int number) { return "version"; case 38: return "versionMap"; + case 39: + return "subPlans"; default: return null; } @@ -685,6 +712,7 @@ public int getFieldNumber(String name) { fieldMap.put("predictions", 36); fieldMap.put("version", 37); fieldMap.put("versionMap", 38); + fieldMap.put("subPlans", 39); } }; diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java index 2631ffa2..d71154b4 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Set; import java.util.TreeMap; +import java.util.Map; import java.util.stream.Collectors; @XmlRootElement(name = "QueryMetricListResponse") @@ -34,7 +35,7 @@ public String getMainContent() { builder.append("\n"); builder.append(""); builder.append(""); - builder.append(""); + builder.append(""); builder.append(""); builder.append(""); builder.append(""); @@ -85,6 +86,8 @@ public String getMainContent() { builder.append(isJexlQuery(parameters) ? ""); builder.append(""); + builder.append(""); builder.append(""); String beginDate = metric.getBeginDate() == null ? "" : sdf.format(metric.getBeginDate()); diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsSubplanResponse.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsSubplanResponse.java new file mode 100644 index 00000000..fbbbc34e --- /dev/null +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsSubplanResponse.java @@ -0,0 +1,15 @@ +package datawave.microservice.querymetric; + +import javax.xml.bind.annotation.XmlAccessOrder; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorOrder; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "QueryMetricListResponse") +@XmlAccessorType(XmlAccessType.NONE) +@XmlAccessorOrder(XmlAccessOrder.ALPHABETICAL) +public class QueryMetricsSubplanResponse extends BaseQueryMetricSubplanResponse { + + private static final long serialVersionUID = 1L; +} diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index f4b85a23..e7e9651b 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -8,6 +8,7 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.querymetric.config.QueryMetricSinkConfiguration.QueryMetricSinkBinding; import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory; +import datawave.microservice.querymetric.factory.BaseQueryMetricSubplanResponseFactory; import datawave.microservice.querymetric.handler.QueryGeometryHandler; import datawave.microservice.querymetric.handler.ShardTableQueryMetricHandler; import datawave.microservice.querymetric.handler.SimpleQueryGeometryHandler; @@ -54,6 +55,7 @@ import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.TimeZone; import static datawave.microservice.querymetric.QueryMetricOperations.DEFAULT_DATETIME.BEGIN; @@ -82,6 +84,7 @@ public class QueryMetricOperations { private Cache lastWrittenQueryMetricCache; private MarkingFunctions markingFunctions; private BaseQueryMetricListResponseFactory queryMetricListResponseFactory; + private BaseQueryMetricSubplanResponseFactory queryMetricSubplanResponseFactory; private MergeLockLifecycleListener mergeLock; private MetricUpdateEntryProcessorFactory entryProcessorFactory; private QueryMetricOperationsStats stats; @@ -127,7 +130,8 @@ enum DEFAULT_DATETIME { @Autowired public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cacheManager, ShardTableQueryMetricHandler handler, QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, BaseQueryMetricListResponseFactory queryMetricListResponseFactory, - MergeLockLifecycleListener mergeLock, MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats) { + BaseQueryMetricSubplanResponseFactory queryMetricSubplanResponseFactory, MergeLockLifecycleListener mergeLock, + MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats) { this.handler = handler; this.geometryHandler = geometryHandler; this.cacheManager = cacheManager; @@ -135,6 +139,7 @@ public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cach this.lastWrittenQueryMetricCache = cacheManager.getCache(LAST_WRITTEN_METRICS); this.markingFunctions = markingFunctions; this.queryMetricListResponseFactory = queryMetricListResponseFactory; + this.queryMetricSubplanResponseFactory = queryMetricSubplanResponseFactory; this.mergeLock = mergeLock; this.entryProcessorFactory = entryProcessorFactory; this.stats = stats; @@ -254,24 +259,7 @@ private void storeMetricUpdate(QueryMetricUpdateHolder metricUpdate) { storeTimer.stop(); } - /** - * Returns metrics for the current users queries that are identified by the id - * - * @param currentUser - * the current user - * @param queryId - * the query id - * @return datawave.webservice.result.QueryMetricListResponse base query metric list response - * @HTTP 200 success - * @HTTP 500 internal server error - */ - @PermitAll - @RequestMapping(path = "/id/{queryId}", method = {RequestMethod.GET}, - produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_HTML_VALUE}) - public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDetails currentUser, - @ApiParam("queryId to return") @PathVariable("queryId") String queryId) { - - BaseQueryMetricListResponse response = this.queryMetricListResponseFactory.createDetailedResponse(); + List getMetricList(String queryId, ProxiedUserDetails currentUser, String blacklistedFields) { List metricList = new ArrayList<>(); try { BaseQueryMetric metric; @@ -279,7 +267,7 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDet if (metricUpdate != null && metricUpdate.isNewMetric()) { metric = metricUpdate.getMetric(); } else { - metric = this.handler.getQueryMetric(queryId); + metric = this.handler.getQueryMetric(queryId, blacklistedFields); } if (metric != null) { boolean allowAllMetrics = false; @@ -309,8 +297,58 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDet } } } catch (Exception e) { - response.addException(new QueryException(e.getMessage(), 500)); + log.error(e.getMessage(), e); + } + return metricList; + } + + /** + * Returns metrics for the current users queries that are identified by the id + * + * @param currentUser + * the current user + * @param queryId + * the query id + * @return datawave.webservice.result.QueryMetricListResponse base query metric list response + * @HTTP 200 success + * @HTTP 500 internal server error + */ + @PermitAll + @RequestMapping(path = "/id/{queryId}", method = {RequestMethod.GET}, + produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_HTML_VALUE}) + public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDetails currentUser, + @ApiParam("queryId to return") @PathVariable("queryId") String queryId) { + + BaseQueryMetricListResponse response = this.queryMetricListResponseFactory.createDetailedResponse(); + List metricList = getMetricList(queryId, currentUser, "SUBPLAN"); + response.setResult(metricList); + if (metricList.isEmpty()) { + response.setHasResults(false); + } else { + response.setGeoQuery(metricList.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery)); + response.setHasResults(true); } + return response; + } + + /** + * Returns subplans for the current users queries that are identified by the id + * + * @param currentUser + * the current user + * @param queryId + * the query id + * @return datawave.webservice.result.QueryMetricListResponse base query metric list response + * @HTTP 200 success + * @HTTP 500 internal server errorsu + */ + @PermitAll + @RequestMapping(path = "/id/{queryId}/subplans", method = {RequestMethod.GET}, + produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_HTML_VALUE}) + public BaseQueryMetricSubplanResponse subplans(@AuthenticationPrincipal ProxiedUserDetails currentUser, + @ApiParam("queryId to return") @PathVariable("queryId") String queryId) { + BaseQueryMetricSubplanResponse response = this.queryMetricSubplanResponseFactory.createSubplanResponse(); + List metricList = getMetricList(queryId, currentUser, "PAGE_METRICS"); response.setResult(metricList); if (metricList.isEmpty()) { response.setHasResults(false); diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java index cad44787..0894e9d1 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java @@ -12,8 +12,10 @@ import datawave.microservice.querymetric.QueryMetricFactory; import datawave.microservice.querymetric.QueryMetricFactoryImpl; import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory; +import datawave.microservice.querymetric.factory.BaseQueryMetricSubplanResponseFactory; import datawave.microservice.querymetric.factory.QueryMetricListResponseFactory; import datawave.microservice.querymetric.factory.QueryMetricQueryLogicFactory; +import datawave.microservice.querymetric.factory.QueryMetricSubplanResponseFactory; import datawave.microservice.querymetric.handler.QueryGeometryHandler; import datawave.microservice.querymetric.handler.QueryMetricCombiner; import datawave.microservice.querymetric.handler.ShardTableQueryMetricHandler; @@ -92,6 +94,11 @@ public BaseQueryMetricListResponseFactory queryMetricListResponseFactory() { return new QueryMetricListResponseFactory(); } + @Bean + public BaseQueryMetricSubplanResponseFactory queryMetricSubplanResponseFactory() { + return new QueryMetricSubplanResponseFactory(); + } + @Bean @ConditionalOnMissingBean public LuceneToJexlQueryParser luceneToJexlQueryParser() { diff --git a/service/src/main/java/datawave/microservice/querymetric/factory/BaseQueryMetricSubplanResponseFactory.java b/service/src/main/java/datawave/microservice/querymetric/factory/BaseQueryMetricSubplanResponseFactory.java new file mode 100644 index 00000000..e5f41ceb --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/factory/BaseQueryMetricSubplanResponseFactory.java @@ -0,0 +1,8 @@ +package datawave.microservice.querymetric.factory; + +import datawave.microservice.querymetric.BaseQueryMetricSubplanResponse; + +public interface BaseQueryMetricSubplanResponseFactory { + + T createSubplanResponse(); +} diff --git a/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricSubplanResponseFactory.java b/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricSubplanResponseFactory.java new file mode 100644 index 00000000..4d73449f --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/factory/QueryMetricSubplanResponseFactory.java @@ -0,0 +1,12 @@ +package datawave.microservice.querymetric.factory; + +import datawave.microservice.querymetric.BaseQueryMetricSubplanResponse; +import datawave.microservice.querymetric.QueryMetricsSubplanResponse; + +public class QueryMetricSubplanResponseFactory implements BaseQueryMetricSubplanResponseFactory { + + @Override + public BaseQueryMetricSubplanResponse createSubplanResponse() { + return new QueryMetricsSubplanResponse(); + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index 8a677d0e..904e0bd2 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -16,7 +16,6 @@ import java.text.SimpleDateFormat; import java.util.Collection; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -297,6 +296,15 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored == null ? -1 : stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(updated.getSourceCount())); } + Map updatedSubPlans = updated.getSubPlans(); + if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { + Map storedSubPlans = stored == null ? null : stored.getSubPlans(); + for (Map.Entry entry : updatedSubPlans.entrySet()) { + if (storedSubPlans == null || !storedSubPlans.containsKey(entry.getKey())) { + fields.put("SUBPLAN", entry.getKey() + " : " + entry.getValue()); + } + } + } if (isFirstWrite(updated.getUser(), stored == null ? null : stored.getUser())) { fields.put("USER", updated.getUser()); } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index f1d3d22b..c141f682 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -188,7 +188,13 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy combinedMetric.setPlan(updatedQueryMetric.getPlan()); } // only update once - if (combinedMetric.getPredictions() == null && updatedQueryMetric.getPredictions() != null) { + if ((combinedMetric.getSubPlans() == null || combinedMetric.getSubPlans().isEmpty()) && updatedQueryMetric.getSubPlans() != null + && !updatedQueryMetric.getSubPlans().isEmpty()) { + combinedMetric.setSubPlans(updatedQueryMetric.getSubPlans()); + } + // only update once + if ((combinedMetric.getPredictions() == null || combinedMetric.getPredictions().isEmpty()) && updatedQueryMetric.getPredictions() != null + && !updatedQueryMetric.getPredictions().isEmpty()) { combinedMetric.setPredictions(updatedQueryMetric.getPredictions()); } // use the max numUpdates diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index a708b483..2b5040b9 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -28,6 +28,7 @@ import datawave.microservice.querymetric.QueryMetricsSummaryResponse; import datawave.microservice.querymetric.config.QueryMetricHandlerProperties; import datawave.microservice.querymetric.factory.QueryMetricQueryLogicFactory; +import datawave.query.QueryParameters; import datawave.query.iterator.QueryOptions; import datawave.query.language.parser.jexl.LuceneToJexlQueryParser; import datawave.security.authorization.DatawavePrincipal; @@ -371,7 +372,12 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy } public T getQueryMetric(final String queryId) throws Exception { - List queryMetrics = getQueryMetrics("QUERY_ID == '" + queryId + "'"); + List queryMetrics = getQueryMetrics("QUERY_ID == '" + queryId + "'", null); + return queryMetrics.isEmpty() ? null : queryMetrics.get(0); + } + + public T getQueryMetric(final String queryId, String blacklistedFields) throws Exception { + List queryMetrics = getQueryMetrics("QUERY_ID == '" + queryId + "'", blacklistedFields); return queryMetrics.isEmpty() ? null : queryMetrics.get(0); } @@ -379,7 +385,7 @@ public Query createQuery() { return new QueryImpl(); } - public List getQueryMetrics(final String query) throws Exception { + public List getQueryMetrics(final String query, String blacklistedFields) throws Exception { Date end = new Date(); Date begin = DateUtils.setYears(end, 2000); Query queryImpl = createQuery(); @@ -394,6 +400,9 @@ public List getQueryMetrics(final String query) throws Exception { queryImpl.setPagesize(1000); queryImpl.setId(UUID.randomUUID()); queryImpl.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true")); + if (StringUtils.isNotBlank(blacklistedFields)) { + queryImpl.addParameter(QueryParameters.BLACKLISTED_FIELDS, blacklistedFields); + } return getQueryMetrics(queryImpl); } @@ -500,6 +509,7 @@ public T toMetric(EventBase event) { List field = event.getFields(); m.setMarkings(event.getMarkings()); TreeMap pageMetrics = Maps.newTreeMap(); + Map subplans = new HashMap<>(); boolean createDateSet = false; for (FieldBase f : field) { @@ -622,6 +632,13 @@ public T toMetric(EventBase event) { } } else if (fieldName.equals("PLAN")) { m.setPlan(fieldValue); + } else if (fieldName.equals("SUBPLAN")) { + if (fieldValue != null) { + String[] arr = fieldValue.split(" : ", 2); + if (arr.length >= 2) { + subplans.put(arr[0], arr[1]); + } + } } else if (fieldName.equals("POSITIVE_SELECTORS")) { List positiveSelectors = m.getPositiveSelectors(); if (positiveSelectors == null) { @@ -703,6 +720,7 @@ public T toMetric(EventBase event) { } } + m.setSubPlans(subplans); m.setPageTimes(new ArrayList<>(pageMetrics.values())); return m; } catch (RuntimeException e) { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 06fadc4b..2a7bca26 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -1,6 +1,7 @@ package datawave.microservice.querymetric; import com.google.common.collect.Multimap; +import datawave.data.hash.UID; import datawave.microservice.querymetric.handler.ContentQueryMetricsIngestHelper; import datawave.microservice.querymetric.persistence.AccumuloMapStore; import datawave.util.StringUtils; @@ -9,6 +10,8 @@ import datawave.webservice.query.result.event.EventBase; import datawave.webservice.query.result.event.FieldBase; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.PartialKey; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.junit.After; import org.junit.Assert; @@ -25,6 +28,7 @@ import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -81,6 +85,56 @@ public void PageMetricTest() throws Exception { assertNoDuplicateFields(queryId); } + public String getRangeKey(Range range) { + Key key = range.getStartKey(); + StringBuilder builder = new StringBuilder(); + builder.append(key.getRow()); + String cf = key.getColumnFamily().toString(); + if (cf.length() > 0) { + String[] parts = StringUtils.split(cf, '\0'); + builder.append('/').append(parts[0]).append('/').append(parts[1]); + } + return builder.toString(); + } + + @Test + public void SubPlanTest() throws Exception { + String queryId = createQueryId(); + BaseQueryMetric m = createMetric(queryId); + + Date now = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + String date = sdf.format(now); + int numShardSubplans = 10; + for (int i = 0; i < numShardSubplans; i++) { + String shard = date + "_" + i; + Key begin = new Key(shard); + Key end = begin.followingKey(PartialKey.ROW); + m.addSubPlan(getRangeKey(new Range(begin, end)), "shardSubplan_" + i); + } + int numDocSubplans = 10; + for (int i = 0; i < numDocSubplans; i++) { + String shard = date + "_" + i; + String uid = UID.builder().newId().toString(); + Key begin = new Key(shard, "datatype\0" + uid); + Key end = begin.followingKey(PartialKey.ROW_COLFAM); + m.addSubPlan(getRangeKey(new Range(begin, end)), "documentSubplan_" + i); + } + int numSubplans = numShardSubplans + numDocSubplans; + + // @formatter:off + client.submit(new QueryMetricClient.Request.Builder() + .withMetric(m) + .withMetricType(QueryMetricType.COMPLETE) + .withUser(this.adminUser) + .build()); + // @formatter:on + this.ensureDataWritten(incomingQueryMetricsCache, lastWrittenQueryMetricCache, queryId); + BaseQueryMetric storedMetric = shardTableQueryMetricHandler.getQueryMetric(queryId); + Assert.assertEquals(numSubplans, storedMetric.getSubPlans().size()); + assertEquals(m, storedMetric); + } + @Test public void OutOfOrderLifecycleTest() throws Exception { int port = this.webServicePort; diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index 3139bae3..68519dad 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -29,6 +29,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.time.DateUtils; import org.apache.hadoop.io.Text; @@ -292,6 +293,7 @@ public static void assertEquals(String message, BaseQueryMetric m1, BaseQueryMet Assert.assertEquals(message + "docRanges", m1.getDocRanges(), m2.getDocRanges()); Assert.assertEquals(message + "fiRanges", m1.getFiRanges(), m2.getFiRanges()); Assert.assertTrue(message + "plan", assertObjectsEqual(m1.getPlan(), m2.getPlan())); + Assert.assertTrue(message + "subPlans", assertObjectsEqual(m1.getSubPlans(), m2.getSubPlans())); Assert.assertEquals(message + "loginTime", m1.getLoginTime(), m2.getLoginTime()); Assert.assertTrue(message + "predictions", assertObjectsEqual(m1.getPredictions(), m2.getPredictions())); Assert.assertEquals(message + "versionMap", m1.getVersionMap(), m2.getVersionMap()); From 6e7b6957601a54975a21dfd435ce7df2b8f771e4 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Fri, 29 Sep 2023 12:33:51 -0400 Subject: [PATCH 2/7] Change structure of stored subplan metrics to map --- .../querymetric/BaseQueryMetric.java | 10 ++-- .../BaseQueryMetricSubplanResponse.java | 5 +- .../microservice/querymetric/QueryMetric.java | 10 +++- .../ContentQueryMetricsIngestHelper.java | 9 ++-- .../handler/ShardTableQueryMetricHandler.java | 10 +++- .../QueryMetricConsistencyTest.java | 51 ++++++++++++++----- 6 files changed, 67 insertions(+), 28 deletions(-) diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index adb26ce0..3e6f8839 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -685,7 +685,7 @@ public int getFieldNumber(String name) { @XmlElement(name = "subplans") @XmlJavaTypeAdapter(StringMapAdapter.class) - protected Map subPlans = new HashMap<>(); + protected Map subPlans = new HashMap<>(); public static final String DATAWAVE = "DATAWAVE"; protected static final Map discoveredVersionMap = BaseQueryMetric.getVersionsFromClasspath(); @@ -696,15 +696,15 @@ public enum Lifecycle { NONE, DEFINED, INITIALIZED, RESULTS, CLOSED, CANCELLED, MAXRESULTS, NEXTTIMEOUT, TIMEOUT, SHUTDOWN, MAXWORK } - public void addSubPlan(String range, String plan) { - subPlans.put(range, plan); + public void addSubPlan(String plan, int[] rangeCounts) { + subPlans.put(plan, rangeCounts); } - public Map getSubPlans() { + public Map getSubPlans() { return subPlans; } - public void setSubPlans(Map subPlans) { + public void setSubPlans(Map subPlans) { this.subPlans = subPlans; } diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java index be2df79b..db1e8652 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java @@ -9,6 +9,7 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; @@ -132,7 +133,7 @@ public String getMainContent() { builder.append(""); if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) { int s = 0; - for (Map.Entry e : metric.getSubPlans().entrySet()) { + for (Map.Entry e : metric.getSubPlans().entrySet()) { // highlight alternating rows if (s % 2 == 0) { builder.append(""); @@ -140,7 +141,7 @@ public String getMainContent() { builder.append(""); } builder.append(""); - builder.append(""); + builder.append(""); builder.append("\n\n"); s++; } diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java index 361d857a..fefbd90e 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java @@ -410,7 +410,7 @@ public void writeTo(Output output, QueryMetric message) throws IOException { } if (message.subPlans != null) { - for (Map.Entry entry : message.subPlans.entrySet()) { + for (Map.Entry entry : message.subPlans.entrySet()) { output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true); } } @@ -570,7 +570,13 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException { String encodedPlans = input.readString(); String[] splitPlans = StringUtils.split(encodedPlans, "\0"); if (splitPlans.length == 2) { - message.subPlans.put(splitPlans[0], splitPlans[1]); + int[] rangeCounts = new int[2]; + int index = 0; + for (String count : splitPlans[1].substring(1, splitPlans[1].length() - 1).split(", ")) { + rangeCounts[index] = Integer.parseInt(count); + index++; + } + message.subPlans.put(splitPlans[0], rangeCounts); } break; default: diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index 904e0bd2..b859969c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -15,6 +15,7 @@ import org.slf4j.LoggerFactory; import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -296,12 +297,12 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored == null ? -1 : stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(updated.getSourceCount())); } - Map updatedSubPlans = updated.getSubPlans(); + Map updatedSubPlans = updated.getSubPlans(); if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { - Map storedSubPlans = stored == null ? null : stored.getSubPlans(); - for (Map.Entry entry : updatedSubPlans.entrySet()) { + Map storedSubPlans = stored == null ? null : stored.getSubPlans(); + for (Map.Entry entry : updatedSubPlans.entrySet()) { if (storedSubPlans == null || !storedSubPlans.containsKey(entry.getKey())) { - fields.put("SUBPLAN", entry.getKey() + " : " + entry.getValue()); + fields.put("SUBPLAN", entry.getKey() + " : " + Arrays.toString(entry.getValue())); } } } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 2b5040b9..213f23d3 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -509,7 +509,7 @@ public T toMetric(EventBase event) { List field = event.getFields(); m.setMarkings(event.getMarkings()); TreeMap pageMetrics = Maps.newTreeMap(); - Map subplans = new HashMap<>(); + Map subplans = new HashMap<>(); boolean createDateSet = false; for (FieldBase f : field) { @@ -635,8 +635,14 @@ public T toMetric(EventBase event) { } else if (fieldName.equals("SUBPLAN")) { if (fieldValue != null) { String[] arr = fieldValue.split(" : ", 2); + int[] rangeCounts = new int[2]; + int index = 0; + for (String count : arr[1].substring(1, arr[1].length() - 1).split(", ")) { + rangeCounts[index] = Integer.parseInt(count); + index++; + } if (arr.length >= 2) { - subplans.put(arr[0], arr[1]); + subplans.put(arr[0], rangeCounts); } } } else if (fieldName.equals("POSITIVE_SELECTORS")) { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 2a7bca26..7c3c2d3d 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -36,6 +36,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -85,23 +88,39 @@ public void PageMetricTest() throws Exception { assertNoDuplicateFields(queryId); } - public String getRangeKey(Range range) { + public void updateRangeCounts(Map shardCounts, Map documentCounts, + String subplan, Range range) { Key key = range.getStartKey(); - StringBuilder builder = new StringBuilder(); - builder.append(key.getRow()); String cf = key.getColumnFamily().toString(); if (cf.length() > 0) { - String[] parts = StringUtils.split(cf, '\0'); - builder.append('/').append(parts[0]).append('/').append(parts[1]); + if (documentCounts.containsKey(subplan)) { + documentCounts.put(subplan, documentCounts.get(subplan) + 1); + } else { + documentCounts.put(subplan, 1); + } + } else { + if (shardCounts.containsKey(subplan)) { + shardCounts.put(subplan, shardCounts.get(subplan) + 1); + } else { + shardCounts.put(subplan, 1); + } } - return builder.toString(); } - + @Test public void SubPlanTest() throws Exception { String queryId = createQueryId(); BaseQueryMetric m = createMetric(queryId); - + Map shardCounts = new HashMap<>(); + Map documentCounts = new HashMap<>(); + List plans = new ArrayList<>(); + plans.add("F1 == 'value1' || F2 == 'value2'"); + plans.add("F3 == 'value3' || F4 == 'value4'"); + plans.add("F2 == 'value2' || F3 == 'value3'"); + plans.add("F1 == 'value1' || F6 == 'value6'"); + plans.add("F1 == 'value1' || F5 == 'value5'"); + Random r = new Random(System.currentTimeMillis()); + Date now = new Date(); SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); String date = sdf.format(now); @@ -110,7 +129,8 @@ public void SubPlanTest() throws Exception { String shard = date + "_" + i; Key begin = new Key(shard); Key end = begin.followingKey(PartialKey.ROW); - m.addSubPlan(getRangeKey(new Range(begin, end)), "shardSubplan_" + i); + String subplan = plans.get(r.nextInt(plans.size())); + updateRangeCounts(shardCounts, documentCounts, subplan, new Range(begin, end)); } int numDocSubplans = 10; for (int i = 0; i < numDocSubplans; i++) { @@ -118,10 +138,15 @@ public void SubPlanTest() throws Exception { String uid = UID.builder().newId().toString(); Key begin = new Key(shard, "datatype\0" + uid); Key end = begin.followingKey(PartialKey.ROW_COLFAM); - m.addSubPlan(getRangeKey(new Range(begin, end)), "documentSubplan_" + i); + String subplan = plans.get(r.nextInt(plans.size())); + updateRangeCounts(shardCounts, documentCounts, subplan, new Range(begin, end)); } - int numSubplans = numShardSubplans + numDocSubplans; - + for (String p : plans) { + Integer shardCount = shardCounts.getOrDefault(p, 0); + Integer documentCount = documentCounts.getOrDefault(p, 0); + m.addSubPlan(p, new int[] {shardCount, documentCount}); + } + // @formatter:off client.submit(new QueryMetricClient.Request.Builder() .withMetric(m) @@ -131,7 +156,7 @@ public void SubPlanTest() throws Exception { // @formatter:on this.ensureDataWritten(incomingQueryMetricsCache, lastWrittenQueryMetricCache, queryId); BaseQueryMetric storedMetric = shardTableQueryMetricHandler.getQueryMetric(queryId); - Assert.assertEquals(numSubplans, storedMetric.getSubPlans().size()); + Assert.assertEquals(m.getSubPlans().size(), storedMetric.getSubPlans().size()); assertEquals(m, storedMetric); } From 92d12dddaaece512ae5ddf32aa7abc40209aad8a Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Fri, 29 Sep 2023 10:48:14 -0400 Subject: [PATCH 3/7] Use a Map instead of Map for subplans to facilitate serialization and comparison --- .../querymetric/BaseQueryMetric.java | 11 ++-- .../BaseQueryMetricSubplanResponse.java | 5 +- .../microservice/querymetric/QueryMetric.java | 14 ++--- .../StringIntegerListMapAdapter.java | 54 +++++++++++++++++++ .../ContentQueryMetricsIngestHelper.java | 8 +-- .../handler/ShardTableQueryMetricHandler.java | 15 +++--- .../QueryMetricConsistencyTest.java | 16 +++--- 7 files changed, 88 insertions(+), 35 deletions(-) create mode 100644 api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index 3e6f8839..d4b11c66 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -1,7 +1,6 @@ package datawave.microservice.querymetric; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.protobuf.Internal; import datawave.marking.MarkingFunctions; import datawave.webservice.query.Query; import datawave.webservice.query.QueryImpl.Parameter; @@ -684,8 +683,8 @@ public int getFieldNumber(String name) { protected Set predictions = new HashSet<>(); @XmlElement(name = "subplans") - @XmlJavaTypeAdapter(StringMapAdapter.class) - protected Map subPlans = new HashMap<>(); + @XmlJavaTypeAdapter(StringIntegerListMapAdapter.class) + protected Map> subPlans = new HashMap<>(); public static final String DATAWAVE = "DATAWAVE"; protected static final Map discoveredVersionMap = BaseQueryMetric.getVersionsFromClasspath(); @@ -696,15 +695,15 @@ public enum Lifecycle { NONE, DEFINED, INITIALIZED, RESULTS, CLOSED, CANCELLED, MAXRESULTS, NEXTTIMEOUT, TIMEOUT, SHUTDOWN, MAXWORK } - public void addSubPlan(String plan, int[] rangeCounts) { + public void addSubPlan(String plan, List rangeCounts) { subPlans.put(plan, rangeCounts); } - public Map getSubPlans() { + public Map> getSubPlans() { return subPlans; } - public void setSubPlans(Map subPlans) { + public void setSubPlans(Map> subPlans) { this.subPlans = subPlans; } diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java index db1e8652..a9f1ff77 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java @@ -9,6 +9,7 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -133,7 +134,7 @@ public String getMainContent() { builder.append(""); if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) { int s = 0; - for (Map.Entry e : metric.getSubPlans().entrySet()) { + for (Map.Entry> e : metric.getSubPlans().entrySet()) { // highlight alternating rows if (s % 2 == 0) { builder.append(""); @@ -141,7 +142,7 @@ public String getMainContent() { builder.append(""); } builder.append(""); - builder.append(""); + builder.append(""); builder.append("\n\n"); s++; } diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java index fefbd90e..926f158f 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java @@ -26,6 +26,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -410,8 +411,8 @@ public void writeTo(Output output, QueryMetric message) throws IOException { } if (message.subPlans != null) { - for (Map.Entry entry : message.subPlans.entrySet()) { - output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true); + for (Map.Entry> entry : message.subPlans.entrySet()) { + output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), StringUtils.join(entry.getValue(), ",")), "\0"), true); } } @@ -570,11 +571,10 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException { String encodedPlans = input.readString(); String[] splitPlans = StringUtils.split(encodedPlans, "\0"); if (splitPlans.length == 2) { - int[] rangeCounts = new int[2]; - int index = 0; - for (String count : splitPlans[1].substring(1, splitPlans[1].length() - 1).split(", ")) { - rangeCounts[index] = Integer.parseInt(count); - index++; + List rangeCounts = new ArrayList<>(); + String[] rangeCountSplit = StringUtils.split(splitPlans[1], ","); + for (String count : rangeCountSplit) { + rangeCounts.add(Integer.parseInt(count)); } message.subPlans.put(splitPlans[0], rangeCounts); } diff --git a/api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java b/api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java new file mode 100644 index 00000000..1dcfddde --- /dev/null +++ b/api/src/main/java/datawave/microservice/querymetric/StringIntegerListMapAdapter.java @@ -0,0 +1,54 @@ +package datawave.microservice.querymetric; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlValue; +import javax.xml.bind.annotation.adapters.XmlAdapter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Provides JAX-B marshalling/unmarshalling of {@link Map} of String to List of Integer. This allows the marshalled type to be in our own namespace rather than + * in the "default" one, which when triggered will then assume the "" prefix and push all of our own elements into the "ns2" prefix. + */ +public class StringIntegerListMapAdapter extends XmlAdapter>> { + + @Override + public Map> unmarshal(StringIntegerListMap v) throws Exception { + HashMap> map = new HashMap<>(); + for (StringIntegerListMapEntry entry : v.entries) { + map.put(entry.key, entry.value); + } + return map; + } + + @Override + public StringIntegerListMap marshal(Map> v) throws Exception { + StringIntegerListMap map = new StringIntegerListMap(); + for (Map.Entry> entry : v.entrySet()) { + map.entries.add(new StringIntegerListMapEntry(entry.getKey(), entry.getValue())); + } + return map; + } + + public static class StringIntegerListMap { + @XmlElement(name = "entry") + private List entries = new ArrayList<>(); + } + + public static class StringIntegerListMapEntry { + @XmlAttribute(name = "name") + private String key; + @XmlValue + private List value; + + public StringIntegerListMapEntry() {} + + public StringIntegerListMapEntry(String key, List value) { + this.key = key; + this.value = value; + } + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index b859969c..09b7614c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -297,12 +297,12 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored == null ? -1 : stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(updated.getSourceCount())); } - Map updatedSubPlans = updated.getSubPlans(); + Map> updatedSubPlans = updated.getSubPlans(); if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { - Map storedSubPlans = stored == null ? null : stored.getSubPlans(); - for (Map.Entry entry : updatedSubPlans.entrySet()) { + Map> storedSubPlans = stored == null ? null : stored.getSubPlans(); + for (Map.Entry> entry : updatedSubPlans.entrySet()) { if (storedSubPlans == null || !storedSubPlans.containsKey(entry.getKey())) { - fields.put("SUBPLAN", entry.getKey() + " : " + Arrays.toString(entry.getValue())); + fields.put("SUBPLAN", entry.getKey() + " : " + StringUtils.join(entry.getValue(), ",")); } } } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 213f23d3..4155d5c9 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -509,7 +509,7 @@ public T toMetric(EventBase event) { List field = event.getFields(); m.setMarkings(event.getMarkings()); TreeMap pageMetrics = Maps.newTreeMap(); - Map subplans = new HashMap<>(); + Map> subplans = new HashMap<>(); boolean createDateSet = false; for (FieldBase f : field) { @@ -635,13 +635,12 @@ public T toMetric(EventBase event) { } else if (fieldName.equals("SUBPLAN")) { if (fieldValue != null) { String[] arr = fieldValue.split(" : ", 2); - int[] rangeCounts = new int[2]; - int index = 0; - for (String count : arr[1].substring(1, arr[1].length() - 1).split(", ")) { - rangeCounts[index] = Integer.parseInt(count); - index++; - } - if (arr.length >= 2) { + if (arr.length == 2) { + List rangeCounts = new ArrayList<>(); + String[] rangeCountSplit = StringUtils.split(arr[1], ","); + for (String count : rangeCountSplit) { + rangeCounts.add(Integer.parseInt(count)); + } subplans.put(arr[0], rangeCounts); } } diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 7c3c2d3d..261bfda1 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -30,6 +30,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -88,8 +89,7 @@ public void PageMetricTest() throws Exception { assertNoDuplicateFields(queryId); } - public void updateRangeCounts(Map shardCounts, Map documentCounts, - String subplan, Range range) { + public void updateRangeCounts(Map shardCounts, Map documentCounts, String subplan, Range range) { Key key = range.getStartKey(); String cf = key.getColumnFamily().toString(); if (cf.length() > 0) { @@ -106,13 +106,13 @@ public void updateRangeCounts(Map shardCounts, Map shardCounts = new HashMap<>(); - Map documentCounts = new HashMap<>(); + Map shardCounts = new HashMap<>(); + Map documentCounts = new HashMap<>(); List plans = new ArrayList<>(); plans.add("F1 == 'value1' || F2 == 'value2'"); plans.add("F3 == 'value3' || F4 == 'value4'"); @@ -120,7 +120,7 @@ public void SubPlanTest() throws Exception { plans.add("F1 == 'value1' || F6 == 'value6'"); plans.add("F1 == 'value1' || F5 == 'value5'"); Random r = new Random(System.currentTimeMillis()); - + Date now = new Date(); SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); String date = sdf.format(now); @@ -144,9 +144,9 @@ public void SubPlanTest() throws Exception { for (String p : plans) { Integer shardCount = shardCounts.getOrDefault(p, 0); Integer documentCount = documentCounts.getOrDefault(p, 0); - m.addSubPlan(p, new int[] {shardCount, documentCount}); + m.addSubPlan(p, Arrays.asList(shardCount, documentCount)); } - + // @formatter:off client.submit(new QueryMetricClient.Request.Builder() .withMetric(m) From e642a197c6f8e4cccc761e59416de602eb6437dc Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 2 Oct 2023 12:52:04 -0400 Subject: [PATCH 4/7] Ensure proper combination and deletion of superseded subPlan values --- .../ContentQueryMetricsIngestHelper.java | 17 +++++++-- .../handler/QueryMetricCombiner.java | 36 +++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index 09b7614c..80fbe0b6 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -301,8 +301,10 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { Map> storedSubPlans = stored == null ? null : stored.getSubPlans(); for (Map.Entry> entry : updatedSubPlans.entrySet()) { - if (storedSubPlans == null || !storedSubPlans.containsKey(entry.getKey())) { - fields.put("SUBPLAN", entry.getKey() + " : " + StringUtils.join(entry.getValue(), ",")); + String subPlan = entry.getKey(); + List updatedRangeCounts = entry.getValue(); + if (storedSubPlans == null || !storedSubPlans.containsKey(subPlan) || !storedSubPlans.get(subPlan).equals(updatedRangeCounts)) { + fields.put("SUBPLAN", subPlan + " : " + StringUtils.join(updatedRangeCounts, ",")); } } } @@ -419,6 +421,17 @@ public Multimap getEventFieldsToDelete(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(stored.getSourceCount())); } + Map> updatedSubPlans = updated.getSubPlans(); + if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { + Map> storedSubPlans = stored.getSubPlans(); + for (Map.Entry> entry : updatedSubPlans.entrySet()) { + String subPlan = entry.getKey(); + List updatedRangeCounts = entry.getValue(); + if (storedSubPlans != null && storedSubPlans.containsKey(subPlan) && !storedSubPlans.get(subPlan).equals(updatedRangeCounts)) { + fields.put("SUBPLAN", subPlan + " : " + StringUtils.join(storedSubPlans.get(subPlan), ",")); + } + } + } if (isChanged(updated.getYieldCount(), stored.getYieldCount())) { fields.put("YIELD_COUNT", Long.toString(stored.getYieldCount())); } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index c141f682..9ef0ff04 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -3,13 +3,18 @@ import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.QueryMetricType; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; public class QueryMetricCombiner implements Serializable { @@ -183,15 +188,32 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges()); combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges()); } - // only update once - if (combinedMetric.getPlan() == null && updatedQueryMetric.getPlan() != null) { - combinedMetric.setPlan(updatedQueryMetric.getPlan()); + Map> newPlanMap = new HashMap<>(); + Map> storedSubPlans = new HashMap<>(); + Map> updatedSubPlans = new HashMap<>(); + if (combinedMetric.getSubPlans() != null) { + storedSubPlans.putAll(combinedMetric.getSubPlans()); } - // only update once - if ((combinedMetric.getSubPlans() == null || combinedMetric.getSubPlans().isEmpty()) && updatedQueryMetric.getSubPlans() != null - && !updatedQueryMetric.getSubPlans().isEmpty()) { - combinedMetric.setSubPlans(updatedQueryMetric.getSubPlans()); + if (updatedQueryMetric.getSubPlans() != null) { + updatedSubPlans.putAll(updatedQueryMetric.getSubPlans()); } + Set allSubPlanKeys = new HashSet<>(); + allSubPlanKeys.addAll(storedSubPlans.keySet()); + allSubPlanKeys.addAll(updatedSubPlans.keySet()); + for (String subplan : allSubPlanKeys) { + List storedCounts = storedSubPlans.getOrDefault(subplan, Collections.emptyList()); + List updatedCounts = updatedSubPlans.getOrDefault(subplan, Collections.emptyList()); + int maxSize = Math.max(storedCounts.size(), updatedCounts.size()); + List newCounts = new ArrayList<>(); + for (int i = 0; i < maxSize; i++) { + int stored = (i < storedCounts.size()) ? storedCounts.get(i) : 0; + int updated = (i < updatedCounts.size()) ? updatedCounts.get(i) : 0; + newCounts.add(Math.max(stored, updated)); + } + newPlanMap.put(subplan, newCounts); + } + combinedMetric.setSubPlans(newPlanMap); + // only update once if ((combinedMetric.getPredictions() == null || combinedMetric.getPredictions().isEmpty()) && updatedQueryMetric.getPredictions() != null && !updatedQueryMetric.getPredictions().isEmpty()) { From 4ed5fdf4beb531efa1ad336d3249649b2773d2fd Mon Sep 17 00:00:00 2001 From: foster33 Date: Thu, 7 Dec 2023 09:46:36 -0500 Subject: [PATCH 5/7] Implement RangeCount class & other refactoring --- .../querymetric/BaseQueryMetric.java | 20 ++++++++++---- .../BaseQueryMetricSubplanResponse.java | 6 ++--- .../microservice/querymetric/QueryMetric.java | 15 +++++++---- .../microservice/querymetric/RangeCounts.java | 23 ++++++++++++++++ .../ContentQueryMetricsIngestHelper.java | 23 ++++++++-------- .../handler/QueryMetricCombiner.java | 27 ++++++++++--------- .../handler/ShardTableQueryMetricHandler.java | 23 +++++++++++----- .../QueryMetricConsistencyTest.java | 7 ++--- 8 files changed, 98 insertions(+), 46 deletions(-) create mode 100644 api/src/main/java/datawave/microservice/querymetric/RangeCounts.java diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index d4b11c66..57211718 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -684,7 +684,7 @@ public int getFieldNumber(String name) { @XmlElement(name = "subplans") @XmlJavaTypeAdapter(StringIntegerListMapAdapter.class) - protected Map> subPlans = new HashMap<>(); + protected Map subPlans = new HashMap<>(); public static final String DATAWAVE = "DATAWAVE"; protected static final Map discoveredVersionMap = BaseQueryMetric.getVersionsFromClasspath(); @@ -695,15 +695,25 @@ public enum Lifecycle { NONE, DEFINED, INITIALIZED, RESULTS, CLOSED, CANCELLED, MAXRESULTS, NEXTTIMEOUT, TIMEOUT, SHUTDOWN, MAXWORK } - public void addSubPlan(String plan, List rangeCounts) { - subPlans.put(plan, rangeCounts); + public void addSubPlan(String plan, RangeCounts rangeCounts) { + synchronized (this.subPlans) { + if (subPlans.containsKey(plan)) { + RangeCounts combinedCounts = new RangeCounts(); + RangeCounts currentCounts = subPlans.get(plan); + combinedCounts.setDocumentRangeCount(currentCounts.getDocumentRangeCount() + rangeCounts.getDocumentRangeCount()); + combinedCounts.setShardRangeCount(currentCounts.getShardRangeCount() + rangeCounts.getShardRangeCount()); + subPlans.put(plan, combinedCounts); + } else { + subPlans.put(plan, rangeCounts); + } + } } - public Map> getSubPlans() { + public Map getSubPlans() { return subPlans; } - public void setSubPlans(Map> subPlans) { + public void setSubPlans(Map subPlans) { this.subPlans = subPlans; } diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java index a9f1ff77..e3768a78 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricSubplanResponse.java @@ -9,8 +9,6 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlTransient; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; @@ -134,7 +132,7 @@ public String getMainContent() { builder.append(""); if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) { int s = 0; - for (Map.Entry> e : metric.getSubPlans().entrySet()) { + for (Map.Entry e : metric.getSubPlans().entrySet()) { // highlight alternating rows if (s % 2 == 0) { builder.append(""); @@ -142,7 +140,7 @@ public String getMainContent() { builder.append(""); } builder.append(""); - builder.append(""); + builder.append(""); builder.append("\n\n"); s++; } diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java index 926f158f..b26c1505 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java @@ -411,7 +411,7 @@ public void writeTo(Output output, QueryMetric message) throws IOException { } if (message.subPlans != null) { - for (Map.Entry> entry : message.subPlans.entrySet()) { + for (Map.Entry entry : message.subPlans.entrySet()) { output.writeString(39, StringUtils.join(Arrays.asList(entry.getKey(), StringUtils.join(entry.getValue(), ",")), "\0"), true); } } @@ -571,10 +571,15 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException { String encodedPlans = input.readString(); String[] splitPlans = StringUtils.split(encodedPlans, "\0"); if (splitPlans.length == 2) { - List rangeCounts = new ArrayList<>(); - String[] rangeCountSplit = StringUtils.split(splitPlans[1], ","); - for (String count : rangeCountSplit) { - rangeCounts.add(Integer.parseInt(count)); + RangeCounts rangeCounts = new RangeCounts(); + int index = 0; + for (String count : StringUtils.split(splitPlans[1], ",")) { + if (index == 0) { + rangeCounts.setDocumentRangeCount(Integer.parseInt(count)); + } else if (index == 1) { + rangeCounts.setShardRangeCount(Integer.parseInt(count)); + } + index++; } message.subPlans.put(splitPlans[0], rangeCounts); } diff --git a/api/src/main/java/datawave/microservice/querymetric/RangeCounts.java b/api/src/main/java/datawave/microservice/querymetric/RangeCounts.java new file mode 100644 index 00000000..b07b67ab --- /dev/null +++ b/api/src/main/java/datawave/microservice/querymetric/RangeCounts.java @@ -0,0 +1,23 @@ +package datawave.microservice.querymetric; + +public class RangeCounts { + + private long documentRangeCount; + private long shardRangeCount; + + public long getDocumentRangeCount() { + return documentRangeCount; + } + + public long getShardRangeCount() { + return shardRangeCount; + } + + public void setDocumentRangeCount(long newDocumentRangeCount) { + this.documentRangeCount = newDocumentRangeCount; + } + + public void setShardRangeCount(long newShardRangeCount) { + this.shardRangeCount = newShardRangeCount; + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index 80fbe0b6..c8887f67 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -9,13 +9,13 @@ import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.BaseQueryMetric.Prediction; +import datawave.microservice.querymetric.RangeCounts; import datawave.webservice.query.util.QueryUtil; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.text.SimpleDateFormat; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -297,14 +297,14 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored == null ? -1 : stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(updated.getSourceCount())); } - Map> updatedSubPlans = updated.getSubPlans(); + Map updatedSubPlans = updated.getSubPlans(); if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { - Map> storedSubPlans = stored == null ? null : stored.getSubPlans(); - for (Map.Entry> entry : updatedSubPlans.entrySet()) { + Map storedSubPlans = stored == null ? null : stored.getSubPlans(); + for (Map.Entry entry : updatedSubPlans.entrySet()) { String subPlan = entry.getKey(); - List updatedRangeCounts = entry.getValue(); + RangeCounts updatedRangeCounts = entry.getValue(); if (storedSubPlans == null || !storedSubPlans.containsKey(subPlan) || !storedSubPlans.get(subPlan).equals(updatedRangeCounts)) { - fields.put("SUBPLAN", subPlan + " : " + StringUtils.join(updatedRangeCounts, ",")); + fields.put("SUBPLAN", subPlan + " : " + updatedRangeCounts.getDocumentRangeCount() + "," + updatedRangeCounts.getShardRangeCount()); } } } @@ -421,14 +421,15 @@ public Multimap getEventFieldsToDelete(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(stored.getSourceCount())); } - Map> updatedSubPlans = updated.getSubPlans(); + Map updatedSubPlans = updated.getSubPlans(); if (updatedSubPlans != null && !updatedSubPlans.isEmpty()) { - Map> storedSubPlans = stored.getSubPlans(); - for (Map.Entry> entry : updatedSubPlans.entrySet()) { + Map storedSubPlans = stored.getSubPlans(); + for (Map.Entry entry : updatedSubPlans.entrySet()) { String subPlan = entry.getKey(); - List updatedRangeCounts = entry.getValue(); + RangeCounts updatedRangeCounts = entry.getValue(); if (storedSubPlans != null && storedSubPlans.containsKey(subPlan) && !storedSubPlans.get(subPlan).equals(updatedRangeCounts)) { - fields.put("SUBPLAN", subPlan + " : " + StringUtils.join(storedSubPlans.get(subPlan), ",")); + fields.put("SUBPLAN", subPlan + " : [" + storedSubPlans.get(subPlan).getDocumentRangeCount() + ", " + + storedSubPlans.get(subPlan).getShardRangeCount() + "]"); } } } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index 9ef0ff04..22df3185 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -3,6 +3,7 @@ import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.QueryMetricType; +import datawave.microservice.querymetric.RangeCounts; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -188,9 +189,9 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges()); combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges()); } - Map> newPlanMap = new HashMap<>(); - Map> storedSubPlans = new HashMap<>(); - Map> updatedSubPlans = new HashMap<>(); + Map newPlanMap = new HashMap<>(); + Map storedSubPlans = new HashMap<>(); + Map updatedSubPlans = new HashMap<>(); if (combinedMetric.getSubPlans() != null) { storedSubPlans.putAll(combinedMetric.getSubPlans()); } @@ -201,15 +202,17 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy allSubPlanKeys.addAll(storedSubPlans.keySet()); allSubPlanKeys.addAll(updatedSubPlans.keySet()); for (String subplan : allSubPlanKeys) { - List storedCounts = storedSubPlans.getOrDefault(subplan, Collections.emptyList()); - List updatedCounts = updatedSubPlans.getOrDefault(subplan, Collections.emptyList()); - int maxSize = Math.max(storedCounts.size(), updatedCounts.size()); - List newCounts = new ArrayList<>(); - for (int i = 0; i < maxSize; i++) { - int stored = (i < storedCounts.size()) ? storedCounts.get(i) : 0; - int updated = (i < updatedCounts.size()) ? updatedCounts.get(i) : 0; - newCounts.add(Math.max(stored, updated)); - } + RangeCounts defaultRangeCount = new RangeCounts(); + RangeCounts storedCounts = storedSubPlans.getOrDefault(subplan, defaultRangeCount); + RangeCounts updatedCounts = updatedSubPlans.getOrDefault(subplan, defaultRangeCount); + + long documentCount = Math.max(storedCounts.getDocumentRangeCount(), updatedCounts.getDocumentRangeCount()); + long shardCount = Math.max(storedCounts.getShardRangeCount(), updatedCounts.getShardRangeCount()); + + RangeCounts newCounts = new RangeCounts(); + newCounts.setDocumentRangeCount(documentCount); + newCounts.setShardRangeCount(shardCount); + newPlanMap.put(subplan, newCounts); } combinedMetric.setSubPlans(newPlanMap); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 4155d5c9..534ae54c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -26,6 +26,7 @@ import datawave.microservice.querymetric.QueryMetricFactory; import datawave.microservice.querymetric.QueryMetricType; import datawave.microservice.querymetric.QueryMetricsSummaryResponse; +import datawave.microservice.querymetric.RangeCounts; import datawave.microservice.querymetric.config.QueryMetricHandlerProperties; import datawave.microservice.querymetric.factory.QueryMetricQueryLogicFactory; import datawave.query.QueryParameters; @@ -509,7 +510,7 @@ public T toMetric(EventBase event) { List field = event.getFields(); m.setMarkings(event.getMarkings()); TreeMap pageMetrics = Maps.newTreeMap(); - Map> subplans = new HashMap<>(); + Map subplans = new HashMap<>(); boolean createDateSet = false; for (FieldBase f : field) { @@ -636,11 +637,7 @@ public T toMetric(EventBase event) { if (fieldValue != null) { String[] arr = fieldValue.split(" : ", 2); if (arr.length == 2) { - List rangeCounts = new ArrayList<>(); - String[] rangeCountSplit = StringUtils.split(arr[1], ","); - for (String count : rangeCountSplit) { - rangeCounts.add(Integer.parseInt(count)); - } + RangeCounts rangeCounts = getRangeCounts(arr); subplans.put(arr[0], rangeCounts); } } @@ -733,6 +730,20 @@ public T toMetric(EventBase event) { } } + private static RangeCounts getRangeCounts(String[] arr) { + RangeCounts ranges = new RangeCounts(); + int index = 0; + for (String count : arr[1].substring(1, arr[1].length() - 1).split(", ")) { + if (index == 0) { + ranges.setDocumentRangeCount(Integer.parseInt(count)); + } else if (index == 1) { + ranges.setShardRangeCount(Integer.parseInt(count)); + } + index++; + } + return ranges; + } + protected void createAndConfigureTablesIfNecessary(String[] tableNames, AccumuloClient accumuloClient, Configuration conf) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { for (String table : tableNames) { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 261bfda1..14c36b19 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -142,9 +142,10 @@ public void SubPlanTest() throws Exception { updateRangeCounts(shardCounts, documentCounts, subplan, new Range(begin, end)); } for (String p : plans) { - Integer shardCount = shardCounts.getOrDefault(p, 0); - Integer documentCount = documentCounts.getOrDefault(p, 0); - m.addSubPlan(p, Arrays.asList(shardCount, documentCount)); + RangeCounts ranges = new RangeCounts(); + ranges.setShardRangeCount(shardCounts.getOrDefault(p, 0)); + ranges.setDocumentRangeCount(documentCounts.getOrDefault(p, 0)); + m.addSubPlan(p, ranges); } // @formatter:off From 8a0815b8cf8e2e8c63b7e90216a0e85b4f501d6b Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 18 Dec 2023 15:49:29 -0500 Subject: [PATCH 6/7] Fix SubPlanTest to compare subplans and whole metric, make RangeCounts Serializable and implemet equals/hashCode, correct toMetric and getRangeCounts to correctly parse rangeCounts from accumulo --- .../microservice/querymetric/RangeCounts.java | 27 ++++++++++++++++++- .../handler/ShardTableQueryMetricHandler.java | 7 +++-- .../QueryMetricConsistencyTest.java | 5 ++-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/datawave/microservice/querymetric/RangeCounts.java b/api/src/main/java/datawave/microservice/querymetric/RangeCounts.java index b07b67ab..c23e0730 100644 --- a/api/src/main/java/datawave/microservice/querymetric/RangeCounts.java +++ b/api/src/main/java/datawave/microservice/querymetric/RangeCounts.java @@ -1,6 +1,11 @@ package datawave.microservice.querymetric; -public class RangeCounts { +import java.io.Serializable; +import java.util.Objects; + +import org.apache.commons.lang3.builder.EqualsBuilder; + +public class RangeCounts implements Serializable { private long documentRangeCount; private long shardRangeCount; @@ -20,4 +25,24 @@ public void setDocumentRangeCount(long newDocumentRangeCount) { public void setShardRangeCount(long newShardRangeCount) { this.shardRangeCount = newShardRangeCount; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + RangeCounts that = (RangeCounts) o; + + return new EqualsBuilder().append(documentRangeCount, that.documentRangeCount).append(shardRangeCount, that.shardRangeCount).isEquals(); + } + + @Override + public int hashCode() { + return Objects.hash(documentRangeCount, shardRangeCount); + } } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index ad295a94..fb1e8c4c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -587,8 +587,7 @@ public T toMetric(EventBase event) { if (fieldValue != null) { String[] arr = fieldValue.split(" : ", 2); if (arr.length == 2) { - RangeCounts rangeCounts = getRangeCounts(arr); - subplans.put(arr[0], rangeCounts); + subplans.put(arr[0], getRangeCounts(arr[1])); } } } else if (fieldName.equals("POSITIVE_SELECTORS")) { @@ -680,10 +679,10 @@ public T toMetric(EventBase event) { } } - private static RangeCounts getRangeCounts(String[] arr) { + private static RangeCounts getRangeCounts(String s) { RangeCounts ranges = new RangeCounts(); int index = 0; - for (String count : arr[1].substring(1, arr[1].length() - 1).split(", ")) { + for (String count : StringUtils.split(s, ",")) { if (index == 0) { ranges.setDocumentRangeCount(Integer.parseInt(count)); } else if (index == 1) { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index c2c411e7..0e70a61c 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -108,6 +108,7 @@ public void updateRangeCounts(Map shardCounts, Map Date: Mon, 18 Dec 2023 20:42:04 -0500 Subject: [PATCH 7/7] Remove unused imports --- .../main/java/datawave/microservice/querymetric/QueryMetric.java | 1 - .../querymetric/handler/ShardTableQueryMetricHandler.java | 1 - .../datawave/microservice/querymetric/QueryMetricTestBase.java | 1 - 3 files changed, 3 deletions(-) diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java index 1f598d4c..eeb449d3 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetric.java @@ -8,7 +8,6 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.TreeMap; diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index fb1e8c4c..9ee5ebeb 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index c3bd2bc1..0c5f948c 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -29,7 +29,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.time.DateUtils; import org.apache.hadoop.io.Text;
VisibilityQuery DateUserUserDNProxy Server(s)Query IDQuery TypeQuery LogicQueryQuery PlanQuery NameBegin DateEnd DateParametersQuery AuthsQuery LogicQueryQuery PlanQuery Subplan(s)Query NameBegin DateEnd DateParametersQuery AuthsServerPredictionsLogin Time (ms)" : "") .append(StringEscapeUtils.escapeHtml4(metric.getQuery())).append("").append(StringEscapeUtils.escapeHtml4(metric.getPlan())).append("").append("Subplan(s)").append("").append(metric.getQueryName()).append("
RangeSub Plan
").append(e.getKey()).append("").append(e.getValue()).append("").append(Arrays.toString(e.getValue())).append("
RangeSub Plan
").append(e.getKey()).append("").append(Arrays.toString(e.getValue())).append("").append(e.getValue()).append("
RangeSub Plan
").append(e.getKey()).append("").append(e.getValue()).append("").append("[" + e.getValue().getDocumentRangeCount() + "," + e.getValue().getShardRangeCount() + "]").append("