From d2d3075026387b6994af388b28cc5945d811fa72 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 13 Jun 2023 16:00:29 -0400 Subject: [PATCH 01/48] Updated to use jexl3 --- .../querymetric/handler/BaseQueryMetricHandler.java | 4 ++-- .../querymetric/handler/SimpleQueryGeometryHandler.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/BaseQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/BaseQueryMetricHandler.java index a3baafaa..e3636356 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/BaseQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/BaseQueryMetricHandler.java @@ -8,8 +8,8 @@ import datawave.query.jexl.visitors.TreeFlatteningRebuildingVisitor; import datawave.query.language.parser.jexl.LuceneToJexlQueryParser; import datawave.query.language.tree.QueryNode; -import org.apache.commons.jexl2.parser.ASTEQNode; -import org.apache.commons.jexl2.parser.ASTJexlScript; +import org.apache.commons.jexl3.parser.ASTEQNode; +import org.apache.commons.jexl3.parser.ASTJexlScript; import org.apache.commons.lang.time.DateUtils; import org.apache.log4j.Logger; diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java index c2879a2a..cae4aa98 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java @@ -9,7 +9,7 @@ import datawave.webservice.query.QueryImpl; import datawave.webservice.query.map.QueryGeometry; import datawave.webservice.query.map.QueryGeometryResponse; -import org.apache.commons.jexl2.parser.JexlNode; +import org.apache.commons.jexl3.parser.JexlNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 87fc4762c2c8dc89d24d7c189002e1fca74c4dd4 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Fri, 17 Nov 2023 16:07:47 -0500 Subject: [PATCH 02/48] Correct combining logic for setupTime, createCallTime, loginTime; default setupTime to -1 instead of 0 --- .../microservice/querymetric/BaseQueryMetric.java | 2 +- .../querymetric/BaseQueryMetricListResponse.java | 2 +- .../querymetric/QueryMetricsDetailListResponse.java | 2 +- .../querymetric/handler/QueryMetricCombiner.java | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index bbe52fc4..ad1dac9c 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -617,7 +617,7 @@ public int getFieldNumber(String name) { @XmlElement protected String queryId = null; @XmlElement - protected long setupTime = 0; + protected long setupTime = -1; @XmlElement protected String query = null; @XmlElement diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java index 3fa12288..2a95eb76 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java @@ -155,7 +155,7 @@ public String getMainContent() { builder.append("").append(queryAuths).append(""); builder.append("").append(metric.getHost()).append(""); - builder.append("").append(metric.getSetupTime()).append(""); + builder.append("").append(numToString(metric.getSetupTime())).append(""); builder.append("").append(numToString(metric.getCreateCallTime())).append("\n"); builder.append("").append(metric.getNumPages()).append(""); builder.append("").append(metric.getNumResults()).append(""); diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java index 04abde70..4a88e359 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java @@ -120,7 +120,7 @@ public String getMainContent() { builder.append(""); } builder.append("").append(numToString(metric.getLoginTime(), 0)).append(""); - builder.append("").append(metric.getSetupTime()).append(""); + builder.append("").append(numToString(metric.getSetupTime(), 0)).append(""); builder.append("").append(numToString(metric.getCreateCallTime(), 0)).append("\n"); builder.append("").append(metric.getNumPages()).append(""); builder.append("").append(metric.getNumResults()).append(""); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index 8a33815d..c570629c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -156,16 +156,16 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy if (combinedMetric.getParameters() == null && updatedQueryMetric.getParameters() != null) { combinedMetric.setParameters(updatedQueryMetric.getParameters()); } - // only update once - if (combinedMetric.getSetupTime() > -1) { + // if updatedQueryMetric.setupTime is greater than combinedMetric.setupTime then update + if (updatedQueryMetric.getSetupTime() > combinedMetric.getSetupTime()) { combinedMetric.setSetupTime(updatedQueryMetric.getSetupTime()); } - // only update once - if (combinedMetric.getCreateCallTime() > -1) { + // if updatedQueryMetric.createCallTime is greater than combinedMetric.createCallTime then update + if (updatedQueryMetric.getCreateCallTime() > combinedMetric.getCreateCallTime()) { combinedMetric.setCreateCallTime(updatedQueryMetric.getCreateCallTime()); } - // only update once - if (combinedMetric.getLoginTime() > -1) { + // if updatedQueryMetric.loginTime is greater than combinedMetric.loginTime then update + if (updatedQueryMetric.getLoginTime() > combinedMetric.getLoginTime()) { combinedMetric.setLoginTime(updatedQueryMetric.getLoginTime()); } From 9f12a71d1990a39cefcde2afd515ad1826258b99 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Fri, 17 Nov 2023 16:07:47 -0500 Subject: [PATCH 03/48] Correct combining logic for setupTime, createCallTime, loginTime; default setupTime to -1 instead of 0 --- .../microservice/querymetric/BaseQueryMetric.java | 2 +- .../querymetric/BaseQueryMetricListResponse.java | 2 +- .../querymetric/QueryMetricsDetailListResponse.java | 2 +- .../querymetric/handler/QueryMetricCombiner.java | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java index 3a1168a0..094b98dd 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java @@ -615,7 +615,7 @@ public int getFieldNumber(String name) { @XmlElement protected String queryId = null; @XmlElement - protected long setupTime = 0; + protected long setupTime = -1; @XmlElement protected String query = null; @XmlElement diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java index 2647d3a0..4f674493 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java @@ -153,7 +153,7 @@ public String getMainContent() { builder.append("").append(queryAuths).append(""); builder.append("").append(metric.getHost()).append(""); - builder.append("").append(metric.getSetupTime()).append(""); + builder.append("").append(numToString(metric.getSetupTime())).append(""); builder.append("").append(numToString(metric.getCreateCallTime())).append("\n"); builder.append("").append(metric.getNumPages()).append(""); builder.append("").append(metric.getNumResults()).append(""); diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java index 2631ffa2..8a81d735 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java @@ -109,7 +109,7 @@ public String getMainContent() { builder.append(""); } builder.append("").append(numToString(metric.getLoginTime(), 0)).append(""); - builder.append("").append(metric.getSetupTime()).append(""); + builder.append("").append(numToString(metric.getSetupTime(), 0)).append(""); builder.append("").append(numToString(metric.getCreateCallTime(), 0)).append("\n"); builder.append("").append(metric.getNumPages()).append(""); builder.append("").append(metric.getNumResults()).append(""); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index f1d3d22b..2c65346f 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -155,16 +155,16 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy if (combinedMetric.getParameters() == null && updatedQueryMetric.getParameters() != null) { combinedMetric.setParameters(updatedQueryMetric.getParameters()); } - // only update once - if (combinedMetric.getSetupTime() > -1) { + // if updatedQueryMetric.setupTime is greater than combinedMetric.setupTime then update + if (updatedQueryMetric.getSetupTime() > combinedMetric.getSetupTime()) { combinedMetric.setSetupTime(updatedQueryMetric.getSetupTime()); } - // only update once - if (combinedMetric.getCreateCallTime() > -1) { + // if updatedQueryMetric.createCallTime is greater than combinedMetric.createCallTime then update + if (updatedQueryMetric.getCreateCallTime() > combinedMetric.getCreateCallTime()) { combinedMetric.setCreateCallTime(updatedQueryMetric.getCreateCallTime()); } - // only update once - if (combinedMetric.getLoginTime() > -1) { + // if updatedQueryMetric.loginTime is greater than combinedMetric.loginTime then update + if (updatedQueryMetric.getLoginTime() > combinedMetric.getLoginTime()) { combinedMetric.setLoginTime(updatedQueryMetric.getLoginTime()); } From f388ed02c0b8fd3fc101bbd7879c50406a234247 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 20 Nov 2023 11:30:36 -0500 Subject: [PATCH 04/48] JexlFormattedStringBuildingVisitor.formatMetrics can throw StackOverflowError; protect with try/catch and add unformatted metric as a backup --- .../querymetric/QueryMetricOperations.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index f4e71737..08739dfd 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -12,6 +12,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.TimeZone; @@ -326,11 +327,27 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal DatawaveUserDe response.addException(new QueryException(e.getMessage(), 500)); } // Set the result to have the formatted query and query plan - response.setResult(JexlFormattedStringBuildingVisitor.formatMetrics(metricList)); - if (metricList.isEmpty()) { + // StackOverflowErrors seen in JexlFormattedStringBuildingVisitor.formatMetrics, so protect + // this call for each metric with try/catch and add original metric if formatMetrics fails + List fmtMetricList = new ArrayList<>(); + for (BaseQueryMetric m : metricList) { + List formatted = null; + try { + formatted = JexlFormattedStringBuildingVisitor.formatMetrics(Collections.singletonList(m)); + } catch (StackOverflowError | Exception e) { + log.warn(String.format("%s while formatting metric %s: %s", e.getClass().getCanonicalName(), m.getQueryId(), e.getMessage())); + } + if (formatted == null || formatted.isEmpty()) { + fmtMetricList.add(m); + } else { + fmtMetricList.addAll(formatted); + } + } + response.setResult(fmtMetricList); + if (fmtMetricList.isEmpty()) { response.setHasResults(false); } else { - response.setGeoQuery(metricList.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery)); + response.setGeoQuery(fmtMetricList.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery)); response.setHasResults(true); } return response; From cf19fbf7112db308ed20f2d89bda71338cc2f633 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 20 Nov 2023 20:16:41 +0000 Subject: [PATCH 05/48] enabled confirm acks and added retry logic. --- .../querymetric/QueryMetricOperations.java | 131 ++++++++++++++++-- .../QueryMetricHandlerConfiguration.java | 2 +- .../config/QueryMetricProperties.java | 79 +++++++++++ .../src/main/resources/config/bootstrap.yml | 9 +- .../src/test/resources/config/application.yml | 1 + 5 files changed, 212 insertions(+), 10 deletions(-) create mode 100644 service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 08739dfd..7c2802a4 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -15,7 +15,12 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.TimeZone; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import javax.annotation.PreDestroy; import javax.annotation.security.PermitAll; @@ -28,10 +33,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.http.MediaType; -import org.springframework.messaging.support.MessageBuilder; +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.integration.annotation.ServiceActivator; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.security.access.annotation.Secured; @@ -45,12 +54,13 @@ import com.codahale.metrics.Timer; import com.hazelcast.core.HazelcastInstanceNotActiveException; -import com.hazelcast.map.EntryProcessor; import com.hazelcast.map.IMap; import com.hazelcast.spring.cache.HazelcastCacheManager; import datawave.marking.MarkingFunctions; import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.querymetric.config.QueryMetricProperties; +import datawave.microservice.querymetric.config.QueryMetricProperties.Retry; import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory; import datawave.microservice.querymetric.function.QueryMetricSupplier; import datawave.microservice.querymetric.handler.QueryGeometryHandler; @@ -78,9 +88,12 @@ @RestController @RequestMapping(path = "/v1") public class QueryMetricOperations { + // Note: This must match 'confirmAckChannel' in the service configuration. Default set in bootstrap.yml. + public static final String CONFIRM_ACK_CHANNEL = "confirmAckChannel"; private Logger log = LoggerFactory.getLogger(QueryMetricOperations.class); + private QueryMetricProperties queryMetricProperties; private ShardTableQueryMetricHandler handler; private QueryGeometryHandler geometryHandler; private CacheManager cacheManager; @@ -95,6 +108,8 @@ public class QueryMetricOperations { private final QueryMetricSupplier queryMetricSupplier; private final DnUtils dnUtils; + private static final Map correlationLatchMap = new ConcurrentHashMap<>(); + /** * The enum Default datetime. */ @@ -112,6 +127,8 @@ enum DEFAULT_DATETIME { /** * Instantiates a new QueryMetricOperations. * + * @param queryMetricProperties + * the query metric properties * @param cacheManager * the CacheManager * @param handler @@ -134,10 +151,12 @@ enum DEFAULT_DATETIME { * the stats */ @Autowired - public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cacheManager, ShardTableQueryMetricHandler handler, - QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, BaseQueryMetricListResponseFactory queryMetricListResponseFactory, - MergeLockLifecycleListener mergeLock, MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats, - QueryMetricSupplier queryMetricSupplier, DnUtils dnUtils) { + public QueryMetricOperations(QueryMetricProperties queryMetricProperties, @Named("queryMetricCacheManager") CacheManager cacheManager, + ShardTableQueryMetricHandler handler, QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, + BaseQueryMetricListResponseFactory queryMetricListResponseFactory, MergeLockLifecycleListener mergeLock, + MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats, QueryMetricSupplier queryMetricSupplier, + DnUtils dnUtils) { + this.queryMetricProperties = queryMetricProperties; this.handler = handler; this.geometryHandler = geometryHandler; this.cacheManager = cacheManager; @@ -186,7 +205,9 @@ public VoidResponse updateMetrics(@RequestBody List queryMetric } else { log.debug("received metric update via REST: " + m.getQueryId()); } - queryMetricSupplier.send(MessageBuilder.withPayload(new QueryMetricUpdate(m, metricType)).build()); + if (!updateMetric(new QueryMetricUpdate(m, metricType))) { + throw new RuntimeException("Unable to process query metric update for query [" + m.getQueryId() + "]"); + } } return response; } @@ -217,10 +238,104 @@ public VoidResponse updateMetric(@RequestBody BaseQueryMetric queryMetric, } else { log.debug("received metric update via REST: " + queryMetric.getQueryId()); } - queryMetricSupplier.send(MessageBuilder.withPayload(new QueryMetricUpdate(queryMetric, metricType)).build()); + if (!updateMetric(new QueryMetricUpdate(queryMetric, metricType))) { + throw new RuntimeException("Unable to process query metric update for query [" + queryMetric.getQueryId() + "]"); + } return new VoidResponse(); } + /** + * Receives producer confirm acks, and disengages the latch associated with the given correlation ID. + * + * @param message + * the confirmation ack message + */ + @ConditionalOnProperty(value = "datawave.query.metric.confirmAckEnabled", havingValue = "true", matchIfMissing = true) + @ServiceActivator(inputChannel = CONFIRM_ACK_CHANNEL) + public void processConfirmAck(Message message) { + Object headerObj = message.getHeaders().get(IntegrationMessageHeaderAccessor.CORRELATION_ID); + + if (headerObj != null) { + String correlationId = headerObj.toString(); + if (correlationLatchMap.containsKey(correlationId)) { + correlationLatchMap.get(correlationId).countDown(); + } else + log.warn("Unable to decrement latch for ID [{}]", correlationId); + } else { + log.warn("No correlation ID found in confirm ack message"); + } + } + + private boolean updateMetric(QueryMetricUpdate update) { + + boolean success; + final long updateStartTime = System.currentTimeMillis(); + long currentTime; + int attempts = 0; + + Retry retry = queryMetricProperties.getRetry(); + + do { + if (attempts++ > 0) { + try { + Thread.sleep(retry.getBackoffIntervalMillis()); + } catch (InterruptedException e) { + // Ignore -- we'll just end up retrying a little too fast + } + } + + if (log.isDebugEnabled()) { + log.debug("Update attempt {} of {} for query {}", attempts, retry.getMaxAttempts(), update.getMetric().getQueryId()); + } + + success = sendMessage(update); + currentTime = System.currentTimeMillis(); + } while (!success && (currentTime - updateStartTime) < retry.getFailTimeoutMillis() && attempts < retry.getMaxAttempts()); + + if (!success) { + log.warn("Update for query {} failed. {attempts = {}, elapsedMillis = {}}", update.getMetric().getQueryId(), attempts, + (currentTime - updateStartTime)); + } else { + log.info("Update for query {} successful. {attempts = {}, elapsedMillis = {}}", update.getMetric().getQueryId(), attempts, + (currentTime - updateStartTime)); + } + + return success; + } + + /** + * Passes query metric messages to the messaging infrastructure. + *

+ * The metric ID is used as a correlation ID in order to ensure that a producer confirm ack is received. If a producer confirm ack is not received within + * the specified amount of time, a 500 Internal Server Error will be returned to the caller. + * + * @param update + * The query metric update to be sent + */ + private boolean sendMessage(QueryMetricUpdate update) { + String correlationId = UUID.randomUUID().toString(); + + CountDownLatch latch = null; + if (queryMetricProperties.isConfirmAckEnabled()) { + latch = new CountDownLatch(1); + correlationLatchMap.put(correlationId, latch); + } + + boolean success = queryMetricSupplier.send(MessageBuilder.withPayload(update).setCorrelationId(correlationId).build()); + + if (queryMetricProperties.isConfirmAckEnabled()) { + try { + success = success && latch.await(queryMetricProperties.getConfirmAckTimeoutMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + success = false; + } finally { + correlationLatchMap.remove(correlationId); + } + } + + return success; + } + /** * Handle event. * diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java index 53374fcf..fe24968b 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java @@ -56,7 +56,7 @@ import datawave.webservice.query.result.event.ResponseObjectFactory; @Configuration -@EnableConfigurationProperties({QueryMetricHandlerProperties.class, TimelyProperties.class}) +@EnableConfigurationProperties({QueryMetricProperties.class, QueryMetricHandlerProperties.class, TimelyProperties.class}) public class QueryMetricHandlerConfiguration { @Bean diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java new file mode 100644 index 00000000..a43af671 --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java @@ -0,0 +1,79 @@ +package datawave.microservice.querymetric.config; + +import java.util.concurrent.TimeUnit; + +import javax.validation.Valid; +import javax.validation.constraints.PositiveOrZero; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Validated +@ConfigurationProperties(prefix = "datawave.query.metric") +public class QueryMetricProperties { + private boolean confirmAckEnabled = true; + private long confirmAckTimeoutMillis = 500L; + + @Valid + private Retry retry = new Retry(); + + public boolean isConfirmAckEnabled() { + return confirmAckEnabled; + } + + public void setConfirmAckEnabled(boolean confirmAckEnabled) { + this.confirmAckEnabled = confirmAckEnabled; + } + + public long getConfirmAckTimeoutMillis() { + return confirmAckTimeoutMillis; + } + + public void setConfirmAckTimeoutMillis(long confirmAckTimeoutMillis) { + this.confirmAckTimeoutMillis = confirmAckTimeoutMillis; + } + + public Retry getRetry() { + return retry; + } + + public void setRetry(Retry retry) { + this.retry = retry; + } + + @Validated + public static class Retry { + @PositiveOrZero + private int maxAttempts = 10; + + @PositiveOrZero + private long failTimeoutMillis = TimeUnit.MINUTES.toMillis(5); + + @PositiveOrZero + private long backoffIntervalMillis = TimeUnit.SECONDS.toMillis(5); + + public int getMaxAttempts() { + return maxAttempts; + } + + public void setMaxAttempts(int maxAttempts) { + this.maxAttempts = maxAttempts; + } + + public long getFailTimeoutMillis() { + return failTimeoutMillis; + } + + public void setFailTimeoutMillis(long failTimeoutMillis) { + this.failTimeoutMillis = failTimeoutMillis; + } + + public long getBackoffIntervalMillis() { + return backoffIntervalMillis; + } + + public void setBackoffIntervalMillis(long backoffIntervalMillis) { + this.backoffIntervalMillis = backoffIntervalMillis; + } + } +} diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index bb3cf0fe..817926f8 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -13,7 +13,14 @@ spring: max-attempts: 60 uri: '${CONFIG_SERVER_URL:http://configuration:8888/configserver}' allow-override: true - + stream: + rabbit: + bindings: + queryMetricSource-out-0: + producer: + producer: + # Note: This must match CONFIRM_ACK_CHANNEL in AuditController.java or producer confirms will not work. + confirmAckChannel: 'confirmAckChannel' datawave: table: cache: diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index ee4768bc..c47433f1 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -96,6 +96,7 @@ datawave: logServiceStatsRateMs: 300000 publishServiceStatsToTimelyRateMs: 60000 publishQueryStatsToTimelyRateMs: 60000 + confirmAckEnabled: false metadata: all-metadata-auths: From 6e48eb7e5a2100c0793aab197b485fb9c5c4b354 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Wed, 22 Nov 2023 11:19:37 -0500 Subject: [PATCH 06/48] Write lock until Hazelcast instance set up, prevent deadlock on subsequent unlocking if previous write lock timed out --- .../MergeLockLifecycleListener.java | 21 +++++++++++++++---- .../HazelcastMetricCacheConfiguration.java | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java b/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java index d052e1e9..04ee8182 100644 --- a/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java +++ b/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java @@ -91,12 +91,12 @@ public void stateChanged(LifecycleEvent event) { switch (event.getState()) { case MERGING: // lock for a maximum time so that we don't lock forever - this.writeLockRunnable.lock(120000, event.getState()); + this.writeLockRunnable.lock(event.getState(), 5, TimeUnit.MINUTES); log.info(event + " [" + getLocalMemberUuid() + "]"); break; case SHUTTING_DOWN: // lock for a maximum time so that we don't lock forever - this.writeLockRunnable.lock(60000, event.getState()); + this.writeLockRunnable.lock(event.getState(), 60, TimeUnit.SECONDS); makeServiceUnavailable(); log.info(event + " [" + getLocalMemberUuid() + "]"); break; @@ -178,6 +178,11 @@ public void run() { } else { try { Thread.sleep(100); + // If a write lock timed out and the clusterLock is no longer locked when requestUnlock + // happens, we should reset requestUnlock to false and allow the requesting thread to continue + if (this.requestUnlock.get() == true && !this.clusterLock.isWriteLocked()) { + this.requestUnlock.set(false); + } } catch (InterruptedException e) { log.error(e.getMessage(), e); } @@ -185,11 +190,19 @@ public void run() { } } - public void lock(long maxLockMilliseconds, LifecycleEvent.LifecycleState state) { + public void lock(LifecycleEvent.LifecycleState state) { + lock(state, -1, TimeUnit.MILLISECONDS); + } + + public void lock(LifecycleEvent.LifecycleState state, long maxDuration, TimeUnit timeUnit) { log.info("locking for write [" + state + "]"); // prompt run() method to lock the writeLock if (this.requestLock.compareAndSet(false, true)) { - this.scheduledUnlockTime = System.currentTimeMillis() + maxLockMilliseconds; + if (maxDuration >= 0) { + this.scheduledUnlockTime = System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(maxDuration, timeUnit); + } else { + this.scheduledUnlockTime = Long.MAX_VALUE; + } // wait until run() method locks the writeLock and sets requestLock to false while (requestLock.get() == true && !isShuttingDown()) { try { diff --git a/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java index 2b76eb4c..b62392ac 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java @@ -66,7 +66,7 @@ HazelcastInstance hazelcastInstance(Config config, @Qualifier("store") AccumuloM MergeLockLifecycleListener lifecycleListener) { // Autowire both the AccumuloMapStore and AccumuloMapLoader so that they both get created // Ensure that the lastWrittenQueryMetricCache is set into the MapStore before the instance is active and the writeLock is released - lifecycleListener.writeLockRunnable.lock(60000, LifecycleEvent.LifecycleState.STARTING); + lifecycleListener.writeLockRunnable.lock(LifecycleEvent.LifecycleState.STARTING); HazelcastInstance instance = Hazelcast.newHazelcastInstance(config); try { From 0e09b61dba0e010e3a6e9f21f778c7f9ae4d973c Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Wed, 22 Nov 2023 11:19:37 -0500 Subject: [PATCH 07/48] Write lock until Hazelcast instance set up, prevent deadlock on subsequent unlocking if previous write lock timed out --- .../MergeLockLifecycleListener.java | 21 +++++++++++++++---- .../HazelcastMetricCacheConfiguration.java | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java b/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java index df51705c..9184bc19 100644 --- a/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java +++ b/service/src/main/java/datawave/microservice/querymetric/MergeLockLifecycleListener.java @@ -89,12 +89,12 @@ public void stateChanged(LifecycleEvent event) { switch (event.getState()) { case MERGING: // lock for a maximum time so that we don't lock forever - this.writeLockRunnable.lock(120000, event.getState()); + this.writeLockRunnable.lock(event.getState(), 5, TimeUnit.MINUTES); log.info(event + " [" + getLocalMemberUuid() + "]"); break; case SHUTTING_DOWN: // lock for a maximum time so that we don't lock forever - this.writeLockRunnable.lock(60000, event.getState()); + this.writeLockRunnable.lock(event.getState(), 60, TimeUnit.SECONDS); makeServiceUnavailable(); log.info(event + " [" + getLocalMemberUuid() + "]"); break; @@ -176,6 +176,11 @@ public void run() { } else { try { Thread.sleep(100); + // If a write lock timed out and the clusterLock is no longer locked when requestUnlock + // happens, we should reset requestUnlock to false and allow the requesting thread to continue + if (this.requestUnlock.get() == true && !this.clusterLock.isWriteLocked()) { + this.requestUnlock.set(false); + } } catch (InterruptedException e) { log.error(e.getMessage(), e); } @@ -183,11 +188,19 @@ public void run() { } } - public void lock(long maxLockMilliseconds, LifecycleEvent.LifecycleState state) { + public void lock(LifecycleEvent.LifecycleState state) { + lock(state, -1, TimeUnit.MILLISECONDS); + } + + public void lock(LifecycleEvent.LifecycleState state, long maxDuration, TimeUnit timeUnit) { log.info("locking for write [" + state + "]"); // prompt run() method to lock the writeLock if (this.requestLock.compareAndSet(false, true)) { - this.scheduledUnlockTime = System.currentTimeMillis() + maxLockMilliseconds; + if (maxDuration >= 0) { + this.scheduledUnlockTime = System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(maxDuration, timeUnit); + } else { + this.scheduledUnlockTime = Long.MAX_VALUE; + } // wait until run() method locks the writeLock and sets requestLock to false while (requestLock.get() == true && !isShuttingDown()) { try { diff --git a/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java index fe92f061..cc29f716 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java @@ -66,7 +66,7 @@ HazelcastInstance hazelcastInstance(Config config, @Qualifier("store") AccumuloM MergeLockLifecycleListener lifecycleListener) { // Autowire both the AccumuloMapStore and AccumuloMapLoader so that they both get created // Ensure that the lastWrittenQueryMetricCache is set into the MapStore before the instance is active and the writeLock is released - lifecycleListener.writeLockRunnable.lock(60000, LifecycleEvent.LifecycleState.STARTING); + lifecycleListener.writeLockRunnable.lock(LifecycleEvent.LifecycleState.STARTING); HazelcastInstance instance = Hazelcast.newHazelcastInstance(config); try { From 517c16f6b9679179dab8983053ceaafd0a266acc Mon Sep 17 00:00:00 2001 From: Keith Ratcliffe Date: Wed, 22 Nov 2023 12:22:34 -0500 Subject: [PATCH 08/48] Closes #13 - Updating DW XMLNS for webservice responses (#14) From 985f0cdaf1b3cba9cad0622260a1a250c6f0333b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 22 Nov 2023 17:58:17 +0000 Subject: [PATCH 09/48] fixed confirm ack configuration, and updated query client response type. --- .../config/QueryMetricHandlerConfiguration.java | 5 +++-- .../RemoteShardTableQueryMetricHandler.java | 14 +++++++++----- service/src/main/resources/config/bootstrap.yml | 5 ++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java index 4d5e9c0d..db54d35b 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java @@ -94,10 +94,11 @@ QueryMetricFactory queryMetricFactory() { public ShardTableQueryMetricHandler shardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, @Qualifier("warehouse") AccumuloClientPool accumuloClientPool, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, - WebClient.Builder webClientBuilder, @Autowired(required = false) JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { + ResponseObjectFactory responseObjectFactory, WebClient.Builder webClientBuilder, + @Autowired(required = false) JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { if (queryMetricHandlerProperties.isUseRemoteQuery()) { return new RemoteShardTableQueryMetricHandler(queryMetricHandlerProperties, accumuloClientPool, logicFactory, metricFactory, markingFunctions, - queryMetricCombiner, luceneToJexlQueryParser, webClientBuilder, jwtTokenHandler, dnUtils); + queryMetricCombiner, luceneToJexlQueryParser, responseObjectFactory, webClientBuilder, jwtTokenHandler, dnUtils); } else { return new LocalShardTableQueryMetricHandler(queryMetricHandlerProperties, accumuloClientPool, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java index c0c67fc1..1203d53b 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java @@ -28,22 +28,26 @@ import datawave.query.language.parser.jexl.LuceneToJexlQueryParser; import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.JWTTokenHandler; +import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.result.BaseQueryResponse; public class RemoteShardTableQueryMetricHandler extends ShardTableQueryMetricHandler { private static final Logger log = LoggerFactory.getLogger(RemoteShardTableQueryMetricHandler.class); private DatawaveUserDetails userDetails; + private final ResponseObjectFactory responseObjectFactory; private final WebClient webClient; private final WebClient authWebClient; private final JWTTokenHandler jwtTokenHandler; public RemoteShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, @Qualifier("warehouse") AccumuloClientPool clientPool, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, - QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, WebClient.Builder webClientBuilder, - JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { + QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, ResponseObjectFactory responseObjectFactory, + WebClient.Builder webClientBuilder, JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { super(queryMetricHandlerProperties, clientPool, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); + this.responseObjectFactory = responseObjectFactory; + this.webClient = webClientBuilder.baseUrl(queryMetricHandlerProperties.getQueryServiceUri()).build(); this.jwtTokenHandler = jwtTokenHandler; @@ -90,7 +94,6 @@ protected BaseQueryResponse createAndNext(Query query) throws Exception { return webClient.post() .uri(uriBuilder -> uriBuilder .path("/" + queryMetricHandlerProperties.getQueryMetricsLogic() + "/createAndNext") - .queryParam(QueryParameters.QUERY_POOL, queryMetricHandlerProperties.getQueryPool()) .queryParam(QueryParameters.QUERY_BEGIN, beginDate) .queryParam(QueryParameters.QUERY_END, endDate) .queryParam(QueryParameters.QUERY_LOGIC_NAME, query.getQueryLogicName()) @@ -103,9 +106,10 @@ protected BaseQueryResponse createAndNext(Query query) throws Exception { .queryParam(QueryParameters.QUERY_PARAMS, query.getParameters().stream().map(p -> String.join(":", p.getParameterName(), p.getParameterValue())).collect(Collectors.joining(";"))) .build()) .header("Authorization", bearerHeader) + .header("Pool", queryMetricHandlerProperties.getQueryPool()) .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) .retrieve() - .bodyToMono(BaseQueryResponse.class) + .bodyToMono(responseObjectFactory.getEventQueryResponse().getClass()) .block(Duration.ofMillis(queryMetricHandlerProperties.getRemoteQueryTimeoutMillis())); // @formatter:on } catch (IllegalStateException e) { @@ -125,7 +129,7 @@ protected BaseQueryResponse next(String queryId) throws Exception { .header("Authorization", bearerHeader) .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) .retrieve() - .bodyToMono(BaseQueryResponse.class) + .bodyToMono(responseObjectFactory.getEventQueryResponse().getClass()) .block(Duration.ofMillis(queryMetricHandlerProperties.getRemoteQueryTimeoutMillis())); // @formatter:on } catch (IllegalStateException e) { diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index 817926f8..8e991771 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -18,9 +18,8 @@ spring: bindings: queryMetricSource-out-0: producer: - producer: - # Note: This must match CONFIRM_ACK_CHANNEL in AuditController.java or producer confirms will not work. - confirmAckChannel: 'confirmAckChannel' + # Note: This must match CONFIRM_ACK_CHANNEL in AuditController.java or producer confirms will not work. + confirmAckChannel: 'confirmAckChannel' datawave: table: cache: From aa8b6a1c191070eef4bf165395371f536dd28235 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 22 Nov 2023 19:12:21 +0000 Subject: [PATCH 10/48] Fixed bootstrap configuration, and fixed rest-based query client. --- .../config/QueryMetricHandlerConfiguration.java | 5 +++-- .../handler/RemoteShardTableQueryMetricHandler.java | 12 ++++++++---- service/src/main/resources/config/bootstrap.yml | 5 ++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java index fe24968b..de656e59 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java @@ -94,10 +94,11 @@ QueryMetricFactory queryMetricFactory() { public ShardTableQueryMetricHandler shardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, @Qualifier("warehouse") AccumuloClientPool accumuloClientPool, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, - WebClient.Builder webClientBuilder, @Autowired(required = false) JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { + ResponseObjectFactory responseObjectFactory, WebClient.Builder webClientBuilder, + @Autowired(required = false) JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { if (queryMetricHandlerProperties.isUseRemoteQuery()) { return new RemoteShardTableQueryMetricHandler(queryMetricHandlerProperties, accumuloClientPool, logicFactory, metricFactory, markingFunctions, - queryMetricCombiner, luceneToJexlQueryParser, webClientBuilder, jwtTokenHandler, dnUtils); + queryMetricCombiner, luceneToJexlQueryParser, responseObjectFactory, webClientBuilder, jwtTokenHandler, dnUtils); } else { return new LocalShardTableQueryMetricHandler(queryMetricHandlerProperties, accumuloClientPool, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java index 35c4361c..6145af2d 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/RemoteShardTableQueryMetricHandler.java @@ -28,22 +28,26 @@ import datawave.webservice.query.Query; import datawave.webservice.query.QueryParameters; import datawave.webservice.query.QueryParametersImpl; +import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.result.BaseQueryResponse; public class RemoteShardTableQueryMetricHandler extends ShardTableQueryMetricHandler { private static final Logger log = LoggerFactory.getLogger(RemoteShardTableQueryMetricHandler.class); private DatawaveUserDetails userDetails; + private final ResponseObjectFactory responseObjectFactory; private final WebClient webClient; private final WebClient authWebClient; private final JWTTokenHandler jwtTokenHandler; public RemoteShardTableQueryMetricHandler(QueryMetricHandlerProperties queryMetricHandlerProperties, @Qualifier("warehouse") AccumuloClientPool clientPool, QueryMetricQueryLogicFactory logicFactory, QueryMetricFactory metricFactory, MarkingFunctions markingFunctions, - QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, WebClient.Builder webClientBuilder, - JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { + QueryMetricCombiner queryMetricCombiner, LuceneToJexlQueryParser luceneToJexlQueryParser, ResponseObjectFactory responseObjectFactory, + WebClient.Builder webClientBuilder, JWTTokenHandler jwtTokenHandler, DnUtils dnUtils) { super(queryMetricHandlerProperties, clientPool, logicFactory, metricFactory, markingFunctions, queryMetricCombiner, luceneToJexlQueryParser, dnUtils); + this.responseObjectFactory = responseObjectFactory; + this.webClient = webClientBuilder.baseUrl(queryMetricHandlerProperties.getQueryServiceUri()).build(); this.jwtTokenHandler = jwtTokenHandler; @@ -104,7 +108,7 @@ protected BaseQueryResponse createAndNext(Query query) throws Exception { .header("Authorization", bearerHeader) .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) .retrieve() - .bodyToMono(BaseQueryResponse.class) + .bodyToMono(responseObjectFactory.getEventQueryResponse().getClass()) .block(Duration.ofMillis(queryMetricHandlerProperties.getRemoteQueryTimeoutMillis())); // @formatter:on } catch (IllegalStateException e) { @@ -124,7 +128,7 @@ protected BaseQueryResponse next(String queryId) throws Exception { .header("Authorization", bearerHeader) .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) .retrieve() - .bodyToMono(BaseQueryResponse.class) + .bodyToMono(responseObjectFactory.getEventQueryResponse().getClass()) .block(Duration.ofMillis(queryMetricHandlerProperties.getRemoteQueryTimeoutMillis())); // @formatter:on } catch (IllegalStateException e) { diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index 817926f8..8e991771 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -18,9 +18,8 @@ spring: bindings: queryMetricSource-out-0: producer: - producer: - # Note: This must match CONFIRM_ACK_CHANNEL in AuditController.java or producer confirms will not work. - confirmAckChannel: 'confirmAckChannel' + # Note: This must match CONFIRM_ACK_CHANNEL in AuditController.java or producer confirms will not work. + confirmAckChannel: 'confirmAckChannel' datawave: table: cache: From 1ec83a314d1fddab31abbba74e5cbfa535cf2a70 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 22 Nov 2023 19:39:18 +0000 Subject: [PATCH 11/48] Added metric client confirm ack configuration --- .../microservice/querymetric/QueryMetricOperations.java | 5 +++-- service/src/test/resources/config/application.yml | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 7c2802a4..3a6b05b8 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -205,7 +205,7 @@ public VoidResponse updateMetrics(@RequestBody List queryMetric } else { log.debug("received metric update via REST: " + m.getQueryId()); } - if (!updateMetric(new QueryMetricUpdate(m, metricType))) { + if (!updateMetric(new QueryMetricUpdate<>(m, metricType))) { throw new RuntimeException("Unable to process query metric update for query [" + m.getQueryId() + "]"); } } @@ -259,8 +259,9 @@ public void processConfirmAck(Message message) { String correlationId = headerObj.toString(); if (correlationLatchMap.containsKey(correlationId)) { correlationLatchMap.get(correlationId).countDown(); - } else + } else { log.warn("Unable to decrement latch for ID [{}]", correlationId); + } } else { log.warn("No correlation ID found in confirm ack message"); } diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index c47433f1..52e1b870 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -86,6 +86,7 @@ datawave: transport: message host: localhost port: ${server.port} + confirmAckEnabled: false timely: enabled: false host: localhost From c78feb04c905015a622741c25bddae356e5bed8a Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Wed, 22 Nov 2023 15:33:09 -0500 Subject: [PATCH 12/48] Correct comment in bootstrap.yml --- service/src/main/resources/config/bootstrap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index 8e991771..e6d49725 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -18,7 +18,7 @@ spring: bindings: queryMetricSource-out-0: producer: - # Note: This must match CONFIRM_ACK_CHANNEL in AuditController.java or producer confirms will not work. + # Note: This must match CONFIRM_ACK_CHANNEL in QueryMetricOperations.java or producer confirms will not work. confirmAckChannel: 'confirmAckChannel' datawave: table: From c52497fc553a4b4f355b39d77806fff9da70dcde Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 22 Nov 2023 22:25:59 +0000 Subject: [PATCH 13/48] formatting --- service/src/main/resources/config/bootstrap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml index 8e991771..e6d49725 100644 --- a/service/src/main/resources/config/bootstrap.yml +++ b/service/src/main/resources/config/bootstrap.yml @@ -18,7 +18,7 @@ spring: bindings: queryMetricSource-out-0: producer: - # Note: This must match CONFIRM_ACK_CHANNEL in AuditController.java or producer confirms will not work. + # Note: This must match CONFIRM_ACK_CHANNEL in QueryMetricOperations.java or producer confirms will not work. confirmAckChannel: 'confirmAckChannel' datawave: table: From a7c85a3eeb0c7818586e029d69bc18a9fb2f1ce6 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 27 Nov 2023 11:03:29 -0500 Subject: [PATCH 14/48] Release datawave-query-metric api/service 2.1.4 --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 68d4ef6d..213b6339 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ query-metric-api - 2.1.4-SNAPSHOT + 2.1.4 https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index 16351cb4..d0c1e99c 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ query-metric-parent - 2.1.4-SNAPSHOT + 2.1.4 pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index e87b113a..205b9f03 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ query-metric-service - 2.1.4-SNAPSHOT + 2.1.4 DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service From c6e064d75cb85a5ee07bdf56b08d9250ba24ab06 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 27 Nov 2023 11:05:06 -0500 Subject: [PATCH 15/48] Set version to 2.1.5-SNAPSHOT --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 213b6339..6b637fc3 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ query-metric-api - 2.1.4 + 2.1.5-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index d0c1e99c..55cb43de 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ query-metric-parent - 2.1.4 + 2.1.5-SNAPSHOT pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 205b9f03..7e918d1b 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ query-metric-service - 2.1.4 + 2.1.5-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service From 823b78b25e18c747d6210780381e48b2f3a9bc93 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 27 Nov 2023 11:13:00 -0500 Subject: [PATCH 16/48] Release datawave-query-metric api/service 3.0.1 --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 8617b310..c5cc4841 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.1-SNAPSHOT + 3.0.1 https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index 0e6c4af5..a22c3f4b 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.1-SNAPSHOT + 3.0.1 pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 9ae1ddea..72389f9d 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.1-SNAPSHOT + 3.0.1 DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service From 7898de892be78181be49a9a8ac21e3d973af0e34 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 27 Nov 2023 11:14:33 -0500 Subject: [PATCH 17/48] Set version to 3.0.2-SNAPSHOT --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index c5cc4841..edfa3411 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.1 + 3.0.2-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index a22c3f4b..f4fde354 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.1 + 3.0.2-SNAPSHOT pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 72389f9d..5ab89a27 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.1 + 3.0.2-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service From a1222931e33d867b733524bbd560ca90b12311a4 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 4 Dec 2023 11:47:12 -0500 Subject: [PATCH 18/48] Specify datatype in ShardTableQueryMetricHandler to enable caching in certain methods in MetadataHelpers --- .../handler/ShardTableQueryMetricHandler.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index a708b483..20fa93f8 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -28,6 +28,7 @@ import datawave.microservice.querymetric.QueryMetricsSummaryResponse; import datawave.microservice.querymetric.config.QueryMetricHandlerProperties; import datawave.microservice.querymetric.factory.QueryMetricQueryLogicFactory; +import datawave.query.QueryParameters; import datawave.query.iterator.QueryOptions; import datawave.query.language.parser.jexl.LuceneToJexlQueryParser; import datawave.security.authorization.DatawavePrincipal; @@ -82,6 +83,7 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -393,7 +395,10 @@ public List getQueryMetrics(final String query) throws Exception { queryImpl.setExpirationDate(DateUtils.addDays(new Date(), 1)); queryImpl.setPagesize(1000); queryImpl.setId(UUID.randomUUID()); - queryImpl.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true")); + Map parameters = new LinkedHashMap<>(); + parameters.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"); + parameters.put(QueryParameters.DATATYPE_FILTER_SET, "querymetrics"); + queryImpl.setParameters(parameters); return getQueryMetrics(queryImpl); } @@ -834,7 +839,10 @@ public QueryMetricsSummaryResponse getQueryMetricsSummary(Date begin, Date end, query.setPagesize(1000); query.setUserDN(datawaveUserShortName); query.setId(UUID.randomUUID()); - query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true")); + Map parameters = new LinkedHashMap<>(); + parameters.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"); + parameters.put(QueryParameters.DATATYPE_FILTER_SET, "querymetrics"); + query.setParameters(parameters); List queryMetrics = getQueryMetrics(query); response = processQueryMetricsSummary(queryMetrics, end); From 8243e4dd9186a3384419aa4715883128aacfc67b Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 4 Dec 2023 13:08:34 -0500 Subject: [PATCH 19/48] Enable the persisting of a changed query plan in query metric --- .../ContentQueryMetricsIngestHelper.java | 8 +++-- .../handler/QueryMetricCombiner.java | 18 ++++++++-- .../QueryMetricConsistencyTest.java | 35 +++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index 8a677d0e..c0499afd 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -16,7 +16,6 @@ import java.text.SimpleDateFormat; import java.util.Collection; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -237,7 +236,7 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isFirstWrite(updated.getParameters(), stored == null ? null : stored.getParameters())) { fields.put("PARAMETERS", QueryUtil.toParametersString(updated.getParameters())); } - if (isFirstWrite(updated.getPlan(), stored == null ? null : stored.getPlan())) { + if (isChanged(updated.getPlan(), stored == null ? null : stored.getPlan())) { fields.put("PLAN", updated.getPlan()); } if (isFirstWrite(updated.getProxyServers(), stored == null ? null : stored.getProxyServers())) { @@ -401,6 +400,11 @@ public Multimap getEventFieldsToDelete(T updated, T stored) { } } } + if (stored.getPlan() != null) { + if (stored.getPlan() != null && isChanged(updated.getPlan(), stored.getPlan())) { + fields.put("PLAN", stored.getPlan()); + } + } if (isChanged(updated.getSeekCount(), stored.getSeekCount())) { fields.put("SEEK_COUNT", Long.toString(stored.getSeekCount())); } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index 2c65346f..22aeaefa 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -3,6 +3,7 @@ import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.QueryMetricType; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +25,11 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy // duplicate cachedQueryMetric so that we leave that object unchanged and return a combined metric combinedMetric = (T) cachedQueryMetric.duplicate(); + boolean inOrderUpdate = true; + if (updatedQueryMetric.getLastUpdated() != null && cachedQueryMetric.getLastUpdated() != null) { + inOrderUpdate = updatedQueryMetric.getLastUpdated().after(cachedQueryMetric.getLastUpdated()); + } + // only update once if (combinedMetric.getQueryType() == null && updatedQueryMetric.getQueryType() != null) { combinedMetric.setQueryType(updatedQueryMetric.getQueryType()); @@ -183,8 +189,8 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges()); combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges()); } - // only update once - if (combinedMetric.getPlan() == null && updatedQueryMetric.getPlan() != null) { + // update if the update is in-order and the value changed + if (inOrderUpdate && isChanged(updatedQueryMetric.getPlan(), combinedMetric.getPlan())) { combinedMetric.setPlan(updatedQueryMetric.getPlan()); } // only update once @@ -249,4 +255,12 @@ protected PageMetric combinePageMetrics(PageMetric updated, PageMetric stored) { } return pm; } + + protected boolean isChanged(String updated, String stored) { + if ((StringUtils.isBlank(stored) && StringUtils.isNotBlank(updated)) || (stored != null && updated != null && !stored.equals(updated))) { + return true; + } else { + return false; + } + } } diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 06fadc4b..fbd3217d 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -125,6 +125,41 @@ public void OutOfOrderLifecycleTest() throws Exception { assertNoDuplicateFields(queryId); } + @Test + public void ChangePlanTest() throws Exception { + int port = this.webServicePort; + String queryId = createQueryId(); + BaseQueryMetric m = createMetric(queryId); + UriComponents metricUri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(port).path(String.format(getMetricsUrl, queryId)) + .build(); + m.setPlan("InitialPlan"); + // @formatter:off + client.submit(new QueryMetricClient.Request.Builder() + .withMetric(m) + .withMetricType(QueryMetricType.COMPLETE) + .withUser(this.adminUser) + .build()); + // @formatter:on + m.setPlan("RevisedPlan"); + m.setLastUpdated(new Date(m.getLastUpdated().getTime() + 1)); + // @formatter:off + client.submit(new QueryMetricClient.Request.Builder() + .withMetric(m) + .withMetricType(QueryMetricType.COMPLETE) + .withUser(this.adminUser) + .build()); + // @formatter:on + + HttpEntity metricRequestEntity = createRequestEntity(null, this.adminUser, null); + ResponseEntity metricResponse = this.restTemplate.exchange(metricUri.toUri(), HttpMethod.GET, metricRequestEntity, + BaseQueryMetricListResponse.class); + + Assert.assertEquals(1, metricResponse.getBody().getNumResults()); + BaseQueryMetric returnedMetric = (BaseQueryMetric) metricResponse.getBody().getResult().get(0); + Assert.assertEquals("plan incorrect", m.getPlan(), returnedMetric.getPlan()); + assertNoDuplicateFields(queryId); + } + @Test public void DistributedUpdateTest() throws Exception { int port = this.webServicePort; From d11bf9bc46aa5194616baf1a3403f1a88c41a145 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 4 Dec 2023 16:20:56 -0500 Subject: [PATCH 20/48] Require that updated Lifecycle is greater than current Lifecycle --- .../microservice/querymetric/handler/QueryMetricCombiner.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index 22aeaefa..14f9b9c8 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -116,7 +116,9 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy combinedMetric.setErrorCode(updatedQueryMetric.getErrorCode()); } // use updated lifecycle unless trying to update a final lifecycle with a non-final lifecycle - if ((combinedMetric.isLifecycleFinal() && !updatedQueryMetric.isLifecycleFinal()) == false) { + // or if updating with a lifecycle that is less than the current + if ((combinedMetric.isLifecycleFinal() && !updatedQueryMetric.isLifecycleFinal()) == false + && updatedQueryMetric.getLifecycle().compareTo(combinedMetric.getLifecycle()) > 0) { combinedMetric.setLifecycle(updatedQueryMetric.getLifecycle()); } // only update once From 74eb77cfc95cbae8ef2b3e2d7d833d915b5e4c8d Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Mon, 4 Dec 2023 09:20:31 -0500 Subject: [PATCH 21/48] Correlate metric updates by queryId to avoid filling hazelcast operation queue with updates from the same queryId --- .../microservice/querymetric/Correlator.java | 79 ++++++++++ .../MetricUpdateEntryProcessor.java | 27 ++-- .../querymetric/QueryMetricOperations.java | 103 +++++++++++- .../querymetric/QueryMetricUpdateHolder.java | 26 ++-- .../config/CorrelatorConfiguration.java | 10 ++ .../config/CorrelatorProperties.java | 35 +++++ .../persistence/AccumuloMapStore.java | 2 +- .../querymetric/CorrelatorTest.java | 146 ++++++++++++++++++ .../config/application-correlator.yml | 5 + .../src/test/resources/config/application.yml | 2 + 10 files changed, 404 insertions(+), 31 deletions(-) create mode 100644 service/src/main/java/datawave/microservice/querymetric/Correlator.java create mode 100644 service/src/main/java/datawave/microservice/querymetric/config/CorrelatorConfiguration.java create mode 100644 service/src/main/java/datawave/microservice/querymetric/config/CorrelatorProperties.java create mode 100644 service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java create mode 100644 service/src/test/resources/config/application-correlator.yml diff --git a/service/src/main/java/datawave/microservice/querymetric/Correlator.java b/service/src/main/java/datawave/microservice/querymetric/Correlator.java new file mode 100644 index 00000000..7e8cae11 --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/Correlator.java @@ -0,0 +1,79 @@ +package datawave.microservice.querymetric; + +import datawave.microservice.querymetric.config.CorrelatorProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Component +public class Correlator { + + private CorrelatorProperties correlatorProperties; + private boolean isShuttingDown = false; + private Map> updates = new HashMap<>(); + private LinkedHashMap created = new LinkedHashMap<>(); + + @Autowired + public Correlator(CorrelatorProperties correlatorProperties) { + this.correlatorProperties = correlatorProperties; + } + + public void addMetricUpdate(QueryMetricUpdate update) { + String queryId = update.getMetric().getQueryId(); + synchronized (this.updates) { + List updatesForQuery = this.updates.get(queryId); + if (updatesForQuery == null) { + updatesForQuery = new ArrayList<>(); + this.created.put(queryId, System.currentTimeMillis()); + this.updates.put(queryId, updatesForQuery); + } + updatesForQuery.add(update); + } + } + + public List getMetricUpdates(Set inProcess) { + long now = System.currentTimeMillis(); + long maxQueueSize = this.correlatorProperties.getMaxCorrelationQueueSize(); + long maxCorrelationTimeMs = this.correlatorProperties.getMaxCorrelationTimeMs(); + String oldestQueryId; + List returnedUpdates = null; + synchronized (this.updates) { + long numEntries = this.created.size(); + // Find the oldest entry that is not inProcessing on this instance + // If shuttingDown, then just return the oldest entry + Map.Entry oldestAvailableEntry = this.created.entrySet().stream().filter(e -> this.isShuttingDown || !inProcess.contains(e.getKey())) + .findFirst().orElse(null); + + // If we have reached the max queue size, then don't filter by !inProcess + if (oldestAvailableEntry == null && numEntries >= maxQueueSize) { + oldestAvailableEntry = this.created.entrySet().stream().findFirst().orElse(null); + } + + if (oldestAvailableEntry != null) { + long maxAge = now - oldestAvailableEntry.getValue(); + oldestQueryId = oldestAvailableEntry.getKey(); + if (numEntries >= maxQueueSize || maxAge > maxCorrelationTimeMs || this.isShuttingDown) { + returnedUpdates = this.updates.remove(oldestQueryId); + this.created.remove(oldestQueryId); + } + } + } + return returnedUpdates; + } + + public boolean isEnabled() { + return this.correlatorProperties.isEnabled(); + } + + public void shutdown(boolean isShuttingDown) { + this.isShuttingDown = isShuttingDown; + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java b/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java index ac8826b4..dbd18c4c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java +++ b/service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java @@ -17,33 +17,34 @@ public MetricUpdateEntryProcessor(QueryMetricUpdateHolder metricUpdate, QueryMet @Override public Long process(Map.Entry entry) { - QueryMetricUpdateHolder updatedHolder; + QueryMetricUpdateHolder storedHolder; QueryMetricType metricType = this.metricUpdate.getMetricType(); BaseQueryMetric updatedMetric = this.metricUpdate.getMetric(); long start = System.currentTimeMillis(); if (entry.getValue() == null) { - updatedHolder = this.metricUpdate; + storedHolder = this.metricUpdate; } else { - updatedHolder = entry.getValue(); - BaseQueryMetric storedMetric = entry.getValue().getMetric(); + storedHolder = entry.getValue(); + BaseQueryMetric storedMetric = storedHolder.getMetric(); BaseQueryMetric combinedMetric; combinedMetric = this.combiner.combineMetrics(updatedMetric, storedMetric, metricType); - updatedHolder.setMetric(combinedMetric); - updatedHolder.setMetricType(metricType); + storedHolder.setMetric(combinedMetric); + storedHolder.setMetricType(metricType); + storedHolder.updateLowestLifecycle(this.metricUpdate.getLowestLifecycle()); } if (metricType.equals(QueryMetricType.DISTRIBUTED) && updatedMetric != null) { // these values are added incrementally in a distributed update. Because we can not be sure // exactly when the incomingQueryMetricCache value is stored, it would otherwise be possible // for updates to be included twice. These values are reset after being used in the AccumuloMapStore - updatedHolder.addValue("sourceCount", updatedMetric.getSourceCount()); - updatedHolder.addValue("nextCount", updatedMetric.getNextCount()); - updatedHolder.addValue("seekCount", updatedMetric.getSeekCount()); - updatedHolder.addValue("yieldCount", updatedMetric.getYieldCount()); - updatedHolder.addValue("docRanges", updatedMetric.getDocRanges()); - updatedHolder.addValue("fiRanges", updatedMetric.getFiRanges()); + storedHolder.addValue("sourceCount", updatedMetric.getSourceCount()); + storedHolder.addValue("nextCount", updatedMetric.getNextCount()); + storedHolder.addValue("seekCount", updatedMetric.getSeekCount()); + storedHolder.addValue("yieldCount", updatedMetric.getYieldCount()); + storedHolder.addValue("docRanges", updatedMetric.getDocRanges()); + storedHolder.addValue("fiRanges", updatedMetric.getFiRanges()); } - entry.setValue(updatedHolder); + entry.setValue(storedHolder); return Long.valueOf(System.currentTimeMillis() - start); } } diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index f4b85a23..5e5f5e7d 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -52,8 +52,11 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.TimeZone; import static datawave.microservice.querymetric.QueryMetricOperations.DEFAULT_DATETIME.BEGIN; @@ -83,8 +86,10 @@ public class QueryMetricOperations { private MarkingFunctions markingFunctions; private BaseQueryMetricListResponseFactory queryMetricListResponseFactory; private MergeLockLifecycleListener mergeLock; + private Correlator correlator; private MetricUpdateEntryProcessorFactory entryProcessorFactory; private QueryMetricOperationsStats stats; + private static Set inProcess = Collections.synchronizedSet(new HashSet<>()); /** * The enum Default datetime. @@ -127,7 +132,8 @@ enum DEFAULT_DATETIME { @Autowired public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cacheManager, ShardTableQueryMetricHandler handler, QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, BaseQueryMetricListResponseFactory queryMetricListResponseFactory, - MergeLockLifecycleListener mergeLock, MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats) { + MergeLockLifecycleListener mergeLock, Correlator correlator, MetricUpdateEntryProcessorFactory entryProcessorFactory, + QueryMetricOperationsStats stats) { this.handler = handler; this.geometryHandler = geometryHandler; this.cacheManager = cacheManager; @@ -136,16 +142,42 @@ public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cach this.markingFunctions = markingFunctions; this.queryMetricListResponseFactory = queryMetricListResponseFactory; this.mergeLock = mergeLock; + this.correlator = correlator; this.entryProcessorFactory = entryProcessorFactory; this.stats = stats; } @PreDestroy public void shutdown() { + if (this.correlator.isEnabled()) { + this.correlator.shutdown(true); + ensureUpdatesProcessed(); + } this.stats.queueAggregatedQueryStatsForTimely(); this.stats.writeQueryStatsToTimely(); } + @Scheduled(fixedDelay = 2000) + public void ensureUpdatesProcessed() { + if (this.correlator.isEnabled()) { + List correlatedUpdates; + do { + correlatedUpdates = this.correlator.getMetricUpdates(QueryMetricOperations.inProcess); + if (correlatedUpdates != null && !correlatedUpdates.isEmpty()) { + try { + String queryId = correlatedUpdates.get(0).getMetric().getQueryId(); + QueryMetricType metricType = correlatedUpdates.get(0).getMetricType(); + QueryMetricUpdateHolder metricUpdate = combineMetricUpdates(correlatedUpdates, metricType); + log.debug("storing correlated updates for {}", queryId); + storeMetricUpdates(metricUpdate); + } catch (Exception e) { + log.error("exception while combining correlated updates: " + e.getMessage(), e); + } + } + } while (correlatedUpdates != null && !correlatedUpdates.isEmpty()); + } + } + /** * Update metrics void response. * @@ -207,6 +239,18 @@ public VoidResponse updateMetric(@RequestBody BaseQueryMetric queryMetric, return new VoidResponse(); } + private boolean shouldCorrelate(QueryMetricUpdate update) { + // add the first update for a metric to get it into the cache + if ((update.getMetric().getLifecycle().ordinal() <= BaseQueryMetric.Lifecycle.DEFINED.ordinal())) { + return false; + } + if (this.correlator.isEnabled()) { + return true; + } else { + return false; + } + } + /** * Handle event. * @@ -216,28 +260,71 @@ public VoidResponse updateMetric(@RequestBody BaseQueryMetric queryMetric, @StreamListener(QueryMetricSinkBinding.SINK_NAME) public void handleEvent(QueryMetricUpdate update) { stats.getMeter(METERS.MESSAGE).mark(); - String queryId = update.getMetric().getQueryId(); - this.stats.queueTimelyMetrics(update); - log.debug("storing update for {}", queryId); - if (update.getMetric().getPositiveSelectors() == null) { - this.handler.populateMetricSelectors(update.getMetric()); + if (shouldCorrelate(update)) { + log.debug("adding update for {} to correlator", update.getMetric().getQueryId()); + this.correlator.addMetricUpdate(update); + } else { + log.debug("storing update for {}", update.getMetric().getQueryId()); + storeMetricUpdates(new QueryMetricUpdateHolder(update)); + } + + if (correlator.isEnabled()) { + List correlatedUpdates; + do { + correlatedUpdates = this.correlator.getMetricUpdates(QueryMetricOperations.inProcess); + if (correlatedUpdates != null && !correlatedUpdates.isEmpty()) { + try { + String queryId = correlatedUpdates.get(0).getMetric().getQueryId(); + QueryMetricType metricType = correlatedUpdates.get(0).getMetricType(); + QueryMetricUpdateHolder metricUpdate = combineMetricUpdates(correlatedUpdates, metricType); + log.debug("storing correlated updates for {}", queryId); + storeMetricUpdates(metricUpdate); + } catch (Exception e) { + log.error("exception while combining correlated updates: " + e.getMessage(), e); + } + } + } while (correlatedUpdates != null && !correlatedUpdates.isEmpty()); } - storeMetricUpdate(new QueryMetricUpdateHolder(update)); } private String getClusterLocalMemberUuid() { return ((HazelcastCacheManager) this.cacheManager).getHazelcastInstance().getCluster().getLocalMember().getUuid(); } - private void storeMetricUpdate(QueryMetricUpdateHolder metricUpdate) { + private QueryMetricUpdateHolder combineMetricUpdates(List updates, QueryMetricType metricType) throws Exception { + BaseQueryMetric combinedMetric = null; + BaseQueryMetric.Lifecycle lowestLifecycle = null; + for (QueryMetricUpdate u : updates) { + this.stats.queueTimelyMetrics(u); + if (combinedMetric == null) { + combinedMetric = u.getMetric(); + lowestLifecycle = u.getMetric().getLifecycle(); + } else { + if (u.getMetric().getLifecycle().ordinal() < lowestLifecycle.ordinal()) { + lowestLifecycle = u.getMetric().getLifecycle(); + } + combinedMetric = this.handler.combineMetrics(u.getMetric(), combinedMetric, metricType); + } + } + QueryMetricUpdateHolder metricUpdateHolder = new QueryMetricUpdateHolder(combinedMetric, metricType); + metricUpdateHolder.updateLowestLifecycle(lowestLifecycle); + return metricUpdateHolder; + } + + private void storeMetricUpdates(QueryMetricUpdateHolder metricUpdate) { Timer.Context storeTimer = this.stats.getTimer(TIMERS.STORE).time(); String queryId = metricUpdate.getMetric().getQueryId(); try { IMap incomingQueryMetricsCacheHz = ((IMap) incomingQueryMetricsCache.getNativeCache()); + if (metricUpdate.getMetric().getPositiveSelectors() == null) { + this.handler.populateMetricSelectors(metricUpdate.getMetric()); + } this.mergeLock.lock(); + QueryMetricOperations.inProcess.add(queryId); try { incomingQueryMetricsCacheHz.executeOnKey(queryId, this.entryProcessorFactory.createEntryProcessor(metricUpdate)); } finally { + QueryMetricOperations.inProcess.remove(queryId); this.mergeLock.unlock(); } } catch (Exception e) { diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricUpdateHolder.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricUpdateHolder.java index c1803a95..d3c830b3 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricUpdateHolder.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricUpdateHolder.java @@ -8,12 +8,12 @@ public class QueryMetricUpdateHolder extends QueryMetricUpdate { private boolean persisted = false; - private Lifecycle lowestLifecycleSincePersist; + private Lifecycle lowestLifecycle; private Map values = new HashMap<>(); public QueryMetricUpdateHolder(T metric, QueryMetricType metricType) { super(metric, metricType); - this.lowestLifecycleSincePersist = this.metric.getLifecycle(); + this.lowestLifecycle = this.metric.getLifecycle(); } public QueryMetricUpdateHolder(T metric) { @@ -27,7 +27,7 @@ public QueryMetricUpdateHolder(QueryMetricUpdate metricUpdate) { // If we know that this metric has been persisted by the AccumuloMapStore, then it is not new // Because the metric can be ejected from the incoming cache, we also track the lowest lifecycle public boolean isNewMetric() { - return !persisted && (lowestLifecycleSincePersist == null || lowestLifecycleSincePersist.equals(Lifecycle.DEFINED)); + return !persisted && (lowestLifecycle == null || lowestLifecycle.equals(Lifecycle.DEFINED)); } public void addValue(String key, Long value) { @@ -46,21 +46,29 @@ public Long getValue(String key) { } } - public void persisted() { + public void setPersisted() { persisted = true; values.clear(); - lowestLifecycleSincePersist = null; + lowestLifecycle = null; } - public Lifecycle getLowestLifecycleSincePersist() { - return lowestLifecycleSincePersist; + public Lifecycle getLowestLifecycle() { + return lowestLifecycle; + } + + public void updateLowestLifecycle(Lifecycle lifecycle) { + if (!persisted && lifecycle != null) { + if (this.lowestLifecycle == null || (lifecycle.ordinal() < this.lowestLifecycle.ordinal())) { + this.lowestLifecycle = lifecycle; + } + } } @Override public void setMetric(T metric) { super.setMetric(metric); - if (this.lowestLifecycleSincePersist == null || this.metric.getLifecycle().ordinal() < this.lowestLifecycleSincePersist.ordinal()) { - this.lowestLifecycleSincePersist = this.metric.getLifecycle(); + if (this.lowestLifecycle == null || this.metric.getLifecycle().ordinal() < this.lowestLifecycle.ordinal()) { + this.lowestLifecycle = this.metric.getLifecycle(); } } } diff --git a/service/src/main/java/datawave/microservice/querymetric/config/CorrelatorConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/CorrelatorConfiguration.java new file mode 100644 index 00000000..d2ba0f95 --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/config/CorrelatorConfiguration.java @@ -0,0 +1,10 @@ +package datawave.microservice.querymetric.config; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableConfigurationProperties({CorrelatorProperties.class}) +public class CorrelatorConfiguration { + +} diff --git a/service/src/main/java/datawave/microservice/querymetric/config/CorrelatorProperties.java b/service/src/main/java/datawave/microservice/querymetric/config/CorrelatorProperties.java new file mode 100644 index 00000000..7e8e6afc --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/config/CorrelatorProperties.java @@ -0,0 +1,35 @@ +package datawave.microservice.querymetric.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "datawave.query.metric.correlator") +public class CorrelatorProperties { + + private long maxCorrelationTimeMs = 30000; + private long maxCorrelationQueueSize = 1000; + private boolean enabled = true; + + public long getMaxCorrelationTimeMs() { + return maxCorrelationTimeMs; + } + + public void setMaxCorrelationTimeMs(long maxCorrelationTimeMs) { + this.maxCorrelationTimeMs = maxCorrelationTimeMs; + } + + public long getMaxCorrelationQueueSize() { + return maxCorrelationQueueSize; + } + + public void setMaxCorrelationQueueSize(long maxCorrelationQueueSize) { + this.maxCorrelationQueueSize = maxCorrelationQueueSize; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public boolean isEnabled() { + return enabled; + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index 14c5aee1..a55ff192 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -160,7 +160,7 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception } lastWrittenQueryMetricCache.set(queryId, new QueryMetricUpdateHolder(updatedMetric)); - queryMetricUpdate.persisted(); + queryMetricUpdate.setPersisted(); failures.invalidate(queryId); } finally { if (queryMetricUpdate.getMetricType().equals(QueryMetricType.DISTRIBUTED)) { diff --git a/service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java b/service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java new file mode 100644 index 00000000..0ffc8165 --- /dev/null +++ b/service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java @@ -0,0 +1,146 @@ +package datawave.microservice.querymetric; + +import datawave.microservice.querymetric.config.CorrelatorProperties; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"CorrelatorTest", "QueryMetricTest", "hazelcast-writebehind", "correlator"}) +public class CorrelatorTest extends QueryMetricTestBase { + + @Autowired + private QueryMetricOperations queryMetricOperations; + + @Autowired + private CorrelatorProperties correlatorProperties; + + @Autowired + private Correlator correlator; + + @Before + public void setup() { + super.setup(); + } + + @After + public void cleanup() { + super.cleanup(); + } + + @Test + public void TestSizeLimitedQueue() throws Exception { + this.correlatorProperties.setMaxCorrelationTimeMs(30000); + this.correlatorProperties.setMaxCorrelationQueueSize(95); + testMetricsCorrelated(100, 100); + } + + @Test + public void TestTimeLimitedQueue() throws Exception { + this.correlatorProperties.setMaxCorrelationTimeMs(500); + this.correlatorProperties.setMaxCorrelationQueueSize(100); + testMetricsCorrelated(100, 100); + } + + @Test + public void TestNoStorageUntilShutdown() throws Exception { + this.correlatorProperties.setMaxCorrelationTimeMs(60000); + this.correlatorProperties.setMaxCorrelationQueueSize(100); + testMetricsCorrelated(100, 10); + } + + @Test + public void TestManyQueries() throws Exception { + this.correlatorProperties.setMaxCorrelationTimeMs(500); + this.correlatorProperties.setMaxCorrelationQueueSize(100); + testMetricsCorrelated(1000, 20); + } + + public void testMetricsCorrelated(int numMetrics, int maxPages) throws Exception { + List updates = new ArrayList<>(); + List metrics = new ArrayList<>(); + for (int x = 0; x < numMetrics; x++) { + BaseQueryMetric m = createMetric(); + metrics.add(m); + updates.add(m.duplicate()); + } + + List shuffledUpdates = new ArrayList<>(); + Random r = new Random(); + for (BaseQueryMetric m : metrics) { + int numPages = r.nextInt(maxPages); + BaseQueryMetric m2 = m; + for (int x = 0; x < numPages; x++) { + m2 = m2.duplicate(); + m.setLifecycle(BaseQueryMetric.Lifecycle.RESULTS); + m2.setLifecycle(BaseQueryMetric.Lifecycle.RESULTS); + BaseQueryMetric.PageMetric pageMetric = new BaseQueryMetric.PageMetric("localhost", 100, 100, 100, 100, -1, -1, -1, -1); + m.addPageMetric(pageMetric); + m2.addPageMetric(pageMetric); + shuffledUpdates.add(m2); + } + } + + // randomize the order of metric updates + Collections.shuffle(shuffledUpdates); + updates.addAll(shuffledUpdates); + + LinkedBlockingDeque updateDeque = new LinkedBlockingDeque<>(); + updateDeque.addAll(updates); + ExecutorService executorService = Executors.newFixedThreadPool(20); + for (int x = 0; x < 20; x++) { + Runnable runnable = () -> { + BaseQueryMetric m; + do { + m = updateDeque.poll(); + if (m != null) { + queryMetricOperations.handleEvent(new QueryMetricUpdate(m, QueryMetricType.COMPLETE)); + } + } while (m != null); + }; + executorService.submit(runnable); + } + log.debug("done submitting metrics"); + executorService.shutdown(); + boolean completed = executorService.awaitTermination(2, TimeUnit.MINUTES); + Assert.assertTrue("executor tasks completed", completed); + // flush the correlator + this.correlator.shutdown(true); + this.queryMetricOperations.ensureUpdatesProcessed(); + this.correlator.shutdown(false); + + long start = System.currentTimeMillis(); + for (BaseQueryMetric m : metrics) { + String queryId = m.getQueryId(); + ensureDataStored(incomingQueryMetricsCache, queryId); + QueryMetricUpdate metricUpdate; + BaseQueryMetric storedMetric = null; + do { + metricUpdate = incomingQueryMetricsCache.get(queryId, QueryMetricUpdate.class); + if (metricUpdate == null && (System.currentTimeMillis() - start) < 5000) { + Thread.sleep(200); + } else { + storedMetric = metricUpdate.getMetric(); + } + } while (storedMetric == null); + Assert.assertNotNull("missing metric " + queryId, storedMetric); + assertEquals("incomingQueryMetricsCache metric wrong for id:" + m.getQueryId(), m, storedMetric); + } + } +} diff --git a/service/src/test/resources/config/application-correlator.yml b/service/src/test/resources/config/application-correlator.yml new file mode 100644 index 00000000..77a0d5a1 --- /dev/null +++ b/service/src/test/resources/config/application-correlator.yml @@ -0,0 +1,5 @@ +datawave: + query: + metric: + correlator: + enabled: true diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index 2ecee9a2..31d6c6b5 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -99,6 +99,8 @@ datawave: logServiceStatsRateMs: 300000 publishServiceStatsToTimelyRateMs: 60000 publishQueryStatsToTimelyRateMs: 60000 + correlator: + enabled: false metadata: all-metadata-auths: From 0d599b158d65537f108ffef2c57852ae7016662c Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Tue, 5 Dec 2023 13:07:14 -0500 Subject: [PATCH 22/48] Change lastWrittenQueryMetrics cache to a Caffeine cache instead of Hazelcast --- .../microservice/querymetric/CacheStats.java | 18 ++------ .../querymetric/QueryMetricOperations.java | 11 ++--- .../QueryMetricOperationsStats.java | 18 ++++---- .../HazelcastMetricCacheConfiguration.java | 17 ++----- .../config/QueryMetricCacheConfiguration.java | 37 +++++++++++++++ .../config/QueryMetricCacheProperties.java | 44 ++++++++++++++++++ .../config/StatsConfiguration.java | 7 ++- .../persistence/AccumuloMapStore.java | 19 +++++--- .../persistence/MetricCacheListener.java | 46 +++++++++++++++++++ .../querymetric/HazelcastCachingTest.java | 17 ------- .../QueryMetricConsistencyTest.java | 2 +- .../querymetric/QueryMetricTestBase.java | 10 ++-- .../application-hazelcast-writebehind.yml | 18 -------- .../application-hazelcast-writethrough.yml | 18 -------- .../src/test/resources/config/application.yml | 4 ++ 15 files changed, 172 insertions(+), 114 deletions(-) create mode 100644 service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java create mode 100644 service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheProperties.java create mode 100644 service/src/main/java/datawave/microservice/querymetric/persistence/MetricCacheListener.java diff --git a/api/src/main/java/datawave/microservice/querymetric/CacheStats.java b/api/src/main/java/datawave/microservice/querymetric/CacheStats.java index 79bbd4a0..a9994119 100644 --- a/api/src/main/java/datawave/microservice/querymetric/CacheStats.java +++ b/api/src/main/java/datawave/microservice/querymetric/CacheStats.java @@ -14,7 +14,7 @@ @XmlRootElement(name = "CacheStats") @XmlAccessorType(XmlAccessType.NONE) -@XmlType(propOrder = {"serviceStats", "incomingQueryMetrics", "lastWrittenQueryMetrics"}) +@XmlType(propOrder = {"serviceStats", "incomingQueryMetrics"}) public class CacheStats implements Serializable { private static final long serialVersionUID = 1L; @@ -33,10 +33,6 @@ public class CacheStats implements Serializable { @XmlJavaTypeAdapter(StringMapAdapter.class) private Map incomingQueryMetrics = new HashMap<>(); - @XmlElement(name = "lastWrittenQueryMetrics") - @XmlJavaTypeAdapter(StringMapAdapter.class) - private Map lastWrittenQueryMetrics = new HashMap<>(); - public CacheStats() { } @@ -65,14 +61,6 @@ public Map getIncomingQueryMetrics() { return incomingQueryMetrics; } - public void setLastWrittenQueryMetrics(Map stats) { - this.lastWrittenQueryMetrics = stats; - } - - public Map getLastWrittenQueryMetrics() { - return lastWrittenQueryMetrics; - } - public void setServiceStats(Map serviceStats) { this.serviceStats = serviceStats; } @@ -89,11 +77,11 @@ public boolean equals(Object o) { return false; CacheStats that = (CacheStats) o; return host.equals(that.host) && memberUuid.equals(that.memberUuid) && incomingQueryMetrics.equals(that.incomingQueryMetrics) - && lastWrittenQueryMetrics.equals(that.lastWrittenQueryMetrics) && serviceStats.equals(that.serviceStats); + && serviceStats.equals(that.serviceStats); } @Override public int hashCode() { - return Objects.hash(host, memberUuid, incomingQueryMetrics, lastWrittenQueryMetrics, serviceStats); + return Objects.hash(host, memberUuid, incomingQueryMetrics, serviceStats); } } diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 5e5f5e7d..faa76ea8 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -62,7 +62,6 @@ import static datawave.microservice.querymetric.QueryMetricOperations.DEFAULT_DATETIME.BEGIN; import static datawave.microservice.querymetric.QueryMetricOperations.DEFAULT_DATETIME.END; import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.INCOMING_METRICS; -import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.LAST_WRITTEN_METRICS; import static datawave.microservice.querymetric.config.QueryMetricSourceConfiguration.QueryMetricSourceBinding.SOURCE_NAME; import static datawave.microservice.querymetric.QueryMetricOperationsStats.METERS; import static datawave.microservice.querymetric.QueryMetricOperationsStats.TIMERS; @@ -82,7 +81,6 @@ public class QueryMetricOperations { private QueryGeometryHandler geometryHandler; private CacheManager cacheManager; private Cache incomingQueryMetricsCache; - private Cache lastWrittenQueryMetricCache; private MarkingFunctions markingFunctions; private BaseQueryMetricListResponseFactory queryMetricListResponseFactory; private MergeLockLifecycleListener mergeLock; @@ -138,7 +136,6 @@ public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cach this.geometryHandler = geometryHandler; this.cacheManager = cacheManager; this.incomingQueryMetricsCache = cacheManager.getCache(INCOMING_METRICS); - this.lastWrittenQueryMetricCache = cacheManager.getCache(LAST_WRITTEN_METRICS); this.markingFunctions = markingFunctions; this.queryMetricListResponseFactory = queryMetricListResponseFactory; this.mergeLock = mergeLock; @@ -363,10 +360,10 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDet try { BaseQueryMetric metric; QueryMetricUpdateHolder metricUpdate = incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class); - if (metricUpdate != null && metricUpdate.isNewMetric()) { - metric = metricUpdate.getMetric(); - } else { + if (metricUpdate == null) { metric = this.handler.getQueryMetric(queryId); + } else { + metric = metricUpdate.getMetric(); } if (metric != null) { boolean allowAllMetrics = false; @@ -546,8 +543,6 @@ public CacheStats getCacheStats() { CacheStats cacheStats = new CacheStats(); IMap incomingCacheHz = ((IMap) incomingQueryMetricsCache.getNativeCache()); cacheStats.setIncomingQueryMetrics(this.stats.getLocalMapStats(incomingCacheHz.getLocalMapStats())); - IMap lastWrittenCacheHz = ((IMap) lastWrittenQueryMetricCache.getNativeCache()); - cacheStats.setLastWrittenQueryMetrics(this.stats.getLocalMapStats(lastWrittenCacheHz.getLocalMapStats())); cacheStats.setServiceStats(this.stats.formatStats(this.stats.getServiceStats(), true)); cacheStats.setMemberUuid(getClusterLocalMemberUuid()); try { diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java index 5fafffbb..a3715ff5 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java @@ -14,6 +14,7 @@ import datawave.util.timely.UdpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; @@ -31,7 +32,6 @@ import java.util.stream.Collectors; import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.INCOMING_METRICS; -import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.LAST_WRITTEN_METRICS; import static java.util.concurrent.TimeUnit.MINUTES; public class QueryMetricOperationsStats { @@ -46,6 +46,7 @@ public class QueryMetricOperationsStats { protected ShardTableQueryMetricHandler handler; protected AccumuloMapStore mapStore; protected CacheManager cacheManager; + protected Cache lastWrittenCache; protected Map hostCountMap = new HashMap<>(); protected Map userCountMap = new HashMap<>(); protected Map logicCountMap = new HashMap<>(); @@ -67,11 +68,12 @@ public enum METERS { */ public QueryMetricOperationsStats(TimelyProperties timelyProperties, ShardTableQueryMetricHandler handler, CacheManager cacheManager, - AccumuloMapStore mapStore) { + @Qualifier("lastWrittenQueryMetrics") Cache lastWrittenCache, AccumuloMapStore mapStore) { this.timelyProperties = timelyProperties; this.handler = handler; this.mapStore = mapStore; this.cacheManager = cacheManager; + this.lastWrittenCache = lastWrittenCache; for (TIMERS name : TIMERS.values()) { this.timerMap.put(name, new Timer(new SlidingTimeWindowArrayReservoir(1, MINUTES))); } @@ -106,18 +108,16 @@ public Meter getMeter(METERS name) { return this.meterMap.get(name); } - protected void addLocalMapStats(Map serviceStats) { + protected void addCacheStats(Map serviceStats) { Cache incomingCache = this.cacheManager.getCache(INCOMING_METRICS); if (incomingCache != null) { IMap incomingQueryMetricsCache = ((IMap) incomingCache.getNativeCache()); serviceStats.put("incomingQueryMetrics.dirtyEntryCount", Double.valueOf(incomingQueryMetricsCache.getLocalMapStats().getDirtyEntryCount())); serviceStats.put("incomingQueryMetrics.ownedEntryCount", Double.valueOf(incomingQueryMetricsCache.getLocalMapStats().getOwnedEntryCount())); } - Cache lastWrittenCache = this.cacheManager.getCache(LAST_WRITTEN_METRICS); - if (lastWrittenCache != null) { - IMap lastWrittenQueryMetricCache = ((IMap) lastWrittenCache.getNativeCache()); - serviceStats.put("lastWrittenQueryMetric.dirtyEntryCount", Double.valueOf(lastWrittenQueryMetricCache.getLocalMapStats().getDirtyEntryCount())); - serviceStats.put("lastWrittenQueryMetric.ownedEntryCount", Double.valueOf(lastWrittenQueryMetricCache.getLocalMapStats().getOwnedEntryCount())); + if (this.lastWrittenCache != null) { + long estimatedSize = ((com.github.benmanes.caffeine.cache.Cache) this.lastWrittenCache.getNativeCache()).estimatedSize(); + serviceStats.put("lastWrittenQueryMetric.estimatedSize", Double.valueOf(estimatedSize)); } } @@ -127,7 +127,7 @@ public void writeServiceStatsToTimely() { long timestamp = System.currentTimeMillis(); Map serviceStatsDouble = getServiceStats(); - addLocalMapStats(serviceStatsDouble); + addCacheStats(serviceStatsDouble); Map serviceStats = formatStats(serviceStatsDouble, false); serviceStats.entrySet().forEach(entry -> { serviceStatsToWriteToTimely diff --git a/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java index cc29f716..3c54b557 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/HazelcastMetricCacheConfiguration.java @@ -1,18 +1,15 @@ package datawave.microservice.querymetric.config; import com.hazelcast.config.Config; -import com.hazelcast.config.ConfigurationException; import com.hazelcast.config.DiscoveryStrategyConfig; import com.hazelcast.config.InMemoryFormat; import com.hazelcast.config.JoinConfig; import com.hazelcast.config.ListenerConfig; import com.hazelcast.config.MapConfig; -import com.hazelcast.config.MapStoreConfig; import com.hazelcast.config.TcpIpConfig; import com.hazelcast.config.XmlConfigBuilder; import com.hazelcast.core.Hazelcast; import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.core.IMap; import com.hazelcast.core.LifecycleEvent; import com.hazelcast.kubernetes.HazelcastKubernetesDiscoveryStrategyFactory; import com.hazelcast.kubernetes.KubernetesProperties; @@ -31,6 +28,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.Cache; import org.springframework.cloud.consul.discovery.ConsulDiscoveryProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -49,7 +47,6 @@ public class HazelcastMetricCacheConfiguration { private Logger log = LoggerFactory.getLogger(HazelcastMetricCacheConfiguration.class); - public static final String LAST_WRITTEN_METRICS = "lastWrittenQueryMetrics"; public static final String INCOMING_METRICS = "incomingQueryMetrics"; @Value("${spring.application.name}") @@ -63,7 +60,7 @@ public HazelcastCacheManager queryMetricCacheManager(@Qualifier("metrics") Hazel @Bean @Qualifier("metrics") HazelcastInstance hazelcastInstance(Config config, @Qualifier("store") AccumuloMapStore mapStore, @Qualifier("loader") AccumuloMapLoader mapLoader, - MergeLockLifecycleListener lifecycleListener) { + MergeLockLifecycleListener lifecycleListener, @Qualifier("lastWrittenQueryMetrics") Cache lastWrittenCache) { // Autowire both the AccumuloMapStore and AccumuloMapLoader so that they both get created // Ensure that the lastWrittenQueryMetricCache is set into the MapStore before the instance is active and the writeLock is released lifecycleListener.writeLockRunnable.lock(LifecycleEvent.LifecycleState.STARTING); @@ -72,18 +69,10 @@ HazelcastInstance hazelcastInstance(Config config, @Qualifier("store") AccumuloM try { HazelcastCacheManager cacheManager = new HazelcastCacheManager(instance); - HazelcastCache lastWrittenQueryMetricsCache = (HazelcastCache) cacheManager.getCache(LAST_WRITTEN_METRICS); - lastWrittenQueryMetricsCache.getNativeCache().addEntryListener(new MetricMapListener(LAST_WRITTEN_METRICS), true); - HazelcastCache incomingMetricsCache = (HazelcastCache) cacheManager.getCache(INCOMING_METRICS); incomingMetricsCache.getNativeCache().addEntryListener(new MetricMapListener(INCOMING_METRICS), true); - MapStoreConfig mapStoreConfig = config.getMapConfigs().get(LAST_WRITTEN_METRICS).getMapStoreConfig(); - if (mapStoreConfig.getInitialLoadMode().equals(MapStoreConfig.InitialLoadMode.LAZY)) { - // prompts loading all keys otherwise we are getting a deadlock - lastWrittenQueryMetricsCache.getNativeCache().size(); - } - mapStore.setLastWrittenQueryMetricCache(lastWrittenQueryMetricsCache); + mapStore.setLastWrittenQueryMetricCache(lastWrittenCache); System.setProperty("hzAddress", instance.getCluster().getLocalMember().getAddress().toString()); System.setProperty("hzUuid", instance.getCluster().getLocalMember().getUuid()); } catch (Exception e) { diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java new file mode 100644 index 00000000..d6d86bf4 --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java @@ -0,0 +1,37 @@ +package datawave.microservice.querymetric.config; + +import com.github.benmanes.caffeine.cache.Caffeine; +import datawave.microservice.querymetric.persistence.MetricCacheListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.Cache; +import org.springframework.cache.caffeine.CaffeineCache; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.concurrent.TimeUnit; + +@Configuration +@EnableConfigurationProperties({QueryMetricCacheProperties.class}) +public class QueryMetricCacheConfiguration { + + private Logger log = LoggerFactory.getLogger(QueryMetricCacheConfiguration.class); + public static final String LAST_WRITTEN_METRICS = "lastWrittenQueryMetrics"; + + @Bean + @Qualifier("lastWrittenQueryMetrics") + public Cache lastWrittenQueryMetrics(QueryMetricCacheProperties cacheProperties) { + // @formatter:off + QueryMetricCacheProperties.Cache lastWrittenQueryMetrics = cacheProperties.getLastWrittenQueryMetrics(); + return new CaffeineCache("lastWrittenQueryMetrics", + Caffeine.newBuilder() + .initialCapacity(lastWrittenQueryMetrics.getMaximumSize()) + .maximumSize(lastWrittenQueryMetrics.getMaximumSize()) + .expireAfterWrite(lastWrittenQueryMetrics.getTtlSeconds(), TimeUnit.SECONDS) + .removalListener(new MetricCacheListener(LAST_WRITTEN_METRICS)) + .build()); + // @formatter:on + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheProperties.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheProperties.java new file mode 100644 index 00000000..05da4fe4 --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheProperties.java @@ -0,0 +1,44 @@ +package datawave.microservice.querymetric.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +import javax.validation.constraints.PositiveOrZero; + +@Validated +@ConfigurationProperties(prefix = "datawave.query.metric.cache") +public class QueryMetricCacheProperties { + + private Cache lastWrittenQueryMetrics = new Cache(); + + public void setLastWrittenQueryMetrics(Cache lastWrittenQueryMetrics) { + this.lastWrittenQueryMetrics = lastWrittenQueryMetrics; + } + + public Cache getLastWrittenQueryMetrics() { + return lastWrittenQueryMetrics; + } + + public class Cache { + @PositiveOrZero + private int maximumSize = 5000; + @PositiveOrZero + private long ttlSeconds = 600; + + public void setMaximumSize(int maximumSize) { + this.maximumSize = maximumSize; + } + + public int getMaximumSize() { + return maximumSize; + } + + public void setTtlSeconds(long ttlSeconds) { + this.ttlSeconds = ttlSeconds; + } + + public long getTtlSeconds() { + return ttlSeconds; + } + } +} diff --git a/service/src/main/java/datawave/microservice/querymetric/config/StatsConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/StatsConfiguration.java index 22492656..bbbcfa11 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/StatsConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/StatsConfiguration.java @@ -3,7 +3,9 @@ import datawave.microservice.querymetric.QueryMetricOperationsStats; import datawave.microservice.querymetric.handler.ShardTableQueryMetricHandler; import datawave.microservice.querymetric.persistence.AccumuloMapStore; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -16,7 +18,8 @@ public class StatsConfiguration { @Bean @ConditionalOnMissingBean QueryMetricOperationsStats queryMetricOperationsStats(TimelyProperties timelyProperties, ShardTableQueryMetricHandler handler, - @Named("queryMetricCacheManager") CacheManager cacheManager, AccumuloMapStore mapStore) { - return new QueryMetricOperationsStats(timelyProperties, handler, cacheManager, mapStore); + @Named("queryMetricCacheManager") CacheManager cacheManager, @Qualifier("lastWrittenQueryMetrics") Cache lastWrittenCache, + AccumuloMapStore mapStore) { + return new QueryMetricOperationsStats(timelyProperties, handler, cacheManager, lastWrittenCache, mapStore); } } diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index a55ff192..6680957c 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -3,7 +3,6 @@ import com.codahale.metrics.SlidingTimeWindowArrayReservoir; import com.codahale.metrics.Timer; import com.google.common.cache.CacheBuilder; -import com.hazelcast.core.IMap; import com.hazelcast.core.MapLoader; import com.hazelcast.core.MapStore; import com.hazelcast.core.MapStoreFactory; @@ -38,7 +37,7 @@ public class AccumuloMapStore extends AccumuloMapLoad private static AccumuloMapStore instance; private Logger log = LoggerFactory.getLogger(AccumuloMapStore.class); - private IMap lastWrittenQueryMetricCache; + private Cache lastWrittenQueryMetricCache; private MergeLockLifecycleListener mergeLock; private com.google.common.cache.Cache failures; private Timer writeTimer = new Timer(new SlidingTimeWindowArrayReservoir(1, MINUTES)); @@ -72,7 +71,7 @@ public void shutdown() { } public void setLastWrittenQueryMetricCache(Cache lastWrittenQueryMetricCache) { - this.lastWrittenQueryMetricCache = (IMap) lastWrittenQueryMetricCache.getNativeCache(); + this.lastWrittenQueryMetricCache = lastWrittenQueryMetricCache; } @Override @@ -106,14 +105,21 @@ public void storeWithRetry(QueryMetricUpdateHolder queryMetricUpdate) { public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception { String queryId = queryMetricUpdate.getMetric().getQueryId(); T updatedMetric = null; - this.mergeLock.lock(); try { updatedMetric = (T) queryMetricUpdate.getMetric().duplicate(); QueryMetricType metricType = queryMetricUpdate.getMetricType(); QueryMetricUpdateHolder lastQueryMetricUpdate = null; if (!queryMetricUpdate.isNewMetric()) { - lastQueryMetricUpdate = (QueryMetricUpdateHolder) lastWrittenQueryMetricCache.get(queryId); + lastQueryMetricUpdate = (QueryMetricUpdateHolder) lastWrittenQueryMetricCache.get(queryId, () -> { + log.debug("getting metric {} from accumulo", queryId); + T m = handler.getQueryMetric(queryId); + if (m == null) { + return null; + } else { + return new QueryMetricUpdateHolder(m, metricType); + } + }); } if (lastQueryMetricUpdate != null) { @@ -159,7 +165,7 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception log.debug("writing metric to accumulo: " + queryId); } - lastWrittenQueryMetricCache.set(queryId, new QueryMetricUpdateHolder(updatedMetric)); + lastWrittenQueryMetricCache.put(queryId, new QueryMetricUpdateHolder(updatedMetric)); queryMetricUpdate.setPersisted(); failures.invalidate(queryId); } finally { @@ -175,7 +181,6 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception queryMetricUpdate.getMetric().setFiRanges(updatedMetric.getFiRanges()); } } - this.mergeLock.unlock(); } } diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/MetricCacheListener.java b/service/src/main/java/datawave/microservice/querymetric/persistence/MetricCacheListener.java new file mode 100644 index 00000000..7852cbbf --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/MetricCacheListener.java @@ -0,0 +1,46 @@ +package datawave.microservice.querymetric.persistence; + +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.RemovalListener; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricUpdate; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.SimpleDateFormat; + +public class MetricCacheListener implements RemovalListener { + + private Logger log = LoggerFactory.getLogger(MetricCacheListener.class); + + private final String cacheName; + + public MetricCacheListener(String cacheName) { + this.cacheName = cacheName; + } + + @Override + public void onRemoval(@Nullable Object key, @Nullable Object value, @NonNull RemovalCause cause) { + if (!cause.equals(RemovalCause.REPLACED)) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd HHmmss"); + String queryId = (String) key; + StringBuilder sb = new StringBuilder(); + sb.append(String.format("removalCause=%s, key=%s", cause, queryId)); + BaseQueryMetric metric = null; + if (value != null && value instanceof QueryMetricUpdate) { + metric = ((QueryMetricUpdate) value).getMetric(); + sb.append(String.format(" metric[createDate=%s, host=%s, lifecycle=%s, numPages=%s, numUpdates=%s]", sdf.format(metric.getCreateDate()), + metric.getHost(), metric.getLifecycle(), metric.getPageTimes().size(), metric.getNumUpdates())); + } + if (cause.equals(RemovalCause.SIZE) && metric != null && !metric.isLifecycleFinal()) { + // reaching max cache size and evicting metrics that are not done writing + log.info(cacheName + " " + sb); + } else { + // more routine - evicting due to expired time + log.debug(cacheName + " " + sb); + } + } + } +} diff --git a/service/src/test/java/datawave/microservice/querymetric/HazelcastCachingTest.java b/service/src/test/java/datawave/microservice/querymetric/HazelcastCachingTest.java index 748e53b2..f818c525 100644 --- a/service/src/test/java/datawave/microservice/querymetric/HazelcastCachingTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/HazelcastCachingTest.java @@ -10,8 +10,6 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; -import java.util.Collections; - @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"HazelcastCachingTest", "QueryMetricTest", "hazelcast-writethrough"}) @@ -27,21 +25,6 @@ public void cleanup() { super.cleanup(); } - @Test - public void TestReadThroughCache() { - - try { - String queryId = createQueryId(); - BaseQueryMetric m = createMetric(queryId); - shardTableQueryMetricHandler.writeMetric(m, Collections.emptyList(), m.getCreateDate().getTime(), false); - BaseQueryMetric metricFromReadThroughCache = lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdate.class).getMetric(); - assertEquals("read through cache failed", m, metricFromReadThroughCache); - } catch (Exception e) { - log.error(e.getMessage(), e); - Assert.fail(e.getMessage()); - } - } - @Test public void TestWriteThroughCache() { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index fbd3217d..428fe8c1 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -344,7 +344,7 @@ public void DuplicateAccumuloEntryTest() throws Exception { String queryId = createQueryId(); QueryMetric storedQueryMetric = (QueryMetric) createMetric(queryId); QueryMetric updatedQueryMetric = (QueryMetric) storedQueryMetric.duplicate(); - updatedQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); + updatedQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.RESULTS); updatedQueryMetric.setNumResults(2000); updatedQueryMetric.setDocRanges(400); updatedQueryMetric.setNextCount(400); diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index 3139bae3..730ea79e 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -64,7 +64,6 @@ import java.util.Set; import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.INCOMING_METRICS; -import static datawave.microservice.querymetric.config.HazelcastMetricCacheConfiguration.LAST_WRITTEN_METRICS; import static datawave.security.authorization.DatawaveUser.UserType.USER; public class QueryMetricTestBase { @@ -110,9 +109,12 @@ public class QueryMetricTestBase { @Autowired private QueryMetricClientProperties queryMetricClientProperties; - protected Cache incomingQueryMetricsCache; + @Autowired + @Qualifier("lastWrittenQueryMetrics") protected Cache lastWrittenQueryMetricCache; + protected Cache incomingQueryMetricsCache; + @LocalServerPort protected int webServicePort; @@ -144,7 +146,6 @@ public void setup() { this.nonAdminUser = new ProxiedUserDetails(Collections.singleton(nonAdminDWUser), nonAdminDWUser.getCreationTime()); QueryMetricTestBase.staticCacheManager = cacheManager; this.incomingQueryMetricsCache = cacheManager.getCache(INCOMING_METRICS); - this.lastWrittenQueryMetricCache = cacheManager.getCache(LAST_WRITTEN_METRICS); this.shardTableQueryMetricHandler.verifyTables(); BaseQueryMetric m = createMetric(); // this is to ensure that the QueryMetrics_m table @@ -412,9 +413,8 @@ protected void ensureDataWritten(Cache incomingCache, Cache lastWrittenCache, St MapStoreConfig mapStoreConfig = config.getMapConfig(incomingCache.getName()).getMapStoreConfig(); int writeDelaySeconds = Math.min(mapStoreConfig.getWriteDelaySeconds(), 1000); boolean found = false; - IMap hzCache = ((IMap) lastWrittenCache.getNativeCache()); while (!found && System.currentTimeMillis() < (now + (1000 * (writeDelaySeconds + 1)))) { - found = hzCache.containsKey(queryId); + found = lastWrittenCache.get(queryId) != null; if (!found) { try { Thread.sleep(250); diff --git a/service/src/test/resources/config/application-hazelcast-writebehind.yml b/service/src/test/resources/config/application-hazelcast-writebehind.yml index 66d5e3c6..c4d3201a 100644 --- a/service/src/test/resources/config/application-hazelcast-writebehind.yml +++ b/service/src/test/resources/config/application-hazelcast-writebehind.yml @@ -33,22 +33,4 @@ hazelcast: 1000 - - OBJECT - - 1 - - 600 - - 3600 - com.hazelcast.spi.merge.LatestUpdateMergePolicy - - 100000 - - LRU - - datawave.microservice.querymetric.persistence.AccumuloMapLoader$Factory - - diff --git a/service/src/test/resources/config/application-hazelcast-writethrough.yml b/service/src/test/resources/config/application-hazelcast-writethrough.yml index e8b9deea..b79853a4 100644 --- a/service/src/test/resources/config/application-hazelcast-writethrough.yml +++ b/service/src/test/resources/config/application-hazelcast-writethrough.yml @@ -33,22 +33,4 @@ hazelcast: 1000 - - OBJECT - - 1 - - 3600 - - 7200 - com.hazelcast.spi.merge.LatestUpdateMergePolicy - - 100000 - - LRU - - datawave.microservice.querymetric.persistence.AccumuloMapLoader$Factory - - diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index 31d6c6b5..3eb56679 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -84,6 +84,10 @@ datawave: queryVisibility: PUBLIC defaultMetricVisibility: (PUBLIC) baseMaps: "{}" + cache: + lastWrittenQueryMetrics: + maximumSize: 100 + ttlSeconds: 60 client: enabled: true transport: message_test From cd480d7a598e1e6acd4abba48bbd57295ce67917 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Tue, 5 Dec 2023 18:09:12 -0500 Subject: [PATCH 23/48] Limit metric fields retrieved from Accumulo and compared for mutations to reduce retrieval time, bandwidth, and memory usage --- .../ContentQueryMetricsIngestHelper.java | 104 ++++++++++++------ .../handler/QueryMetricCombiner.java | 1 + .../handler/QueryMetricHandler.java | 3 +- .../handler/ShardTableQueryMetricHandler.java | 29 +++-- .../persistence/AccumuloMapStore.java | 28 ++++- ...ernateContentQueryMetricsIngestHelper.java | 11 +- ...AlternateShardTableQueryMetricHandler.java | 5 +- 7 files changed, 129 insertions(+), 52 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java index c0499afd..f635071f 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java @@ -16,6 +16,7 @@ import java.text.SimpleDateFormat; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -30,8 +31,8 @@ public class ContentQueryMetricsIngestHelper extends CSVIngestHelper implements private Set contentIndexFields = new HashSet<>(); private HelperDelegate delegate; - public ContentQueryMetricsIngestHelper(boolean deleteMode) { - this(deleteMode, new HelperDelegate<>()); + public ContentQueryMetricsIngestHelper(boolean deleteMode, Collection ignoredFields) { + this(deleteMode, new HelperDelegate<>(ignoredFields)); } public ContentQueryMetricsIngestHelper(boolean deleteMode, HelperDelegate delegate) { @@ -90,6 +91,13 @@ public int getFieldSizeThreshold() { } public static class HelperDelegate { + private Collection ignoredFields = Collections.EMPTY_LIST; + + public HelperDelegate() {} + + public HelperDelegate(Collection ignoredFields) { + this.ignoredFields = ignoredFields; + } protected boolean isChanged(String updated, String stored) { if ((StringUtils.isBlank(stored) && StringUtils.isNotBlank(updated)) || (stored != null && updated != null && !stored.equals(updated))) { @@ -162,17 +170,25 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { SimpleDateFormat sdf_date_time1 = new SimpleDateFormat("yyyyMMdd HHmmss"); SimpleDateFormat sdf_date_time2 = new SimpleDateFormat("yyyyMMdd HHmmss"); - if (isFirstWrite(updated.getPositiveSelectors(), stored == null ? null : stored.getPositiveSelectors())) { - fields.putAll("POSITIVE_SELECTORS", updated.getPositiveSelectors()); + if (!ignoredFields.contains("POSITIVE_SELECTORS")) { + if (isFirstWrite(updated.getPositiveSelectors(), stored == null ? null : stored.getPositiveSelectors())) { + fields.putAll("POSITIVE_SELECTORS", updated.getPositiveSelectors()); + } } - if (isFirstWrite(updated.getNegativeSelectors(), stored == null ? null : stored.getNegativeSelectors())) { - fields.putAll("NEGATIVE_SELECTORS", updated.getNegativeSelectors()); + if (!ignoredFields.contains("NEGATIVE_SELECTORS")) { + if (isFirstWrite(updated.getNegativeSelectors(), stored == null ? null : stored.getNegativeSelectors())) { + fields.putAll("NEGATIVE_SELECTORS", updated.getNegativeSelectors()); + } } - if (isFirstWrite(updated.getQueryAuthorizations(), stored == null ? null : stored.getQueryAuthorizations())) { - fields.put("AUTHORIZATIONS", updated.getQueryAuthorizations()); + if (!ignoredFields.contains("AUTHORIZATIONS")) { + if (isFirstWrite(updated.getQueryAuthorizations(), stored == null ? null : stored.getQueryAuthorizations())) { + fields.put("AUTHORIZATIONS", updated.getQueryAuthorizations()); + } } - if (isFirstWrite(updated.getBeginDate(), stored == null ? null : stored.getBeginDate())) { - fields.put("BEGIN_DATE", sdf_date_time1.format(updated.getBeginDate())); + if (!ignoredFields.contains("BEGIN_DATE")) { + if (isFirstWrite(updated.getBeginDate(), stored == null ? null : stored.getBeginDate())) { + fields.put("BEGIN_DATE", sdf_date_time1.format(updated.getBeginDate())); + } } if (isChanged(updated.getCreateCallTime(), stored == null ? -1 : stored.getCreateCallTime())) { fields.put("CREATE_CALL_TIME", Long.toString(updated.getCreateCallTime())); @@ -186,8 +202,10 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getElapsedTime(), stored == null ? -1 : stored.getElapsedTime())) { fields.put("ELAPSED_TIME", Long.toString(updated.getElapsedTime())); } - if (isFirstWrite(updated.getEndDate(), stored == null ? null : stored.getEndDate())) { - fields.put("END_DATE", sdf_date_time1.format(updated.getEndDate())); + if (!ignoredFields.contains("END_DATE")) { + if (isFirstWrite(updated.getEndDate(), stored == null ? null : stored.getEndDate())) { + fields.put("END_DATE", sdf_date_time1.format(updated.getEndDate())); + } } if (isChanged(updated.getErrorCode(), stored == null ? null : stored.getErrorCode())) { fields.put("ERROR_CODE", updated.getErrorCode()); @@ -233,14 +251,18 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getNumUpdates(), stored == null ? -1 : stored.getNumUpdates())) { fields.put("NUM_UPDATES", Long.toString(updated.getNumUpdates())); } - if (isFirstWrite(updated.getParameters(), stored == null ? null : stored.getParameters())) { - fields.put("PARAMETERS", QueryUtil.toParametersString(updated.getParameters())); + if (!ignoredFields.contains("PARAMETERS")) { + if (isFirstWrite(updated.getParameters(), stored == null ? null : stored.getParameters())) { + fields.put("PARAMETERS", QueryUtil.toParametersString(updated.getParameters())); + } } if (isChanged(updated.getPlan(), stored == null ? null : stored.getPlan())) { fields.put("PLAN", updated.getPlan()); } - if (isFirstWrite(updated.getProxyServers(), stored == null ? null : stored.getProxyServers())) { - fields.put("PROXY_SERVERS", StringUtils.join(updated.getProxyServers(), ",")); + if (!ignoredFields.contains("PROXY_SERVERS")) { + if (isFirstWrite(updated.getProxyServers(), stored == null ? null : stored.getProxyServers())) { + fields.put("PROXY_SERVERS", StringUtils.join(updated.getProxyServers(), ",")); + } } Map storedPageMetricMap = new HashMap<>(); @@ -272,20 +294,28 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { } } } - if (isFirstWrite(updated.getQuery(), stored == null ? null : stored.getQuery())) { - fields.put("QUERY", updated.getQuery()); + if (!ignoredFields.contains("QUERY")) { + if (isFirstWrite(updated.getQuery(), stored == null ? null : stored.getQuery())) { + fields.put("QUERY", updated.getQuery()); + } } if (isFirstWrite(updated.getQueryId(), stored == null ? null : stored.getQueryId())) { fields.put("QUERY_ID", updated.getQueryId()); } - if (isFirstWrite(updated.getQueryLogic(), stored == null ? null : stored.getQueryLogic())) { - fields.put("QUERY_LOGIC", updated.getQueryLogic()); + if (!ignoredFields.contains("QUERY_LOGIC")) { + if (isFirstWrite(updated.getQueryLogic(), stored == null ? null : stored.getQueryLogic())) { + fields.put("QUERY_LOGIC", updated.getQueryLogic()); + } } - if (isFirstWrite(updated.getQueryName(), stored == null ? null : stored.getQueryName())) { - fields.put("QUERY_NAME", updated.getQueryName()); + if (!ignoredFields.contains("QUERY_NAME")) { + if (isFirstWrite(updated.getQueryName(), stored == null ? null : stored.getQueryName())) { + fields.put("QUERY_NAME", updated.getQueryName()); + } } - if (isFirstWrite(updated.getQueryType(), stored == null ? null : stored.getQueryType())) { - fields.put("QUERY_TYPE", updated.getQueryType()); + if (!ignoredFields.contains("QUERY_TYPE")) { + if (isFirstWrite(updated.getQueryType(), stored == null ? null : stored.getQueryType())) { + fields.put("QUERY_TYPE", updated.getQueryType()); + } } if (isFirstWrite(updated.getSetupTime(), stored == null ? 0 : stored.getSetupTime(), 0)) { fields.put("SETUP_TIME", Long.toString(updated.getSetupTime())); @@ -296,18 +326,24 @@ public Multimap getEventFieldsToWrite(T updated, T stored) { if (isChanged(updated.getSourceCount(), stored == null ? -1 : stored.getSourceCount())) { fields.put("SOURCE_COUNT", Long.toString(updated.getSourceCount())); } - if (isFirstWrite(updated.getUser(), stored == null ? null : stored.getUser())) { - fields.put("USER", updated.getUser()); + if (!ignoredFields.contains("USER")) { + if (isFirstWrite(updated.getUser(), stored == null ? null : stored.getUser())) { + fields.put("USER", updated.getUser()); + } } - if (isFirstWrite(updated.getUserDN(), stored == null ? null : stored.getUserDN())) { - fields.put("USER_DN", updated.getUserDN()); + if (!ignoredFields.contains("USER_DN")) { + if (isFirstWrite(updated.getUserDN(), stored == null ? null : stored.getUserDN())) { + fields.put("USER_DN", updated.getUserDN()); + } } - if (isFirstWrite(updated.getVersionMap(), stored == null ? null : stored.getVersionMap())) { - Map versionMap = updated.getVersionMap(); - if (versionMap != null) { - versionMap.entrySet().stream().forEach(e -> { - fields.put("VERSION." + e.getKey().toUpperCase(), e.getValue()); - }); + if (!ignoredFields.contains("VERSION")) { + if (isFirstWrite(updated.getVersionMap(), stored == null ? null : stored.getVersionMap())) { + Map versionMap = updated.getVersionMap(); + if (versionMap != null) { + versionMap.entrySet().stream().forEach(e -> { + fields.put("VERSION." + e.getKey().toUpperCase(), e.getValue()); + }); + } } } if (isChanged(updated.getYieldCount(), stored == null ? -1 : stored.getYieldCount())) { diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java index 14f9b9c8..b4684729 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java @@ -54,6 +54,7 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy if (combinedMetric.getQuery() == null && updatedQueryMetric.getQuery() != null) { combinedMetric.setQuery(updatedQueryMetric.getQuery()); } + // only update once if (combinedMetric.getHost() == null && updatedQueryMetric.getHost() != null) { combinedMetric.setHost(updatedQueryMetric.getHost()); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricHandler.java index b0c6bb0a..9af3dcca 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricHandler.java @@ -6,6 +6,7 @@ import datawave.microservice.querymetric.QueryMetricsSummaryResponse; import datawave.webservice.query.Query; +import java.util.Collection; import java.util.Date; import java.util.Map; @@ -17,7 +18,7 @@ public interface QueryMetricHandler { Map getEventFields(BaseQueryMetric queryMetric); - ContentQueryMetricsIngestHelper getQueryMetricsIngestHelper(boolean deleteMode); + ContentQueryMetricsIngestHelper getQueryMetricsIngestHelper(boolean deleteMode, Collection ignoredFields); Query createQuery(); diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java index 20fa93f8..c107f025 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java @@ -1,6 +1,5 @@ package datawave.microservice.querymetric.handler; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -198,7 +197,7 @@ public void verifyTables() { AbstractColumnBasedHandler handler = new ContentIndexingColumnBasedHandler() { @Override public AbstractContentIngestHelper getContentIndexingDataTypeHelper() { - return getQueryMetricsIngestHelper(false); + return getQueryMetricsIngestHelper(false, Collections.EMPTY_LIST); } }; Map trackingMap = AccumuloClientTracking.getTrackingMap(Thread.currentThread().getStackTrace()); @@ -228,6 +227,11 @@ private void writeMetric(T updated, T stored, long timestamp, boolean delete, Co } public void writeMetric(T updatedQueryMetric, List storedQueryMetrics, long timestamp, boolean delete) throws Exception { + writeMetric(updatedQueryMetric, storedQueryMetrics, timestamp, delete, Collections.EMPTY_LIST); + } + + public void writeMetric(T updatedQueryMetric, List storedQueryMetrics, long timestamp, boolean delete, Collection ignoredFields) + throws Exception { try { TaskAttemptID taskId = new TaskAttemptID(new TaskID(new JobID(JOB_ID, 1), TaskType.MAP, 1), 1); this.accumuloRecordWriterLock.readLock().lock(); @@ -237,7 +241,7 @@ public void writeMetric(T updatedQueryMetric, List storedQueryMetrics, long t ContentIndexingColumnBasedHandler handler = new ContentIndexingColumnBasedHandler() { @Override public AbstractContentIngestHelper getContentIndexingDataTypeHelper() { - return getQueryMetricsIngestHelper(delete); + return getQueryMetricsIngestHelper(delete, ignoredFields); } }; handler.setup(context); @@ -277,7 +281,7 @@ private Mutation getMutation(Key key, Value value) { public Map getEventFields(BaseQueryMetric queryMetric) { // ignore duplicates as none are expected Map eventFields = new HashMap<>(); - ContentQueryMetricsIngestHelper ingestHelper = getQueryMetricsIngestHelper(false); + ContentQueryMetricsIngestHelper ingestHelper = getQueryMetricsIngestHelper(false, Collections.EMPTY_LIST); ingestHelper.setup(conf); Multimap fieldsToWrite = ingestHelper.getEventFieldsToWrite(queryMetric, null); for (Entry entry : fieldsToWrite.entries()) { @@ -373,7 +377,11 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy } public T getQueryMetric(final String queryId) throws Exception { - List queryMetrics = getQueryMetrics("QUERY_ID == '" + queryId + "'"); + return getQueryMetric(queryId, Collections.emptySet()); + } + + public T getQueryMetric(final String queryId, Collection ignoredFields) throws Exception { + List queryMetrics = getQueryMetrics("QUERY_ID == '" + queryId + "'", ignoredFields); return queryMetrics.isEmpty() ? null : queryMetrics.get(0); } @@ -381,7 +389,7 @@ public Query createQuery() { return new QueryImpl(); } - public List getQueryMetrics(final String query) throws Exception { + public List getQueryMetrics(final String query, Collection ignoredFields) throws Exception { Date end = new Date(); Date begin = DateUtils.setYears(end, 2000); Query queryImpl = createQuery(); @@ -397,7 +405,10 @@ public List getQueryMetrics(final String query) throws Exception { queryImpl.setId(UUID.randomUUID()); Map parameters = new LinkedHashMap<>(); parameters.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"); - parameters.put(QueryParameters.DATATYPE_FILTER_SET, "querymetrics"); + parameters.put(QueryOptions.DATATYPE_FILTER, "querymetrics"); + if (ignoredFields != null && !ignoredFields.isEmpty()) { + parameters.put(QueryOptions.BLACKLISTED_FIELDS, StringUtils.join(ignoredFields, ",")); + } queryImpl.setParameters(parameters); return getQueryMetrics(queryImpl); } @@ -803,8 +814,8 @@ public void reload() { } @Override - public ContentQueryMetricsIngestHelper getQueryMetricsIngestHelper(boolean deleteMode) { - return new ContentQueryMetricsIngestHelper(deleteMode); + public ContentQueryMetricsIngestHelper getQueryMetricsIngestHelper(boolean deleteMode, Collection ignoredFields) { + return new ContentQueryMetricsIngestHelper(deleteMode, ignoredFields); } @Override diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index 6680957c..308bb3d4 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -20,10 +20,13 @@ import org.springframework.stereotype.Component; import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutionException; @@ -34,6 +37,18 @@ @Component("store") @ConditionalOnProperty(name = "hazelcast.server.enabled") public class AccumuloMapStore extends AccumuloMapLoader implements MapStore> { + // A list of fields that won't change once written. There is no need to retrieve these fields from Accumulo + // Any change here must be accounted for in ContentQueryMetricIngestHelper. + public static final List ignoreFieldsOnQuery = Arrays.asList("POSITIVE_SELECTORS", "NEGATIVE_SELECTORS", "AUTHORIZATIONS", "BEGIN_DATE", "END_DATE", + "PARAMETERS", "PROXY_SERVERS", "PREDICTION", "QUERY", "QUERY_LOGIC", "QUERY_NAME", "QUERY_TYPE", "USER", "USER_DN", "VERSION", + "PAGE_METRICS"); + // Exclude PREDICTION, PAGE_METRICS which we don't want to pull from Accumulo but which can change after query creation + public static final List ignoreFieldsOnWrite = new ArrayList<>(); + + static { + AccumuloMapStore.ignoreFieldsOnWrite.addAll(ignoreFieldsOnQuery); + AccumuloMapStore.ignoreFieldsOnWrite.removeAll(Arrays.asList("PREDICTION", "PAGE_METRICS")); + } private static AccumuloMapStore instance; private Logger log = LoggerFactory.getLogger(AccumuloMapStore.class); @@ -103,6 +118,7 @@ public void storeWithRetry(QueryMetricUpdateHolder queryMetricUpdate) { } public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception { + String queryId = queryMetricUpdate.getMetric().getQueryId(); T updatedMetric = null; try { @@ -110,13 +126,17 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception QueryMetricType metricType = queryMetricUpdate.getMetricType(); QueryMetricUpdateHolder lastQueryMetricUpdate = null; + final List ignoredFields = new ArrayList<>(); if (!queryMetricUpdate.isNewMetric()) { lastQueryMetricUpdate = (QueryMetricUpdateHolder) lastWrittenQueryMetricCache.get(queryId, () -> { log.debug("getting metric {} from accumulo", queryId); - T m = handler.getQueryMetric(queryId); + T m = handler.getQueryMetric(queryId, ignoreFieldsOnQuery); if (m == null) { return null; } else { + // these fields will not be populated in the returned metric, + // so we should not compare them later for writing mutations + ignoredFields.addAll(ignoreFieldsOnWrite); return new QueryMetricUpdateHolder(m, metricType); } }); @@ -152,12 +172,12 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception updatedMetric.setLastUpdated(new Date(updatedMetric.getLastUpdated().getTime() + 1)); if (lastQueryMetric.getLastUpdated() != null) { - handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), deleteTimestamp, true); + handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), deleteTimestamp, true, ignoredFields); } - handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), writeTimestamp, false); + handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), writeTimestamp, false, ignoredFields); } else { updatedMetric.setLastUpdated(updatedMetric.getCreateDate()); - handler.writeMetric(updatedMetric, Collections.emptyList(), updatedMetric.getCreateDate().getTime(), false); + handler.writeMetric(updatedMetric, Collections.emptyList(), updatedMetric.getCreateDate().getTime(), false, ignoredFields); } if (log.isTraceEnabled()) { log.trace("writing metric to accumulo: " + queryId + " - " + queryMetricUpdate.getMetric()); diff --git a/service/src/test/java/datawave/microservice/querymetric/config/AlternateContentQueryMetricsIngestHelper.java b/service/src/test/java/datawave/microservice/querymetric/config/AlternateContentQueryMetricsIngestHelper.java index 6cdd388e..14553481 100644 --- a/service/src/test/java/datawave/microservice/querymetric/config/AlternateContentQueryMetricsIngestHelper.java +++ b/service/src/test/java/datawave/microservice/querymetric/config/AlternateContentQueryMetricsIngestHelper.java @@ -3,13 +3,20 @@ import com.google.common.collect.Multimap; import datawave.microservice.querymetric.handler.ContentQueryMetricsIngestHelper; +import java.util.Collection; + public class AlternateContentQueryMetricsIngestHelper extends ContentQueryMetricsIngestHelper { - public AlternateContentQueryMetricsIngestHelper(boolean deleteMode) { - super(deleteMode, new HelperDelegate()); + public AlternateContentQueryMetricsIngestHelper(boolean deleteMode, Collection ignoredFields) { + super(deleteMode, new HelperDelegate(ignoredFields)); } private static class HelperDelegate extends ContentQueryMetricsIngestHelper.HelperDelegate { + + public HelperDelegate(Collection ignoredFields) { + super(ignoredFields); + } + @Override protected void putExtendedFieldsToWrite(AlternateQueryMetric updated, AlternateQueryMetric stored, Multimap fields) { if (isFirstWrite(updated.getExtraField(), stored == null ? null : stored.getExtraField())) { diff --git a/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java b/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java index ff58750f..22f9263d 100644 --- a/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java +++ b/service/src/test/java/datawave/microservice/querymetric/config/AlternateShardTableQueryMetricHandler.java @@ -12,6 +12,7 @@ import datawave.webservice.query.result.event.FieldBase; import org.springframework.beans.factory.annotation.Qualifier; +import java.util.Collection; import java.util.List; public class AlternateShardTableQueryMetricHandler extends ShardTableQueryMetricHandler { @@ -23,8 +24,8 @@ public AlternateShardTableQueryMetricHandler(QueryMetricHandlerProperties queryM } @Override - public ContentQueryMetricsIngestHelper getQueryMetricsIngestHelper(boolean deleteMode) { - return new AlternateContentQueryMetricsIngestHelper(deleteMode); + public ContentQueryMetricsIngestHelper getQueryMetricsIngestHelper(boolean deleteMode, Collection ignoredFields) { + return new AlternateContentQueryMetricsIngestHelper(deleteMode, ignoredFields); } @Override From 0e858afdf1dcf15c761238a100458f936e3b391d Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 7 Dec 2023 12:03:15 -0500 Subject: [PATCH 24/48] Add retry logic on sending message to rabbit queue, add timer on rest call and message send --- .../querymetric/QueryMetricOperations.java | 117 ++++++++++++++---- .../QueryMetricOperationsStats.java | 9 +- .../QueryMetricHandlerConfiguration.java | 2 +- .../config/QueryMetricProperties.java | 61 +++++++++ 4 files changed, 163 insertions(+), 26 deletions(-) create mode 100644 service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index faa76ea8..faf95c7f 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -6,6 +6,8 @@ import com.hazelcast.spring.cache.HazelcastCacheManager; import datawave.marking.MarkingFunctions; import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.querymetric.config.QueryMetricProperties; +import datawave.microservice.querymetric.config.QueryMetricProperties.Retry; import datawave.microservice.querymetric.config.QueryMetricSinkConfiguration.QueryMetricSinkBinding; import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory; import datawave.microservice.querymetric.handler.QueryGeometryHandler; @@ -77,6 +79,7 @@ public class QueryMetricOperations { private Logger log = LoggerFactory.getLogger(QueryMetricOperations.class); + private QueryMetricProperties queryMetricProperties; private ShardTableQueryMetricHandler handler; private QueryGeometryHandler geometryHandler; private CacheManager cacheManager; @@ -110,6 +113,8 @@ enum DEFAULT_DATETIME { /** * Instantiates a new QueryMetricOperations. * + * @param queryMetricProperties + * the query metric properties * @param cacheManager * the CacheManager * @param handler @@ -128,10 +133,11 @@ enum DEFAULT_DATETIME { * the stats */ @Autowired - public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cacheManager, ShardTableQueryMetricHandler handler, - QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, BaseQueryMetricListResponseFactory queryMetricListResponseFactory, - MergeLockLifecycleListener mergeLock, Correlator correlator, MetricUpdateEntryProcessorFactory entryProcessorFactory, - QueryMetricOperationsStats stats) { + public QueryMetricOperations(QueryMetricProperties queryMetricProperties, @Named("queryMetricCacheManager") CacheManager cacheManager, + ShardTableQueryMetricHandler handler, QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, + BaseQueryMetricListResponseFactory queryMetricListResponseFactory, MergeLockLifecycleListener mergeLock, Correlator correlator, + MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats) { + this.queryMetricProperties = queryMetricProperties; this.handler = handler; this.geometryHandler = geometryHandler; this.cacheManager = cacheManager; @@ -194,17 +200,27 @@ public VoidResponse updateMetrics(@RequestBody List queryMetric if (!this.mergeLock.isAllowedReadLock()) { throw new IllegalStateException("service unavailable"); } - stats.getMeter(METERS.REST).mark(queryMetrics.size()); - VoidResponse response = new VoidResponse(); - for (BaseQueryMetric m : queryMetrics) { - if (log.isTraceEnabled()) { - log.trace("received metric update via REST: " + m.toString()); - } else { - log.debug("received metric update via REST: " + m.getQueryId()); + Timer.Context restTimer = this.stats.getTimer(TIMERS.REST).time(); + try { + for (BaseQueryMetric m : queryMetrics) { + if (log.isTraceEnabled()) { + log.trace("received metric update via REST: " + m.toString()); + } else { + log.debug("received metric update via REST: " + m.getQueryId()); + } + Timer.Context messageSendTimer = this.stats.getTimer(TIMERS.MESSAGE_SEND).time(); + try { + if (!updateMetric(new QueryMetricUpdate<>(m, metricType))) { + throw new RuntimeException("Unable to process query metric update for query [" + m.getQueryId() + "]"); + } + } finally { + messageSendTimer.stop(); + } } - output.send(MessageBuilder.withPayload(new QueryMetricUpdate(m, metricType)).build()); + } finally { + restTimer.stop(); } - return response; + return new VoidResponse(); } /** @@ -226,16 +242,74 @@ public VoidResponse updateMetric(@RequestBody BaseQueryMetric queryMetric, if (!this.mergeLock.isAllowedReadLock()) { throw new IllegalStateException("service unavailable"); } - stats.getMeter(METERS.REST).mark(); - if (log.isTraceEnabled()) { - log.trace("received metric update via REST: " + queryMetric.toString()); - } else { - log.debug("received metric update via REST: " + queryMetric.getQueryId()); + Timer.Context restTimer = this.stats.getTimer(TIMERS.REST).time(); + try { + if (log.isTraceEnabled()) { + log.trace("received metric update via REST: " + queryMetric.toString()); + } else { + log.debug("received metric update via REST: " + queryMetric.getQueryId()); + } + Timer.Context messageSendTimer = this.stats.getTimer(TIMERS.MESSAGE_SEND).time(); + try { + if (!updateMetric(new QueryMetricUpdate(queryMetric, metricType))) { + throw new RuntimeException("Unable to process query metric update for query [" + queryMetric.getQueryId() + "]"); + } + } finally { + messageSendTimer.stop(); + } + } finally { + restTimer.stop(); } - output.send(MessageBuilder.withPayload(new QueryMetricUpdate(queryMetric, metricType)).build()); return new VoidResponse(); } + private boolean updateMetric(QueryMetricUpdate update) { + + boolean success; + final long updateStartTime = System.currentTimeMillis(); + long currentTime; + int attempts = 0; + + Retry retry = queryMetricProperties.getRetry(); + + do { + if (attempts++ > 0) { + try { + Thread.sleep(retry.getBackoffIntervalMillis()); + } catch (InterruptedException e) { + // Ignore -- we'll just end up retrying a little too fast + } + } + + if (log.isDebugEnabled()) { + log.debug("Update attempt {} of {} for query {}", attempts, retry.getMaxAttempts(), update.getMetric().getQueryId()); + } + + success = output.send(MessageBuilder.withPayload(update).build()); + currentTime = System.currentTimeMillis(); + } while (!success && (currentTime - updateStartTime) < retry.getFailTimeoutMillis() && attempts < retry.getMaxAttempts()); + + if (!success) { + log.warn("Update for query {} failed. {attempts = {}, elapsedMillis = {}}", update.getMetric().getQueryId(), attempts, + (currentTime - updateStartTime)); + } else { + log.info("Update for query {} successful. {attempts = {}, elapsedMillis = {}}", update.getMetric().getQueryId(), attempts, + (currentTime - updateStartTime)); + } + + return success; + } + + /** + * Passes query metric messages to the messaging infrastructure. + *

+ * The metric ID is used as a correlation ID in order to ensure that a producer confirm ack is received. If a producer confirm ack is not received within + * the specified amount of time, a 500 Internal Server Error will be returned to the caller. + * + * @param update + * The query metric update to be sent + */ + private boolean shouldCorrelate(QueryMetricUpdate update) { // add the first update for a metric to get it into the cache if ((update.getMetric().getLifecycle().ordinal() <= BaseQueryMetric.Lifecycle.DEFINED.ordinal())) { @@ -256,7 +330,7 @@ private boolean shouldCorrelate(QueryMetricUpdate update) { */ @StreamListener(QueryMetricSinkBinding.SINK_NAME) public void handleEvent(QueryMetricUpdate update) { - stats.getMeter(METERS.MESSAGE).mark(); + stats.getMeter(METERS.MESSAGE_RECEIVE).mark(); if (shouldCorrelate(update)) { log.debug("adding update for {} to correlator", update.getMetric().getQueryId()); this.correlator.addMetricUpdate(update); @@ -334,8 +408,9 @@ private void storeMetricUpdates(QueryMetricUpdateHolder metricUpdate) { } // fail the handling of the message throw new RuntimeException(e.getMessage()); + } finally { + storeTimer.stop(); } - storeTimer.stop(); } /** diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java index a3715ff5..e55ce950 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java @@ -54,11 +54,11 @@ public class QueryMetricOperationsStats { protected Map staticTags = new LinkedHashMap<>(); public enum TIMERS { - STORE + MESSAGE_SEND, REST, STORE } public enum METERS { - REST, MESSAGE + MESSAGE_RECEIVE } /* @@ -232,8 +232,9 @@ public Map getServiceStats() { Map stats = new LinkedHashMap<>(); addTimerStats("store", getTimer(TIMERS.STORE), stats); addTimerStats("accumulo", this.mapStore.getWriteTimer(), stats); - addMeterStats("message", getMeter(METERS.MESSAGE), stats); - addMeterStats("rest", getMeter(METERS.REST), stats); + addTimerStats("message.send", getTimer(TIMERS.MESSAGE_SEND), stats); + addTimerStats("rest", getTimer(TIMERS.REST), stats); + addMeterStats("message.receive", getMeter(METERS.MESSAGE_RECEIVE), stats); return stats; } diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java index cad44787..25cccab7 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerConfiguration.java @@ -38,7 +38,7 @@ import java.util.Set; @Configuration -@EnableConfigurationProperties({QueryMetricHandlerProperties.class, TimelyProperties.class}) +@EnableConfigurationProperties({QueryMetricProperties.class, QueryMetricHandlerProperties.class, TimelyProperties.class}) public class QueryMetricHandlerConfiguration { @Bean diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java new file mode 100644 index 00000000..4cbce44e --- /dev/null +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricProperties.java @@ -0,0 +1,61 @@ +package datawave.microservice.querymetric.config; + +import java.util.concurrent.TimeUnit; + +import javax.validation.Valid; +import javax.validation.constraints.PositiveOrZero; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Validated +@ConfigurationProperties(prefix = "datawave.query.metric") +public class QueryMetricProperties { + + @Valid + private Retry retry = new Retry(); + + public Retry getRetry() { + return retry; + } + + public void setRetry(Retry retry) { + this.retry = retry; + } + + @Validated + public static class Retry { + @PositiveOrZero + private int maxAttempts = 10; + + @PositiveOrZero + private long failTimeoutMillis = TimeUnit.MINUTES.toMillis(5); + + @PositiveOrZero + private long backoffIntervalMillis = TimeUnit.SECONDS.toMillis(5); + + public int getMaxAttempts() { + return maxAttempts; + } + + public void setMaxAttempts(int maxAttempts) { + this.maxAttempts = maxAttempts; + } + + public long getFailTimeoutMillis() { + return failTimeoutMillis; + } + + public void setFailTimeoutMillis(long failTimeoutMillis) { + this.failTimeoutMillis = failTimeoutMillis; + } + + public long getBackoffIntervalMillis() { + return backoffIntervalMillis; + } + + public void setBackoffIntervalMillis(long backoffIntervalMillis) { + this.backoffIntervalMillis = backoffIntervalMillis; + } + } +} From 74c8aea4228f317c054e3690cc20477d9f4006bf Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 7 Dec 2023 18:56:10 +0000 Subject: [PATCH 25/48] Updated producer confirm acks to perform better with bulk updates. --- .../querymetric/QueryMetricOperations.java | 113 +++++++++++++----- 1 file changed, 86 insertions(+), 27 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 3a6b05b8..4b8417b5 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -14,6 +14,8 @@ import java.util.Calendar; import java.util.Collections; import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -21,6 +23,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.annotation.PreDestroy; import javax.annotation.security.PermitAll; @@ -53,6 +56,7 @@ import org.springframework.web.bind.annotation.RestController; import com.codahale.metrics.Timer; +import com.google.common.collect.Lists; import com.hazelcast.core.HazelcastInstanceNotActiveException; import com.hazelcast.map.IMap; import com.hazelcast.spring.cache.HazelcastCacheManager; @@ -201,14 +205,14 @@ public VoidResponse updateMetrics(@RequestBody List queryMetric VoidResponse response = new VoidResponse(); for (BaseQueryMetric m : queryMetrics) { if (log.isTraceEnabled()) { - log.trace("received metric update via REST: " + m.toString()); + log.trace("received bulk metric update via REST: " + m.toString()); } else { - log.debug("received metric update via REST: " + m.getQueryId()); - } - if (!updateMetric(new QueryMetricUpdate<>(m, metricType))) { - throw new RuntimeException("Unable to process query metric update for query [" + m.getQueryId() + "]"); + log.debug("received bulk metric update via REST: " + m.getQueryId()); } } + if (!updateMetrics(queryMetrics.stream().map(m -> new QueryMetricUpdate<>(m, metricType)).collect(Collectors.toList()))) { + throw new RuntimeException("Unable to process bulk query metric update"); + } return response; } @@ -267,7 +271,9 @@ public void processConfirmAck(Message message) { } } - private boolean updateMetric(QueryMetricUpdate update) { + private boolean updateMetrics(List updates) { + List failedUpdates = new ArrayList<>(updates.size()); + Map updatesById = new LinkedHashMap<>(); boolean success; final long updateStartTime = System.currentTimeMillis(); @@ -286,54 +292,107 @@ private boolean updateMetric(QueryMetricUpdate update) { } if (log.isDebugEnabled()) { - log.debug("Update attempt {} of {} for query {}", attempts, retry.getMaxAttempts(), update.getMetric().getQueryId()); + log.debug("Bulk update attempt {} of {}", attempts, retry.getMaxAttempts()); } - success = sendMessage(update); + // send all of the remaining metric updates + success = sendMessages(updates, failedUpdates, updatesById) && awaitConfirmAcks(updatesById, failedUpdates); currentTime = System.currentTimeMillis(); } while (!success && (currentTime - updateStartTime) < retry.getFailTimeoutMillis() && attempts < retry.getMaxAttempts()); if (!success) { - log.warn("Update for query {} failed. {attempts = {}, elapsedMillis = {}}", update.getMetric().getQueryId(), attempts, - (currentTime - updateStartTime)); + log.warn("Bulk update failed. {attempts = {}, elapsedMillis = {}}", attempts, (currentTime - updateStartTime)); } else { - log.info("Update for query {} successful. {attempts = {}, elapsedMillis = {}}", update.getMetric().getQueryId(), attempts, - (currentTime - updateStartTime)); + log.info("Bulk update successful. {attempts = {}, elapsedMillis = {}}", attempts, (currentTime - updateStartTime)); } return success; } + private boolean updateMetric(QueryMetricUpdate update) { + return updateMetrics(Lists.newArrayList(update)); + } + /** * Passes query metric messages to the messaging infrastructure. - *

- * The metric ID is used as a correlation ID in order to ensure that a producer confirm ack is received. If a producer confirm ack is not received within - * the specified amount of time, a 500 Internal Server Error will be returned to the caller. * - * @param update - * The query metric update to be sent + * @param updates + * The query metric updates to be sent, not null + * @param failedUpdates + * A list that will be populated with the failed metric updates, not null + * @param updatesById + * A map that will be populated with the correlation ids and associated metric updates, not null + * @return true if all messages were successfully sent, false otherwise */ - private boolean sendMessage(QueryMetricUpdate update) { - String correlationId = UUID.randomUUID().toString(); + private boolean sendMessages(List updates, List failedUpdates, Map updatesById) { + failedUpdates.clear(); - CountDownLatch latch = null; - if (queryMetricProperties.isConfirmAckEnabled()) { - latch = new CountDownLatch(1); - correlationLatchMap.put(correlationId, latch); + boolean success = true; + // send all of the remaining metric updates + for (QueryMetricUpdate update : updates) { + String correlationId = UUID.randomUUID().toString(); + if (sendMessage(correlationId, update)) { + if (queryMetricProperties.isConfirmAckEnabled()) { + updatesById.put(correlationId, update); + } + } else { + // if it failed, add it to the failed list + failedUpdates.add(update); + success = false; + } } - boolean success = queryMetricSupplier.send(MessageBuilder.withPayload(update).setCorrelationId(correlationId).build()); + updates.retainAll(failedUpdates); + return success; + } + + private boolean sendMessage(String correlationId, QueryMetricUpdate update) { + boolean success = false; + if (queryMetricSupplier.send(MessageBuilder.withPayload(update).setCorrelationId(correlationId).build())) { + success = true; + if (queryMetricProperties.isConfirmAckEnabled()) { + correlationLatchMap.put(correlationId, new CountDownLatch(1)); + } + } + return success; + } + + /** + * Waits for the producer confirm acks to be received for the updates that were sent. If a producer confirm ack is not received within the specified amount + * of time, a 500 Internal Server Error will be returned to the caller. + * + * @param updatesById + * A map of query metric updates keyed by their correlation id, not null + * @param failedUpdates + * A list that will be populated with the failed metric updates, not null + * @return true if all confirm acks were successfully received, false otherwise + */ + private boolean awaitConfirmAcks(Map updatesById, List failedUpdates) { + boolean success = true; + // wait for the confirm acks only after all sends are successful + if (queryMetricProperties.isConfirmAckEnabled()) { + for (String correlationId : updatesById.keySet()) { + if (!awaitConfirmAck(correlationId)) { + failedUpdates.add(updatesById.remove(correlationId)); + success = false; + } + } + } + return success; + } + + private boolean awaitConfirmAck(String correlationId) { + boolean success = false; if (queryMetricProperties.isConfirmAckEnabled()) { try { - success = success && latch.await(queryMetricProperties.getConfirmAckTimeoutMillis(), TimeUnit.MILLISECONDS); + success = correlationLatchMap.get(correlationId).await(queryMetricProperties.getConfirmAckTimeoutMillis(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { - success = false; + log.warn("Interrupted waiting for confirm ack {}", correlationId); } finally { correlationLatchMap.remove(correlationId); } } - return success; } From e0ac9621d5906b19155c4eab69170d09b78c598b Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Fri, 8 Dec 2023 11:22:41 -0500 Subject: [PATCH 26/48] Increase cache size and ttl for lastWrittenQueryMetrics cache in testing --- service/src/test/resources/config/application.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index 3eb56679..2beed9c0 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -86,8 +86,8 @@ datawave: baseMaps: "{}" cache: lastWrittenQueryMetrics: - maximumSize: 100 - ttlSeconds: 60 + maximumSize: 5000 + ttlSeconds: 600 client: enabled: true transport: message_test From c7a873985c4da2cec91770eac8816360a8ccb4e9 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Fri, 8 Dec 2023 12:10:50 -0500 Subject: [PATCH 27/48] Add timer to record the time it takes in AccumuloMapStore to fetch lastWrittenQuertMetric when necessary --- .../QueryMetricOperationsStats.java | 3 +- .../persistence/AccumuloMapStore.java | 46 +++++++++++-------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java index e55ce950..4f58c7bb 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java @@ -231,7 +231,8 @@ public Map formatStats(Map stats, boolean useSepar public Map getServiceStats() { Map stats = new LinkedHashMap<>(); addTimerStats("store", getTimer(TIMERS.STORE), stats); - addTimerStats("accumulo", this.mapStore.getWriteTimer(), stats); + addTimerStats("accumulo.write", this.mapStore.getWriteTimer(), stats); + addTimerStats("accumulo.read", this.mapStore.getReadTimer(), stats); addTimerStats("message.send", getTimer(TIMERS.MESSAGE_SEND), stats); addTimerStats("rest", getTimer(TIMERS.REST), stats); addMeterStats("message.receive", getMeter(METERS.MESSAGE_RECEIVE), stats); diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index 308bb3d4..8f92c89e 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -2,12 +2,11 @@ import com.codahale.metrics.SlidingTimeWindowArrayReservoir; import com.codahale.metrics.Timer; -import com.google.common.cache.CacheBuilder; +import com.github.benmanes.caffeine.cache.Caffeine; import com.hazelcast.core.MapLoader; import com.hazelcast.core.MapStore; import com.hazelcast.core.MapStoreFactory; import datawave.microservice.querymetric.BaseQueryMetric; -import datawave.microservice.querymetric.MergeLockLifecycleListener; import datawave.microservice.querymetric.QueryMetricType; import datawave.microservice.querymetric.QueryMetricUpdateHolder; import datawave.microservice.querymetric.handler.ShardTableQueryMetricHandler; @@ -29,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static java.util.concurrent.TimeUnit.MINUTES; @@ -53,9 +51,9 @@ public class AccumuloMapStore extends AccumuloMapLoad private static AccumuloMapStore instance; private Logger log = LoggerFactory.getLogger(AccumuloMapStore.class); private Cache lastWrittenQueryMetricCache; - private MergeLockLifecycleListener mergeLock; - private com.google.common.cache.Cache failures; + private com.github.benmanes.caffeine.cache.Cache failures; private Timer writeTimer = new Timer(new SlidingTimeWindowArrayReservoir(1, MINUTES)); + private Timer readTimer = new Timer(new SlidingTimeWindowArrayReservoir(1, MINUTES)); private boolean shuttingDown = false; public static class Factory implements MapStoreFactory { @@ -66,10 +64,13 @@ public MapLoader newMapStore(String mapName, Properties } @Autowired - public AccumuloMapStore(ShardTableQueryMetricHandler handler, MergeLockLifecycleListener mergeLock) { + public AccumuloMapStore(ShardTableQueryMetricHandler handler) { this.handler = handler; - this.mergeLock = mergeLock; - this.failures = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.MINUTES).build(); + // @formatter:off + this.failures = Caffeine.newBuilder() + .expireAfterWrite(60, TimeUnit.MINUTES) + .build(); + // @formatter:on AccumuloMapStore.instance = this; } @@ -130,14 +131,19 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception if (!queryMetricUpdate.isNewMetric()) { lastQueryMetricUpdate = (QueryMetricUpdateHolder) lastWrittenQueryMetricCache.get(queryId, () -> { log.debug("getting metric {} from accumulo", queryId); - T m = handler.getQueryMetric(queryId, ignoreFieldsOnQuery); - if (m == null) { - return null; - } else { - // these fields will not be populated in the returned metric, - // so we should not compare them later for writing mutations - ignoredFields.addAll(ignoreFieldsOnWrite); - return new QueryMetricUpdateHolder(m, metricType); + Timer.Context readTimerContext = readTimer.time(); + try { + T m = handler.getQueryMetric(queryId, ignoreFieldsOnQuery); + if (m == null) { + return null; + } else { + // these fields will not be populated in the returned metric, + // so we should not compare them later for writing mutations + ignoredFields.addAll(ignoreFieldsOnWrite); + return new QueryMetricUpdateHolder(m, metricType); + } + } finally { + readTimerContext.stop(); } }); } @@ -208,8 +214,8 @@ private boolean retryOnException(QueryMetricUpdate update, Exception e) { String queryId = update.getMetric().getQueryId(); Integer numFailures = 1; try { - numFailures = (Integer) this.failures.get(queryId, () -> 0) + 1; - } catch (ExecutionException e1) { + numFailures = (Integer) this.failures.get(queryId, o -> 0) + 1; + } catch (Exception e1) { log.error(e1.getMessage(), e1); } if (numFailures < 3) { @@ -265,4 +271,8 @@ public void deleteAll(Collection keys) { public Timer getWriteTimer() { return writeTimer; } + + public Timer getReadTimer() { + return readTimer; + } } From 9326be116a0b5ba73868b1c5f2dd62f1c0474386 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Dec 2023 17:58:09 +0000 Subject: [PATCH 28/48] bumped query metric api to 3.0.2 --- service/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/pom.xml b/service/pom.xml index 5ab89a27..7c0a8655 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -28,7 +28,7 @@ 3.0.1 3.0.0 3.0.0 - 3.0.0 + 3.0.2 3.0.0 2.0.0 2.0.0 From 2aea1d32f49706d6ad39801076823c9fbd7c5538 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Dec 2023 18:01:49 +0000 Subject: [PATCH 29/48] [maven-release-plugin] prepare release 3.0.2 --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f4fde354..f4225f93 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.2-SNAPSHOT + 3.0.2 pom https://code.nsa.gov/datawave-query-metric-service @@ -17,6 +17,7 @@ scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git + 3.0.2 https://github.com/NationalSecurityAgency/datawave-query-metric-service From 799e7b98c3b2f04eb6d3a21196110248175a5074 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Dec 2023 18:01:50 +0000 Subject: [PATCH 30/48] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f4225f93..7852fd59 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.2 + 3.0.3-SNAPSHOT pom https://code.nsa.gov/datawave-query-metric-service @@ -17,7 +17,7 @@ scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - 3.0.2 + HEAD https://github.com/NationalSecurityAgency/datawave-query-metric-service From d46debaeffcedda206745bd62038822d93631ce9 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Dec 2023 21:07:54 +0000 Subject: [PATCH 31/48] [maven-release-plugin] prepare release api-3.0.2 --- api/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index edfa3411..7f584491 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,12 +8,12 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.2-SNAPSHOT + 3.0.2 https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - HEAD + api-3.0.2 https://github.com/NationalSecurityAgency/datawave-query-metric-service From ca577ad8b3bf1afabd7209aa54d4804fe1d1c8fc Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Dec 2023 21:07:55 +0000 Subject: [PATCH 32/48] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 7f584491..62dd9682 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,12 +8,12 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.2 + 3.0.3-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - api-3.0.2 + HEAD https://github.com/NationalSecurityAgency/datawave-query-metric-service From 8619aa65e8d2269d7f67140f4e225ca429806111 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Dec 2023 21:11:09 +0000 Subject: [PATCH 33/48] [maven-release-plugin] prepare release service-3.0.2 --- service/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 7c0a8655..f6792a06 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,13 +8,13 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.2-SNAPSHOT + 3.0.2 DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - HEAD + service-3.0.2 https://github.com/NationalSecurityAgency/datawave-query-metric-service From dd97acf30eca8e11e0e56e63950ae3b3cb69ea29 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Dec 2023 21:11:11 +0000 Subject: [PATCH 34/48] [maven-release-plugin] prepare for next development iteration --- service/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index f6792a06..221f4e7d 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,13 +8,13 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.2 + 3.0.3-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - service-3.0.2 + HEAD https://github.com/NationalSecurityAgency/datawave-query-metric-service From aab01178fa84df7d8066116c9ddc87cb898f7080 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Tue, 12 Dec 2023 14:24:50 -0500 Subject: [PATCH 35/48] Fix intermittent test failures in CorrelatorTest, QueryMetricOperationsHazelcastWriteBehindTest --- .../microservice/querymetric/Correlator.java | 4 +++ .../querymetric/QueryMetricOperations.java | 34 +++++++++++++++++-- .../config/QueryMetricCacheConfiguration.java | 2 +- .../persistence/AccumuloMapStore.java | 13 +++---- .../querymetric/CorrelatorTest.java | 28 +++++++-------- .../QueryMetricOperationsTest.java | 12 +++---- .../querymetric/QueryMetricTestBase.java | 6 ++-- .../src/test/resources/config/application.yml | 1 - 8 files changed, 64 insertions(+), 36 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/Correlator.java b/service/src/main/java/datawave/microservice/querymetric/Correlator.java index 7e8cae11..c9f10087 100644 --- a/service/src/main/java/datawave/microservice/querymetric/Correlator.java +++ b/service/src/main/java/datawave/microservice/querymetric/Correlator.java @@ -76,4 +76,8 @@ public boolean isEnabled() { public void shutdown(boolean isShuttingDown) { this.isShuttingDown = isShuttingDown; } + + public boolean isShuttingDown() { + return isShuttingDown; + } } diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index faf95c7f..f42b8f23 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -60,6 +60,7 @@ import java.util.List; import java.util.Set; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; import static datawave.microservice.querymetric.QueryMetricOperations.DEFAULT_DATETIME.BEGIN; import static datawave.microservice.querymetric.QueryMetricOperations.DEFAULT_DATETIME.END; @@ -88,6 +89,7 @@ public class QueryMetricOperations { private BaseQueryMetricListResponseFactory queryMetricListResponseFactory; private MergeLockLifecycleListener mergeLock; private Correlator correlator; + private AtomicBoolean timedCorrelationInProgress = new AtomicBoolean(false); private MetricUpdateEntryProcessorFactory entryProcessorFactory; private QueryMetricOperationsStats stats; private static Set inProcess = Collections.synchronizedSet(new HashSet<>()); @@ -154,14 +156,40 @@ public QueryMetricOperations(QueryMetricProperties queryMetricProperties, @Named public void shutdown() { if (this.correlator.isEnabled()) { this.correlator.shutdown(true); - ensureUpdatesProcessed(); + // we've locked out the timer thread, but need to + // wait for it to complete the last write + while (isTimedCorrelationInProgress()) { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + + } + } + ensureUpdatesProcessed(false); } this.stats.queueAggregatedQueryStatsForTimely(); this.stats.writeQueryStatsToTimely(); } + public boolean isTimedCorrelationInProgress() { + return this.timedCorrelationInProgress.get(); + } + @Scheduled(fixedDelay = 2000) - public void ensureUpdatesProcessed() { + public void ensureUpdatesProcessedScheduled() { + // don't write metrics from this thread while shutting down, + // so we can make sure that the process completes + if (!this.correlator.isShuttingDown()) { + this.timedCorrelationInProgress.set(true); + try { + ensureUpdatesProcessed(true); + } finally { + this.timedCorrelationInProgress.set(false); + } + } + } + + public void ensureUpdatesProcessed(boolean scheduled) { if (this.correlator.isEnabled()) { List correlatedUpdates; do { @@ -177,7 +205,7 @@ public void ensureUpdatesProcessed() { log.error("exception while combining correlated updates: " + e.getMessage(), e); } } - } while (correlatedUpdates != null && !correlatedUpdates.isEmpty()); + } while (!(scheduled && this.correlator.isShuttingDown()) && correlatedUpdates != null && !correlatedUpdates.isEmpty()); } } diff --git a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java index d6d86bf4..edcb74b4 100644 --- a/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java +++ b/service/src/main/java/datawave/microservice/querymetric/config/QueryMetricCacheConfiguration.java @@ -31,7 +31,7 @@ public Cache lastWrittenQueryMetrics(QueryMetricCacheProperties cacheProperties) .maximumSize(lastWrittenQueryMetrics.getMaximumSize()) .expireAfterWrite(lastWrittenQueryMetrics.getTtlSeconds(), TimeUnit.SECONDS) .removalListener(new MetricCacheListener(LAST_WRITTEN_METRICS)) - .build()); + .build(), false); // @formatter:on } } diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index 8f92c89e..fa036114 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -129,23 +129,24 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception final List ignoredFields = new ArrayList<>(); if (!queryMetricUpdate.isNewMetric()) { - lastQueryMetricUpdate = (QueryMetricUpdateHolder) lastWrittenQueryMetricCache.get(queryId, () -> { + lastQueryMetricUpdate = lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdateHolder.class); + if (lastQueryMetricUpdate == null) { log.debug("getting metric {} from accumulo", queryId); Timer.Context readTimerContext = readTimer.time(); try { T m = handler.getQueryMetric(queryId, ignoreFieldsOnQuery); - if (m == null) { - return null; - } else { + if (m != null) { // these fields will not be populated in the returned metric, // so we should not compare them later for writing mutations ignoredFields.addAll(ignoreFieldsOnWrite); - return new QueryMetricUpdateHolder(m, metricType); + lastQueryMetricUpdate = new QueryMetricUpdateHolder(m, metricType); } + } catch (Exception e) { + log.error(e.getMessage(), e); } finally { readTimerContext.stop(); } - }); + } } if (lastQueryMetricUpdate != null) { diff --git a/service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java b/service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java index 0ffc8165..13310b56 100644 --- a/service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/CorrelatorTest.java @@ -42,6 +42,7 @@ public void setup() { @After public void cleanup() { super.cleanup(); + this.correlator.shutdown(false); } @Test @@ -122,25 +123,20 @@ public void testMetricsCorrelated(int numMetrics, int maxPages) throws Exception Assert.assertTrue("executor tasks completed", completed); // flush the correlator this.correlator.shutdown(true); - this.queryMetricOperations.ensureUpdatesProcessed(); - this.correlator.shutdown(false); - - long start = System.currentTimeMillis(); + while (this.queryMetricOperations.isTimedCorrelationInProgress()) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { + + } + } + this.queryMetricOperations.ensureUpdatesProcessed(false); for (BaseQueryMetric m : metrics) { String queryId = m.getQueryId(); ensureDataStored(incomingQueryMetricsCache, queryId); - QueryMetricUpdate metricUpdate; - BaseQueryMetric storedMetric = null; - do { - metricUpdate = incomingQueryMetricsCache.get(queryId, QueryMetricUpdate.class); - if (metricUpdate == null && (System.currentTimeMillis() - start) < 5000) { - Thread.sleep(200); - } else { - storedMetric = metricUpdate.getMetric(); - } - } while (storedMetric == null); - Assert.assertNotNull("missing metric " + queryId, storedMetric); - assertEquals("incomingQueryMetricsCache metric wrong for id:" + m.getQueryId(), m, storedMetric); + QueryMetricUpdate metricUpdate = incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class); + Assert.assertNotNull("missing metric " + queryId, metricUpdate); + assertEquals("incomingQueryMetricsCache metric wrong for id:" + m.getQueryId(), m, metricUpdate.getMetric()); } } } diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricOperationsTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricOperationsTest.java index 5efa0ae6..e98941b2 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricOperationsTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricOperationsTest.java @@ -35,8 +35,8 @@ public void MetricStoredCorrectlyInCachesAndAccumulo() throws Exception { .build()); // @formatter:on ensureDataWritten(incomingQueryMetricsCache, lastWrittenQueryMetricCache, queryId); - assertEquals("lastWrittenQueryMetricCache metric wrong", m, lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdate.class).getMetric()); - assertEquals("incomingQueryMetricsCache metric wrong", m, incomingQueryMetricsCache.get(queryId, QueryMetricUpdate.class).getMetric()); + assertEquals("lastWrittenQueryMetricCache metric wrong", m, lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdateHolder.class).getMetric()); + assertEquals("incomingQueryMetricsCache metric wrong", m, incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class).getMetric()); assertEquals("accumulo metric wrong", m, shardTableQueryMetricHandler.getQueryMetric(queryId)); } @@ -59,14 +59,14 @@ public void MultipleMetricsStoredCorrectlyInCachesAndAccumulo() throws Exception metrics.forEach((m) -> { String queryId = m.getQueryId(); ensureDataWritten(incomingQueryMetricsCache, lastWrittenQueryMetricCache, queryId); - assertEquals("lastWrittenQueryMetricCache metric wrong", m, lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdate.class).getMetric()); + assertEquals("lastWrittenQueryMetricCache metric wrong", m, lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdateHolder.class).getMetric()); try { assertEquals("accumulo metric wrong", m, shardTableQueryMetricHandler.getQueryMetric(queryId)); } catch (Exception e) { log.error(e.getMessage(), e); Assert.fail(e.getMessage()); } - assertEquals("incomingQueryMetricsCache metric wrong", m, incomingQueryMetricsCache.get(queryId, QueryMetricUpdate.class).getMetric()); + assertEquals("incomingQueryMetricsCache metric wrong", m, incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class).getMetric()); }); } @@ -89,8 +89,8 @@ public void MultipleMetricsAsListStoredCorrectlyInCachesAndAccumulo() throws Exc metrics.forEach((m) -> { String queryId = m.getQueryId(); ensureDataWritten(incomingQueryMetricsCache, lastWrittenQueryMetricCache, queryId); - assertEquals("lastWrittenQueryMetricCache metric wrong", m, lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdate.class).getMetric()); - assertEquals("incomingQueryMetricsCache metric wrong", m, incomingQueryMetricsCache.get(queryId, QueryMetricUpdate.class).getMetric()); + assertEquals("lastWrittenQueryMetricCache metric wrong", m, lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdateHolder.class).getMetric()); + assertEquals("incomingQueryMetricsCache metric wrong", m, incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class).getMetric()); try { assertEquals("accumulo metric wrong", m, shardTableQueryMetricHandler.getQueryMetric(queryId)); } catch (Exception e) { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index 730ea79e..bab85d72 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -401,7 +401,7 @@ protected void ensureDataStored(Cache incomingCache, String queryId) { found = hzCache.containsKey(queryId); if (!found) { try { - Thread.sleep(250); + Thread.sleep(50); } catch (InterruptedException e) {} } } @@ -414,10 +414,10 @@ protected void ensureDataWritten(Cache incomingCache, Cache lastWrittenCache, St int writeDelaySeconds = Math.min(mapStoreConfig.getWriteDelaySeconds(), 1000); boolean found = false; while (!found && System.currentTimeMillis() < (now + (1000 * (writeDelaySeconds + 1)))) { - found = lastWrittenCache.get(queryId) != null; + found = lastWrittenCache.get(queryId, QueryMetricUpdateHolder.class) != null; if (!found) { try { - Thread.sleep(250); + Thread.sleep(50); } catch (InterruptedException e) {} } } diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index 2beed9c0..1356e82b 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -68,7 +68,6 @@ datawave: username: ${warehouse-cluster.accumulo.username} password: ${warehouse-cluster.accumulo.password} accumuloClientPoolSize: 16 - mapStoreWriteThreads: 1 numShards: 10 fieldLengthThreshold: 4049 shardTableName: QueryMetrics_e From cac73bf8a0204342f19a89fcd9bc0c543be391fd Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Tue, 12 Dec 2023 14:48:51 -0500 Subject: [PATCH 36/48] Release datawave-query-metric api/service 3.0.3 --- api/pom.xml | 4 ++-- pom.xml | 2 +- service/pom.xml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 62dd9682..c187b7cd 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,12 +8,12 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.3-SNAPSHOT + 3.0.3 https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - HEAD + api-3.0.3 https://github.com/NationalSecurityAgency/datawave-query-metric-service diff --git a/pom.xml b/pom.xml index 7852fd59..b866bdf3 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.3-SNAPSHOT + 3.0.3 pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 221f4e7d..9013f3ac 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,13 +8,13 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.3-SNAPSHOT + 3.0.3 DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - HEAD + service-3.0.3 https://github.com/NationalSecurityAgency/datawave-query-metric-service @@ -28,7 +28,7 @@ 3.0.1 3.0.0 3.0.0 - 3.0.2 + 3.0.3 3.0.0 2.0.0 2.0.0 From d4b1808ab749b486eb540f6eb19015ce726c01a1 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Tue, 12 Dec 2023 14:53:46 -0500 Subject: [PATCH 37/48] Set version to 3.0.4-SNAPSHOT --- api/pom.xml | 4 ++-- pom.xml | 2 +- service/pom.xml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index c187b7cd..96fcc525 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,12 +8,12 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.3 + 3.0.4-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - api-3.0.3 + HEAD https://github.com/NationalSecurityAgency/datawave-query-metric-service diff --git a/pom.xml b/pom.xml index b866bdf3..92edb22f 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.3 + 3.0.4-SNAPSHOT pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 9013f3ac..286107cd 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,13 +8,13 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.3 + 3.0.4-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git scm:git:git@github.com:NationalSecurityAgency/datawave-query-metric-service.git - service-3.0.3 + HEAD https://github.com/NationalSecurityAgency/datawave-query-metric-service From 24d3917e8eb675a60188519c556e41e7d5c6b485 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Tue, 12 Dec 2023 14:57:52 -0500 Subject: [PATCH 38/48] Release datawave-query-metric api/service 2.1.5 --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 6b637fc3..88240bf2 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ query-metric-api - 2.1.5-SNAPSHOT + 2.1.5 https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index 55cb43de..47ff696b 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ query-metric-parent - 2.1.5-SNAPSHOT + 2.1.5 pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 7e918d1b..a5d876d8 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ query-metric-service - 2.1.5-SNAPSHOT + 2.1.5 DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service From 26c79e57e0bc9194f0b39a461eadf4daef1dbb14 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Tue, 12 Dec 2023 14:59:06 -0500 Subject: [PATCH 39/48] Set version to 2.1.6-SNAPSHOT --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 88240bf2..f8df0d08 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ query-metric-api - 2.1.5 + 2.1.6-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index 47ff696b..2f343819 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ query-metric-parent - 2.1.5 + 2.1.6-SNAPSHOT pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index a5d876d8..8b3c1fd7 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ query-metric-service - 2.1.5 + 2.1.6-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service From 2324adaaf07420852203eddda8debc7bfe744e08 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 14 Dec 2023 11:57:38 -0500 Subject: [PATCH 40/48] elapsedTime=0 if lastUpdated set to createDate in AccumuloMapStore; ensure lastUpdated does not get changed while metric is being written --- .../persistence/AccumuloMapStore.java | 2 -- .../QueryMetricConsistencyTest.java | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index fa036114..a6f7581f 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -176,14 +176,12 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception } long writeTimestamp = deleteTimestamp + 1; updatedMetric.setNumUpdates(numUpdates + 1); - updatedMetric.setLastUpdated(new Date(updatedMetric.getLastUpdated().getTime() + 1)); if (lastQueryMetric.getLastUpdated() != null) { handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), deleteTimestamp, true, ignoredFields); } handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), writeTimestamp, false, ignoredFields); } else { - updatedMetric.setLastUpdated(updatedMetric.getCreateDate()); handler.writeMetric(updatedMetric, Collections.emptyList(), updatedMetric.getCreateDate().getTime(), false, ignoredFields); } if (log.isTraceEnabled()) { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 428fe8c1..4bcd4bde 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -33,6 +33,8 @@ import java.util.List; import java.util.Map; +import static org.junit.Assert.assertNotNull; + @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryMetricConsistencyTest", "QueryMetricTest", "hazelcast-writethrough"}) @@ -275,6 +277,26 @@ public void ToMetricTest() { QueryMetricTestBase.assertEquals("metrics are not equal", queryMetric, newMetric); } + /* + * Check that the last updated time (which is used to calculate the elapsed time) does not get changed when being written to Accumulo + */ + @Test + public void LastUpdatedTest() { + QueryMetric queryMetric = (QueryMetric) createMetric(); + String queryId = queryMetric.getQueryId(); + Date lastUpdated = new Date(queryMetric.getCreateDate().getTime() + 60000); + queryMetric.setLastUpdated(lastUpdated); + queryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); + incomingQueryMetricsCache.put(queryId, new QueryMetricUpdateHolder(queryMetric.duplicate())); + ensureDataWritten(incomingQueryMetricsCache, lastWrittenQueryMetricCache, queryId); + + QueryMetricUpdateHolder storedMetricHolder = lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdateHolder.class); + assertNotNull("storedQueryMetric is null", storedMetricHolder); + QueryMetricTestBase.assertEquals("metric should not change", queryMetric, storedMetricHolder.getMetric()); + Assert.assertEquals("Elapsed time incorrect", 60000, storedMetricHolder.getMetric().getElapsedTime()); + Assert.assertEquals("Last updated incorrect", lastUpdated, storedMetricHolder.getMetric().getLastUpdated()); + } + @Test public void CombineMetricsTest() throws Exception { QueryMetric storedQueryMetric = (QueryMetric) createMetric(); From ffc13a3d4e2a21a15ef5d3568b8eefe8dceacc53 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 14 Dec 2023 12:55:27 -0500 Subject: [PATCH 41/48] Move geo response classes to query-metric-api, ensure json/xml/html responses and map link work from webservice/metric-service --- .../BaseQueryMetricListResponse.java | 37 ++++- .../querymetric/QueryGeometry.java | 73 ++++++++++ .../querymetric/QueryGeometryResponse.java | 133 ++++++++++++++++++ .../QueryMetricsDetailListResponse.java | 12 +- .../querymetric/QueryMetricOperations.java | 13 +- .../handler/QueryGeometryHandler.java | 2 +- .../handler/SimpleQueryGeometryHandler.java | 11 +- .../SimpleQueryGeometryHandlerTest.java | 4 +- 8 files changed, 270 insertions(+), 15 deletions(-) create mode 100644 api/src/main/java/datawave/microservice/querymetric/QueryGeometry.java create mode 100644 api/src/main/java/datawave/microservice/querymetric/QueryGeometryResponse.java diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java index 4f674493..f6c30949 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java @@ -1,5 +1,6 @@ package datawave.microservice.querymetric; +import com.fasterxml.jackson.annotation.JsonIgnore; import datawave.webservice.HtmlProvider; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.webservice.result.BaseResponse; @@ -12,7 +13,9 @@ import java.text.SimpleDateFormat; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.TreeMap; public abstract class BaseQueryMetricListResponse extends BaseResponse implements HtmlProvider { @@ -25,10 +28,27 @@ public abstract class BaseQueryMetricListResponse ext protected List result = null; @XmlElement protected int numResults = 0; + @XmlElement + protected boolean isGeoQuery = false; @XmlTransient private boolean administratorMode = false; - @XmlTransient - private boolean isGeoQuery = false; + private String JQUERY_INCLUDES; + protected String BASE_URL = "/DataWave/Query/Metrics"; + + public BaseQueryMetricListResponse() { + setHtmlIncludePaths(new HashMap<>()); + } + + public void setHtmlIncludePaths(Map pathMap) { + // @formatter:off + JQUERY_INCLUDES = + "\n"; + // @formatter:on + } + + public void setBaseUrl(String baseUrl) { + this.BASE_URL = baseUrl; + } private static String numToString(long number) { return (number == -1 || number == 0) ? "" : Long.toString(number); @@ -67,21 +87,27 @@ public void setGeoQuery(boolean geoQuery) { isGeoQuery = geoQuery; } + @JsonIgnore + @XmlTransient @Override public String getTitle() { return TITLE; } + @JsonIgnore + @XmlTransient @Override public String getPageHeader() { return getTitle(); } + @JsonIgnore + @XmlTransient @Override public String getHeadContent() { if (isGeoQuery) { // @formatter:off - return "" + + return JQUERY_INCLUDES + "\n"; + JQUERY_INCLUDES = + "\n"; + MAP_INCLUDES = + "\n" + + ""; + // @formatter:on + } + + @XmlElement(name = "queryId", nillable = true) + protected String queryId = null; + + @JsonIgnore + @XmlTransient + protected String basemaps = null; + + @XmlElementWrapper(name = "features") + @XmlElement(name = "feature") + protected List result = null; + + @JsonIgnore + @XmlTransient + @Override + public String getTitle() { + if (queryId != null) + return TITLE + " - " + queryId; + return TITLE; + } + + @JsonIgnore + @XmlTransient + @Override + public String getHeadContent() { + String basemapData = "\n"; + String featureData = "\n"; + return String.join("\n", featureData, JQUERY_INCLUDES, LEAFLET_INCLUDES, basemapData, MAP_INCLUDES); + } + + @JsonIgnore + @XmlTransient + @Override + public String getPageHeader() { + return getTitle(); + } + + @JsonIgnore + @XmlTransient + @Override + public String getMainContent() { + return "

"; + } + + private String toGeoJsonFeatures() { + if (!this.result.isEmpty()) + return "[ " + this.result.stream().map(QueryGeometry::toGeoJsonFeature).collect(Collectors.joining(", ")) + " ]"; + else + return "undefined"; + } + + public String getQueryId() { + return queryId; + } + + public void setQueryId(String queryId) { + this.queryId = queryId; + } + + public List getResult() { + return result; + } + + public void setResult(List result) { + this.result = result; + } + + public String getBasemaps() { + return basemaps; + } + + public void setBasemaps(String basemaps) { + this.basemaps = basemaps; + } +} diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java index 8a81d735..77ff4815 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java @@ -1,5 +1,6 @@ package datawave.microservice.querymetric; +import com.fasterxml.jackson.annotation.JsonIgnore; import datawave.webservice.query.QueryImpl.Parameter; import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.BaseQueryMetric.Prediction; @@ -11,6 +12,7 @@ import javax.xml.bind.annotation.XmlAccessorOrder; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; @@ -27,6 +29,8 @@ public class QueryMetricsDetailListResponse extends QueryMetricListResponse { private static final long serialVersionUID = 1L; + @JsonIgnore + @XmlTransient @Override public String getMainContent() { StringBuilder builder = new StringBuilder(), pageTimesBuilder = new StringBuilder(); @@ -79,7 +83,13 @@ public String getMainContent() { builder.append("").append(userDN == null ? "" : userDN).append(""); String proxyServers = metric.getProxyServers() == null ? "" : StringUtils.join(metric.getProxyServers(), "
"); builder.append("").append(proxyServers).append(""); - builder.append("").append(metric.getQueryId()).append(""); + if (this.isAdministratorMode()) { + builder.append("").append(metric.getQueryId()).append(""); + } else { + builder.append("").append(metric.getQueryId()) + .append(""); + } builder.append("").append(metric.getQueryType()).append(""); builder.append("").append(metric.getQueryLogic()).append(""); builder.append(isJexlQuery(parameters) ? "" : "") diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index f42b8f23..2bb05531 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -17,7 +17,6 @@ import datawave.security.util.DnUtils; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; -import datawave.webservice.query.map.QueryGeometryResponse; import datawave.webservice.result.VoidResponse; import io.swagger.annotations.ApiParam; import org.apache.accumulo.core.security.Authorizations; @@ -57,6 +56,7 @@ import java.util.Collections; import java.util.Date; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Set; import java.util.TimeZone; @@ -93,6 +93,7 @@ public class QueryMetricOperations { private MetricUpdateEntryProcessorFactory entryProcessorFactory; private QueryMetricOperationsStats stats; private static Set inProcess = Collections.synchronizedSet(new HashSet<>()); + private final LinkedHashMap pathPrefixMap = new LinkedHashMap<>(); /** * The enum Default datetime. @@ -150,6 +151,10 @@ public QueryMetricOperations(QueryMetricProperties queryMetricProperties, @Named this.correlator = correlator; this.entryProcessorFactory = entryProcessorFactory; this.stats = stats; + this.pathPrefixMap.put("jquery", "/querymetric/webjars/jquery"); + this.pathPrefixMap.put("leaflet", "/querymetric/webjars/leaflet"); + this.pathPrefixMap.put("css", "/querymetric/css"); + this.pathPrefixMap.put("js", "/querymetric/js"); } @PreDestroy @@ -459,6 +464,8 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDet @ApiParam("queryId to return") @PathVariable("queryId") String queryId) { BaseQueryMetricListResponse response = this.queryMetricListResponseFactory.createDetailedResponse(); + response.setHtmlIncludePaths(this.pathPrefixMap); + response.setBaseUrl("/querymetric/v1"); List metricList = new ArrayList<>(); try { BaseQueryMetric metric; @@ -526,7 +533,9 @@ public QueryGeometryResponse map(@AuthenticationPrincipal ProxiedUserDetails cur QueryGeometryResponse queryGeometryResponse = new QueryGeometryResponse(); BaseQueryMetricListResponse metricResponse = query(currentUser, queryId); if (metricResponse.getExceptions() == null || metricResponse.getExceptions().isEmpty()) { - return geometryHandler.getQueryGeometryResponse(queryId, metricResponse.getResult()); + QueryGeometryResponse response = geometryHandler.getQueryGeometryResponse(queryId, metricResponse.getResult()); + response.setHtmlIncludePaths(pathPrefixMap); + return response; } else { metricResponse.getExceptions().forEach(e -> queryGeometryResponse.addException(new QueryException(e.getMessage(), e.getCause(), e.getCode()))); return queryGeometryResponse; diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java index 46ee03ba..9d7865bd 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java @@ -1,7 +1,7 @@ package datawave.microservice.querymetric.handler; import datawave.microservice.querymetric.BaseQueryMetric; -import datawave.webservice.query.map.QueryGeometryResponse; +import datawave.microservice.querymetric.QueryGeometryResponse; import java.util.List; diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java index 2c3fd29f..7392364b 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java @@ -1,22 +1,24 @@ package datawave.microservice.querymetric.handler; import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryGeometry; +import datawave.microservice.querymetric.QueryGeometryResponse; import datawave.microservice.querymetric.config.QueryMetricHandlerProperties; import datawave.query.jexl.JexlASTHelper; import datawave.query.jexl.visitors.GeoFeatureVisitor; import datawave.query.language.parser.ParseException; import datawave.query.language.parser.jexl.LuceneToJexlQueryParser; import datawave.webservice.query.QueryImpl; -import datawave.webservice.query.map.QueryGeometry; -import datawave.webservice.query.map.QueryGeometryResponse; import org.apache.commons.jexl2.parser.JexlNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import static datawave.query.QueryParameters.QUERY_SYNTAX; @@ -39,7 +41,7 @@ public SimpleQueryGeometryHandler(QueryMetricHandlerProperties queryMetricHandle @Override public QueryGeometryResponse getQueryGeometryResponse(String id, List metrics) { - QueryGeometryResponse response = new QueryMetricGeometryResponse(id, basemaps); + QueryGeometryResponse response = new QueryGeometryResponse(id, basemaps); if (metrics != null) { Set queryGeometries = new LinkedHashSet<>(); @@ -48,7 +50,8 @@ public QueryGeometryResponse getQueryGeometryResponse(String id, List features = GeoFeatureVisitor.getGeoFeatures(queryNode, isLuceneQuery); + queryGeometries.addAll(features.stream().map(f -> new QueryGeometry(f.getFunction(), f.getGeometry())).collect(Collectors.toList())); } catch (Exception e) { log.error(e.getMessage(), e); response.addException(new Exception("Unable to parse the geo features")); diff --git a/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java b/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java index 04d64330..687e4a58 100644 --- a/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java @@ -1,11 +1,11 @@ package datawave.microservice.querymetric.handler; +import datawave.microservice.querymetric.QueryGeometry; +import datawave.microservice.querymetric.QueryGeometryResponse; import datawave.microservice.querymetric.QueryMetric; import datawave.microservice.querymetric.config.QueryMetricHandlerProperties; import datawave.webservice.query.QueryImpl; import datawave.webservice.query.exception.QueryExceptionType; -import datawave.webservice.query.map.QueryGeometry; -import datawave.webservice.query.map.QueryGeometryResponse; import org.junit.Assert; import org.junit.Before; import org.junit.Test; From e780cccd1c10bfeebc3692ab40ed5fa98ea8c3cf Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 14 Dec 2023 14:07:32 -0500 Subject: [PATCH 42/48] Remove errant test configuration setting --- service/src/test/resources/config/application.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index 46508a65..bbc16258 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -89,7 +89,6 @@ datawave: transport: message host: localhost port: ${server.port} - confirmAckEnabled: false timely: enabled: false host: localhost From 653e170acb1e2f9a2a7f4adf00bb2d481e426e49 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 14 Dec 2023 11:57:38 -0500 Subject: [PATCH 43/48] elapsedTime=0 if lastUpdated set to createDate in AccumuloMapStore; ensure lastUpdated does not get changed while metric is being written --- .../persistence/AccumuloMapStore.java | 2 -- .../QueryMetricConsistencyTest.java | 21 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java index c504d5b8..4433b356 100644 --- a/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java +++ b/service/src/main/java/datawave/microservice/querymetric/persistence/AccumuloMapStore.java @@ -179,14 +179,12 @@ public void store(QueryMetricUpdateHolder queryMetricUpdate) throws Exception } long writeTimestamp = deleteTimestamp + 1; updatedMetric.setNumUpdates(numUpdates + 1); - updatedMetric.setLastUpdated(new Date(updatedMetric.getLastUpdated().getTime() + 1)); if (lastQueryMetric.getLastUpdated() != null) { handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), deleteTimestamp, true, ignoredFields); } handler.writeMetric(updatedMetric, Collections.singletonList(lastQueryMetric), writeTimestamp, false, ignoredFields); } else { - updatedMetric.setLastUpdated(updatedMetric.getCreateDate()); handler.writeMetric(updatedMetric, Collections.emptyList(), updatedMetric.getCreateDate().getTime(), false, ignoredFields); } if (log.isTraceEnabled()) { diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java index 61456ed2..d67e8fdd 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricConsistencyTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; import java.util.ArrayList; @@ -279,6 +280,26 @@ public void ToMetricTest() { metricAssertEquals("metrics are not equal", queryMetric, newMetric); } + /* + * Check that the last updated time (which is used to calculate the elapsed time) does not get changed when being written to Accumulo + */ + @Test + public void LastUpdatedTest() { + QueryMetric queryMetric = (QueryMetric) createMetric(); + String queryId = queryMetric.getQueryId(); + Date lastUpdated = new Date(queryMetric.getCreateDate().getTime() + 60000); + queryMetric.setLastUpdated(lastUpdated); + queryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); + incomingQueryMetricsCache.put(queryId, new QueryMetricUpdateHolder(queryMetric.duplicate())); + ensureDataWritten(incomingQueryMetricsCache, lastWrittenQueryMetricCache, queryId); + + QueryMetricUpdateHolder storedMetricHolder = lastWrittenQueryMetricCache.get(queryId, QueryMetricUpdateHolder.class); + assertNotNull(storedMetricHolder, "storedQueryMetric is null"); + metricAssertEquals("metric should not change", queryMetric, storedMetricHolder.getMetric()); + assertEquals(60000, storedMetricHolder.getMetric().getElapsedTime(), "Elapsed time incorrect"); + assertEquals(lastUpdated, storedMetricHolder.getMetric().getLastUpdated(), "Last updated incorrect"); + } + @Test public void CombineMetricsTest() throws Exception { QueryMetric storedQueryMetric = (QueryMetric) createMetric(); From b2a8fa6b125302b3b768a381ee309c8c32d00272 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 14 Dec 2023 13:23:31 -0500 Subject: [PATCH 44/48] Move geo response classes to query-metric-api, ensure json/xml/html responses and map link work from webservice/metric-service --- .../BaseQueryMetricListResponse.java | 38 ++++- .../querymetric/QueryGeometry.java | 73 ++++++++++ .../querymetric/QueryGeometryResponse.java | 134 ++++++++++++++++++ .../QueryMetricsDetailListResponse.java | 13 +- service/pom.xml | 2 +- .../querymetric/QueryMetricOperations.java | 17 ++- .../handler/QueryGeometryHandler.java | 2 +- .../handler/SimpleQueryGeometryHandler.java | 10 +- .../SimpleQueryGeometryHandlerTest.java | 5 +- 9 files changed, 275 insertions(+), 19 deletions(-) create mode 100644 api/src/main/java/datawave/microservice/querymetric/QueryGeometry.java create mode 100644 api/src/main/java/datawave/microservice/querymetric/QueryGeometryResponse.java diff --git a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java index 2a95eb76..809118f5 100644 --- a/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/BaseQueryMetricListResponse.java @@ -3,7 +3,9 @@ import java.text.SimpleDateFormat; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.TreeMap; import javax.xml.bind.annotation.XmlElement; @@ -13,6 +15,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.webservice.HtmlProvider; import datawave.webservice.result.BaseResponse; @@ -27,10 +31,27 @@ public abstract class BaseQueryMetricListResponse ext protected List result = null; @XmlElement protected int numResults = 0; + @XmlElement + protected boolean isGeoQuery = false; @XmlTransient private boolean administratorMode = false; - @XmlTransient - private boolean isGeoQuery = false; + private String JQUERY_INCLUDES; + protected String BASE_URL = "/DataWave/Query/Metrics"; + + public BaseQueryMetricListResponse() { + setHtmlIncludePaths(new HashMap<>()); + } + + public void setHtmlIncludePaths(Map pathMap) { + // @formatter:off + JQUERY_INCLUDES = + "\n"; + // @formatter:on + } + + public void setBaseUrl(String baseUrl) { + this.BASE_URL = baseUrl; + } private static String numToString(long number) { return (number == -1 || number == 0) ? "" : Long.toString(number); @@ -69,21 +90,27 @@ public void setGeoQuery(boolean geoQuery) { isGeoQuery = geoQuery; } + @JsonIgnore + @XmlTransient @Override public String getTitle() { return TITLE; } + @JsonIgnore + @XmlTransient @Override public String getPageHeader() { return getTitle(); } + @JsonIgnore + @XmlTransient @Override public String getHeadContent() { if (isGeoQuery) { // @formatter:off - return "" + + return JQUERY_INCLUDES + "\n"; + JQUERY_INCLUDES = + "\n"; + MAP_INCLUDES = + "\n" + + ""; + // @formatter:on + } + + @XmlElement(name = "queryId", nillable = true) + protected String queryId = null; + + @JsonIgnore + @XmlTransient + protected String basemaps = null; + + @XmlElementWrapper(name = "features") + @XmlElement(name = "feature") + protected List result = null; + + @JsonIgnore + @XmlTransient + @Override + public String getTitle() { + if (queryId != null) + return TITLE + " - " + queryId; + return TITLE; + } + + @JsonIgnore + @XmlTransient + @Override + public String getHeadContent() { + String basemapData = "\n"; + String featureData = "\n"; + return String.join("\n", featureData, JQUERY_INCLUDES, LEAFLET_INCLUDES, basemapData, MAP_INCLUDES); + } + + @JsonIgnore + @XmlTransient + @Override + public String getPageHeader() { + return getTitle(); + } + + @JsonIgnore + @XmlTransient + @Override + public String getMainContent() { + return "
"; + } + + private String toGeoJsonFeatures() { + if (!this.result.isEmpty()) + return "[ " + this.result.stream().map(QueryGeometry::toGeoJsonFeature).collect(Collectors.joining(", ")) + " ]"; + else + return "undefined"; + } + + public String getQueryId() { + return queryId; + } + + public void setQueryId(String queryId) { + this.queryId = queryId; + } + + public List getResult() { + return result; + } + + public void setResult(List result) { + this.result = result; + } + + public String getBasemaps() { + return basemaps; + } + + public void setBasemaps(String basemaps) { + this.basemaps = basemaps; + } +} diff --git a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java index 4a88e359..ed388e58 100644 --- a/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java +++ b/api/src/main/java/datawave/microservice/querymetric/QueryMetricsDetailListResponse.java @@ -14,10 +14,13 @@ import javax.xml.bind.annotation.XmlAccessorOrder; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import datawave.microservice.querymetric.BaseQueryMetric.PageMetric; import datawave.microservice.querymetric.BaseQueryMetric.Prediction; import datawave.webservice.query.QueryImpl.Parameter; @@ -36,6 +39,8 @@ public String getHeadContent() { // metric page } + @JsonIgnore + @XmlTransient @Override public String getMainContent() { StringBuilder builder = new StringBuilder(), pageTimesBuilder = new StringBuilder(); @@ -88,7 +93,13 @@ public String getMainContent() { builder.append("").append(userDN == null ? "" : userDN).append(""); String proxyServers = metric.getProxyServers() == null ? "" : StringUtils.join(metric.getProxyServers(), "
"); builder.append("").append(proxyServers).append(""); - builder.append("").append(metric.getQueryId()).append(""); + if (this.isAdministratorMode()) { + builder.append("").append(metric.getQueryId()).append(""); + } else { + builder.append("").append(metric.getQueryId()) + .append(""); + } builder.append("").append(metric.getQueryType()).append(""); builder.append("").append(metric.getQueryLogic()).append(""); // Note the query and query plan are added to the table later (see the javascript at the end of this for loop) diff --git a/service/pom.xml b/service/pom.xml index 286107cd..81cf01a3 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -28,7 +28,7 @@ 3.0.1 3.0.0 3.0.0 - 3.0.3 + 3.0.4-SNAPSHOT 3.0.0 2.0.0 2.0.0 diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index f692fc45..488aa764 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -62,6 +62,7 @@ import datawave.marking.MarkingFunctions; import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.querymetric.QueryGeometryResponse; import datawave.microservice.querymetric.config.QueryMetricProperties; import datawave.microservice.querymetric.config.QueryMetricProperties.Retry; import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory; @@ -74,7 +75,6 @@ import datawave.security.authorization.DatawaveUser; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; -import datawave.webservice.query.map.QueryGeometryResponse; import datawave.webservice.result.VoidResponse; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; @@ -109,6 +109,7 @@ public class QueryMetricOperations { private MetricUpdateEntryProcessorFactory entryProcessorFactory; private QueryMetricOperationsStats stats; private static Set inProcess = Collections.synchronizedSet(new HashSet<>()); + private final LinkedHashMap pathPrefixMap = new LinkedHashMap<>(); private final QueryMetricSupplier queryMetricSupplier; private final DnUtils dnUtils; @@ -174,6 +175,10 @@ public QueryMetricOperations(QueryMetricProperties queryMetricProperties, @Named this.stats = stats; this.queryMetricSupplier = queryMetricSupplier; this.dnUtils = dnUtils; + this.pathPrefixMap.put("jquery", "/querymetric/webjars/jquery"); + this.pathPrefixMap.put("leaflet", "/querymetric/webjars/leaflet"); + this.pathPrefixMap.put("css", "/querymetric/css"); + this.pathPrefixMap.put("js", "/querymetric/js"); } @PreDestroy @@ -544,6 +549,8 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal DatawaveUserDe @Parameter(description = "queryId to return") @PathVariable("queryId") String queryId) { BaseQueryMetricListResponse response = this.queryMetricListResponseFactory.createDetailedResponse(); + response.setHtmlIncludePaths(this.pathPrefixMap); + response.setBaseUrl("/querymetric/v1"); List metricList = new ArrayList<>(); try { BaseQueryMetric metric; @@ -628,11 +635,13 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal DatawaveUserDe public QueryGeometryResponse map(@AuthenticationPrincipal DatawaveUserDetails currentUser, @PathVariable("queryId") String queryId) { QueryGeometryResponse queryGeometryResponse = new QueryGeometryResponse(); BaseQueryMetricListResponse metricResponse = query(currentUser, queryId); - if (!metricResponse.getExceptions().isEmpty()) { + if (metricResponse.getExceptions() == null || metricResponse.getExceptions().isEmpty()) { + QueryGeometryResponse response = geometryHandler.getQueryGeometryResponse(queryId, metricResponse.getResult()); + response.setHtmlIncludePaths(pathPrefixMap); + return response; + } else { metricResponse.getExceptions().forEach(e -> queryGeometryResponse.addException(new QueryException(e.getMessage(), e.getCause(), e.getCode()))); return queryGeometryResponse; - } else { - return geometryHandler.getQueryGeometryResponse(queryId, metricResponse.getResult()); } } diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java index 2d3e06ae..79cb1acb 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/QueryGeometryHandler.java @@ -3,7 +3,7 @@ import java.util.List; import datawave.microservice.querymetric.BaseQueryMetric; -import datawave.webservice.query.map.QueryGeometryResponse; +import datawave.microservice.querymetric.QueryGeometryResponse; public interface QueryGeometryHandler { diff --git a/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java b/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java index 064cd6c7..28649318 100644 --- a/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java +++ b/service/src/main/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandler.java @@ -6,20 +6,21 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.jexl2.parser.JexlNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryGeometry; +import datawave.microservice.querymetric.QueryGeometryResponse; import datawave.microservice.querymetric.config.QueryMetricHandlerProperties; import datawave.query.jexl.JexlASTHelper; import datawave.query.jexl.visitors.GeoFeatureVisitor; import datawave.query.language.parser.ParseException; import datawave.query.language.parser.jexl.LuceneToJexlQueryParser; import datawave.webservice.query.QueryImpl; -import datawave.webservice.query.map.QueryGeometry; -import datawave.webservice.query.map.QueryGeometryResponse; /** * This class is used to extract query geometries from the query metrics in an effort to provide those geometries for subsequent display to the user. @@ -40,7 +41,7 @@ public SimpleQueryGeometryHandler(QueryMetricHandlerProperties queryMetricHandle @Override public QueryGeometryResponse getQueryGeometryResponse(String id, List metrics) { - QueryGeometryResponse response = new QueryMetricGeometryResponse(id, basemaps); + QueryGeometryResponse response = new QueryGeometryResponse(id, basemaps); if (metrics != null) { Set queryGeometries = new LinkedHashSet<>(); @@ -49,7 +50,8 @@ public QueryGeometryResponse getQueryGeometryResponse(String id, List features = GeoFeatureVisitor.getGeoFeatures(queryNode, isLuceneQuery); + queryGeometries.addAll(features.stream().map(f -> new QueryGeometry(f.getFunction(), f.getGeometry())).collect(Collectors.toList())); } catch (Exception e) { log.error(e.getMessage(), e); response.addException(new Exception("Unable to parse the geo features")); diff --git a/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java b/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java index 0fed7e70..19a2587d 100644 --- a/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java +++ b/service/src/test/java/datawave/microservice/querymetric/handler/SimpleQueryGeometryHandlerTest.java @@ -9,16 +9,15 @@ import java.util.List; import java.util.Set; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import datawave.microservice.querymetric.QueryGeometry; +import datawave.microservice.querymetric.QueryGeometryResponse; import datawave.microservice.querymetric.QueryMetric; import datawave.microservice.querymetric.config.QueryMetricHandlerProperties; import datawave.webservice.query.QueryImpl; import datawave.webservice.query.exception.QueryExceptionType; -import datawave.webservice.query.map.QueryGeometry; -import datawave.webservice.query.map.QueryGeometryResponse; public class SimpleQueryGeometryHandlerTest { From 48e85722a4ec2904e6c8b932d0c7a08bb7fd1c6f Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 14 Dec 2023 14:36:42 -0500 Subject: [PATCH 45/48] Release datawave-query-metric api/service 3.0.4 --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 96fcc525..a3e201b1 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.4-SNAPSHOT + 3.0.4 https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index 92edb22f..a7d64064 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.4-SNAPSHOT + 3.0.4 pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index 81cf01a3..d5c3e64f 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.4-SNAPSHOT + 3.0.4 DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service @@ -28,7 +28,7 @@ 3.0.1 3.0.0 3.0.0 - 3.0.4-SNAPSHOT + 3.0.4 3.0.0 2.0.0 2.0.0 From 01c174971f26cfff4f5efb95516681edef8adc64 Mon Sep 17 00:00:00 2001 From: Bill Oley Date: Thu, 14 Dec 2023 14:37:52 -0500 Subject: [PATCH 46/48] Set version to 3.0.5-SNAPSHOT --- api/pom.xml | 2 +- pom.xml | 2 +- service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index a3e201b1..89bc7988 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-parent/pom.xml query-metric-api - 3.0.4 + 3.0.5-SNAPSHOT https://code.nsa.gov/datawave-query-metric-service scm:git:https://github.com/NationalSecurityAgency/datawave-query-metric-service.git diff --git a/pom.xml b/pom.xml index a7d64064..50310158 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ../../microservice-parent/pom.xml query-metric-parent - 3.0.4 + 3.0.5-SNAPSHOT pom https://code.nsa.gov/datawave-query-metric-service diff --git a/service/pom.xml b/service/pom.xml index d5c3e64f..34489b6b 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -8,7 +8,7 @@ ../../../microservice-service-parent/pom.xml query-metric-service - 3.0.4 + 3.0.5-SNAPSHOT DATAWAVE Query Metric Microservice https://code.nsa.gov/datawave-query-metric-service From 62991a1ac5622bff68018f445a0cc77f9f61599d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 15 Dec 2023 16:44:48 +0000 Subject: [PATCH 47/48] Fixed concurrent modification exception --- .../microservice/querymetric/QueryMetricOperations.java | 2 +- service/src/test/resources/config/application.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 488aa764..72d5cca0 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -446,7 +446,7 @@ private boolean awaitConfirmAcks(Map updatesById, List boolean success = true; // wait for the confirm acks only after all sends are successful if (queryMetricProperties.isConfirmAckEnabled()) { - for (String correlationId : updatesById.keySet()) { + for (String correlationId : new HashSet<>(updatesById.keySet())) { if (!awaitConfirmAck(correlationId)) { failedUpdates.add(updatesById.remove(correlationId)); success = false; diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml index bbc16258..46508a65 100644 --- a/service/src/test/resources/config/application.yml +++ b/service/src/test/resources/config/application.yml @@ -89,6 +89,7 @@ datawave: transport: message host: localhost port: ${server.port} + confirmAckEnabled: false timely: enabled: false host: localhost From 7c8da1cffe34954eb4a8dec5e352c6d6fc9e2ff4 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 17 Jan 2024 19:46:17 +0000 Subject: [PATCH 48/48] updated with latest from integration --- .../querymetric/QueryMetricOperations.java | 116 ++++++++++------- .../QueryMetricOperationsStats.java | 121 ++++++++++-------- .../function/QueryMetricConsumer.java | 7 +- .../querymetric/QueryMetricTestBase.java | 2 +- 4 files changed, 139 insertions(+), 107 deletions(-) diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java index 72d5cca0..b8e456bf 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperations.java @@ -23,6 +23,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import javax.annotation.PreDestroy; import javax.annotation.security.PermitAll; @@ -62,7 +63,6 @@ import datawave.marking.MarkingFunctions; import datawave.microservice.authorization.user.DatawaveUserDetails; -import datawave.microservice.querymetric.QueryGeometryResponse; import datawave.microservice.querymetric.config.QueryMetricProperties; import datawave.microservice.querymetric.config.QueryMetricProperties.Retry; import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory; @@ -229,7 +229,7 @@ public void ensureUpdatesProcessed(boolean scheduled) { QueryMetricType metricType = correlatedUpdates.get(0).getMetricType(); QueryMetricUpdateHolder metricUpdate = combineMetricUpdates(correlatedUpdates, metricType); log.debug("storing correlated updates for {}", queryId); - storeMetricUpdates(metricUpdate); + storeMetricUpdate(metricUpdate); } catch (Exception e) { log.error("exception while combining correlated updates: " + e.getMessage(), e); } @@ -258,22 +258,17 @@ public VoidResponse updateMetrics(@RequestBody List queryMetric if (!this.mergeLock.isAllowedReadLock()) { throw new IllegalStateException("service unavailable"); } + for (BaseQueryMetric m : queryMetrics) { + if (log.isTraceEnabled()) { + log.trace("received bulk metric update via REST: " + m.toString()); + } else { + log.debug("received bulk metric update via REST: " + m.getQueryId()); + } + } Timer.Context restTimer = this.stats.getTimer(TIMERS.REST).time(); try { - for (BaseQueryMetric m : queryMetrics) { - if (log.isTraceEnabled()) { - log.trace("received metric update via REST: " + m.toString()); - } else { - log.debug("received metric update via REST: " + m.getQueryId()); - } - Timer.Context messageSendTimer = this.stats.getTimer(TIMERS.MESSAGE_SEND).time(); - try { - if (!updateMetric(new QueryMetricUpdate<>(m, metricType))) { - throw new RuntimeException("Unable to process query metric update for query [" + m.getQueryId() + "]"); - } - } finally { - messageSendTimer.stop(); - } + if (!updateMetrics(queryMetrics.stream().map(m -> new QueryMetricUpdate<>(m, metricType)).collect(Collectors.toList()))) { + throw new RuntimeException("Unable to process bulk query metric update"); } } finally { restTimer.stop(); @@ -308,13 +303,8 @@ public VoidResponse updateMetric(@RequestBody BaseQueryMetric queryMetric, } else { log.debug("received metric update via REST: " + queryMetric.getQueryId()); } - Timer.Context messageSendTimer = this.stats.getTimer(TIMERS.MESSAGE_SEND).time(); - try { - if (!updateMetric(new QueryMetricUpdate(queryMetric, metricType))) { - throw new RuntimeException("Unable to process query metric update for query [" + queryMetric.getQueryId() + "]"); - } - } finally { - messageSendTimer.stop(); + if (!updateMetrics(Lists.newArrayList(new QueryMetricUpdate(queryMetric, metricType)))) { + throw new RuntimeException("Unable to process query metric update for query [" + queryMetric.getQueryId() + "]"); } } finally { restTimer.stop(); @@ -346,8 +336,8 @@ public void processConfirmAck(Message message) { } private boolean updateMetrics(List updates) { - List failedUpdates = new ArrayList<>(updates.size()); Map updatesById = new LinkedHashMap<>(); + Map timersById = new LinkedHashMap<>(); boolean success; final long updateStartTime = System.currentTimeMillis(); @@ -356,6 +346,7 @@ private boolean updateMetrics(List updates) { Retry retry = queryMetricProperties.getRetry(); + List failedConfirmAck = new ArrayList<>(updates.size()); do { if (attempts++ > 0) { try { @@ -363,6 +354,10 @@ private boolean updateMetrics(List updates) { } catch (InterruptedException e) { // Ignore -- we'll just end up retrying a little too fast } + + // perform some retry upkeep + updates.addAll(failedConfirmAck); + failedConfirmAck.clear(); } if (log.isDebugEnabled()) { @@ -370,53 +365,66 @@ private boolean updateMetrics(List updates) { } // send all of the remaining metric updates - success = sendMessages(updates, failedUpdates, updatesById) && awaitConfirmAcks(updatesById, failedUpdates); + success = sendMessages(updates, updatesById, timersById) && awaitConfirmAcks(updatesById, failedConfirmAck, timersById); currentTime = System.currentTimeMillis(); } while (!success && (currentTime - updateStartTime) < retry.getFailTimeoutMillis() && attempts < retry.getMaxAttempts()); + // stop any timers that remain + timersById.values().stream().forEach(timer -> timer.stop()); + if (!success) { log.warn("Bulk update failed. {attempts = {}, elapsedMillis = {}}", attempts, (currentTime - updateStartTime)); } else { - log.info("Bulk update successful. {attempts = {}, elapsedMillis = {}}", attempts, (currentTime - updateStartTime)); + log.debug("Bulk update successful. {attempts = {}, elapsedMillis = {}}", attempts, (currentTime - updateStartTime)); } return success; } - private boolean updateMetric(QueryMetricUpdate update) { - return updateMetrics(Lists.newArrayList(update)); - } - /** * Passes query metric messages to the messaging infrastructure. * * @param updates * The query metric updates to be sent, not null - * @param failedUpdates - * A list that will be populated with the failed metric updates, not null * @param updatesById * A map that will be populated with the correlation ids and associated metric updates, not null + * @param timersById + * A map of dropwizard timers to record the time to send or send/ack each message * @return true if all messages were successfully sent, false otherwise */ - private boolean sendMessages(List updates, List failedUpdates, Map updatesById) { - failedUpdates.clear(); + private boolean sendMessages(List updates, Map updatesById, Map timersById) { + + List failedSend = new ArrayList<>(updates.size()); boolean success = true; // send all of the remaining metric updates for (QueryMetricUpdate update : updates) { + Timer.Context timer = this.stats.getTimer(TIMERS.MESSAGE_SEND).time(); String correlationId = UUID.randomUUID().toString(); - if (sendMessage(correlationId, update)) { - if (queryMetricProperties.isConfirmAckEnabled()) { - updatesById.put(correlationId, update); + boolean sendSuccessful = false; + try { + sendSuccessful = sendMessage(correlationId, update); + if (sendSuccessful) { + if (queryMetricProperties.isConfirmAckEnabled()) { + updatesById.put(correlationId, update); + } + } else { + // if it failed, add it to the failed list + failedSend.add(update); + success = false; + } + } finally { + if (sendSuccessful && queryMetricProperties.isConfirmAckEnabled()) { + // message send successful but waiting for confirmAck, so store the timer by correlationId + timersById.put(correlationId, timer); + } else { + // either message send successful and not waiting for confirmAck or message send failed + timer.stop(); } - } else { - // if it failed, add it to the failed list - failedUpdates.add(update); - success = false; } } - updates.retainAll(failedUpdates); + updates.retainAll(failedSend); return success; } @@ -438,18 +446,29 @@ private boolean sendMessage(String correlationId, QueryMetricUpdate update) { * * @param updatesById * A map of query metric updates keyed by their correlation id, not null - * @param failedUpdates + * @param failedConfirmAck * A list that will be populated with the failed metric updates, not null + * @param timersById + * A map of dropwizard timers to record the time to send or send/ack each message * @return true if all confirm acks were successfully received, false otherwise */ - private boolean awaitConfirmAcks(Map updatesById, List failedUpdates) { + private boolean awaitConfirmAcks(Map updatesById, List failedConfirmAck, + Map timersById) { boolean success = true; // wait for the confirm acks only after all sends are successful if (queryMetricProperties.isConfirmAckEnabled()) { for (String correlationId : new HashSet<>(updatesById.keySet())) { - if (!awaitConfirmAck(correlationId)) { - failedUpdates.add(updatesById.remove(correlationId)); - success = false; + try { + if (!awaitConfirmAck(correlationId)) { + failedConfirmAck.add(updatesById.remove(correlationId)); + success = false; + } + } finally { + // either ack confirmed or we need to resend, so stop the timer for this update + Timer.Context timer = timersById.get(correlationId); + if (timer != null) { + timer.stop(); + } } } } @@ -478,7 +497,6 @@ public QueryMetricUpdateHolder combineMetricUpdates(List upda BaseQueryMetric combinedMetric = null; BaseQueryMetric.Lifecycle lowestLifecycle = null; for (QueryMetricUpdate u : updates) { - this.stats.queueTimelyMetrics(u); if (combinedMetric == null) { combinedMetric = u.getMetric(); lowestLifecycle = u.getMetric().getLifecycle(); @@ -494,7 +512,7 @@ public QueryMetricUpdateHolder combineMetricUpdates(List upda return metricUpdateHolder; } - public void storeMetricUpdates(QueryMetricUpdateHolder metricUpdate) { + public void storeMetricUpdate(QueryMetricUpdateHolder metricUpdate) { Timer.Context storeTimer = this.stats.getTimer(TIMERS.STORE).time(); String queryId = metricUpdate.getMetric().getQueryId(); try { diff --git a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java index 8e706780..d3bed256 100644 --- a/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java +++ b/service/src/main/java/datawave/microservice/querymetric/QueryMetricOperationsStats.java @@ -40,6 +40,17 @@ public class QueryMetricOperationsStats { private Logger log = LoggerFactory.getLogger(getClass()); + private static final String RatePerSec_1_Min_Avg = ".RatePerSec_1_Min_Avg"; + private static final String RatePerSec_5_Min_Avg = ".RatePerSec_5_Min_Avg"; + private static final String RatePerSec_15_Min_Avg = ".RatePerSec_15_Min_Avg"; + private static final String Latency_Mean = ".Latency_Mean"; + private static final String Latency_Median = ".Latency_Median"; + private static final String Latency_Max = ".Latency_Max"; + private static final String Latency_Min = ".Latency_Min"; + private static final String Latency_75 = ".Latency_75"; + private static final String Latency_95 = ".Latency_95"; + private static final String Latency_99 = ".Latency_99"; + private static final String Latency_999 = ".Latency_999"; private Map timerMap = new HashMap<>(); private Map meterMap = new HashMap<>(); private TcpClient timelyTcpClient; @@ -247,38 +258,40 @@ public void queueTimelyMetrics(QueryMetricUpdate update) { } public void queueTimelyMetrics(BaseQueryMetric queryMetric) { - String queryType = queryMetric.getQueryType(); - if (this.timelyProperties.isEnabled() && queryType != null && queryType.equalsIgnoreCase("RunningQuery")) { - BaseQueryMetric.Lifecycle lifecycle = queryMetric.getLifecycle(); - String host = queryMetric.getHost(); - String user = queryMetric.getUser(); - String logic = queryMetric.getQueryLogic(); - if (lifecycle.equals(BaseQueryMetric.Lifecycle.CLOSED) || lifecycle.equals(BaseQueryMetric.Lifecycle.CANCELLED)) { - long createDate = queryMetric.getCreateDate().getTime(); - // write ELAPSED_TIME - this.queryStatsToWriteToTimely.add("put dw.query.metrics.ELAPSED_TIME " + createDate + " " + queryMetric.getElapsedTime() + " HOST=" + host - + getCommonTags() + "\n"); - this.queryStatsToWriteToTimely.add("put dw.query.metrics.ELAPSED_TIME " + createDate + " " + queryMetric.getElapsedTime() + " USER=" + user - + getCommonTags() + "\n"); - this.queryStatsToWriteToTimely.add("put dw.query.metrics.ELAPSED_TIME " + createDate + " " + queryMetric.getElapsedTime() + " QUERY_LOGIC=" - + logic + getCommonTags() + "\n"); - - // write NUM_RESULTS - this.queryStatsToWriteToTimely.add("put dw.query.metrics.NUM_RESULTS " + createDate + " " + queryMetric.getNumResults() + " HOST=" + host - + getCommonTags() + "\n"); - this.queryStatsToWriteToTimely.add("put dw.query.metrics.NUM_RESULTS " + createDate + " " + queryMetric.getNumResults() + " USER=" + user - + getCommonTags() + "\n"); - this.queryStatsToWriteToTimely.add("put dw.query.metrics.NUM_RESULTS " + createDate + " " + queryMetric.getNumResults() + " QUERY_LOGIC=" - + logic + getCommonTags() + "\n"); - } else if (lifecycle.equals(BaseQueryMetric.Lifecycle.INITIALIZED)) { - // aggregate these metrics for later writing to timely - synchronized (this.hostCountMap) { - Long hostCount = this.hostCountMap.get(host); - this.hostCountMap.put(host, hostCount == null ? 1l : hostCount + 1); - Long userCount = this.userCountMap.get(user); - this.userCountMap.put(user, userCount == null ? 1l : userCount + 1); - Long logicCount = this.logicCountMap.get(logic); - this.logicCountMap.put(logic, logicCount == null ? 1l : logicCount + 1); + if (this.timelyProperties.isEnabled()) { + String queryType = queryMetric.getQueryType(); + if (queryType != null && queryType.equalsIgnoreCase("RunningQuery")) { + BaseQueryMetric.Lifecycle lifecycle = queryMetric.getLifecycle(); + String host = queryMetric.getHost(); + String user = queryMetric.getUser(); + String logic = queryMetric.getQueryLogic(); + if (lifecycle.equals(BaseQueryMetric.Lifecycle.CLOSED) || lifecycle.equals(BaseQueryMetric.Lifecycle.CANCELLED)) { + long createDate = queryMetric.getCreateDate().getTime(); + // write ELAPSED_TIME + this.queryStatsToWriteToTimely.add("put dw.query.metrics.ELAPSED_TIME " + createDate + " " + queryMetric.getElapsedTime() + " HOST=" + host + + getCommonTags() + "\n"); + this.queryStatsToWriteToTimely.add("put dw.query.metrics.ELAPSED_TIME " + createDate + " " + queryMetric.getElapsedTime() + " USER=" + user + + getCommonTags() + "\n"); + this.queryStatsToWriteToTimely.add("put dw.query.metrics.ELAPSED_TIME " + createDate + " " + queryMetric.getElapsedTime() + " QUERY_LOGIC=" + + logic + getCommonTags() + "\n"); + + // write NUM_RESULTS + this.queryStatsToWriteToTimely.add("put dw.query.metrics.NUM_RESULTS " + createDate + " " + queryMetric.getNumResults() + " HOST=" + host + + getCommonTags() + "\n"); + this.queryStatsToWriteToTimely.add("put dw.query.metrics.NUM_RESULTS " + createDate + " " + queryMetric.getNumResults() + " USER=" + user + + getCommonTags() + "\n"); + this.queryStatsToWriteToTimely.add("put dw.query.metrics.NUM_RESULTS " + createDate + " " + queryMetric.getNumResults() + " QUERY_LOGIC=" + + logic + getCommonTags() + "\n"); + } else if (lifecycle.equals(BaseQueryMetric.Lifecycle.INITIALIZED)) { + // aggregate these metrics for later writing to timely + synchronized (this.hostCountMap) { + Long hostCount = this.hostCountMap.get(host); + this.hostCountMap.put(host, hostCount == null ? 1l : hostCount + 1); + Long userCount = this.userCountMap.get(user); + this.userCountMap.put(user, userCount == null ? 1l : userCount + 1); + Long logicCount = this.logicCountMap.get(logic); + this.logicCountMap.put(logic, logicCount == null ? 1l : logicCount + 1); + } } } } @@ -298,15 +311,15 @@ protected String getCommonTags() { public void logServiceStats() { Map stats = getServiceStats(); Map serviceStats = formatStats(stats, true); - String storeRate1 = serviceStats.get("storeRatePerSec_1_Min_Avg"); - String storeRate5 = serviceStats.get("storeRatePerSec_5_Min_Avg"); - String storeRate15 = serviceStats.get("storeRatePerSec_15_Min_Avg"); - String storeLat = serviceStats.get("storeLatency_Mean"); + String storeRate1 = serviceStats.get("store" + RatePerSec_1_Min_Avg); + String storeRate5 = serviceStats.get("store" + RatePerSec_5_Min_Avg); + String storeRate15 = serviceStats.get("store" + RatePerSec_15_Min_Avg); + String storeLat = serviceStats.get("store" + Latency_Mean); log.info("storeMetric rates/sec 1m={} 5m={} 15m={} opLatMs={}", storeRate1, storeRate5, storeRate15, storeLat); - String accumuloRate1 = serviceStats.get("accumuloRatePerSec_1_Min_Avg"); - String accumuloRate5 = serviceStats.get("accumuloRatePerSec_5_Min_Avg"); - String accumuloRate15 = serviceStats.get("accumuloRatePerSec_15_Min_Avg"); - String accumuloLat = serviceStats.get("accumuloLatency_Mean"); + String accumuloRate1 = serviceStats.get("accumulo.write" + RatePerSec_1_Min_Avg); + String accumuloRate5 = serviceStats.get("accumulo.write" + RatePerSec_5_Min_Avg); + String accumuloRate15 = serviceStats.get("accumulo.write" + RatePerSec_15_Min_Avg); + String accumuloLat = serviceStats.get("accumulo.write" + Latency_Mean); log.info("accumulo rates/sec 1m={} 5m={} 15m={} opLatMs={}", accumuloRate1, accumuloRate5, accumuloRate15, accumuloLat); } @@ -335,22 +348,22 @@ public void queueAggregatedQueryStatsForTimely() { private void addTimerStats(String baseName, Timer timer, Map stats) { Snapshot snapshot = timer.getSnapshot(); - stats.put(baseName + "Latency_Mean", snapshot.getMean() / 1000000); - stats.put(baseName + "Latency_Median", snapshot.getMedian() / 1000000); - stats.put(baseName + "Latency_Max", ((Number) snapshot.getMax()).doubleValue() / 1000000); - stats.put(baseName + "Latency_Min", ((Number) snapshot.getMin()).doubleValue() / 1000000); - stats.put(baseName + "Latency_75", snapshot.get75thPercentile() / 1000000); - stats.put(baseName + "Latency_95", snapshot.get95thPercentile() / 1000000); - stats.put(baseName + "Latency_99", snapshot.get99thPercentile() / 1000000); - stats.put(baseName + "Latency_999", snapshot.get999thPercentile() / 1000000); - stats.put(baseName + "RatePerSec_1_Min_Avg", timer.getOneMinuteRate()); - stats.put(baseName + "RatePerSec_5_Min_Avg", timer.getFiveMinuteRate()); - stats.put(baseName + "RatePerSec_15_Min_Avg", timer.getFifteenMinuteRate()); + stats.put(baseName + Latency_Mean, snapshot.getMean() / 1000000); + stats.put(baseName + Latency_Median, snapshot.getMedian() / 1000000); + stats.put(baseName + Latency_Max, ((Number) snapshot.getMax()).doubleValue() / 1000000); + stats.put(baseName + Latency_Min, ((Number) snapshot.getMin()).doubleValue() / 1000000); + stats.put(baseName + Latency_75, snapshot.get75thPercentile() / 1000000); + stats.put(baseName + Latency_95, snapshot.get95thPercentile() / 1000000); + stats.put(baseName + Latency_99, snapshot.get99thPercentile() / 1000000); + stats.put(baseName + Latency_999, snapshot.get999thPercentile() / 1000000); + stats.put(baseName + RatePerSec_1_Min_Avg, timer.getOneMinuteRate()); + stats.put(baseName + RatePerSec_5_Min_Avg, timer.getFiveMinuteRate()); + stats.put(baseName + RatePerSec_15_Min_Avg, timer.getFifteenMinuteRate()); } private void addMeterStats(String baseName, Metered meter, Map stats) { - stats.put(baseName + "RatePerSec_1_Min_Avg", meter.getOneMinuteRate()); - stats.put(baseName + "RatePerSec_5_Min_Avg", meter.getFiveMinuteRate()); - stats.put(baseName + "RatePerSec_15_Min_Avg", meter.getFifteenMinuteRate()); + stats.put(baseName + RatePerSec_1_Min_Avg, meter.getOneMinuteRate()); + stats.put(baseName + RatePerSec_5_Min_Avg, meter.getFiveMinuteRate()); + stats.put(baseName + RatePerSec_15_Min_Avg, meter.getFifteenMinuteRate()); } } diff --git a/service/src/main/java/datawave/microservice/querymetric/function/QueryMetricConsumer.java b/service/src/main/java/datawave/microservice/querymetric/function/QueryMetricConsumer.java index da141276..3ef1e6b8 100644 --- a/service/src/main/java/datawave/microservice/querymetric/function/QueryMetricConsumer.java +++ b/service/src/main/java/datawave/microservice/querymetric/function/QueryMetricConsumer.java @@ -30,13 +30,14 @@ public QueryMetricConsumer(QueryMetricOperations queryMetricOperations, Correlat @Override public void accept(QueryMetricUpdate queryMetricUpdate) { try { - stats.getMeter(QueryMetricOperationsStats.METERS.MESSAGE_RECEIVE).mark(); + this.stats.queueTimelyMetrics(queryMetricUpdate); + this.stats.getMeter(QueryMetricOperationsStats.METERS.MESSAGE_RECEIVE).mark(); if (shouldCorrelate(queryMetricUpdate)) { log.debug("adding update for {} to correlator", queryMetricUpdate.getMetric().getQueryId()); this.correlator.addMetricUpdate(queryMetricUpdate); } else { log.debug("storing update for {}", queryMetricUpdate.getMetric().getQueryId()); - queryMetricOperations.storeMetricUpdates(new QueryMetricUpdateHolder(queryMetricUpdate)); + this.queryMetricOperations.storeMetricUpdate(new QueryMetricUpdateHolder(queryMetricUpdate)); } if (this.correlator.isEnabled()) { @@ -49,7 +50,7 @@ public void accept(QueryMetricUpdate queryMetricUpdate) { QueryMetricType metricType = correlatedUpdates.get(0).getMetricType(); QueryMetricUpdateHolder metricUpdate = this.queryMetricOperations.combineMetricUpdates(correlatedUpdates, metricType); log.debug("storing correlated updates for {}", queryId); - queryMetricOperations.storeMetricUpdates(metricUpdate); + queryMetricOperations.storeMetricUpdate(metricUpdate); } catch (Exception e) { log.error("exception while combining correlated updates: " + e.getMessage(), e); } diff --git a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java index ecc97e74..2bff3ab3 100644 --- a/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java +++ b/service/src/test/java/datawave/microservice/querymetric/QueryMetricTestBase.java @@ -627,7 +627,7 @@ public QueryMetricSupplier testQueryMetricSource(@Lazy QueryMetricOperations que return new QueryMetricSupplier() { @Override public boolean send(Message queryMetricUpdate) { - queryMetricOperations.storeMetricUpdates(new QueryMetricUpdateHolder(queryMetricUpdate.getPayload())); + queryMetricOperations.storeMetricUpdate(new QueryMetricUpdateHolder(queryMetricUpdate.getPayload())); return true; } };