Skip to content

Commit

Permalink
Corrects the implementation of subplans for microservice
Browse files Browse the repository at this point in the history
  • Loading branch information
foster33 committed Sep 19, 2023
1 parent 89f24b5 commit feb95a3
Show file tree
Hide file tree
Showing 10 changed files with 220 additions and 83 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
package datawave.microservice.querymetric;

import datawave.webservice.HtmlProvider;
import datawave.webservice.result.BaseResponse;

import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlTransient;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

public class BaseQueryMetricSubplanResponse<T extends BaseQueryMetric> extends BaseResponse implements HtmlProvider {

private static final long serialVersionUID = 1L;
private static final String TITLE = "Query Metrics";
private static final String EMPTY = "";
@XmlElementWrapper(name = "queryMetrics")
@XmlElement(name = "queryMetric")
protected List<T> result = null;
@XmlElement
protected int numResults = 0;
@XmlTransient
private boolean administratorMode = false;
@XmlTransient
private boolean isGeoQuery = false;

private static String numToString(long number) {
return (number == -1 || number == 0) ? "" : Long.toString(number);
}

public List<T> getResult() {
return result;
}

public int getNumResults() {
return numResults;
}

public void setResult(List<T> result) {
this.result = result;
this.numResults = this.result.size();
}

public void setNumResults(int numResults) {
this.numResults = numResults;
}

public boolean isAdministratorMode() {
return administratorMode;
}

public void setAdministratorMode(boolean administratorMode) {
this.administratorMode = administratorMode;
}

public boolean isGeoQuery() {
return isGeoQuery;
}

public void setGeoQuery(boolean geoQuery) {
isGeoQuery = geoQuery;
}

@Override
public String getTitle() {
return TITLE;
}

@Override
public String getPageHeader() {
return getTitle();
}

@Override
public String getHeadContent() {
if (isGeoQuery) {
// @formatter:off
return "<script type='text/javascript' src='/jquery.min.js'></script>" +
"<script type='text/javascript'>" +
"$(document).ready(function() {" +
" var currentUrl = window.location.href.replace(/\\/+$/, '');" +
" var queryHeader = document.getElementById(\"query-header\").innerHTML;" +
" queryHeader = queryHeader + '<br>(<a href=\"' + currentUrl + '/map\">map</a>)';" +
" document.getElementById(\"query-header\").innerHTML = queryHeader;" +
"});" +
"</script>";
// @formatter: on
} else {
return EMPTY;
}
}

@Override
public String getMainContent() {
StringBuilder builder = new StringBuilder(), pageTimesBuilder = new StringBuilder();

builder.append("<table>\n");
builder.append("<tr>");
builder.append("<th>Subplan(s)</th>");

pageTimesBuilder.append("<table>\n");

builder.append(pageTimesBuilder);

TreeMap<Date,T> metricMap = new TreeMap<>(Collections.reverseOrder());

for (T metric : this.getResult()) {
metricMap.put(metric.getCreateDate(), metric);
}

int x = 0;
for (T metric : metricMap.values()) {
// highlight alternating rows
if (x % 2 == 0) {
builder.append("<tr class=\"highlight\">\n");
} else {
builder.append("<tr>\n");
}
x++;
if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) {
for (Map.Entry<String,String> subplan : metric.getSubPlans().entrySet()) {
builder.append("<td>").append(subplan.getKey()).append(subplan.getValue()).append("</td>");
}
} else {
builder.append("<td>NONE<td/>");
}

builder.append("\n</tr>\n");
}

builder.append("</table>\n<br/>\n");
pageTimesBuilder.append("</table>\n");

builder.append(pageTimesBuilder);

return builder.toString();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import datawave.microservice.authorization.user.ProxiedUserDetails;
import datawave.microservice.querymetric.config.QueryMetricSinkConfiguration.QueryMetricSinkBinding;
import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory;
import datawave.microservice.querymetric.factory.BaseQueryMetricSubplanResponseFactory;
import datawave.microservice.querymetric.handler.QueryGeometryHandler;
import datawave.microservice.querymetric.handler.ShardTableQueryMetricHandler;
import datawave.microservice.querymetric.handler.SimpleQueryGeometryHandler;
Expand Down Expand Up @@ -83,6 +84,7 @@ public class QueryMetricOperations {
private Cache lastWrittenQueryMetricCache;
private MarkingFunctions markingFunctions;
private BaseQueryMetricListResponseFactory queryMetricListResponseFactory;
private BaseQueryMetricSubplanResponseFactory queryMetricSubplanResponseFactory;
private MergeLockLifecycleListener mergeLock;
private MetricUpdateEntryProcessorFactory entryProcessorFactory;
private QueryMetricOperationsStats stats;
Expand Down Expand Up @@ -127,7 +129,7 @@ enum DEFAULT_DATETIME {
*/
@Autowired
public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cacheManager, ShardTableQueryMetricHandler handler,
QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, BaseQueryMetricListResponseFactory queryMetricListResponseFactory,
QueryGeometryHandler geometryHandler, MarkingFunctions markingFunctions, BaseQueryMetricListResponseFactory queryMetricListResponseFactory, BaseQueryMetricSubplanResponseFactory queryMetricSubplanResponseFactory,
MergeLockLifecycleListener mergeLock, MetricUpdateEntryProcessorFactory entryProcessorFactory, QueryMetricOperationsStats stats) {
this.handler = handler;
this.geometryHandler = geometryHandler;
Expand All @@ -136,6 +138,7 @@ public QueryMetricOperations(@Named("queryMetricCacheManager") CacheManager cach
this.lastWrittenQueryMetricCache = cacheManager.getCache(LAST_WRITTEN_METRICS);
this.markingFunctions = markingFunctions;
this.queryMetricListResponseFactory = queryMetricListResponseFactory;
this.queryMetricSubplanResponseFactory = queryMetricSubplanResponseFactory;
this.mergeLock = mergeLock;
this.entryProcessorFactory = entryProcessorFactory;
this.stats = stats;
Expand Down Expand Up @@ -254,33 +257,16 @@ private void storeMetricUpdate(QueryMetricUpdateHolder metricUpdate) {
}
storeTimer.stop();
}

/**
* Returns metrics for the current users queries that are identified by the id
*
* @param currentUser
* the current user
* @param queryId
* the query id
* @return datawave.webservice.result.QueryMetricListResponse base query metric list response
* @HTTP 200 success
* @HTTP 500 internal server error
*/
@PermitAll
@RequestMapping(path = "/id/{queryId}", method = {RequestMethod.GET},
produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_HTML_VALUE})
public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDetails currentUser,
@ApiParam("queryId to return") @PathVariable("queryId") String queryId) {

BaseQueryMetricListResponse response = this.queryMetricListResponseFactory.createDetailedResponse();

List<BaseQueryMetric> getMetricList(String queryId, ProxiedUserDetails currentUser, String blacklistedFields) {
List<BaseQueryMetric> metricList = new ArrayList<>();
try {
BaseQueryMetric metric;
QueryMetricUpdateHolder metricUpdate = incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class);
if (metricUpdate != null && metricUpdate.isNewMetric()) {
metric = metricUpdate.getMetric();
} else {
metric = this.handler.getQueryMetric(queryId);
metric = this.handler.getQueryMetric(queryId, blacklistedFields);
}
if (metric != null) {
boolean allowAllMetrics = false;
Expand Down Expand Up @@ -310,8 +296,30 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDet
}
}
} catch (Exception e) {
response.addException(new QueryException(e.getMessage(), 500));
log.error(e.getMessage(), e);
}
return metricList;
}

/**
* Returns metrics for the current users queries that are identified by the id
*
* @param currentUser
* the current user
* @param queryId
* the query id
* @return datawave.webservice.result.QueryMetricListResponse base query metric list response
* @HTTP 200 success
* @HTTP 500 internal server error
*/
@PermitAll
@RequestMapping(path = "/id/{queryId}", method = {RequestMethod.GET},
produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_HTML_VALUE})
public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDetails currentUser,
@ApiParam("queryId to return") @PathVariable("queryId") String queryId) {

BaseQueryMetricListResponse response = this.queryMetricListResponseFactory.createDetailedResponse();
List<BaseQueryMetric> metricList = getMetricList(queryId, currentUser, "SUBPLAN");
response.setResult(metricList);
if (metricList.isEmpty()) {
response.setHasResults(false);
Expand All @@ -336,34 +344,17 @@ public BaseQueryMetricListResponse query(@AuthenticationPrincipal ProxiedUserDet
@PermitAll
@RequestMapping(path = "/id/{queryId}/subplans", method = {RequestMethod.GET},
produces = {MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.TEXT_HTML_VALUE})
public BaseQueryMetricListResponse subplans(@AuthenticationPrincipal ProxiedUserDetails currentUser,
public BaseQueryMetricSubplanResponse subplans(@AuthenticationPrincipal ProxiedUserDetails currentUser,
@ApiParam("queryId to return") @PathVariable("queryId") String queryId) {

BaseQueryMetricListResponse response = this.queryMetricListResponseFactory.createDetailedResponse();
List<String> subplanList = new ArrayList<>();
try {
BaseQueryMetric metric;
QueryMetricUpdateHolder metricUpdate = incomingQueryMetricsCache.get(queryId, QueryMetricUpdateHolder.class);
if (metricUpdate != null && metricUpdate.isNewMetric()) {
metric = metricUpdate.getMetric();
if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) {
for (Map.Entry<String,String> subplan : metric.getSubPlans().entrySet()) {
subplanList.add(subplan.getKey() + " : " + subplan.getValue());
}
}
} else {
metric = this.handler.getQueryMetric(queryId);
if (metric.getSubPlans() != null && !metric.getSubPlans().isEmpty()) {
for (Map.Entry<String,String> subplan : metric.getSubPlans().entrySet()) {
subplanList.add(subplan.getKey() + " : " + subplan.getValue());
}
}
}
} catch (Exception e) {
response.addException(new QueryException(e.getMessage(), 500));
BaseQueryMetricSubplanResponse response = this.queryMetricSubplanResponseFactory.createSubplanResponse();
List<BaseQueryMetric> metricList = getMetricList(queryId, currentUser, "PAGE_METRICS");
response.setResult(metricList);
if (metricList.isEmpty()) {
response.setHasResults(false);
} else {
response.setGeoQuery(metricList.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery));
response.setHasResults(true);
}
response.setResult(subplanList);
response.setHasResults(!subplanList.isEmpty());
return response;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
import datawave.microservice.querymetric.QueryMetricFactory;
import datawave.microservice.querymetric.QueryMetricFactoryImpl;
import datawave.microservice.querymetric.factory.BaseQueryMetricListResponseFactory;
import datawave.microservice.querymetric.factory.BaseQueryMetricSubplanResponseFactory;
import datawave.microservice.querymetric.factory.QueryMetricListResponseFactory;
import datawave.microservice.querymetric.factory.QueryMetricQueryLogicFactory;
import datawave.microservice.querymetric.factory.QueryMetricSubplanResponseFactory;
import datawave.microservice.querymetric.handler.QueryGeometryHandler;
import datawave.microservice.querymetric.handler.QueryMetricCombiner;
import datawave.microservice.querymetric.handler.ShardTableQueryMetricHandler;
Expand Down Expand Up @@ -91,6 +93,11 @@ public QueryGeometryHandler geometryHandler(QueryMetricHandlerProperties queryMe
public BaseQueryMetricListResponseFactory queryMetricListResponseFactory() {
return new QueryMetricListResponseFactory();
}

@Bean
public BaseQueryMetricSubplanResponseFactory queryMetricSubplanResponseFactory() {
return new QueryMetricSubplanResponseFactory();
}

@Bean
@ConditionalOnMissingBean
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package datawave.microservice.querymetric.factory;

import datawave.microservice.querymetric.BaseQueryMetricSubplanResponse;

public interface BaseQueryMetricSubplanResponseFactory<T extends BaseQueryMetricSubplanResponse> {

T createSubplanResponse();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package datawave.microservice.querymetric.factory;

import datawave.microservice.querymetric.BaseQueryMetricSubplanResponse;

public class QueryMetricSubplanResponseFactory implements BaseQueryMetricSubplanResponseFactory {

@Override
public BaseQueryMetricSubplanResponse createSubplanResponse() {
return new BaseQueryMetricSubplanResponse();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -370,16 +370,16 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy
return (T) queryMetricCombiner.combineMetrics(updatedQueryMetric, cachedQueryMetric, metricType);
}

public T getQueryMetric(final String queryId) throws Exception {
List<T> queryMetrics = getQueryMetrics("QUERY_ID == '" + queryId + "'");
public T getQueryMetric(final String queryId, String blacklistedFields) throws Exception {
List<T> queryMetrics = getQueryMetrics("QUERY_ID == '" + queryId + "'", blacklistedFields);
return queryMetrics.isEmpty() ? null : queryMetrics.get(0);
}

public Query createQuery() {
return new QueryImpl();
}

public List<T> getQueryMetrics(final String query) throws Exception {
public List<T> getQueryMetrics(final String query, String blacklistedFields) throws Exception {
Date end = new Date();
Date begin = DateUtils.setYears(end, 2000);
Query queryImpl = createQuery();
Expand All @@ -394,6 +394,9 @@ public List<T> getQueryMetrics(final String query) throws Exception {
queryImpl.setPagesize(1000);
queryImpl.setId(UUID.randomUUID());
queryImpl.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"));
if (!blacklistedFields.isEmpty()) {
queryImpl.addParameter(QueryParameters.BLACKLISTED_FIELDS, blacklistedFields);
}
return getQueryMetrics(queryImpl);
}

Expand Down Expand Up @@ -488,31 +491,6 @@ public List<T> getQueryMetrics(Query query) throws Exception {
return queryMetrics;
}

public QueryMetricsSubplanResponse subplan(String queryId) throws Exception {
QueryMetricsSubplanResponse response = new QueryMetricsSubplanResponse();

Date end = new Date();
Date begin = DateUtils.setYears(end, 2000);
QueryImpl query = new QueryImpl();
query.setBeginDate(begin);
query.setEndDate(end);
query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME);
// QueryMetricQueryLogic now enforces that you must be a QueryMetricsAdministrator to query metrics that do not belong to you
query.setQuery("QUERY_ID == '" + queryId + "'");
query.setQueryName(QUERY_METRICS_LOGIC_NAME);
query.setColumnVisibility(queryMetricHandlerProperties.getQueryVisibility());
query.setExpirationDate(DateUtils.addDays(new Date(), 1));
query.setPagesize(1000);
query.setId(UUID.randomUUID());
query.addParameter(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true");
query.addParameter(QueryParameters.BLACKLISTED_FIELDS, "PAGE_METRICS");
List<T> queryMetrics = getQueryMetrics(query);

response.setGeoQuery(queryMetrics.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery));

return response;
}

public T toMetric(EventBase event) {
SimpleDateFormat sdf_date_time1 = new SimpleDateFormat("yyyyMMdd HHmmss");
SimpleDateFormat sdf_date_time2 = new SimpleDateFormat("yyyyMMdd HHmmss");
Expand Down
Loading

0 comments on commit feb95a3

Please sign in to comment.