From bff1ce392c616fda317beab90763757f7937e329 Mon Sep 17 00:00:00 2001 From: Meet Shah Date: Mon, 25 Jan 2021 18:30:24 +0530 Subject: [PATCH] Changed master total PendingQueueSize to PendingQueueSize per task type --- .../metrics/AllMetrics.java | 19 ++++++++++ .../model/MetricsModel.java | 3 +- .../reader/MetricPropertiesConfig.java | 3 +- .../reader/ReaderMetricsProcessorTests.java | 38 ++++++++++++------- 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java index aad4d465d..51b846b1f 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java @@ -821,6 +821,25 @@ public static class Constants { } } + public enum MasterPendingTaskDimension implements MetricDimension { + MASTER_PENDING_TASK_TYPE(Constants.PENDING_TASK_TYPE); + + private final String value; + + MasterPendingTaskDimension(String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } + + public static class Constants { + public static final String PENDING_TASK_TYPE = "Master_PendingTaskType"; + } + } + public enum MasterThrottlingValue implements MetricValue { /** * Sum of total pending tasks throttled by master node. diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java index f899dd81d..35617c094 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java @@ -32,6 +32,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPValue; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.LatencyDimension; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingTaskDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricUnits; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics; @@ -328,7 +329,7 @@ public class MetricsModel { // Master Metrics allMetricsInitializer.put( MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString(), - new MetricAttributes(MetricUnits.COUNT.toString(), EmptyDimension.values())); + new MetricAttributes(MetricUnits.COUNT.toString(), MasterPendingTaskDimension.values())); allMetricsInitializer.put( AllMetrics.MasterMetricValues.MASTER_TASK_QUEUE_TIME.toString(), diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java index dc60749b1..a789e3d46 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java @@ -28,6 +28,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapValue; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPValue; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingTaskDimension; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsDerivedDimension; @@ -238,7 +239,7 @@ private MetricPropertiesConfig() { metricName2Property.put( MetricName.MASTER_PENDING, new MetricProperties( - MetricProperties.EMPTY_DIMENSION, + MasterPendingTaskDimension.values(), MasterPendingValue.values(), createFileHandler( metricPathMap.get(MetricName.MASTER_PENDING), diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java index c27038e52..821e49ea9 100644 --- a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java @@ -145,20 +145,20 @@ private NavigableMap setUpAligningWindow(long lastUpdate new MemoryDBSnapshot(conn, MetricName.MASTER_PENDING, 6001L); long lastUpdateTime1 = 2000L; masterPendingSnap1.setLastUpdatedTime(lastUpdateTime1); - Object[][] values1 = {{0}}; + Object[][] values1 = {{"delete-index",0}}; masterPendingSnap1.insertMultiRows(values1); MemoryDBSnapshot masterPendingSnap2 = new MemoryDBSnapshot(conn, MetricName.MASTER_PENDING, 11001L); long lastUpdateTime2 = 7000L; masterPendingSnap2.setLastUpdatedTime(lastUpdateTime2); - Object[][] values2 = {{1}}; + Object[][] values2 = {{"create-index",1}}; masterPendingSnap2.insertMultiRows(values2); MemoryDBSnapshot masterPendingSnap3 = new MemoryDBSnapshot(conn, MetricName.MASTER_PENDING, 16001L); masterPendingSnap2.setLastUpdatedTime(lastUpdateTime3); - Object[][] values3 = {{3}}; + Object[][] values3 = {{"updateSnapshot",3}}; masterPendingSnap3.insertMultiRows(values3); NavigableMap metricMap = new TreeMap<>(); @@ -222,11 +222,15 @@ public void testAlignNodeMetrics() throws Exception { masterPendingFinal); Result res = alignedWindow.fetchAll(); - assertTrue(1 == res.size()); + assertTrue(2 == res.size()); + Field valueField = DSL.field(MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString(), Double.class); Double pending = Double.parseDouble(res.get(0).get(valueField).toString()); - assertEquals(2.2d, pending, 0.001); + assertEquals(1.0d,pending,0.001); + pending = Double.parseDouble(res.get(1).get(valueField).toString()); + assertEquals(3.0d, pending, 0.001); + } @Test @@ -251,14 +255,20 @@ public void testEmitNodeMetrics() throws Exception { db); Result res = db.queryMetric(MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString()); + assertTrue(2 == res.size()); - assertTrue(1 == res.size()); + Record row = res.get(0); + for (int i = 1; i < row.size(); i++) { + Double pending = Double.parseDouble(row.get(i).toString()); + assertEquals(1.0d, pending, 0.001); + } - Record row0 = res.get(0); - for (int i = 0; i < row0.size(); i++) { - Double pending = Double.parseDouble(row0.get(i).toString()); - assertEquals(2.2d, pending, 0.001); + row = res.get(1); + for (int i = 1; i < row.size(); i++) { + Double pending = Double.parseDouble(row.get(i).toString()); + assertEquals(3.0d, pending, 0.001); } + db.remove(); } @@ -295,10 +305,10 @@ public void testMissingUpperWriterWindow() throws Exception { assertTrue(1 == res.size()); - Record row0 = res.get(0); - for (int i = 0; i < row0.size(); i++) { - Double pending = Double.parseDouble(row0.get(i).toString()); - assertEquals(3.0, pending, 0.001); + Record row = res.get(0); + for (int i = 1; i < row.size(); i++) { + Double pending = Double.parseDouble(row.get(i).toString()); + assertEquals(3.0d, pending, 0.001); } // db tables should not be deleted