Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Changed master total PendingQueueSize to PendingQueueSize per task type #552

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Changed master total PendingQueueSize to PendingQueueSize per task type #552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bff1ce3
Changed master total PendingQueueSize to PendingQueueSize per task type
meetshah777 Jan 25, 2021
f537c02
Merge branch 'main' into pendingTask
meetshah777 Mar 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions ...main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/metrics/AllMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,25 @@ public static class Constants {
}
}

public enum MasterPendingTaskDimension implements MetricDimension {
MASTER_PENDING_TASK_TYPE(Constants.PENDING_TASK_TYPE);

private final String value;

MasterPendingTaskDimension(String value) {
this.value = value;
}

@Override
public String toString() {
return value;
}

public static class Constants {
public static final String PENDING_TASK_TYPE = "Master_PendingTaskType";
}
}

public enum ClusterApplierServiceStatsValue implements MetricValue {
CLUSTER_APPLIER_SERVICE_LATENCY(ClusterApplierServiceStatsValue.Constants.CLUSTER_APPLIER_SERVICE_LATENCY),
CLUSTER_APPLIER_SERVICE_FAILURE(ClusterApplierServiceStatsValue.Constants.CLUSTER_APPLIER_SERVICE_FAILURE);
Expand Down
3 changes: 2 additions & 1 deletion ...main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/model/MetricsModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPDimension;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPValue;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.LatencyDimension;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingTaskDimension;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricUnits;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.OSMetrics;
Expand Down Expand Up @@ -330,7 +331,7 @@ public class MetricsModel {
// Master Metrics
allMetricsInitializer.put(
MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString(),
new MetricAttributes(MetricUnits.COUNT.toString(), EmptyDimension.values()));
new MetricAttributes(MetricUnits.COUNT.toString(), MasterPendingTaskDimension.values()));

allMetricsInitializer.put(
AllMetrics.MasterMetricValues.MASTER_TASK_QUEUE_TIME.toString(),
Expand Down
3 changes: 2 additions & 1 deletion ...om/amazon/opendistro/elasticsearch/performanceanalyzer/reader/MetricPropertiesConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.HeapValue;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPDimension;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.IPValue;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingTaskDimension;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MasterPendingValue;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.MetricName;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ShardStatsDerivedDimension;
Expand Down Expand Up @@ -247,7 +248,7 @@ private MetricPropertiesConfig() {
metricName2Property.put(
MetricName.MASTER_PENDING,
new MetricProperties(
MetricProperties.EMPTY_DIMENSION,
MasterPendingTaskDimension.values(),
MasterPendingValue.values(),
createFileHandler(
metricPathMap.get(MetricName.MASTER_PENDING),
Expand Down
38 changes: 24 additions & 14 deletions ...azon/opendistro/elasticsearch/performanceanalyzer/reader/ReaderMetricsProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,20 +145,20 @@ private NavigableMap<Long, MemoryDBSnapshot> setUpAligningWindow(long lastUpdate
new MemoryDBSnapshot(conn, MetricName.MASTER_PENDING, 6001L);
long lastUpdateTime1 = 2000L;
masterPendingSnap1.setLastUpdatedTime(lastUpdateTime1);
Object[][] values1 = {{0}};
Object[][] values1 = {{"delete-index",0}};
masterPendingSnap1.insertMultiRows(values1);

MemoryDBSnapshot masterPendingSnap2 =
new MemoryDBSnapshot(conn, MetricName.MASTER_PENDING, 11001L);
long lastUpdateTime2 = 7000L;
masterPendingSnap2.setLastUpdatedTime(lastUpdateTime2);
Object[][] values2 = {{1}};
Object[][] values2 = {{"create-index",1}};
masterPendingSnap2.insertMultiRows(values2);

MemoryDBSnapshot masterPendingSnap3 =
new MemoryDBSnapshot(conn, MetricName.MASTER_PENDING, 16001L);
masterPendingSnap2.setLastUpdatedTime(lastUpdateTime3);
Object[][] values3 = {{3}};
Object[][] values3 = {{"updateSnapshot",3}};
masterPendingSnap3.insertMultiRows(values3);

NavigableMap<Long, MemoryDBSnapshot> metricMap = new TreeMap<>();
Expand Down Expand Up @@ -222,11 +222,15 @@ public void testAlignNodeMetrics() throws Exception {
masterPendingFinal);

Result<Record> res = alignedWindow.fetchAll();
assertTrue(1 == res.size());
assertTrue(2 == res.size());

Field<Double> valueField =
DSL.field(MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString(), Double.class);
Double pending = Double.parseDouble(res.get(0).get(valueField).toString());
assertEquals(2.2d, pending, 0.001);
assertEquals(1.0d,pending,0.001);
pending = Double.parseDouble(res.get(1).get(valueField).toString());
assertEquals(3.0d, pending, 0.001);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add tests to assert pending task type as well


}

@Test
Expand All @@ -251,14 +255,20 @@ public void testEmitNodeMetrics() throws Exception {
db);

Result<Record> res = db.queryMetric(MasterPendingValue.MASTER_PENDING_QUEUE_SIZE.toString());
assertTrue(2 == res.size());

assertTrue(1 == res.size());
Record row = res.get(0);
for (int i = 1; i < row.size(); i++) {
Double pending = Double.parseDouble(row.get(i).toString());
assertEquals(1.0d, pending, 0.001);
}

Record row0 = res.get(0);
for (int i = 0; i < row0.size(); i++) {
Double pending = Double.parseDouble(row0.get(i).toString());
assertEquals(2.2d, pending, 0.001);
row = res.get(1);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above

for (int i = 1; i < row.size(); i++) {
Double pending = Double.parseDouble(row.get(i).toString());
assertEquals(3.0d, pending, 0.001);
}

db.remove();
}

Expand Down Expand Up @@ -295,10 +305,10 @@ public void testMissingUpperWriterWindow() throws Exception {

assertTrue(1 == res.size());

Record row0 = res.get(0);
for (int i = 0; i < row0.size(); i++) {
Double pending = Double.parseDouble(row0.get(i).toString());
assertEquals(3.0, pending, 0.001);
Record row = res.get(0);
for (int i = 1; i < row.size(); i++) {
Double pending = Double.parseDouble(row.get(i).toString());
assertEquals(3.0d, pending, 0.001);
}

// db tables should not be deleted
Expand Down