From 3f82b5acff1783fb8e507c61974c8800c34e0044 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 14 Aug 2024 17:33:50 +0530 Subject: [PATCH 1/6] Adding timestamp rounding support in star tree Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 16 +- .../java/org/opensearch/common/Rounding.java | 51 ++- .../datacube/DateDimension.java | 66 +++- .../compositeindex/datacube/Dimension.java | 24 ++ .../datacube/DimensionFactory.java | 7 +- .../datacube/NumericDimension.java | 18 + .../datacube/startree/StarTreeField.java | 33 ++ .../builder/AbstractDocumentsFileManager.java | 8 +- .../startree/builder/BaseStarTreeBuilder.java | 52 ++- .../builder/OffHeapStarTreeBuilder.java | 29 +- .../builder/OnHeapStarTreeBuilder.java | 23 +- .../builder/SegmentDocsFileManager.java | 12 +- .../builder/StarTreeDocsFileManager.java | 17 +- .../index/mapper/StarTreeMapper.java | 68 +++- .../datacube/startree/DateDimensionTests.java | 243 ++++++++++++++ .../builder/AbstractStarTreeBuilderTests.java | 315 ++++++++++++++++-- .../index/mapper/ObjectMapperTests.java | 4 +- .../index/mapper/StarTreeMapperTests.java | 249 ++++++++++++-- 18 files changed, 1081 insertions(+), 154 deletions(-) create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 1cabb8b617ce3..f1392e9bf06f8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -47,10 +47,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .startObject("startree-1") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() + .startObject("date_dimension") .field("name", "timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", getDim(invalidDim, keywordDim)) .endObject() @@ -97,14 +97,14 @@ private static XContentBuilder createMaxDimTestMapping() { .startObject("startree-1") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() + .startObject("date_dimension") .field("name", "timestamp") .startArray("calendar_intervals") .value("day") .value("month") .endArray() .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", "dim2") .endObject() @@ -139,7 +139,7 @@ private static XContentBuilder createMaxDimTestMapping() { } } - private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { + private static XContentBuilder createTestMappingWithoutStarTree() { try { return jsonBuilder().startObject() .startObject("properties") @@ -176,10 +176,10 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .startObject(sameStarTree ? "startree-1" : "startree-2") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() + .startObject("date_dimension") .field("name", "timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", changeDim ? "numeric_new" : getDim(false, false)) .endObject() @@ -295,7 +295,7 @@ public void testUpdateIndexWithAdditionOfStarTree() { } public void testUpdateIndexWithNewerStarTree() { - prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree(false, false, false)).get(); + prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree()).get(); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, diff --git a/server/src/main/java/org/opensearch/common/Rounding.java b/server/src/main/java/org/opensearch/common/Rounding.java index 6f5f1e4328758..c7871e9fb31b1 100644 --- a/server/src/main/java/org/opensearch/common/Rounding.java +++ b/server/src/main/java/org/opensearch/common/Rounding.java @@ -62,11 +62,15 @@ import java.time.temporal.TemporalQueries; import java.time.zone.ZoneOffsetTransition; import java.time.zone.ZoneRules; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.OptionalLong; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * A strategy for rounding milliseconds since epoch. @@ -95,7 +99,7 @@ public enum DateTimeUnit { WEEK_OF_WEEKYEAR((byte) 1, "week", IsoFields.WEEK_OF_WEEK_BASED_YEAR, true, TimeUnit.DAYS.toMillis(7)) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(7); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundWeekOfWeekYear(utcMillis); } @@ -107,7 +111,7 @@ long extraLocalOffsetLookup() { YEAR_OF_CENTURY((byte) 2, "year", ChronoField.YEAR_OF_ERA, false, 12) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(366); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundYear(utcMillis); } @@ -118,7 +122,7 @@ long extraLocalOffsetLookup() { QUARTER_OF_YEAR((byte) 3, "quarter", IsoFields.QUARTER_OF_YEAR, false, 3) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(92); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundQuarterOfYear(utcMillis); } @@ -129,7 +133,7 @@ long extraLocalOffsetLookup() { MONTH_OF_YEAR((byte) 4, "month", ChronoField.MONTH_OF_YEAR, false, 1) { private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(31); - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundMonthOfYear(utcMillis); } @@ -138,7 +142,7 @@ long extraLocalOffsetLookup() { } }, DAY_OF_MONTH((byte) 5, "day", ChronoField.DAY_OF_MONTH, true, ChronoField.DAY_OF_MONTH.getBaseUnit().getDuration().toMillis()) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, this.ratio); } @@ -147,7 +151,7 @@ long extraLocalOffsetLookup() { } }, HOUR_OF_DAY((byte) 6, "hour", ChronoField.HOUR_OF_DAY, true, ChronoField.HOUR_OF_DAY.getBaseUnit().getDuration().toMillis()) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, ratio); } @@ -162,7 +166,7 @@ long extraLocalOffsetLookup() { true, ChronoField.MINUTE_OF_HOUR.getBaseUnit().getDuration().toMillis() ) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, ratio); } @@ -177,7 +181,7 @@ long extraLocalOffsetLookup() { true, ChronoField.SECOND_OF_MINUTE.getBaseUnit().getDuration().toMillis() ) { - long roundFloor(long utcMillis) { + public long roundFloor(long utcMillis) { return DateUtils.roundFloor(utcMillis, ratio); } @@ -210,7 +214,7 @@ public long extraLocalOffsetLookup() { * @param utcMillis the milliseconds since the epoch * @return the rounded down milliseconds since the epoch */ - abstract long roundFloor(long utcMillis); + public abstract long roundFloor(long utcMillis); /** * When looking up {@link LocalTimeOffset} go this many milliseconds @@ -260,6 +264,35 @@ public static DateTimeUnit resolve(byte id) { } } + /** + * DateTimeUnit Comparator which tracks dateTimeUnits from second unit to year unit + */ + public static class DateTimeUnitComparator implements Comparator { + public static final Map ORDERED_DATE_TIME_UNIT = new HashMap<>(); + + static { + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.SECOND_OF_MINUTE, 1); + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MINUTES_OF_HOUR, 2); + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.HOUR_OF_DAY, 3); + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.DAY_OF_MONTH, 4); + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.WEEK_OF_WEEKYEAR, 5); + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MONTH_OF_YEAR, 6); + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.QUARTER_OF_YEAR, 7); + ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.YEAR_OF_CENTURY, 8); + } + + @Override + public int compare(DateTimeUnit unit1, DateTimeUnit unit2) { + return Integer.compare(ORDERED_DATE_TIME_UNIT.get(unit1), ORDERED_DATE_TIME_UNIT.get(unit2)); + } + } + + public static List getSortedDateTimeUnits(List dateTimeUnits) { + return dateTimeUnits.stream() + .sorted(Comparator.comparingInt(DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT::get)) + .collect(Collectors.toList()); + } + public abstract void innerWriteTo(StreamOutput out) throws IOException; @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java index 074016db2aed7..923d3ca699907 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java @@ -10,10 +10,13 @@ import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.time.DateUtils; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.mapper.DateFieldMapper; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -28,19 +31,73 @@ public class DateDimension implements Dimension { public static final String CALENDAR_INTERVALS = "calendar_intervals"; public static final String DATE = "date"; private final String field; + private final List sortedCalendarIntervals; + private final DateFieldMapper.Resolution resolution; - public DateDimension(String field, List calendarIntervals) { + public DateDimension(String field, List calendarIntervals, DateFieldMapper.Resolution resolution) { this.field = field; this.calendarIntervals = calendarIntervals; + // Sort from the lowest unit to the highest unit + this.sortedCalendarIntervals = Rounding.getSortedDateTimeUnits(calendarIntervals); + if (resolution == null) { + this.resolution = DateFieldMapper.Resolution.MILLISECONDS; + } else { + this.resolution = resolution; + } } public List getIntervals() { return calendarIntervals; } + public List getSortedCalendarIntervals() { + return sortedCalendarIntervals; + } + + /** + * Sets the dimension values in sorted order in the provided array starting from the given index. + * + * @param val The value to be set + * @param dims The dimensions array to set the values in + * @param index The starting index in the array + * @return The next available index in the array + */ + @Override + public int setDimensionValues(final Long val, final Long[] dims, int index) { + for (Rounding.DateTimeUnit dateTimeUnit : sortedCalendarIntervals) { + if (val == null) { + dims[index++] = null; + continue; + } + dims[index++] = dateTimeUnit.roundFloor(maybeConvertNanosToMillis(val)); + } + return index; + } + + /** + * Converts nanoseconds to milliseconds based on the resolution of the field + */ + private long maybeConvertNanosToMillis(long nanoSecondsSinceEpoch) { + if (resolution.equals(DateFieldMapper.Resolution.NANOSECONDS)) return DateUtils.toMilliSeconds(nanoSecondsSinceEpoch); + return nanoSecondsSinceEpoch; + } + + /** + * Returns the list of fields that represent the dimension + */ + @Override + public List getDimensionFieldsNames() { + List fields = new ArrayList<>(calendarIntervals.size()); + for (Rounding.DateTimeUnit interval : sortedCalendarIntervals) { + // TODO : revisit this post file format changes + fields.add(field + "_" + interval.shortName()); + } + return fields; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); + builder.startObject("date_dimension"); builder.field(CompositeDataCubeFieldType.NAME, this.getField()); builder.field(CompositeDataCubeFieldType.TYPE, DATE); builder.startArray(CALENDAR_INTERVALS); @@ -69,4 +126,9 @@ public int hashCode() { public String getField() { return field; } + + @Override + public int getNumSubDimensions() { + return calendarIntervals.size(); + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java index 0151a474579be..82e11ec24a4a8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java @@ -11,6 +11,8 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; +import java.util.List; + /** * Base interface for data-cube dimensions * @@ -18,5 +20,27 @@ */ @ExperimentalApi public interface Dimension extends ToXContent { + String getField(); + + /** + * Returns the number of dimension values that gets added to star tree document + * as part of this dimension + */ + int getNumSubDimensions(); + + /** + * Sets the dimension values in the provided array starting from the given index. + * + * @param value The value to be set + * @param dims The dimensions array to set the values in + * @param index The starting index in the array + * @return The next available index in the array + */ + int setDimensionValues(Long value, Long[] dims, int index); + + /** + * Returns the list of dimension fields that represent the dimension + */ + List getDimensionFieldsNames(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index 6a09e947217f5..3fa9208f13728 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -94,6 +94,11 @@ private static DateDimension parseAndCreateDateDimension( calendarIntervals = new ArrayList<>(calendarIntervals); } dimensionMap.remove(CALENDAR_INTERVALS); - return new DateDimension(name, calendarIntervals); + DateFieldMapper.Resolution resolution = null; + if (c != null && c.mapperService() != null && c.mapperService().fieldType(name) != null) { + resolution = ((DateFieldMapper.DateFieldType) c.mapperService().fieldType(name)).resolution(); + } + + return new DateDimension(name, calendarIntervals, resolution); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java index 9c25ef5b25503..d88b6aac5c171 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java @@ -13,6 +13,7 @@ import org.opensearch.index.mapper.CompositeDataCubeFieldType; import java.io.IOException; +import java.util.List; import java.util.Objects; /** @@ -33,6 +34,23 @@ public String getField() { return field; } + @Override + public int getNumSubDimensions() { + return 1; + } + + @Override + public int setDimensionValues(final Long val, final Long[] dims, int index) { + dims[index++] = val; + return index; + } + + @Override + public List getDimensionFieldsNames() { + // TODO : revisit this post file format changes + return List.of(field); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java index 922ddcbea4fe2..36e6a9c539f42 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java @@ -11,10 +11,13 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -29,12 +32,25 @@ public class StarTreeField implements ToXContent { private final List dimensionsOrder; private final List metrics; private final StarTreeFieldConfiguration starTreeConfig; + private final List dimensionNames; + private final List metricNames; public StarTreeField(String name, List dimensions, List metrics, StarTreeFieldConfiguration starTreeConfig) { this.name = name; this.dimensionsOrder = dimensions; this.metrics = metrics; this.starTreeConfig = starTreeConfig; + dimensionNames = new ArrayList<>(); + for (Dimension dimension : dimensions) { + dimensionNames.addAll(dimension.getDimensionFieldsNames()); + } + metricNames = new ArrayList<>(); + for (Metric metric : metrics) { + for (MetricStat metricStat : metric.getMetrics()) { + // TODO : revisit this post file formats + metricNames.add(metric.getField() + "_" + metricStat.name()); + } + } } public String getName() { @@ -45,6 +61,14 @@ public List getDimensionsOrder() { return dimensionsOrder; } + public List getDimensionNames() { + return dimensionNames; + } + + public List getMetricNames() { + return metricNames; + } + public List getMetrics() { return metrics; } @@ -57,13 +81,22 @@ public StarTreeFieldConfiguration getStarTreeConfig() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("name", name); + DateDimension dateDim = null; if (dimensionsOrder != null && !dimensionsOrder.isEmpty()) { builder.startArray("ordered_dimensions"); for (Dimension dimension : dimensionsOrder) { + // Handle dateDimension for later + if (dimension instanceof DateDimension) { + dateDim = (DateDimension) dimension; + continue; + } dimension.toXContent(builder, params); } builder.endArray(); } + if (dateDim != null) { + dateDim.toXContent(builder, params); + } if (metrics != null && !metrics.isEmpty()) { builder.startArray("metrics"); for (Metric metric : metrics) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java index 4214a46b2fc1c..c8cbee47049e1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractDocumentsFileManager.java @@ -40,17 +40,20 @@ public abstract class AbstractDocumentsFileManager implements Closeable { protected final TrackingDirectoryWrapper tmpDirectory; protected final SegmentWriteState state; protected int docSizeInBytes = -1; + protected final int numDimensions; public AbstractDocumentsFileManager( SegmentWriteState state, StarTreeField starTreeField, - List metricAggregatorInfos + List metricAggregatorInfos, + int numDimensions ) { this.starTreeField = starTreeField; this.tmpDirectory = new TrackingDirectoryWrapper(state.directory); this.metricAggregatorInfos = metricAggregatorInfos; this.state = state; numMetrics = metricAggregatorInfos.size(); + this.numDimensions = numDimensions; } private void setDocSizeInBytes(int numBytes) { @@ -123,8 +126,7 @@ protected int writeMetrics(StarTreeDocument starTreeDocument, IndexOutput output * @throws IOException IOException in case of I/O errors */ protected StarTreeDocument readStarTreeDocument(RandomAccessInput input, long offset, boolean isAggregatedDoc) throws IOException { - int dimSize = starTreeField.getDimensionsOrder().size(); - Long[] dimensions = new Long[dimSize]; + Long[] dimensions = new Long[numDimensions]; long initialOffset = offset; offset = readDimensions(dimensions, input, offset); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 872826aa6db06..6ad332d34e6d8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; @@ -43,6 +44,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode.ALL; @@ -69,7 +71,7 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected int totalSegmentDocs; protected int numStarTreeNodes; protected final int maxLeafDocuments; - + List dimensionsSplitOrder = new ArrayList<>(); protected final TreeNode rootNode = getNewNode(); protected final StarTreeField starTreeField; @@ -88,9 +90,12 @@ protected BaseStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState sta this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); - - List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); - this.numDimensions = dimensionsSplitOrder.size(); + int numDims = 0; + for (Dimension dim : starTreeField.getDimensionsOrder()) { + numDims += dim.getNumSubDimensions(); + dimensionsSplitOrder.add(dim); + } + this.numDimensions = numDims; this.skipStarNodeCreationForDimensions = new HashSet<>(); this.totalSegmentDocs = state.segmentInfo.maxDoc(); @@ -98,9 +103,12 @@ protected BaseStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState sta Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); - for (int i = 0; i < numDimensions; i++) { + for (int i = 0; i < dimensionsSplitOrder.size(); i++) { if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) { - this.skipStarNodeCreationForDimensions.add(i); + // add the dimension indices + for (int dimIndex = 0; dimIndex < dimensionsSplitOrder.get(i).getNumSubDimensions(); dimIndex++) { + this.skipStarNodeCreationForDimensions.add(i + dimIndex); + } } } @@ -170,6 +178,30 @@ protected StarTreeDocument getStarTreeDocument( return new StarTreeDocument(dims, metrics); } + /** + * Sets dimensions / metric readers nnd numSegmentDocs + */ + protected void setReadersAndNumSegmentDocs( + SequentialDocValuesIterator[] dimensionReaders, + List metricReaders, + AtomicInteger numSegmentDocs, + StarTreeValues starTreeValues + ) { + + List dimensionNames = starTreeValues.getStarTreeField().getDimensionNames(); + for (int i = 0; i < numDimensions; i++) { + dimensionReaders[i] = new SequentialDocValuesIterator( + starTreeValues.getDimensionDocValuesIteratorMap().get(dimensionNames.get(i)) + ); + } + for (String metric : starTreeValues.getStarTreeField().getMetricNames()) { + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocValuesIteratorMap().get(metric))); + } + numSegmentDocs.set( + Integer.parseInt(starTreeValues.getAttributes().getOrDefault(NUM_SEGMENT_DOCS, String.valueOf(DocIdSetIterator.NO_MORE_DOCS))) + ); + } + /** * Adds a document to the star-tree. * @@ -248,7 +280,8 @@ protected StarTreeDocument getSegmentStarTreeDocument( */ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIterator[] dimensionReaders) throws IOException { Long[] dimensions = new Long[numDimensions]; - for (int i = 0; i < numDimensions; i++) { + int dimIndex = 0; + for (int i = 0; i < dimensionReaders.length; i++) { if (dimensionReaders[i] != null) { try { dimensionReaders[i].nextDoc(currentDocId); @@ -259,7 +292,7 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte logger.error("unable to read the dimension values from the segment", e); throw new IllegalStateException("unable to read the dimension values from the segment", e); } - dimensions[i] = dimensionReaders[i].value(currentDocId); + dimIndex = dimensionsSplitOrder.get(i).setDimensionValues(dimensionReaders[i].value(currentDocId), dimensions, dimIndex); } else { throw new IllegalStateException("dimension readers are empty"); } @@ -422,8 +455,9 @@ public void build(Map fieldProducerMap) throws IOExce List metricReaders = getMetricReaders(state, fieldProducerMap); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; - for (int i = 0; i < numDimensions; i++) { + for (int i = 0; i < dimensionReaders.length; i++) { String dimension = dimensionsSplitOrder.get(i).getField(); + FieldInfo dimensionFieldInfo = state.fieldInfos.fieldInfo(dimension); if (dimensionFieldInfo == null) { dimensionFieldInfo = getFieldInfo(dimension); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index f63b0cb0cc77d..ef1ee52842c68 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -11,11 +11,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; @@ -29,8 +27,8 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; /** * Off-heap implementation of the star tree builder. @@ -52,9 +50,9 @@ public class OffHeapStarTreeBuilder extends BaseStarTreeBuilder { */ protected OffHeapStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState state, MapperService mapperService) throws IOException { super(starTreeField, state, mapperService); - segmentDocumentFileManager = new SegmentDocsFileManager(state, starTreeField, metricAggregatorInfos); + segmentDocumentFileManager = new SegmentDocsFileManager(state, starTreeField, metricAggregatorInfos, numDimensions); try { - starTreeDocumentFileManager = new StarTreeDocsFileManager(state, starTreeField, metricAggregatorInfos); + starTreeDocumentFileManager = new StarTreeDocsFileManager(state, starTreeField, metricAggregatorInfos, numDimensions); } catch (IOException e) { IOUtils.closeWhileHandlingException(segmentDocumentFileManager); throw e; @@ -127,23 +125,12 @@ Iterator mergeStarTrees(List starTreeValuesSub int[] docIds; try { for (StarTreeValues starTreeValues : starTreeValuesSubs) { - List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[starTreeValues.getStarTreeField() - .getDimensionsOrder() - .size()]; - for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocValuesIteratorMap().get(dimension)); - } + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); - for (Map.Entry metricDocValuesEntry : starTreeValues.getMetricDocValuesIteratorMap().entrySet()) { - metricReaders.add(new SequentialDocValuesIterator(metricDocValuesEntry.getValue())); - } + AtomicInteger numSegmentDocs = new AtomicInteger(); + setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); int currentDocId = 0; - int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(NUM_SEGMENT_DOCS, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) - ); - while (currentDocId < numSegmentDocs) { + while (currentDocId < numSegmentDocs.get()) { StarTreeDocument starTreeDocument = getStarTreeDocument(currentDocId, dimensionReaders, metricReaders); segmentDocumentFileManager.writeStarTreeDocument(starTreeDocument, true); numDocs++; @@ -289,7 +276,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start int docId = 1; private boolean hasSameDimensions(StarTreeDocument document1, StarTreeDocument document2) { - for (int i = dimensionId + 1; i < starTreeField.getDimensionsOrder().size(); i++) { + for (int i = dimensionId + 1; i < numDimensions; i++) { if (!Objects.equals(document1.dimensions[i], document2.dimensions[i])) { return false; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 8ff111d3b41d9..37554dec18d81 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -8,10 +8,8 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; @@ -22,8 +20,8 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; /** * On heap single tree builder @@ -114,24 +112,13 @@ Iterator mergeStarTrees(List starTreeValuesSub StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeValuesSubs) throws IOException { List starTreeDocuments = new ArrayList<>(); for (StarTreeValues starTreeValues : starTreeValuesSubs) { - List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; - - for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocValuesIteratorMap().get(dimension)); - } - + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); - for (Map.Entry metricDocValuesEntry : starTreeValues.getMetricDocValuesIteratorMap().entrySet()) { - metricReaders.add(new SequentialDocValuesIterator(metricDocValuesEntry.getValue())); - } + AtomicInteger numSegmentDocs = new AtomicInteger(); + setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); int currentDocId = 0; - int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(NUM_SEGMENT_DOCS, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) - ); - while (currentDocId < numSegmentDocs) { + while (currentDocId < numSegmentDocs.get()) { starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders)); currentDocId++; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java index fe94df57d9535..363e02b52d5e3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SegmentDocsFileManager.java @@ -40,9 +40,13 @@ public class SegmentDocsFileManager extends AbstractDocumentsFileManager impleme private RandomAccessInput segmentRandomInput; final IndexOutput segmentDocsFileOutput; - public SegmentDocsFileManager(SegmentWriteState state, StarTreeField starTreeField, List metricAggregatorInfos) - throws IOException { - super(state, starTreeField, metricAggregatorInfos); + public SegmentDocsFileManager( + SegmentWriteState state, + StarTreeField starTreeField, + List metricAggregatorInfos, + int numDimensions + ) throws IOException { + super(state, starTreeField, metricAggregatorInfos, numDimensions); try { segmentDocsFileOutput = tmpDirectory.createTempOutput(SEGMENT_DOC_FILE_NAME, state.segmentSuffix, state.context); } catch (IOException e) { @@ -78,7 +82,7 @@ public StarTreeDocument readStarTreeDocument(int docId, boolean isAggregatedDoc) @Override public Long[] readDimensions(int docId) throws IOException { maybeInitializeSegmentInput(); - Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; + Long[] dims = new Long[numDimensions]; readDimensions(dims, segmentRandomInput, (long) docId * docSizeInBytes); return dims; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 779ed77b0540a..36138cd618528 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -63,18 +63,23 @@ public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implem private final int fileCountMergeThreshold; private int numStarTreeDocs = 0; - public StarTreeDocsFileManager(SegmentWriteState state, StarTreeField starTreeField, List metricAggregatorInfos) - throws IOException { - this(state, starTreeField, metricAggregatorInfos, DEFAULT_FILE_COUNT_MERGE_THRESHOLD); + public StarTreeDocsFileManager( + SegmentWriteState state, + StarTreeField starTreeField, + List metricAggregatorInfos, + int numDimensions + ) throws IOException { + this(state, starTreeField, metricAggregatorInfos, DEFAULT_FILE_COUNT_MERGE_THRESHOLD, numDimensions); } public StarTreeDocsFileManager( SegmentWriteState state, StarTreeField starTreeField, List metricAggregatorInfos, - int fileCountThreshold + int fileCountThreshold, + int numDimensions ) throws IOException { - super(state, starTreeField, metricAggregatorInfos); + super(state, starTreeField, metricAggregatorInfos, numDimensions); fileToEndDocIdMap = new LinkedHashMap<>(); try { starTreeDocsFileOutput = createStarTreeDocumentsFileOutput(); @@ -118,7 +123,7 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { @Override public Long[] readDimensions(int docId) throws IOException { ensureDocumentReadable(docId); - Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; + Long[] dims = new Long[numDimensions]; readDimensions(dims, starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId)); return dims; } diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java index d9539f9dc0c82..25e810b99c8cf 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.Query; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.DimensionFactory; import org.opensearch.index.compositeindex.datacube.Metric; @@ -42,8 +43,8 @@ public class StarTreeMapper extends ParametrizedFieldMapper { public static final String CONFIG = "config"; public static final String MAX_LEAF_DOCS = "max_leaf_docs"; public static final String SKIP_STAR_NODE_IN_DIMS = "skip_star_node_creation_for_dimensions"; - public static final String BUILD_MODE = "build_mode"; public static final String ORDERED_DIMENSIONS = "ordered_dimensions"; + public static final String DATE_DIMENSION = "date_dimension"; public static final String METRICS = "metrics"; public static final String STATS = "stats"; @@ -61,8 +62,7 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { public static class Builder extends ParametrizedFieldMapper.Builder { private ObjectMapper.Builder objbuilder; private static final Set> ALLOWED_DIMENSION_MAPPER_BUILDERS = Set.of( - NumberFieldMapper.Builder.class, - DateFieldMapper.Builder.class + NumberFieldMapper.Builder.class ); private static final Set> ALLOWED_METRIC_MAPPER_BUILDERS = Set.of(NumberFieldMapper.Builder.class); @@ -87,6 +87,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { StarTreeFieldConfiguration.StarTreeBuildMode buildMode = StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP; List dimensions = buildDimensions(name, paramMap, context); + paramMap.remove(DATE_DIMENSION); paramMap.remove(ORDERED_DIMENSIONS); List metrics = buildMetrics(name, paramMap, context); paramMap.remove(METRICS); @@ -122,16 +123,21 @@ public static class Builder extends ParametrizedFieldMapper.Builder { */ @SuppressWarnings("unchecked") private List buildDimensions(String fieldName, Map map, Mapper.TypeParser.ParserContext context) { + List dimensions = new LinkedList<>(); + DateDimension dateDim = buildDateDimension(fieldName, map, context); + if (dateDim != null) { + dimensions.add(dateDim); + } Object dims = XContentMapValues.extractValue("ordered_dimensions", map); if (dims == null) { throw new IllegalArgumentException( String.format(Locale.ROOT, "ordered_dimensions is required for star tree field [%s]", fieldName) ); } - List dimensions = new LinkedList<>(); + if (dims instanceof List) { - List dimList = (List) dims; - if (dimList.size() > context.getSettings() + List orderedDimensionsList = (List) dims; + if (orderedDimensionsList.size() + dimensions.size() > context.getSettings() .getAsInt( StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_DEFAULT @@ -149,12 +155,12 @@ private List buildDimensions(String fieldName, Map ma ) ); } - if (dimList.size() < 2) { + if (dimensions.size() + orderedDimensionsList.size() < 2) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Atleast two dimensions are required to build star tree index field [%s]", fieldName) ); } - for (Object dim : dimList) { + for (Object dim : orderedDimensionsList) { dimensions.add(getDimension(fieldName, dim, context)); } } else { @@ -165,6 +171,52 @@ private List buildDimensions(String fieldName, Map ma return dimensions; } + private DateDimension buildDateDimension(String fieldName, Map map, Mapper.TypeParser.ParserContext context) { + Object dims = XContentMapValues.extractValue("date_dimension", map); + if (dims == null) { + return null; + } + return getDateDimension(fieldName, dims, context); + } + + /** + * Get dimension based on mapping + */ + @SuppressWarnings("unchecked") + private DateDimension getDateDimension(String fieldName, Object dimensionMapping, Mapper.TypeParser.ParserContext context) { + DateDimension dimension; + Map dimensionMap = (Map) dimensionMapping; + String name = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.NAME, dimensionMap); + dimensionMap.remove(CompositeDataCubeFieldType.NAME); + if (this.objbuilder == null || this.objbuilder.mappersBuilders == null) { + String type = (String) XContentMapValues.extractValue(CompositeDataCubeFieldType.TYPE, dimensionMap); + dimensionMap.remove(CompositeDataCubeFieldType.TYPE); + if (type == null || type.equals(DateDimension.DATE) == false) { + throw new MapperParsingException( + String.format(Locale.ROOT, "unable to parse date dimension for star tree field [%s]", fieldName) + ); + } + return (DateDimension) DimensionFactory.parseAndCreateDimension(name, type, dimensionMap, context); + } else { + Optional dimBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders); + if (dimBuilder.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "unknown date dimension field [%s]", name)); + } + if (dimBuilder.get() instanceof DateFieldMapper.Builder == false) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "date_dimension [%s] should be of type date for star tree field [%s]", name, fieldName) + ); + } + dimension = (DateDimension) DimensionFactory.parseAndCreateDimension(name, dimBuilder.get(), dimensionMap, context); + } + DocumentMapperParser.checkNoRemainingFields( + dimensionMap, + context.indexVersionCreated(), + "Star tree mapping definition has unsupported parameters: " + ); + return dimension; + } + /** * Get dimension based on mapping */ diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java new file mode 100644 index 0000000000000..10c990ec72bc9 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java @@ -0,0 +1,243 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.common.Rounding; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import static org.opensearch.common.Rounding.DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT; + +public class DateDimensionTests extends OpenSearchTestCase { + public void testDateDimension() { + String field = "timestamp"; + List intervals = Arrays.asList( + Rounding.DateTimeUnit.YEAR_OF_CENTURY, + Rounding.DateTimeUnit.HOUR_OF_DAY, + Rounding.DateTimeUnit.MONTH_OF_YEAR + ); + DateDimension dateDimension = new DateDimension(field, intervals, DateFieldMapper.Resolution.MILLISECONDS); + + assertEquals(field, dateDimension.getField()); + assertEquals(intervals, dateDimension.getIntervals()); + assertEquals( + Arrays.asList(Rounding.DateTimeUnit.HOUR_OF_DAY, Rounding.DateTimeUnit.MONTH_OF_YEAR, Rounding.DateTimeUnit.YEAR_OF_CENTURY), + dateDimension.getSortedCalendarIntervals() + ); + } + + public void testSetDimensionValuesWithMultipleIntervalsYear() { + List intervals = Arrays.asList( + Rounding.DateTimeUnit.YEAR_OF_CENTURY, + Rounding.DateTimeUnit.MONTH_OF_YEAR, + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Long[] dims = new Long[4]; + Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC + + int nextIndex = dateDimension.setDimensionValues(testValue, dims, 0); + + assertEquals(4, nextIndex); + assertEquals(1609459200000L, (long) dims[0]); // Hour rounded + assertEquals(1609459200000L, (long) dims[1]); // Day rounded + assertEquals(1609459200000L, (long) dims[2]); // Month rounded + assertEquals(1609459200000L, (long) dims[3]); // Year rounded + assertEquals(4, dateDimension.getNumSubDimensions()); + } + + public void testRoundingAndSortingAllDateTimeUnitsNanos() { + List allUnits = Arrays.asList( + Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, + Rounding.DateTimeUnit.MONTH_OF_YEAR, + Rounding.DateTimeUnit.QUARTER_OF_YEAR, + Rounding.DateTimeUnit.YEAR_OF_CENTURY, + Rounding.DateTimeUnit.SECOND_OF_MINUTE, + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY, + Rounding.DateTimeUnit.DAY_OF_MONTH + ); + + DateDimension dateDimension = new DateDimension("timestamp", allUnits, DateFieldMapper.Resolution.NANOSECONDS); + + // Test sorting + List sortedUnits = dateDimension.getSortedCalendarIntervals(); + assertEquals(allUnits.size(), sortedUnits.size()); + for (int i = 0; i < sortedUnits.size() - 1; i++) { + assertTrue( + "Units should be sorted in ascending order", + ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i)).compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1))) <= 0 + ); + } + + // Test rounding + long testValueNanos = 1655287972382719622L; // 2022-06-15T10:12:52.382719622Z + Long[] dims = new Long[allUnits.size()]; + dateDimension.setDimensionValues(testValueNanos, dims, 0); + + // Expected rounded values (in nanoseconds) + long secondRounded = 1655287972000L; // 2022-06-15T10:12:52 + long minuteRounded = 1655287920000L; // 2022-06-15T10:12:00 + long hourRounded = 1655287200000L; // 2022-06-15T10:00:00 + long dayRounded = 1655251200000L; // 2022-06-15T00:00:00 + long weekRounded = 1655078400000L; // 2022-06-13T00:00:00 // Monday + long monthRounded = 1654041600000L; // 2022-06-01T00:00:00 + long quarterRounded = 1648771200000L; // 2022-04-01T00:00:00 + long yearRounded = 1640995200000L; // 2022-01-01T00:00:00 + + for (int i = 0; i < sortedUnits.size(); i++) { + Rounding.DateTimeUnit unit = sortedUnits.get(i); + switch (unit) { + case SECOND_OF_MINUTE: + assertEquals(secondRounded, (long) dims[i]); + break; + case MINUTES_OF_HOUR: + assertEquals(minuteRounded, (long) dims[i]); + break; + case HOUR_OF_DAY: + assertEquals(hourRounded, (long) dims[i]); + break; + case DAY_OF_MONTH: + assertEquals(dayRounded, (long) dims[i]); + break; + case WEEK_OF_WEEKYEAR: + assertEquals(weekRounded, (long) dims[i]); + break; + case MONTH_OF_YEAR: + assertEquals(monthRounded, (long) dims[i]); + break; + case QUARTER_OF_YEAR: + assertEquals(quarterRounded, (long) dims[i]); + break; + case YEAR_OF_CENTURY: + assertEquals(yearRounded, (long) dims[i]); + break; + default: + fail("Unexpected DateTimeUnit: " + unit); + } + } + } + + public void testRoundingAndSortingAllDateTimeUnitsMillis() { + List allUnits = Arrays.asList( + Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, + Rounding.DateTimeUnit.MONTH_OF_YEAR, + Rounding.DateTimeUnit.QUARTER_OF_YEAR, + Rounding.DateTimeUnit.YEAR_OF_CENTURY, + Rounding.DateTimeUnit.SECOND_OF_MINUTE, + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY, + Rounding.DateTimeUnit.DAY_OF_MONTH + ); + + DateDimension dateDimension = new DateDimension("timestamp", allUnits, DateFieldMapper.Resolution.MILLISECONDS); + + // Test sorting + List sortedUnits = dateDimension.getSortedCalendarIntervals(); + assertEquals(allUnits.size(), sortedUnits.size()); + for (int i = 0; i < sortedUnits.size() - 1; i++) { + assertTrue( + "Units should be sorted in ascending order", + ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i)).compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1))) <= 0 + ); + } + + // Test rounding + long testValueNanos = 1655287972382L; // 2022-06-15T10:12:52.382Z + + Long[] dims = new Long[allUnits.size()]; + dateDimension.setDimensionValues(testValueNanos, dims, 0); + + // Expected rounded values (in millis) + long secondRounded = 1655287972000L; // 2022-06-15T10:12:52 + long minuteRounded = 1655287920000L; // 2022-06-15T10:12:00 + long hourRounded = 1655287200000L; // 2022-06-15T10:00:00 + long dayRounded = 1655251200000L; // 2022-06-15T00:00:00 + long weekRounded = 1655078400000L; // 2022-06-13T00:00:00 // Monday + long monthRounded = 1654041600000L; // 2022-06-01T00:00:00 + long quarterRounded = 1648771200000L; // 2022-04-01T00:00:00 + long yearRounded = 1640995200000L; // 2022-01-01T00:00:00 + + for (int i = 0; i < sortedUnits.size(); i++) { + Rounding.DateTimeUnit unit = sortedUnits.get(i); + switch (unit) { + case SECOND_OF_MINUTE: + assertEquals(secondRounded, (long) dims[i]); + break; + case MINUTES_OF_HOUR: + assertEquals(minuteRounded, (long) dims[i]); + break; + case HOUR_OF_DAY: + assertEquals(hourRounded, (long) dims[i]); + break; + case DAY_OF_MONTH: + assertEquals(dayRounded, (long) dims[i]); + break; + case WEEK_OF_WEEKYEAR: + assertEquals(weekRounded, (long) dims[i]); + break; + case MONTH_OF_YEAR: + assertEquals(monthRounded, (long) dims[i]); + break; + case QUARTER_OF_YEAR: + assertEquals(quarterRounded, (long) dims[i]); + break; + case YEAR_OF_CENTURY: + assertEquals(yearRounded, (long) dims[i]); + break; + default: + fail("Unexpected DateTimeUnit: " + unit); + } + } + } + + public void testGetDimensionFieldsNames() { + DateDimension dateDimension = new DateDimension( + "timestamp", + Arrays.asList(Rounding.DateTimeUnit.HOUR_OF_DAY, Rounding.DateTimeUnit.DAY_OF_MONTH), + DateFieldMapper.Resolution.MILLISECONDS + ); + + List fields = dateDimension.getDimensionFieldsNames(); + + assertEquals(2, fields.size()); + assertEquals("timestamp_hour", fields.get(0)); + assertEquals("timestamp_day", fields.get(1)); + } + + public void testSetDimensionValues() { + DateDimension dateDimension = new DateDimension( + "timestamp", + Arrays.asList(Rounding.DateTimeUnit.YEAR_OF_CENTURY, Rounding.DateTimeUnit.HOUR_OF_DAY), + DateFieldMapper.Resolution.MILLISECONDS + ); + Long[] dims = new Long[2]; + Long testValue = 1609459200000L; // 2021-01-01 00:00:00 UTC + + int nextIndex = dateDimension.setDimensionValues(testValue, dims, 0); + + assertEquals(2, nextIndex); + assertEquals(1609459200000L, (long) dims[0]); // Hour rounded + assertEquals(1609459200000L, (long) dims[1]); // Year rounded + } + + public void testDateTimeUnitComparator() { + Comparator comparator = new Rounding.DateTimeUnitComparator(); + assertTrue(comparator.compare(Rounding.DateTimeUnit.SECOND_OF_MINUTE, Rounding.DateTimeUnit.MINUTES_OF_HOUR) < 0); + assertTrue(comparator.compare(Rounding.DateTimeUnit.HOUR_OF_DAY, Rounding.DateTimeUnit.DAY_OF_MONTH) < 0); + assertTrue(comparator.compare(Rounding.DateTimeUnit.YEAR_OF_CENTURY, Rounding.DateTimeUnit.MONTH_OF_YEAR) > 0); + assertEquals(0, comparator.compare(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR)); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index d1a85949da7fe..31c448e91f320 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -27,8 +27,10 @@ import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Version; +import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -39,6 +41,7 @@ import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode; import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; @@ -488,11 +491,13 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics( SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(compositeField, writeState, mapperService); + List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - + segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); + segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); @@ -506,7 +511,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics( assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); } - builder.build(segmentStarTreeDocumentIterator); + builder.build(starTreeDocumentList.iterator()); validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } @@ -564,11 +569,13 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetri SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(compositeField, writeState, mapperService); + List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - + segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); + segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); @@ -582,7 +589,7 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetri assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); } - builder.build(segmentStarTreeDocumentIterator); + builder.build(starTreeDocumentList.iterator()); validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } @@ -634,10 +641,13 @@ public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOEx SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(compositeField, writeState, mapperService); + List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); + segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); + segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -652,6 +662,8 @@ public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOEx assertEquals(expectedStarTreeDocument.metrics[3], resultStarTreeDocument.metrics[3]); assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); } + builder.build(starTreeDocumentList.iterator()); + validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { @@ -702,10 +714,13 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(compositeField, writeState, mapperService); + List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); + segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); + segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -726,6 +741,10 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + // TODO + // builder.build(starTreeDocumentList.iterator()); + // validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); + } public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() throws IOException { @@ -767,10 +786,13 @@ public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(compositeField, writeState, mapperService); + List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); + segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); + segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -790,8 +812,9 @@ public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() } assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); - builder.build(segmentStarTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 3, 1, builder.getStarTreeDocuments()); + // TODO + // builder.build(starTreeDocumentList.iterator()); + // validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } @@ -910,7 +933,9 @@ public void test_build_halfFloatMetrics() throws IOException { Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); - builder.build(expectedStarTreeDocumentIterator); + // TODO + // builder.build(starTreeDocumentList.iterator()); + // validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } public void test_build_floatMetrics() throws IOException { @@ -986,6 +1011,9 @@ public void test_build_floatMetrics() throws IOException { Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + // TODO + // builder.build(starTreeDocumentList.iterator()); + // validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } public void test_build_longMetrics() throws IOException { @@ -1348,6 +1376,10 @@ public void testFlushFlow() throws IOException { [null, 2] | [20.0, 1] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1358,8 +1390,8 @@ public void testFlushFlow() throws IOException { assertEquals(1L, starTreeDocument.metrics[1]); } assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + builder.build(starTreeDocumentsList.iterator()); + validateStarTree(builder.getRootNode(), 2, 10, builder.getStarTreeDocuments()); } public void testFlushFlowDimsReverse() throws IOException { @@ -1400,6 +1432,10 @@ public void testFlushFlowDimsReverse() throws IOException { [null, 0] | [0.0, 1] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1412,8 +1448,8 @@ public void testFlushFlowDimsReverse() throws IOException { assertEquals(1L, starTreeDocument.metrics[1]); } assertEquals(6, count); - builder.build(starTreeDocumentIterator); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); + builder.build(starTreeDocumentsList.iterator()); + validateStarTree(builder.getRootNode(), 2, 10, builder.getStarTreeDocuments()); } public void testFlushFlowBuild() throws IOException { @@ -1500,7 +1536,7 @@ private static StarTreeField getStarTreeFieldWithMultipleMetrics() { List dims = List.of(d1, d2); List metrics = List.of(m1, m2); StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( - 1000, + 10, new HashSet<>(), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP ); @@ -1556,6 +1592,10 @@ public void testMergeFlowWithSum() throws IOException { * [6,-1] | [120.0] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1565,7 +1605,7 @@ public void testMergeFlowWithSum() throws IOException { ); } assertEquals(6, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -1608,13 +1648,17 @@ public void testMergeFlowWithCount() throws IOException { [6,-1] | [12] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); assertEquals(starTreeDocument.dimensions[0] != null ? starTreeDocument.dimensions[0] * 2 : 4, starTreeDocument.metrics[0]); } assertEquals(6, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -1630,7 +1674,10 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d2sndv = dimList2; SortedNumericDocValues m1sndv = metricsList; Map dimDocIdSetIterators = Map.of("field1", d1sndv, "field3", d2sndv); - Map metricDocIdSetIterators = Map.of("field2", m1sndv); + Map metricDocIdSetIterators = Map.of( + "field2_" + sf.getMetrics().get(0).getMetrics().get(0).name(), + m1sndv + ); StarTreeValues starTreeValues = new StarTreeValues( sf, null, @@ -1689,6 +1736,10 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { [null, 7] | [7] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1699,7 +1750,7 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { } } assertEquals(9, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -1751,6 +1802,9 @@ public void testMergeFlowNumSegmentsDocs() throws IOException { [8, 8] | [8] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1761,6 +1815,8 @@ public void testMergeFlowNumSegmentsDocs() throws IOException { } } assertEquals(9, count); + builder.build(starTreeDocumentsList.iterator()); + validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } public void testMergeFlowWithMissingDocs() throws IOException { @@ -1812,6 +1868,10 @@ public void testMergeFlowWithMissingDocs() throws IOException { [null, 7] | [7] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1821,7 +1881,7 @@ public void testMergeFlowWithMissingDocs() throws IOException { assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } assertEquals(10, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -1870,6 +1930,10 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { [null, null] | [12] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1884,7 +1948,7 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { } } assertEquals(6, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -1934,6 +1998,10 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException [null, null] | [19] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -1951,7 +2019,7 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException } } assertEquals(7, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -2004,6 +2072,10 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { [null, 7] | [7] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -2014,7 +2086,7 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { } } assertEquals(10, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -2067,6 +2139,10 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { [null, 7] | [7] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -2076,7 +2152,7 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } assertEquals(10, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -2117,6 +2193,10 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { [null, 5] | [5] */ int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -2126,7 +2206,7 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { assertEquals(starTreeDocument.dimensions[1], starTreeDocument.metrics[0]); } assertEquals(6, count); - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } @@ -2350,7 +2430,10 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); Map dimDocIdSetIterators = Map.of("field1", d1sndv, "field3", d2sndv, "field5", d3sndv, "field8", d4sndv); - Map metricDocIdSetIterators = Map.of("field2", m1sndv); + Map metricDocIdSetIterators = Map.of( + "field2_" + sf.getMetrics().get(0).getMetrics().get(0).name(), + m1sndv + ); StarTreeValues starTreeValues = new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, getAttributes(500)); return starTreeValues; } @@ -2662,7 +2745,7 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); Map dimDocIdSetIterators = Map.of("field1", d1sndv, "field3", d2sndv, "field5", d3sndv, "field8", d4sndv); - Map metricDocIdSetIterators = Map.of("field2", m1sndv); + Map metricDocIdSetIterators = Map.of("field2_" + m1.getMetrics().get(0).name(), m1sndv); StarTreeValues starTreeValues = new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, getAttributes(1000)); SortedNumericDocValues f2d1sndv = getSortedNumericMock(dimList1, docsWithField1); @@ -2680,7 +2763,7 @@ public void testMergeFlow() throws IOException { "field8", f2d4sndv ); - Map f2metricDocIdSetIterators = Map.of("field2", f2m1sndv); + Map f2metricDocIdSetIterators = Map.of("field2_" + m1.getMetrics().get(0).name(), f2m1sndv); StarTreeValues starTreeValues2 = new StarTreeValues( sf, null, @@ -2691,6 +2774,8 @@ public void testMergeFlow() throws IOException { builder = getStarTreeBuilder(sf, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); /** [0, 0, 0, 0] | [0.0] [1, 1, 1, 1] | [20.0] @@ -2701,15 +2786,189 @@ public void testMergeFlow() throws IOException { ... [999, 999, 999, 999] | [19980.0] */ - for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { + for (StarTreeDocument starTreeDocument : starTreeDocumentsList) { assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); } - builder.build(starTreeDocumentIterator); + builder.build(starTreeDocumentsList.iterator()); // Validate the star tree structure validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } + public void testFlushFlowWithTimestamps() throws IOException { + List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L); + List docsWithField = List.of(0, 1, 3, 4, 5); + List dimList2 = List.of(0L, 1L, 2L, 3L, 4L, 5L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 5); + + List metricsList = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0) + ); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5); + + StarTreeField sf = getStarTreeFieldWithDateDimension(); + SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); + SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); + SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + + builder = getStarTreeBuilder(sf, getWriteState(6), mapperService); + SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( + dimDvs, + List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + ); + /** + * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] + [1655287920000, 1655287200000, 4] | [40.0, 1] + [1655287980000, 1655287200000, 3] | [30.0, 1] + [1655288040000, 1655287200000, 1] | [10.0, 1] + [1655288040000, 1655287200000, 5] | [50.0, 1] + [1655288100000, 1655287200000, 0] | [0.0, 1] + [null, null, 2] | [20.0, 1] + */ + int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + starTreeDocumentIterator = starTreeDocumentsList.iterator(); + while (starTreeDocumentIterator.hasNext()) { + count++; + StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); + assertEquals(starTreeDocument.dimensions[2] * 1 * 10.0, starTreeDocument.metrics[0]); + assertEquals(1L, starTreeDocument.metrics[1]); + } + assertEquals(6, count); + builder.build(starTreeDocumentsList.iterator()); + validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); + } + + public void testMergeFlowWithTimestamps() throws IOException { + List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L); + List docsWithField = List.of(0, 1, 2, 3, 4, 6); + List dimList2 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); + List docsWithField2 = List.of(0, 1, 2, 3, 4, 6); + + List dimList5 = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List docsWithField5 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList1 = List.of( + getLongFromDouble(0.0), + getLongFromDouble(10.0), + getLongFromDouble(20.0), + getLongFromDouble(30.0), + getLongFromDouble(40.0), + getLongFromDouble(50.0), + getLongFromDouble(60.0) + ); + List metricsWithField1 = List.of(0, 1, 2, 3, 4, 5, 6); + List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); + List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); + + List dimList3 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField3 = List.of(0, 1, 3, 4); + List dimList4 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField4 = List.of(0, 1, 3, 4); + + List dimList6 = List.of(5L, 6L, 7L, 8L); + List docsWithField6 = List.of(0, 1, 2, 3); + List metricsList21 = List.of( + getLongFromDouble(50.0), + getLongFromDouble(60.0), + getLongFromDouble(70.0), + getLongFromDouble(80.0), + getLongFromDouble(90.0) + ); + List metricsWithField21 = List.of(0, 1, 2, 3, 4); + List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); + List metricsWithField2 = List.of(0, 1, 2, 3, 4); + + StarTreeField sf = getStarTreeFieldWithDateDimension(); + StarTreeValues starTreeValues = getStarTreeValuesWithDates( + getSortedNumericMock(dimList, docsWithField), + getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(dimList5, docsWithField5), + getSortedNumericMock(metricsList, metricsWithField), + getSortedNumericMock(metricsList1, metricsWithField1), + sf, + "6" + ); + + StarTreeValues starTreeValues2 = getStarTreeValuesWithDates( + getSortedNumericMock(dimList3, docsWithField3), + getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(dimList6, docsWithField6), + getSortedNumericMock(metricsList2, metricsWithField2), + getSortedNumericMock(metricsList21, metricsWithField21), + sf, + "4" + ); + builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** + [[1655287972000, 1655287972000, 3] | [30.0, 3] + [1655288032000, 1655288032000, 2] | [20.0, 2] + [1655288032000, 1655288032000, 8] | [80.0, 8] + [1655288092000, 1655288092000, 1] | [10.0, 1] + [1655288092000, 1655288092000, 4] | [40.0, 4] + [1655288092000, 1655288092000, 6] | [60.0, 6] + [1655288152000, 1655288152000, 0] | [0.0, 0] + [1655288152000, 1655288152000, 5] | [50.0, 5] + [null, null, 5] | [50.0, 5] + [null, null, 7] | [70.0, 7] + */ + int count = 0; + List starTreeDocumentsList = new ArrayList<>(); + starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); + + starTreeDocumentIterator = starTreeDocumentsList.iterator(); + while (starTreeDocumentIterator.hasNext()) { + count++; + StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); + assertEquals(starTreeDocument.dimensions[2] * 10.0, (double) starTreeDocument.metrics[0], 0); + assertEquals(starTreeDocument.dimensions[2], starTreeDocument.metrics[1]); + } + assertEquals(10, count); + builder.build(starTreeDocumentsList.iterator()); + validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); + } + + private StarTreeValues getStarTreeValuesWithDates( + SortedNumericDocValues dimList, + SortedNumericDocValues dimList2, + SortedNumericDocValues dimList3, + SortedNumericDocValues metricsList, + SortedNumericDocValues metricsList1, + StarTreeField sf, + String number + ) { + Map dimDocIdSetIterators = Map.of("field1_minute", dimList, "field1_hour", dimList2, "field3", dimList3); + Map metricDocIdSetIterators = Map.of("field2_COUNT", metricsList, "field2_SUM", metricsList1); + return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of("numSegmentDocs", number)); + } + + private static StarTreeField getStarTreeFieldWithDateDimension() { + List intervals = new ArrayList<>(); + intervals.add(Rounding.DateTimeUnit.MINUTES_OF_HOUR); + intervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + Dimension d1 = new DateDimension("field1", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Dimension d2 = new NumericDimension("field3"); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); + Metric m2 = new Metric("field2", List.of(MetricStat.COUNT)); + List dims = List.of(d1, d2); + List metrics = List.of(m1, m2); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( + 10, + new HashSet<>(), + StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP + ); + StarTreeField sf = new StarTreeField("sf", dims, metrics, c); + return sf; + } + private void validateStarTree(TreeNode root, int totalDimensions, int maxLeafDocuments, List starTreeDocuments) { Queue queue = new LinkedList<>(); queue.offer(new Object[] { root, false }); @@ -2891,7 +3150,7 @@ private static StarTreeField getStarTreeField(MetricStat count) { List dims = List.of(d1, d2); List metrics = List.of(m1); StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( - 1000, + 1, new HashSet<>(), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP ); diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index 504bc622ec12e..8f14543e87db7 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -498,10 +498,10 @@ public void testCompositeFields() throws Exception { .startObject("startree") .field("type", "star_tree") .startObject("config") - .startArray("ordered_dimensions") - .startObject() + .startObject("date_dimension") .field("name", "@timestamp") .endObject() + .startArray("ordered_dimensions") .startObject() .field("name", "status") .endObject() diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 132d2ff5a566a..b4abeaf62d6e1 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -35,6 +35,7 @@ import java.util.Set; import static org.hamcrest.Matchers.containsString; +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; /** * Tests for {@link StarTreeMapper}. @@ -77,6 +78,30 @@ public void testValidStarTree() throws IOException { } } + public void testValidStarTreeWithNoDateDim() throws IOException { + MapperService mapperService = createMapperService(getMinMappingWithDateDims(false, true, true)); + Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); + for (CompositeMappedFieldType type : compositeFieldTypes) { + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; + assertEquals("status", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof NumericDimension); + assertEquals("metric_field", starTreeFieldType.getDimensions().get(1).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof NumericDimension); + assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList( + MetricStat.AVG, + MetricStat.COUNT, + MetricStat.SUM, + MetricStat.MAX, + MetricStat.MIN + ); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); + assertEquals(new HashSet<>(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + public void testValidStarTreeDefaults() throws IOException { MapperService mapperService = createMapperService(getMinMapping()); Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); @@ -85,6 +110,8 @@ public void testValidStarTreeDefaults() throws IOException { assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedDimensionFields = Arrays.asList("@timestamp_minute", "@timestamp_hour"); + assertEquals(expectedDimensionFields, dateDim.getDimensionFieldsNames()); List expectedTimeUnits = Arrays.asList( Rounding.DateTimeUnit.MINUTES_OF_HOUR, Rounding.DateTimeUnit.HOUR_OF_DAY @@ -106,6 +133,60 @@ public void testValidStarTreeDefaults() throws IOException { } } + public void testValidStarTreeDateDims() throws IOException { + MapperService mapperService = createMapperService(getMinMappingWithDateDims(false, false, false)); + Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); + for (CompositeMappedFieldType type : compositeFieldTypes) { + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedDimensionFields = Arrays.asList("@timestamp_week", "@timestamp_month", "@timestamp_quarter"); + assertEquals(expectedDimensionFields, dateDim.getDimensionFieldsNames()); + List expectedTimeUnits = Arrays.asList( + Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, + Rounding.DateTimeUnit.MONTH_OF_YEAR, + Rounding.DateTimeUnit.QUARTER_OF_YEAR + ); + assertEquals(expectedTimeUnits, dateDim.getSortedCalendarIntervals()); + assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList( + MetricStat.AVG, + MetricStat.COUNT, + MetricStat.SUM, + MetricStat.MAX, + MetricStat.MIN + ); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + + public void testInValidStarTreeMinDims() throws IOException { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMappingWithDateDims(false, true, false)) + ); + assertEquals( + "Failed to parse mapping [_doc]: Atleast two dimensions are required to build star tree index field [startree]", + ex.getMessage() + ); + } + + public void testInvalidStarTreeDateDims() throws IOException { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMappingWithDateDims(true, false, false)) + ); + assertEquals( + "Failed to parse mapping [_doc]: At most [3] calendar intervals are allowed in dimension [@timestamp]", + ex.getMessage() + ); + } + public void testInvalidDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, @@ -125,7 +206,7 @@ public void testInvalidMetric() { public void testNoMetrics() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(false, true, false, false)) + () -> createMapperService(getMinMapping(false, true, false, false, false)) ); assertThat( ex.getMessage(), @@ -136,7 +217,7 @@ public void testNoMetrics() { public void testInvalidParam() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getInvalidMapping(false, false, false, false, true)) + () -> createMapperService(getInvalidMapping(false, false, false, false, true, false)) ); assertEquals( "Failed to parse mapping [_doc]: Star tree mapping definition has unsupported parameters: [invalid : {invalid=invalid}]", @@ -147,7 +228,7 @@ public void testInvalidParam() { public void testNoDims() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(true, false, false, false)) + () -> createMapperService(getMinMapping(true, false, false, false, false)) ); assertThat( ex.getMessage(), @@ -155,18 +236,26 @@ public void testNoDims() { ); } + public void testMissingDateDims() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMapping(false, false, false, false, true)) + ); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown date dimension field [@timestamp]")); + } + public void testMissingDims() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(false, false, true, false)) + () -> createMapperService(getMinMapping(false, false, true, false, false)) ); - assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [@timestamp]")); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [status]")); } public void testMissingMetrics() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getMinMapping(false, false, false, true)) + () -> createMapperService(getMinMapping(false, false, false, true, false)) ); assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown metric field [metric_field]")); } @@ -188,7 +277,18 @@ public void testInvalidDimType() { () -> createMapperService(getInvalidMapping(false, false, true, false)) ); assertEquals( - "Failed to parse mapping [_doc]: unsupported field type associated with dimension [@timestamp] as part of star tree field [startree]", + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [status] as part of star tree field [startree]", + ex.getMessage() + ); + } + + public void testInvalidDateDimType() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getInvalidMapping(false, false, true, false, false, true)) + ); + assertEquals( + "Failed to parse mapping [_doc]: date_dimension [@timestamp] should be of type date for star tree field [startree]", ex.getMessage() ); } @@ -239,15 +339,15 @@ public void testMetric() { public void testDimensions() { List d1CalendarIntervals = new ArrayList<>(); d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - DateDimension d1 = new DateDimension("name", d1CalendarIntervals); - DateDimension d2 = new DateDimension("name", d1CalendarIntervals); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); + DateDimension d2 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertEquals(d1, d2); - d2 = new DateDimension("name1", d1CalendarIntervals); + d2 = new DateDimension("name1", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertNotEquals(d1, d2); List d2CalendarIntervals = new ArrayList<>(); d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - d2 = new DateDimension("name", d2CalendarIntervals); + d2 = new DateDimension("name", d2CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertNotEquals(d1, d2); NumericDimension n1 = new NumericDimension("name"); NumericDimension n2 = new NumericDimension("name"); @@ -262,7 +362,7 @@ public void testStarTreeField() { Metric metric1 = new Metric("name", m1); List d1CalendarIntervals = new ArrayList<>(); d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - DateDimension d1 = new DateDimension("name", d1CalendarIntervals); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); NumericDimension n1 = new NumericDimension("numeric"); NumericDimension n2 = new NumericDimension("name1"); @@ -383,14 +483,14 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE b.value("status"); } b.endArray(); - b.startArray("ordered_dimensions"); - b.startObject(); + b.startObject("date_dimension"); b.field("name", "@timestamp"); b.startArray("calendar_intervals"); b.value("day"); b.value("month"); b.endArray(); b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", dim); b.endObject(); @@ -422,21 +522,94 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE } private XContentBuilder getMinMapping() throws IOException { - return getMinMapping(false, false, false, false); + return getMinMapping(false, false, false, false, false); } - private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetrics, boolean missingDim, boolean missingMetric) + private XContentBuilder getMinMappingWithDateDims(boolean calendarIntervalsExceeded, boolean dateDimsAbsent, boolean additionalDim) throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); - if (!isEmptyDims) { - b.startArray("ordered_dimensions"); + if (!dateDimsAbsent) { + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value(getRandom().nextBoolean() ? "week" : "1w"); + if (calendarIntervalsExceeded) { + b.value(getRandom().nextBoolean() ? "day" : "1d"); + b.value(getRandom().nextBoolean() ? "second" : "1s"); + b.value(getRandom().nextBoolean() ? "hour" : "1h"); + b.value(getRandom().nextBoolean() ? "minute" : "1m"); + b.value(getRandom().nextBoolean() ? "year" : "1y"); + } + b.value(getRandom().nextBoolean() ? "month" : "1M"); + b.value(getRandom().nextBoolean() ? "quarter" : "1q"); + b.endArray(); + b.endObject(); + } + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + if (additionalDim) { b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + } + b.endArray(); + + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + b.endArray(); + + b.endObject(); + + b.endObject(); + b.endObject(); + b.startObject("properties"); + + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + + b.startObject("metric_field"); + b.field("type", "integer"); + b.endObject(); + + b.endObject(); + }); + } + + private XContentBuilder getMinMapping( + boolean isEmptyDims, + boolean isEmptyMetrics, + boolean missingDim, + boolean missingMetric, + boolean missingDateDim + ) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + if (!isEmptyDims) { + b.startObject("date_dimension"); b.field("name", "@timestamp"); b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", "status"); b.endObject(); @@ -456,14 +629,16 @@ private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetric b.endObject(); b.endObject(); b.startObject("properties"); - if (!missingDim) { + if (!missingDateDim) { b.startObject("@timestamp"); b.field("type", "date"); b.endObject(); } - b.startObject("status"); - b.field("type", "integer"); - b.endObject(); + if (!missingDim) { + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + } if (!missingMetric) { b.startObject("metric_field"); b.field("type", "integer"); @@ -546,14 +721,14 @@ private XContentBuilder getInvalidMapping( boolean invalidSkipDims, boolean invalidDimType, boolean invalidMetricType, - boolean invalidParam + boolean invalidParam, + boolean invalidDate ) throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); b.field("type", "star_tree"); b.startObject("config"); - b.startArray("skip_star_node_creation_for_dimensions"); { if (invalidSkipDims) { @@ -562,6 +737,9 @@ private XContentBuilder getInvalidMapping( b.value("status"); } b.endArray(); + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.endObject(); if (invalidParam) { b.startObject("invalid"); b.field("invalid", "invalid"); @@ -570,12 +748,9 @@ private XContentBuilder getInvalidMapping( b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "status"); b.endObject(); } - b.startObject(); - b.field("name", "status"); - b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -590,7 +765,7 @@ private XContentBuilder getInvalidMapping( b.endObject(); b.startObject("properties"); b.startObject("@timestamp"); - if (!invalidDimType) { + if (!invalidDate) { b.field("type", "date"); } else { b.field("type", "keyword"); @@ -598,7 +773,11 @@ private XContentBuilder getInvalidMapping( b.endObject(); b.startObject("status"); - b.field("type", "integer"); + if (!invalidDimType) { + b.field("type", "integer"); + } else { + b.field("type", "keyword"); + } b.endObject(); b.startObject("metric_field"); if (invalidMetricType) { @@ -631,15 +810,15 @@ private XContentBuilder getInvalidMappingWithDv( b.value("status"); } b.endArray(); + b.startObject("date_dimension"); + b.field("name", "@timestamp"); + b.endObject(); b.startArray("ordered_dimensions"); if (!singleDim) { b.startObject(); - b.field("name", "@timestamp"); + b.field("name", "status"); b.endObject(); } - b.startObject(); - b.field("name", "status"); - b.endObject(); b.endArray(); b.startArray("metrics"); b.startObject(); @@ -681,7 +860,7 @@ private XContentBuilder getInvalidMappingWithDv( private XContentBuilder getInvalidMapping(boolean singleDim, boolean invalidSkipDims, boolean invalidDimType, boolean invalidMetricType) throws IOException { - return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false); + return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false, false); } protected boolean supportsOrIgnoresBoost() { From fc9614fc09a03144d98f04bc26f92d07ec90592e Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 19 Aug 2024 18:47:20 +0530 Subject: [PATCH 2/6] adding half hour and quarter hour calendar intervals Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 168 ++++++++++- .../java/org/opensearch/common/Rounding.java | 33 --- .../datacube/DateDimension.java | 59 +++- .../datacube/DimensionFactory.java | 12 +- .../startree/StarTreeIndexSettings.java | 17 +- .../utils/date/DateTimeUnitAdapter.java | 62 ++++ .../utils/date/DateTimeUnitRounding.java | 29 ++ .../utils/date/ExtendedDateTimeUnit.java | 75 +++++ .../startree/utils/date/package-info.java | 14 + .../datacube/startree/DateDimensionTests.java | 275 +++++++++++------- .../builder/AbstractStarTreeBuilderTests.java | 68 +++-- .../index/mapper/StarTreeMapperTests.java | 56 ++-- 12 files changed, 655 insertions(+), 213 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/ExtendedDateTimeUnit.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index f1392e9bf06f8..179eef9ec6a9c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -20,6 +20,9 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; @@ -90,6 +93,61 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool } } + private static XContentBuilder createDateTestMapping(boolean duplicate) { + try { + return jsonBuilder().startObject() + .startObject("composite") + .startObject("startree-1") + .field("type", "star_tree") + .startObject("config") + .startObject("date_dimension") + .field("name", "timestamp") + .startArray("calendar_intervals") + .value("day") + .value("quarter-hour") + .value(duplicate ? "quarter-hour" : "half-hour") + .endArray() + .endObject() + .startArray("ordered_dimensions") + .startObject() + .field("name", "numeric_dv") + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "numeric_dv") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + private static XContentBuilder createMaxDimTestMapping() { try { return jsonBuilder().startObject() @@ -102,6 +160,7 @@ private static XContentBuilder createMaxDimTestMapping() { .startArray("calendar_intervals") .value("day") .value("month") + .value("half-hour") .endArray() .endObject() .startArray("ordered_dimensions") @@ -258,11 +317,100 @@ public void testValidCompositeIndex() { assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + for (int i = 0; i < dateDim.getSortedCalendarIntervals().size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList( + MetricStat.AVG, + MetricStat.COUNT, + MetricStat.SUM, + MetricStat.MAX, + MetricStat.MIN + ); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals( + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, + starTreeFieldType.getStarTreeConfig().getBuildMode() + ); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + } + } + + public void testValidCompositeIndexWithDates() { + prepareCreate(TEST_INDEX).setMapping(createDateTestMapping(false)).get(); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + final Index index = resolveIndex("test"); + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + Set fts = indexService.mapperService().getCompositeFieldTypes(); + + for (CompositeMappedFieldType ft : fts) { + assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY, + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ); + for (int i = 0; i < dateDim.getIntervals().size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList( + MetricStat.AVG, + MetricStat.COUNT, + MetricStat.SUM, + MetricStat.MAX, + MetricStat.MIN + ); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals( + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, + starTreeFieldType.getStarTreeConfig().getBuildMode() + ); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + } + } + + public void testValidCompositeIndexWithDuplicateDates() { + prepareCreate(TEST_INDEX).setMapping(createDateTestMapping(true)).get(); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + final Index index = resolveIndex("test"); + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + Set fts = indexService.mapperService().getCompositeFieldTypes(); + + for (CompositeMappedFieldType ft : fts) { + assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ); + for (int i = 0; i < dateDim.getIntervals().size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( @@ -342,11 +490,14 @@ public void testUpdateIndexWhenMappingIsSame() { assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); + } + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( @@ -383,6 +534,7 @@ public void testMaxDimsCompositeIndex() { MapperParsingException ex = expectThrows( MapperParsingException.class, () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) + // Date dimension is considered as one dimension regardless of number of actual calendar intervals .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2)) .get() ); diff --git a/server/src/main/java/org/opensearch/common/Rounding.java b/server/src/main/java/org/opensearch/common/Rounding.java index c7871e9fb31b1..12a399635046e 100644 --- a/server/src/main/java/org/opensearch/common/Rounding.java +++ b/server/src/main/java/org/opensearch/common/Rounding.java @@ -62,15 +62,11 @@ import java.time.temporal.TemporalQueries; import java.time.zone.ZoneOffsetTransition; import java.time.zone.ZoneRules; -import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Objects; import java.util.OptionalLong; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; /** * A strategy for rounding milliseconds since epoch. @@ -264,35 +260,6 @@ public static DateTimeUnit resolve(byte id) { } } - /** - * DateTimeUnit Comparator which tracks dateTimeUnits from second unit to year unit - */ - public static class DateTimeUnitComparator implements Comparator { - public static final Map ORDERED_DATE_TIME_UNIT = new HashMap<>(); - - static { - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.SECOND_OF_MINUTE, 1); - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MINUTES_OF_HOUR, 2); - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.HOUR_OF_DAY, 3); - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.DAY_OF_MONTH, 4); - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.WEEK_OF_WEEKYEAR, 5); - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MONTH_OF_YEAR, 6); - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.QUARTER_OF_YEAR, 7); - ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.YEAR_OF_CENTURY, 8); - } - - @Override - public int compare(DateTimeUnit unit1, DateTimeUnit unit2) { - return Integer.compare(ORDERED_DATE_TIME_UNIT.get(unit1), ORDERED_DATE_TIME_UNIT.get(unit2)); - } - } - - public static List getSortedDateTimeUnits(List dateTimeUnits) { - return dateTimeUnits.stream() - .sorted(Comparator.comparingInt(DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT::get)) - .collect(Collectors.toList()); - } - public abstract void innerWriteTo(StreamOutput out) throws IOException; @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java index 923d3ca699907..00e6162afa16e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java @@ -12,13 +12,19 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.time.DateUtils; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.index.mapper.CompositeDataCubeFieldType; import org.opensearch.index.mapper.DateFieldMapper; import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; /** * Date dimension class @@ -27,18 +33,18 @@ */ @ExperimentalApi public class DateDimension implements Dimension { - private final List calendarIntervals; + private final List calendarIntervals; public static final String CALENDAR_INTERVALS = "calendar_intervals"; public static final String DATE = "date"; private final String field; - private final List sortedCalendarIntervals; + private final List sortedCalendarIntervals; private final DateFieldMapper.Resolution resolution; - public DateDimension(String field, List calendarIntervals, DateFieldMapper.Resolution resolution) { + public DateDimension(String field, List calendarIntervals, DateFieldMapper.Resolution resolution) { this.field = field; this.calendarIntervals = calendarIntervals; // Sort from the lowest unit to the highest unit - this.sortedCalendarIntervals = Rounding.getSortedDateTimeUnits(calendarIntervals); + this.sortedCalendarIntervals = getSortedDateTimeUnits(calendarIntervals); if (resolution == null) { this.resolution = DateFieldMapper.Resolution.MILLISECONDS; } else { @@ -46,11 +52,11 @@ public DateDimension(String field, List calendarIntervals } } - public List getIntervals() { + public List getIntervals() { return calendarIntervals; } - public List getSortedCalendarIntervals() { + public List getSortedCalendarIntervals() { return sortedCalendarIntervals; } @@ -64,7 +70,7 @@ public List getSortedCalendarIntervals() { */ @Override public int setDimensionValues(final Long val, final Long[] dims, int index) { - for (Rounding.DateTimeUnit dateTimeUnit : sortedCalendarIntervals) { + for (DateTimeUnitRounding dateTimeUnit : sortedCalendarIntervals) { if (val == null) { dims[index++] = null; continue; @@ -88,7 +94,7 @@ private long maybeConvertNanosToMillis(long nanoSecondsSinceEpoch) { @Override public List getDimensionFieldsNames() { List fields = new ArrayList<>(calendarIntervals.size()); - for (Rounding.DateTimeUnit interval : sortedCalendarIntervals) { + for (DateTimeUnitRounding interval : sortedCalendarIntervals) { // TODO : revisit this post file format changes fields.add(field + "_" + interval.shortName()); } @@ -101,7 +107,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CompositeDataCubeFieldType.NAME, this.getField()); builder.field(CompositeDataCubeFieldType.TYPE, DATE); builder.startArray(CALENDAR_INTERVALS); - for (Rounding.DateTimeUnit interval : calendarIntervals) { + for (DateTimeUnitRounding interval : calendarIntervals) { builder.value(interval.shortName()); } builder.endArray(); @@ -131,4 +137,39 @@ public String getField() { public int getNumSubDimensions() { return calendarIntervals.size(); } + + /** + * DateTimeUnit Comparator which tracks dateTimeUnits in sorted order from second unit to year unit + */ + public static class DateTimeUnitComparator implements Comparator { + public static final Map ORDERED_DATE_TIME_UNIT = new HashMap<>(); + + static { + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.SECOND_OF_MINUTE.shortName(), 1); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), 2); + ORDERED_DATE_TIME_UNIT.put(ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY.shortName(), 3); + ORDERED_DATE_TIME_UNIT.put(ExtendedDateTimeUnit.HALF_HOUR_OF_DAY.shortName(), 4); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.HOUR_OF_DAY.shortName(), 5); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.DAY_OF_MONTH.shortName(), 6); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR.shortName(), 7); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.MONTH_OF_YEAR.shortName(), 8); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.QUARTER_OF_YEAR.shortName(), 9); + ORDERED_DATE_TIME_UNIT.put(Rounding.DateTimeUnit.YEAR_OF_CENTURY.shortName(), 10); + } + + @Override + public int compare(DateTimeUnitRounding unit1, DateTimeUnitRounding unit2) { + return Integer.compare( + ORDERED_DATE_TIME_UNIT.getOrDefault(unit1.shortName(), Integer.MAX_VALUE), + ORDERED_DATE_TIME_UNIT.getOrDefault(unit2.shortName(), Integer.MAX_VALUE) + ); + } + } + + /** + * Returns a sorted list of dateTimeUnits based on the DateTimeUnitComparator + */ + public static List getSortedDateTimeUnits(List dateTimeUnits) { + return dateTimeUnits.stream().sorted(new DateTimeUnitComparator()).collect(Collectors.toList()); + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index 3fa9208f13728..f2bc8069df863 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -8,18 +8,20 @@ package org.opensearch.index.compositeindex.datacube; -import org.opensearch.common.Rounding; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.NumberFieldMapper; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS; @@ -70,13 +72,13 @@ private static DateDimension parseAndCreateDateDimension( Map dimensionMap, Mapper.TypeParser.ParserContext c ) { - List calendarIntervals = new ArrayList<>(); + Set calendarIntervals; List intervalStrings = XContentMapValues.extractRawValues(CALENDAR_INTERVALS, dimensionMap) .stream() .map(Object::toString) .collect(Collectors.toList()); if (intervalStrings == null || intervalStrings.isEmpty()) { - calendarIntervals = StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings()); + calendarIntervals = new LinkedHashSet<>(StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings())); } else { if (intervalStrings.size() > StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.get(c.getSettings())) { throw new IllegalArgumentException( @@ -88,10 +90,10 @@ private static DateDimension parseAndCreateDateDimension( ) ); } + calendarIntervals = new LinkedHashSet<>(); for (String interval : intervalStrings) { calendarIntervals.add(StarTreeIndexSettings.getTimeUnit(interval)); } - calendarIntervals = new ArrayList<>(calendarIntervals); } dimensionMap.remove(CALENDAR_INTERVALS); DateFieldMapper.Resolution resolution = null; @@ -99,6 +101,6 @@ private static DateDimension parseAndCreateDateDimension( resolution = ((DateFieldMapper.DateFieldType) c.mapperService().fieldType(name)).resolution(); } - return new DateDimension(name, calendarIntervals, resolution); + return new DateDimension(name, new ArrayList<>(calendarIntervals), resolution); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java index a2ac545be3cc9..da0f46075ba48 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java @@ -11,6 +11,9 @@ import org.opensearch.common.Rounding; import org.opensearch.common.settings.Setting; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import java.util.Arrays; @@ -82,9 +85,9 @@ public class StarTreeIndexSettings { /** * Default intervals for date dimension as part of star tree fields */ - public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( "index.composite_index.star_tree.field.default.date_intervals", - Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), ExtendedDateTimeUnit.HALF_HOUR_OF_DAY.shortName()), StarTreeIndexSettings::getTimeUnit, Setting.Property.IndexScope, Setting.Property.Final @@ -107,10 +110,12 @@ public class StarTreeIndexSettings { Setting.Property.Final ); - public static Rounding.DateTimeUnit getTimeUnit(String expression) { - if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { - throw new IllegalArgumentException("unknown calendar intervals specified in star tree index mapping"); + public static DateTimeUnitRounding getTimeUnit(String expression) { + if (DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + return new DateTimeUnitAdapter(DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression)); + } else if (ExtendedDateTimeUnit.DATE_FIELD_UNITS.containsKey(expression)) { + return ExtendedDateTimeUnit.DATE_FIELD_UNITS.get(expression); } - return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + throw new IllegalArgumentException("unknown calendar intervals specified in star tree index mapping"); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java new file mode 100644 index 0000000000000..b18ac33a8f657 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitAdapter.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.date; + +import org.opensearch.common.Rounding; + +import java.util.Objects; + +/** + * Adapter class to convert {@link Rounding.DateTimeUnit} to {@link DateTimeUnitRounding} + * + * @opensearch.experimental + */ +public class DateTimeUnitAdapter implements DateTimeUnitRounding { + private final Rounding.DateTimeUnit dateTimeUnit; + + public DateTimeUnitAdapter(Rounding.DateTimeUnit dateTimeUnit) { + this.dateTimeUnit = dateTimeUnit; + } + + @Override + public String shortName() { + return dateTimeUnit.shortName(); + } + + @Override + public long roundFloor(long utcMillis) { + return dateTimeUnit.roundFloor(utcMillis); + } + + /** + * Checks if this DateTimeUnitRounding is equal to another object. + * Two DateTimeUnitRounding instances are considered equal if they have the same short name. + * + * @param obj The object to compare with + * @return true if the objects are equal, false otherwise + */ + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!(obj instanceof DateTimeUnitRounding)) return false; + DateTimeUnitRounding other = (DateTimeUnitRounding) obj; + return Objects.equals(shortName(), other.shortName()); + } + + /** + * Returns a hash code value for the object. + * This method is implemented to be consistent with equals. + * + * @return a hash code value for this object + */ + @Override + public int hashCode() { + return Objects.hash(shortName()); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java new file mode 100644 index 0000000000000..93212a2424e46 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/DateTimeUnitRounding.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.date; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Interface for rounding time units in starTree + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface DateTimeUnitRounding { + /** + * Returns the short name of the time unit + */ + String shortName(); + + /** + * rounds down the given utcMillis to the nearest unit of time + */ + long roundFloor(long utcMillis); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/ExtendedDateTimeUnit.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/ExtendedDateTimeUnit.java new file mode 100644 index 0000000000000..d8bab994e5aaa --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/ExtendedDateTimeUnit.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.date; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.unmodifiableMap; + +/** + * Enum representing the extended date time units supported for star tree index as part of index mapping. + * The enum values are: + *
    + *
  • HALF_HOUR_OF_DAY: Represents half hour of day rounding
    • + *
  • QUARTER_HOUR_OF_DAY: Represents quarter hour of day rounding
    • + *
+ *

+ * The enum also provides a static map of date field units to their corresponding ExtendedDateTimeUnit instances. + * + * @see org.opensearch.common.Rounding.DateTimeUnit for more information on the dateTimeUnit enum and rounding logic. + * + * @opensearch.experimental + */ +public enum ExtendedDateTimeUnit implements DateTimeUnitRounding { + HALF_HOUR_OF_DAY("half-hour") { + @Override + public long roundFloor(long utcMillis) { + return utcMillis - (utcMillis % TimeUnit.MINUTES.toMillis(30)); + } + }, + QUARTER_HOUR_OF_DAY("quarter-hour") { + @Override + public long roundFloor(long utcMillis) { + return utcMillis - (utcMillis % TimeUnit.MINUTES.toMillis(15)); + } + }; + + public static final Map DATE_FIELD_UNITS; + static { + Map dateFieldUnits = new HashMap<>(); + dateFieldUnits.put("30m", ExtendedDateTimeUnit.HALF_HOUR_OF_DAY); + dateFieldUnits.put("half-hour", ExtendedDateTimeUnit.HALF_HOUR_OF_DAY); + dateFieldUnits.put("15m", ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY); + dateFieldUnits.put("quarter-hour", ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY); + DATE_FIELD_UNITS = unmodifiableMap(dateFieldUnits); + } + + private final String shortName; + + ExtendedDateTimeUnit(String shortName) { + this.shortName = shortName; + } + + /** + * This rounds down the supplied milliseconds since the epoch down to the next unit. In order to retain performance this method + * should be as fast as possible and not try to convert dates to java-time objects if possible + * + * @param utcMillis the milliseconds since the epoch + * @return the rounded down milliseconds since the epoch + */ + @Override + public abstract long roundFloor(long utcMillis); + + @Override + public String shortName() { + return shortName; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java new file mode 100644 index 0000000000000..bb285c894e3e0 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/date/package-info.java @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Classes and utilities for working with dateTimeUnits + * + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.utils.date; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java index 10c990ec72bc9..1437e1e46dce2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java @@ -10,6 +10,9 @@ import org.opensearch.common.Rounding; import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.test.OpenSearchTestCase; @@ -17,32 +20,31 @@ import java.util.Comparator; import java.util.List; -import static org.opensearch.common.Rounding.DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT; +import static org.opensearch.index.compositeindex.datacube.DateDimension.DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT; public class DateDimensionTests extends OpenSearchTestCase { public void testDateDimension() { String field = "timestamp"; - List intervals = Arrays.asList( - Rounding.DateTimeUnit.YEAR_OF_CENTURY, - Rounding.DateTimeUnit.HOUR_OF_DAY, - Rounding.DateTimeUnit.MONTH_OF_YEAR + List intervals = Arrays.asList( + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY) ); DateDimension dateDimension = new DateDimension(field, intervals, DateFieldMapper.Resolution.MILLISECONDS); assertEquals(field, dateDimension.getField()); assertEquals(intervals, dateDimension.getIntervals()); - assertEquals( - Arrays.asList(Rounding.DateTimeUnit.HOUR_OF_DAY, Rounding.DateTimeUnit.MONTH_OF_YEAR, Rounding.DateTimeUnit.YEAR_OF_CENTURY), - dateDimension.getSortedCalendarIntervals() - ); + for (int i = 0; i < intervals.size(); i++) { + assertEquals(intervals.get(i).shortName(), dateDimension.getSortedCalendarIntervals().get(i).shortName()); + } } public void testSetDimensionValuesWithMultipleIntervalsYear() { - List intervals = Arrays.asList( - Rounding.DateTimeUnit.YEAR_OF_CENTURY, - Rounding.DateTimeUnit.MONTH_OF_YEAR, - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.HOUR_OF_DAY + List intervals = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY) ); DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); Long[] dims = new Long[4]; @@ -59,144 +61,166 @@ public void testSetDimensionValuesWithMultipleIntervalsYear() { } public void testRoundingAndSortingAllDateTimeUnitsNanos() { - List allUnits = Arrays.asList( - Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, - Rounding.DateTimeUnit.MONTH_OF_YEAR, - Rounding.DateTimeUnit.QUARTER_OF_YEAR, - Rounding.DateTimeUnit.YEAR_OF_CENTURY, - Rounding.DateTimeUnit.SECOND_OF_MINUTE, - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY, - Rounding.DateTimeUnit.DAY_OF_MONTH - ); + List allUnits = getAllTimeUnits(); DateDimension dateDimension = new DateDimension("timestamp", allUnits, DateFieldMapper.Resolution.NANOSECONDS); // Test sorting - List sortedUnits = dateDimension.getSortedCalendarIntervals(); + List sortedUnits = dateDimension.getSortedCalendarIntervals(); assertEquals(allUnits.size(), sortedUnits.size()); for (int i = 0; i < sortedUnits.size() - 1; i++) { assertTrue( "Units should be sorted in ascending order", - ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i)).compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1))) <= 0 + ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i).shortName()) + .compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1).shortName())) <= 0 ); } // Test rounding - long testValueNanos = 1655287972382719622L; // 2022-06-15T10:12:52.382719622Z + long testValueNanos = 1655293505382719622L; // 2022-06-15T11:45:05.382719622Z Long[] dims = new Long[allUnits.size()]; dateDimension.setDimensionValues(testValueNanos, dims, 0); // Expected rounded values (in nanoseconds) - long secondRounded = 1655287972000L; // 2022-06-15T10:12:52 - long minuteRounded = 1655287920000L; // 2022-06-15T10:12:00 - long hourRounded = 1655287200000L; // 2022-06-15T10:00:00 - long dayRounded = 1655251200000L; // 2022-06-15T00:00:00 - long weekRounded = 1655078400000L; // 2022-06-13T00:00:00 // Monday - long monthRounded = 1654041600000L; // 2022-06-01T00:00:00 - long quarterRounded = 1648771200000L; // 2022-04-01T00:00:00 - long yearRounded = 1640995200000L; // 2022-01-01T00:00:00 + long secondRounded = 1655293505000L; // 2022-06-15T11:45:05Z + long minuteRounded = 1655293500000L; // 2022-06-15T11:45:00Z + long quarterHourRounded = 1655293500000L; // 2022-06-15T11:45:00Z + long halfHourRounded = 1655292600000L; // 2022-06-15T11:30:00Z + long hourRounded = 1655290800000L; // 2022-06-15T11:00:00Z + long dayRounded = 1655251200000L; // 2022-06-15T00:00:00Z + long weekRounded = 1655078400000L; // 2022-06-13T00:00:00Z (Monday) + long monthRounded = 1654041600000L; // 2022-06-01T00:00:00Z + long quarterRounded = 1648771200000L; // 2022-04-01T00:00:00Z + long yearRounded = 1640995200000L; // 2022-01-01T00:00:00Z - for (int i = 0; i < sortedUnits.size(); i++) { - Rounding.DateTimeUnit unit = sortedUnits.get(i); - switch (unit) { - case SECOND_OF_MINUTE: - assertEquals(secondRounded, (long) dims[i]); - break; - case MINUTES_OF_HOUR: - assertEquals(minuteRounded, (long) dims[i]); - break; - case HOUR_OF_DAY: - assertEquals(hourRounded, (long) dims[i]); - break; - case DAY_OF_MONTH: - assertEquals(dayRounded, (long) dims[i]); - break; - case WEEK_OF_WEEKYEAR: - assertEquals(weekRounded, (long) dims[i]); - break; - case MONTH_OF_YEAR: - assertEquals(monthRounded, (long) dims[i]); - break; - case QUARTER_OF_YEAR: - assertEquals(quarterRounded, (long) dims[i]); - break; - case YEAR_OF_CENTURY: - assertEquals(yearRounded, (long) dims[i]); - break; - default: - fail("Unexpected DateTimeUnit: " + unit); - } - } + assertTimeUnits( + sortedUnits, + dims, + secondRounded, + minuteRounded, + hourRounded, + dayRounded, + weekRounded, + monthRounded, + quarterRounded, + yearRounded, + halfHourRounded, + quarterHourRounded + ); } public void testRoundingAndSortingAllDateTimeUnitsMillis() { - List allUnits = Arrays.asList( - Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, - Rounding.DateTimeUnit.MONTH_OF_YEAR, - Rounding.DateTimeUnit.QUARTER_OF_YEAR, - Rounding.DateTimeUnit.YEAR_OF_CENTURY, - Rounding.DateTimeUnit.SECOND_OF_MINUTE, - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY, - Rounding.DateTimeUnit.DAY_OF_MONTH - ); + List allUnits = getAllTimeUnits(); DateDimension dateDimension = new DateDimension("timestamp", allUnits, DateFieldMapper.Resolution.MILLISECONDS); // Test sorting - List sortedUnits = dateDimension.getSortedCalendarIntervals(); + List sortedUnits = dateDimension.getSortedCalendarIntervals(); assertEquals(allUnits.size(), sortedUnits.size()); for (int i = 0; i < sortedUnits.size() - 1; i++) { assertTrue( "Units should be sorted in ascending order", - ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i)).compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1))) <= 0 + ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i).shortName()) + .compareTo(ORDERED_DATE_TIME_UNIT.get(sortedUnits.get(i + 1).shortName())) <= 0 ); } // Test rounding - long testValueNanos = 1655287972382L; // 2022-06-15T10:12:52.382Z - + long testValueNanos = 1724114825234L; // 2024-08-20T00:47:05.234Z Long[] dims = new Long[allUnits.size()]; dateDimension.setDimensionValues(testValueNanos, dims, 0); // Expected rounded values (in millis) - long secondRounded = 1655287972000L; // 2022-06-15T10:12:52 - long minuteRounded = 1655287920000L; // 2022-06-15T10:12:00 - long hourRounded = 1655287200000L; // 2022-06-15T10:00:00 - long dayRounded = 1655251200000L; // 2022-06-15T00:00:00 - long weekRounded = 1655078400000L; // 2022-06-13T00:00:00 // Monday - long monthRounded = 1654041600000L; // 2022-06-01T00:00:00 - long quarterRounded = 1648771200000L; // 2022-04-01T00:00:00 - long yearRounded = 1640995200000L; // 2022-01-01T00:00:00 + long secondRounded = 1724114825000L; // 2024-08-20T00:47:05.000Z + long minuteRounded = 1724114820000L; // 2024-08-20T00:47:00.000Z + long quarterHourRounded = 1724114700000L; // 2024-08-20T00:45:00.000Z + long halfHourRounded = 1724113800000L; // 2024-08-20T00:30:00.000Z + long hourRounded = 1724112000000L; // 2024-08-20T00:00:00.000Z + long dayRounded = 1724112000000L; // 2024-08-20T00:00:00.000Z + long weekRounded = 1724025600000L; // 2024-08-15T00:00:00.000Z (Monday) + long monthRounded = 1722470400000L; // 2024-08-01T00:00:00.000Z + long quarterRounded = 1719792000000L; // 2024-07-01T00:00:00.000Z + long yearRounded = 1704067200000L; // 2024-01-01T00:00:00.000Z + assertTimeUnits( + sortedUnits, + dims, + secondRounded, + minuteRounded, + hourRounded, + dayRounded, + weekRounded, + monthRounded, + quarterRounded, + yearRounded, + halfHourRounded, + quarterHourRounded + ); + } + + private static List getAllTimeUnits() { + return Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.QUARTER_OF_YEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.SECOND_OF_MINUTE), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY + ); + } + + private static void assertTimeUnits( + List sortedUnits, + Long[] dims, + long secondRounded, + long minuteRounded, + long hourRounded, + long dayRounded, + long weekRounded, + long monthRounded, + long quarterRounded, + long yearRounded, + long halfHourRounded, + long quarterHourRounded + ) { for (int i = 0; i < sortedUnits.size(); i++) { - Rounding.DateTimeUnit unit = sortedUnits.get(i); - switch (unit) { - case SECOND_OF_MINUTE: + DateTimeUnitRounding unit = sortedUnits.get(i); + String unitName = unit.shortName(); + switch (unitName) { + case "second": assertEquals(secondRounded, (long) dims[i]); break; - case MINUTES_OF_HOUR: + case "minute": assertEquals(minuteRounded, (long) dims[i]); break; - case HOUR_OF_DAY: + case "hour": assertEquals(hourRounded, (long) dims[i]); break; - case DAY_OF_MONTH: + case "day": assertEquals(dayRounded, (long) dims[i]); break; - case WEEK_OF_WEEKYEAR: + case "week": assertEquals(weekRounded, (long) dims[i]); break; - case MONTH_OF_YEAR: + case "month": assertEquals(monthRounded, (long) dims[i]); break; - case QUARTER_OF_YEAR: + case "quarter": assertEquals(quarterRounded, (long) dims[i]); break; - case YEAR_OF_CENTURY: + case "year": assertEquals(yearRounded, (long) dims[i]); break; + case "half-hour": + assertEquals(halfHourRounded, (long) dims[i]); + break; + case "quarter-hour": + assertEquals(quarterHourRounded, (long) dims[i]); + break; default: fail("Unexpected DateTimeUnit: " + unit); } @@ -206,7 +230,10 @@ public void testRoundingAndSortingAllDateTimeUnitsMillis() { public void testGetDimensionFieldsNames() { DateDimension dateDimension = new DateDimension( "timestamp", - Arrays.asList(Rounding.DateTimeUnit.HOUR_OF_DAY, Rounding.DateTimeUnit.DAY_OF_MONTH), + Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ), DateFieldMapper.Resolution.MILLISECONDS ); @@ -217,10 +244,27 @@ public void testGetDimensionFieldsNames() { assertEquals("timestamp_day", fields.get(1)); } + public void testGetExtendedTimeUnitFieldsNames() { + DateDimension dateDimension = new DateDimension( + "timestamp", + Arrays.asList(ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY), + DateFieldMapper.Resolution.MILLISECONDS + ); + + List fields = dateDimension.getDimensionFieldsNames(); + + assertEquals(2, fields.size()); + assertEquals("timestamp_quarter-hour", fields.get(0)); + assertEquals("timestamp_half-hour", fields.get(1)); + } + public void testSetDimensionValues() { DateDimension dateDimension = new DateDimension( "timestamp", - Arrays.asList(Rounding.DateTimeUnit.YEAR_OF_CENTURY, Rounding.DateTimeUnit.HOUR_OF_DAY), + Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY) + ), DateFieldMapper.Resolution.MILLISECONDS ); Long[] dims = new Long[2]; @@ -234,10 +278,31 @@ public void testSetDimensionValues() { } public void testDateTimeUnitComparator() { - Comparator comparator = new Rounding.DateTimeUnitComparator(); - assertTrue(comparator.compare(Rounding.DateTimeUnit.SECOND_OF_MINUTE, Rounding.DateTimeUnit.MINUTES_OF_HOUR) < 0); - assertTrue(comparator.compare(Rounding.DateTimeUnit.HOUR_OF_DAY, Rounding.DateTimeUnit.DAY_OF_MONTH) < 0); - assertTrue(comparator.compare(Rounding.DateTimeUnit.YEAR_OF_CENTURY, Rounding.DateTimeUnit.MONTH_OF_YEAR) > 0); - assertEquals(0, comparator.compare(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR)); + Comparator comparator = new DateDimension.DateTimeUnitComparator(); + assertTrue( + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.SECOND_OF_MINUTE), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR) + ) < 0 + ); + assertTrue( + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH) + ) < 0 + ); + assertTrue( + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.YEAR_OF_CENTURY), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR) + ) > 0 + ); + assertEquals( + 0, + comparator.compare( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR) + ) + ); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 31c448e91f320..c2ba02209af7f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -40,6 +40,9 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DateFieldMapper; import org.opensearch.index.mapper.DocumentMapper; @@ -2825,12 +2828,12 @@ public void testFlushFlowWithTimestamps() throws IOException { ); /** * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] - [1655287920000, 1655287200000, 4] | [40.0, 1] - [1655287980000, 1655287200000, 3] | [30.0, 1] - [1655288040000, 1655287200000, 1] | [10.0, 1] - [1655288040000, 1655287200000, 5] | [50.0, 1] - [1655288100000, 1655287200000, 0] | [0.0, 1] - [null, null, 2] | [20.0, 1] + [1655287920000, 1655287200000, 1655287200000, 4] | [40.0, 1] + [1655287980000, 1655287200000, 1655287200000, 3] | [30.0, 1] + [1655288040000, 1655287200000, 1655287200000, 1] | [10.0, 1] + [1655288040000, 1655287200000, 1655287200000, 5] | [50.0, 1] + [1655288100000, 1655287200000, 1655287200000, 0] | [0.0, 1] + [null, null, null, 2] | [20.0, 1] */ int count = 0; List starTreeDocumentsList = new ArrayList<>(); @@ -2839,7 +2842,7 @@ public void testFlushFlowWithTimestamps() throws IOException { while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - assertEquals(starTreeDocument.dimensions[2] * 1 * 10.0, starTreeDocument.metrics[0]); + assertEquals(starTreeDocument.dimensions[3] * 1 * 10.0, starTreeDocument.metrics[0]); assertEquals(1L, starTreeDocument.metrics[1]); } assertEquals(6, count); @@ -2852,6 +2855,8 @@ public void testMergeFlowWithTimestamps() throws IOException { List docsWithField = List.of(0, 1, 2, 3, 4, 6); List dimList2 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); List docsWithField2 = List.of(0, 1, 2, 3, 4, 6); + List dimList7 = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L, 1655288092000L, -1L); + List docsWithField7 = List.of(0, 1, 2, 3, 4, 6); List dimList5 = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); List docsWithField5 = List.of(0, 1, 2, 3, 4, 5, 6); @@ -2872,6 +2877,8 @@ public void testMergeFlowWithTimestamps() throws IOException { List docsWithField3 = List.of(0, 1, 3, 4); List dimList4 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); List docsWithField4 = List.of(0, 1, 3, 4); + List dimList8 = List.of(1655288152000L, 1655288092000L, 1655288032000L, -1L); + List docsWithField8 = List.of(0, 1, 3, 4); List dimList6 = List.of(5L, 6L, 7L, 8L); List docsWithField6 = List.of(0, 1, 2, 3); @@ -2890,6 +2897,7 @@ public void testMergeFlowWithTimestamps() throws IOException { StarTreeValues starTreeValues = getStarTreeValuesWithDates( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), + getSortedNumericMock(dimList7, docsWithField7), getSortedNumericMock(dimList5, docsWithField5), getSortedNumericMock(metricsList, metricsWithField), getSortedNumericMock(metricsList1, metricsWithField1), @@ -2900,6 +2908,7 @@ public void testMergeFlowWithTimestamps() throws IOException { StarTreeValues starTreeValues2 = getStarTreeValuesWithDates( getSortedNumericMock(dimList3, docsWithField3), getSortedNumericMock(dimList4, docsWithField4), + getSortedNumericMock(dimList8, docsWithField8), getSortedNumericMock(dimList6, docsWithField6), getSortedNumericMock(metricsList2, metricsWithField2), getSortedNumericMock(metricsList21, metricsWithField21), @@ -2909,16 +2918,16 @@ public void testMergeFlowWithTimestamps() throws IOException { builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** - [[1655287972000, 1655287972000, 3] | [30.0, 3] - [1655288032000, 1655288032000, 2] | [20.0, 2] - [1655288032000, 1655288032000, 8] | [80.0, 8] - [1655288092000, 1655288092000, 1] | [10.0, 1] - [1655288092000, 1655288092000, 4] | [40.0, 4] - [1655288092000, 1655288092000, 6] | [60.0, 6] - [1655288152000, 1655288152000, 0] | [0.0, 0] - [1655288152000, 1655288152000, 5] | [50.0, 5] - [null, null, 5] | [50.0, 5] - [null, null, 7] | [70.0, 7] + [1655287972000, 1655287972000, 1655287972000, 3] | [30.0, 3] + [1655288032000, 1655288032000, 1655288032000, 2] | [20.0, 2] + [1655288032000, 1655288032000, 1655288032000, 8] | [80.0, 8] + [1655288092000, 1655288092000, 1655288092000, 1] | [10.0, 1] + [1655288092000, 1655288092000, 1655288092000, 4] | [40.0, 4] + [1655288092000, 1655288092000, 1655288092000, 6] | [60.0, 6] + [1655288152000, 1655288152000, 1655288152000, 0] | [0.0, 0] + [1655288152000, 1655288152000, 1655288152000, 5] | [50.0, 5] + [null, null, null, 5] | [50.0, 5] + [null, null, null, 7] | [70.0, 7] */ int count = 0; List starTreeDocumentsList = new ArrayList<>(); @@ -2928,32 +2937,43 @@ public void testMergeFlowWithTimestamps() throws IOException { while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - assertEquals(starTreeDocument.dimensions[2] * 10.0, (double) starTreeDocument.metrics[0], 0); - assertEquals(starTreeDocument.dimensions[2], starTreeDocument.metrics[1]); + assertEquals(starTreeDocument.dimensions[3] * 10.0, (double) starTreeDocument.metrics[0], 0); + assertEquals(starTreeDocument.dimensions[3], starTreeDocument.metrics[1]); } assertEquals(10, count); builder.build(starTreeDocumentsList.iterator()); - validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); + validateStarTree(builder.getRootNode(), 4, 10, builder.getStarTreeDocuments()); } private StarTreeValues getStarTreeValuesWithDates( SortedNumericDocValues dimList, SortedNumericDocValues dimList2, + SortedNumericDocValues dimList4, SortedNumericDocValues dimList3, SortedNumericDocValues metricsList, SortedNumericDocValues metricsList1, StarTreeField sf, String number ) { - Map dimDocIdSetIterators = Map.of("field1_minute", dimList, "field1_hour", dimList2, "field3", dimList3); + Map dimDocIdSetIterators = Map.of( + "field1_minute", + dimList, + "field1_half-hour", + dimList4, + "field1_hour", + dimList2, + "field3", + dimList3 + ); Map metricDocIdSetIterators = Map.of("field2_COUNT", metricsList, "field2_SUM", metricsList1); return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of("numSegmentDocs", number)); } private static StarTreeField getStarTreeFieldWithDateDimension() { - List intervals = new ArrayList<>(); - intervals.add(Rounding.DateTimeUnit.MINUTES_OF_HOUR); - intervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + List intervals = new ArrayList<>(); + intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR)); + intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + intervals.add(ExtendedDateTimeUnit.HALF_HOUR_OF_DAY); Dimension d1 = new DateDimension("field1", intervals, DateFieldMapper.Resolution.MILLISECONDS); Dimension d2 = new NumericDimension("field3"); Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index b4abeaf62d6e1..3925968e6cab9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -23,6 +23,9 @@ import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitRounding; +import org.opensearch.index.compositeindex.datacube.startree.utils.date.ExtendedDateTimeUnit; import org.junit.After; import org.junit.Before; @@ -60,11 +63,13 @@ public void testValidStarTree() throws IOException { assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.DAY_OF_MONTH, - Rounding.DateTimeUnit.MONTH_OF_YEAR + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.DAY_OF_MONTH), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR) ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); + } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList(MetricStat.SUM, MetricStat.AVG); @@ -110,13 +115,15 @@ public void testValidStarTreeDefaults() throws IOException { assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedDimensionFields = Arrays.asList("@timestamp_minute", "@timestamp_hour"); + List expectedDimensionFields = Arrays.asList("@timestamp_minute", "@timestamp_half-hour"); assertEquals(expectedDimensionFields, dateDim.getDimensionFieldsNames()); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.MINUTES_OF_HOUR, - Rounding.DateTimeUnit.HOUR_OF_DAY + List expectedTimeUnits = Arrays.asList( + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR), + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getIntervals().get(i).shortName()); + } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( @@ -141,14 +148,16 @@ public void testValidStarTreeDateDims() throws IOException { assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); - List expectedDimensionFields = Arrays.asList("@timestamp_week", "@timestamp_month", "@timestamp_quarter"); + List expectedDimensionFields = Arrays.asList("@timestamp_half-hour", "@timestamp_week", "@timestamp_month"); assertEquals(expectedDimensionFields, dateDim.getDimensionFieldsNames()); - List expectedTimeUnits = Arrays.asList( - Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR, - Rounding.DateTimeUnit.MONTH_OF_YEAR, - Rounding.DateTimeUnit.QUARTER_OF_YEAR + List expectedTimeUnits = Arrays.asList( + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + new DateTimeUnitAdapter(Rounding.DateTimeUnit.WEEK_OF_WEEKYEAR), + new DateTimeUnitAdapter(Rounding.DateTimeUnit.MONTH_OF_YEAR) ); - assertEquals(expectedTimeUnits, dateDim.getSortedCalendarIntervals()); + for (int i = 0; i < expectedTimeUnits.size(); i++) { + assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); + } assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( @@ -337,16 +346,16 @@ public void testMetric() { } public void testDimensions() { - List d1CalendarIntervals = new ArrayList<>(); - d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); DateDimension d2 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertEquals(d1, d2); d2 = new DateDimension("name1", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertNotEquals(d1, d2); - List d2CalendarIntervals = new ArrayList<>(); - d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); - d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + List d2CalendarIntervals = new ArrayList<>(); + d2CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); + d2CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); d2 = new DateDimension("name", d2CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); assertNotEquals(d1, d2); NumericDimension n1 = new NumericDimension("name"); @@ -360,8 +369,8 @@ public void testStarTreeField() { List m1 = new ArrayList<>(); m1.add(MetricStat.MAX); Metric metric1 = new Metric("name", m1); - List d1CalendarIntervals = new ArrayList<>(); - d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); DateDimension d1 = new DateDimension("name", d1CalendarIntervals, DateFieldMapper.Resolution.MILLISECONDS); NumericDimension n1 = new NumericDimension("numeric"); NumericDimension n2 = new NumericDimension("name1"); @@ -545,9 +554,10 @@ private XContentBuilder getMinMappingWithDateDims(boolean calendarIntervalsExcee b.value(getRandom().nextBoolean() ? "hour" : "1h"); b.value(getRandom().nextBoolean() ? "minute" : "1m"); b.value(getRandom().nextBoolean() ? "year" : "1y"); + b.value(getRandom().nextBoolean() ? "quarter-hour" : "15m"); } b.value(getRandom().nextBoolean() ? "month" : "1M"); - b.value(getRandom().nextBoolean() ? "quarter" : "1q"); + b.value(getRandom().nextBoolean() ? "half-hour" : "30m"); b.endArray(); b.endObject(); } From 014f123ddaeee5173a281adc7433a929fa5619c3 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 21 Aug 2024 14:50:09 +0530 Subject: [PATCH 3/6] adding tests Signed-off-by: Bharathwaj G --- .../datacube/startree/DateDimensionTests.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java index 1437e1e46dce2..026e0259a5a84 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/DateDimensionTests.java @@ -60,6 +60,47 @@ public void testSetDimensionValuesWithMultipleIntervalsYear() { assertEquals(4, dateDimension.getNumSubDimensions()); } + public void testSetDimensionValuesForHalfAndQuarterHour() { + List intervals = Arrays.asList( + ExtendedDateTimeUnit.HALF_HOUR_OF_DAY, + ExtendedDateTimeUnit.QUARTER_HOUR_OF_DAY + ); + DateDimension dateDimension = new DateDimension("timestamp", intervals, DateFieldMapper.Resolution.MILLISECONDS); + Long[] dims = new Long[2]; + long testValue = 1724230620123L; // August 21, 2024 8:57:00.123 UTC + + int nextIndex = dateDimension.setDimensionValues(testValue, dims, 0); + + assertEquals(2, nextIndex); + assertEquals(1724229900000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:45:00 UTC + assertEquals(1724229000000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + + dims = new Long[2]; + testValue = 1724229899234L; // Wed, 21 Aug 2024 08:44:59 GMT + dateDimension.setDimensionValues(testValue, dims, 0); + assertEquals(2, nextIndex); + assertEquals(1724229000000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(1724229000000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + + dims = new Long[2]; + testValue = 1724229000123L; // Wed, 21 Aug 2024 08:30:00 GMT + dateDimension.setDimensionValues(testValue, dims, 0); + assertEquals(2, nextIndex); + assertEquals(1724229000000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(1724229000000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:30:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + + dims = new Long[2]; + testValue = 1724228940000L; // Wed, 21 Aug 2024 08:29:00 GMT + dateDimension.setDimensionValues(testValue, dims, 0); + assertEquals(2, nextIndex); + assertEquals(1724228100000L, (long) dims[0]); // Quarter Hour rounded - Wed, 21 Aug 2024 08:15:00 UTC + assertEquals(1724227200000L, (long) dims[1]); // Half hour rounded - Wed, 21 Aug 2024 08:00:00 UTC + assertEquals(2, dateDimension.getNumSubDimensions()); + } + public void testRoundingAndSortingAllDateTimeUnitsNanos() { List allUnits = getAllTimeUnits(); From a52b29c2741639dad1451bbb2f40d36bb9455c96 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 3 Sep 2024 16:30:15 +0530 Subject: [PATCH 4/6] resolving conflicts Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 12 +- .../datacube/ReadDimension.java | 17 ++ .../startree/builder/BaseStarTreeBuilder.java | 50 ++-- .../builder/OffHeapStarTreeBuilder.java | 29 +-- .../builder/OnHeapStarTreeBuilder.java | 30 +-- .../meta/StarTreeMetadataWriter.java | 8 +- .../startree/index/StarTreeValues.java | 10 +- .../datacube/startree/StarTreeTestUtils.java | 15 +- .../builder/AbstractStarTreeBuilderTests.java | 213 +++++++++++------- .../index/mapper/StarTreeMapperTests.java | 33 ++- 10 files changed, 231 insertions(+), 186 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 2d22e10d92c1d..7834588533074 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -327,11 +327,9 @@ public void testValidCompositeIndex() { assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( - MetricStat.AVG, - MetricStat.COUNT, + MetricStat.VALUE_COUNT, MetricStat.SUM, - MetricStat.MAX, - MetricStat.MIN + MetricStat.AVG ); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); @@ -371,11 +369,9 @@ public void testValidCompositeIndexWithDates() { assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( - MetricStat.AVG, - MetricStat.COUNT, + MetricStat.VALUE_COUNT, MetricStat.SUM, - MetricStat.MAX, - MetricStat.MIN + MetricStat.AVG ); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java index 4264ec87d2c74..1e288bec7da3a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube; +import java.util.List; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.CompositeDataCubeFieldType; @@ -31,6 +32,22 @@ public String getField() { return field; } + @Override + public int getNumSubDimensions() { + return 1; + } + + @Override + public int setDimensionValues(Long value, Long[] dims, int index) { + dims[index++] = value; + return index; + } + + @Override + public List getDimensionFieldsNames() { + return List.of(field); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 24816381a69fd..7be0531074ce2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -52,6 +52,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; @@ -229,7 +230,7 @@ public void build( List metricReaders = getMetricReaders(writeState, fieldProducerMap); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; - for (int i = 0; i < numDimensions; i++) { + for (int i = 0; i < dimensionReaders.length; i++) { String dimension = dimensionsSplitOrder.get(i).getField(); FieldInfo dimensionFieldInfo = writeState.fieldInfos.fieldInfo(dimension); if (dimensionFieldInfo == null) { @@ -318,19 +319,19 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A throws IOException { List dimensionWriters = new ArrayList<>(); List metricWriters = new ArrayList<>(); - FieldInfo[] dimensionFieldInfoList = new FieldInfo[starTreeField.getDimensionsOrder().size()]; + FieldInfo[] dimensionFieldInfoList = new FieldInfo[numDimensions]; FieldInfo[] metricFieldInfoList = new FieldInfo[metricAggregatorInfos.size()]; - for (int i = 0; i < dimensionFieldInfoList.length; i++) { - final FieldInfo fi = getFieldInfo( - fullyQualifiedFieldNameForStarTreeDimensionsDocValues( - starTreeField.getName(), - starTreeField.getDimensionsOrder().get(i).getField() - ), - DocValuesType.SORTED_NUMERIC, - fieldNumberAcrossStarTrees.getAndIncrement() - ); - dimensionFieldInfoList[i] = fi; - dimensionWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + int dimIndex = 0; + for(Dimension dim : dimensionsSplitOrder) { + for(String name : dim.getDimensionFieldsNames()) { + final FieldInfo fi = getFieldInfo( + fullyQualifiedFieldNameForStarTreeDimensionsDocValues(starTreeField.getName(), + name), DocValuesType.SORTED_NUMERIC, + fieldNumberAcrossStarTrees.getAndIncrement()); + dimensionFieldInfoList[dimIndex] = fi; + dimensionWriters.add(new SortedNumericDocValuesWriterWrapper(fi, Counter.newCounter())); + dimIndex++; + } } for (int i = 0; i < metricAggregatorInfos.size(); i++) { @@ -379,7 +380,7 @@ private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer, A } } - addStarTreeDocValueFields(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, starTreeField.getDimensionsOrder().size()); + addStarTreeDocValueFields(docValuesConsumer, dimensionWriters, dimensionFieldInfoList, numDimensions); addStarTreeDocValueFields(docValuesConsumer, metricWriters, metricFieldInfoList, metricAggregatorInfos.size()); } @@ -441,19 +442,26 @@ protected void setReadersAndNumSegmentDocs( AtomicInteger numSegmentDocs, StarTreeValues starTreeValues ) { - List dimensionNames = starTreeValues.getStarTreeField().getDimensionNames(); for (int i = 0; i < numDimensions; i++) { dimensionReaders[i] = new SequentialDocValuesIterator( - starTreeValues.getDimensionDocValuesIteratorMap().get(dimensionNames.get(i)) + starTreeValues.getDimensionDocIdSetIterator(dimensionNames.get(i)) ); } - for (String metric : starTreeValues.getStarTreeField().getMetricNames()) { - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocValuesIteratorMap().get(metric))); + // get doc id set iterators for metrics + for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { + String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( + starTreeValues.getStarTreeField().getName(), + metric.getField(), + metricStat.getTypeName() + ); + metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + } } - numSegmentDocs.set( - Integer.parseInt(starTreeValues.getAttributes().getOrDefault(NUM_SEGMENT_DOCS, String.valueOf(DocIdSetIterator.NO_MORE_DOCS))) - ); + + numSegmentDocs.set(Integer.parseInt( + starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)))); } /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index f559094af18a7..fd8cb91764603 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -19,7 +19,6 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -149,31 +148,13 @@ Iterator mergeStarTrees(List starTreeValuesSub int[] docIds; try { for (StarTreeValues starTreeValues : starTreeValuesSubs) { - List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[starTreeValues.getStarTreeField() - .getDimensionsOrder() - .size()]; - for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); - } + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); - // get doc id set iterators for metrics - for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { - String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( - starTreeValues.getStarTreeField().getName(), - metric.getField(), - metricStat.getTypeName() - ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); - } - } + AtomicInteger numSegmentDocs = new AtomicInteger(); + setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); + int currentDocId = 0; - int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) - ); - while (currentDocId < numSegmentDocs) { + while (currentDocId < numSegmentDocs.get()) { StarTreeDocument starTreeDocument = getStarTreeDocument(currentDocId, dimensionReaders, metricReaders); segmentDocumentFileManager.writeStarTreeDocument(starTreeDocument, true); numDocs++; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 8888896f15e5b..0e60d9b07dff7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -15,7 +15,6 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; @@ -135,33 +134,12 @@ Iterator mergeStarTrees(List starTreeValuesSub StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeValuesSubs) throws IOException { List starTreeDocuments = new ArrayList<>(); for (StarTreeValues starTreeValues : starTreeValuesSubs) { - List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; - - for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); - } - + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; List metricReaders = new ArrayList<>(); - // get doc id set iterators for metrics - for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { - String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( - starTreeValues.getStarTreeField().getName(), - metric.getField(), - metricStat.getTypeName() - ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); - - } - } - + AtomicInteger numSegmentDocs = new AtomicInteger(); + setReadersAndNumSegmentDocs(dimensionReaders, metricReaders, numSegmentDocs, starTreeValues); int currentDocId = 0; - int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) - ); - while (currentDocId < numSegmentDocs) { + while (currentDocId < numSegmentDocs.get()) { starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders)); currentDocId++; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java index 1c04350e25047..08ac5afa9eba0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java @@ -128,13 +128,11 @@ private static void writeMeta( metaOut.writeVInt(numNodes); // number of dimensions - // TODO: Revisit the number of dimensions for timestamps (as we will split timestamp into min, hour, etc.) - metaOut.writeVInt(starTreeField.getDimensionsOrder().size()); + metaOut.writeVInt(starTreeField.getDimensionNames().size()); // dimensions - // TODO: Add sub-dimensions for timestamps (as we will split timestamp into min, hour, etc.) - for (Dimension dimension : starTreeField.getDimensionsOrder()) { - metaOut.writeString(dimension.getField()); + for (String dim : starTreeField.getDimensionNames()) { + metaOut.writeString(dim); } // number of metrics diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index 8181a69f49bd4..edd34508874a2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -8,6 +8,7 @@ package org.opensearch.index.compositeindex.datacube.startree.index; +import java.util.Locale; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; @@ -243,8 +244,9 @@ public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { if (dimensionDocValuesIteratorMap.containsKey(dimension)) { return dimensionDocValuesIteratorMap.get(dimension).get(); } - - return DocValues.emptySortedNumeric(); + else { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension %s not present", dimension)); + } } /** @@ -257,9 +259,9 @@ public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricNam if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); + } else { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Metric %s not present", fullyQualifiedMetricName)); } - - return DocValues.emptySortedNumeric(); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index 53c9e05ccb490..394ef54f59474 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -50,11 +50,20 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( List starTreeDocuments = new ArrayList<>(); for (StarTreeValues starTreeValues : starTreeValuesSubs) { List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + int numDimensions = 0; + for(Dimension dimension : dimensionsSplitOrder) { + numDimensions += dimension.getNumSubDimensions(); + } + SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[numDimensions]; + int dimIndex = 0; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + Dimension dimension = dimensionsSplitOrder.get(i); + for(String name : dimension.getDimensionFieldsNames()) { + dimensionReaders[dimIndex] = + new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(name)); + dimIndex++; + } } List metricReaders = new ArrayList<>(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 593f4a1aff5fc..7e61ea3cef2cc 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -88,6 +88,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.StarTreeTestUtils.validateFileFormats; import static org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter.VERSION_CURRENT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; @@ -682,13 +683,11 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics( SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); - segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); @@ -761,13 +760,11 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetri SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); - segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); @@ -835,13 +832,10 @@ public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOEx SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); - segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -857,8 +851,6 @@ public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOEx assertEquals(expectedStarTreeDocument.metrics[4], resultStarTreeDocument.metrics[4]); assertEquals(expectedStarTreeDocument.metrics[5], resultStarTreeDocument.metrics[5]); } - builder.build(starTreeDocumentList.iterator()); - validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); } public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { @@ -909,13 +901,10 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); - segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -937,10 +926,6 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); - // TODO - // builder.build(starTreeDocumentList.iterator()); - // validateStarTree(builder.getRootNode(), 4, 1, builder.getStarTreeDocuments()); - } public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() throws IOException { @@ -992,13 +977,10 @@ public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - List starTreeDocumentList = new ArrayList<>(); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, metricsIterators ); - segmentStarTreeDocumentIterator.forEachRemaining(starTreeDocumentList::add); - segmentStarTreeDocumentIterator = starTreeDocumentList.iterator(); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -1885,7 +1867,7 @@ private List getStarTreeDimensionNames(List dimensionsOrder) List dimensionNames = new ArrayList<>(); for (Dimension dimension : dimensionsOrder) { - dimensionNames.add(dimension.getField()); + dimensionNames.addAll(dimension.getDimensionFieldsNames()); } return dimensionNames; @@ -2296,7 +2278,7 @@ private StarTreeField getStarTreeFieldWithMultipleMetrics() { Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); List dims = List.of(d1, d2); List metrics = List.of(m1, m2); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(10, new HashSet<>(), getBuildMode()); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); return new StarTreeField("sf", dims, metrics, c); } @@ -2705,9 +2687,6 @@ public void testMergeFlowNumSegmentsDocs() throws IOException { [8, 8] | [8] */ int count = 0; - List starTreeDocumentsList = new ArrayList<>(); - starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); - starTreeDocumentIterator = starTreeDocumentsList.iterator(); while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); @@ -2718,8 +2697,6 @@ public void testMergeFlowNumSegmentsDocs() throws IOException { } } assertEquals(9, count); - builder.build(starTreeDocumentsList.iterator()); - validateStarTree(builder.getRootNode(), 2, 1, builder.getStarTreeDocuments()); } public void testMergeFlowWithMissingDocs() throws IOException { @@ -3605,26 +3582,26 @@ public void testMergeFlowWithMaxLeafDocs() throws IOException { } else if (starTreeDocument.dimensions[1] != null && starTreeDocument.dimensions[2] != null && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null - && starTreeDocument.dimensions[2] != null - && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null - && starTreeDocument.dimensions[1] != null - && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[1] != null) { - assertTrue((long) starTreeDocument.metrics[1] == 20L || (long) starTreeDocument.metrics[1] == 40L); - } else if (starTreeDocument.dimensions[1] != null && starTreeDocument.dimensions[3] != null) { - assertEquals(10L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[1] != null) { - assertEquals(100L, starTreeDocument.metrics[1]); - } else if (starTreeDocument.dimensions[0] != null) { - assertEquals(40L, starTreeDocument.metrics[1]); - } + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null + && starTreeDocument.dimensions[2] != null + && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null + && starTreeDocument.dimensions[1] != null + && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null && starTreeDocument.dimensions[1] != null) { + assertTrue((long) starTreeDocument.metrics[1] == 20L || (long) starTreeDocument.metrics[1] == 40L); + } else if (starTreeDocument.dimensions[1] != null && starTreeDocument.dimensions[3] != null) { + assertEquals(10L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[1] != null) { + assertEquals(100L, starTreeDocument.metrics[1]); + } else if (starTreeDocument.dimensions[0] != null) { + assertEquals(40L, starTreeDocument.metrics[1]); + } } validateStarTree(builder.getRootNode(), 4, compositeField.getStarTreeConfig().maxLeafDocs(), builder.getStarTreeDocuments()); metaOut.close(); @@ -4118,7 +4095,19 @@ public void testMergeFlow() throws IOException { () -> d4sndv ); - Map> metricDocIdSetIterators = Map.of("field2_" + m1.getMetrics().get(0).name(), () -> m1sndv, "_doc_count", () -> m2sndv); + Map> metricDocIdSetIterators = Map.of( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + compositeField.getName(), + "field2", + compositeField.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> m1sndv, + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + compositeField.getName(), + "_doc_count", + compositeField.getMetrics().get(1).getMetrics().get(0).getTypeName() + ), + () -> m2sndv); StarTreeValues starTreeValues = new StarTreeValues( compositeField, @@ -4145,7 +4134,19 @@ public void testMergeFlow() throws IOException { () -> f2d4sndv ); - Map> f2metricDocIdSetIterators = Map.of("field2_" + m1.getMetrics().get(0).name(), () -> f2m1sndv, "_doc_count", () -> f2m2sndv); + Map> f2metricDocIdSetIterators = Map.of( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + compositeField.getName(), + "field2", + compositeField.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> f2m1sndv, + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + compositeField.getName(), + "_doc_count", + compositeField.getMetrics().get(1).getMetrics().get(0).getTypeName() + ), + () -> f2m2sndv); StarTreeValues starTreeValues2 = new StarTreeValues( compositeField, null, @@ -4163,8 +4164,6 @@ public void testMergeFlow() throws IOException { ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); - List starTreeDocumentsList = new ArrayList<>(); - starTreeDocumentIterator.forEachRemaining(starTreeDocumentsList::add); /** [0, 0, 0, 0] | [0.0, 2] [1, 1, 1, 1] | [20.0, 2] @@ -4175,7 +4174,7 @@ public void testMergeFlow() throws IOException { ... [999, 999, 999, 999] | [19980.0] */ - for (StarTreeDocument starTreeDocument : starTreeDocumentsList) { + for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); assertEquals(2L, starTreeDocument.metrics[1]); } @@ -4189,7 +4188,7 @@ public void testMergeFlow() throws IOException { docValuesConsumer.close(); StarTreeMetadata starTreeMetadata = new StarTreeMetadata( - "sf", + compositeField.getName(), STAR_TREE, mock(IndexInput.class), VERSION_CURRENT, @@ -4213,6 +4212,7 @@ public void testMergeFlow() throws IOException { ); } + public void testFlushFlowWithTimestamps() throws IOException { List dimList = List.of(1655288152000L, 1655288092000L, 1655288032000L, 1655287972000L, 1655288092000L); List docsWithField = List.of(0, 1, 3, 4, 5); @@ -4229,13 +4229,19 @@ public void testFlushFlowWithTimestamps() throws IOException { ); List metricsWithField = List.of(0, 1, 2, 3, 4, 5); - StarTreeField sf = getStarTreeFieldWithDateDimension(); + compositeField = getStarTreeFieldWithDateDimension(); SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); - - builder = getStarTreeBuilder(sf, getWriteState(6), mapperService); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(6, writeState.segmentInfo.getId()), mapperService); SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimDvs, @@ -4261,7 +4267,7 @@ public void testFlushFlowWithTimestamps() throws IOException { assertEquals(1L, starTreeDocument.metrics[1]); } assertEquals(6, count); - builder.build(starTreeDocumentsList.iterator()); + builder.build(starTreeDocumentsList.iterator(), new AtomicInteger(), docValuesConsumer); validateStarTree(builder.getRootNode(), 3, 10, builder.getStarTreeDocuments()); } @@ -4308,7 +4314,7 @@ public void testMergeFlowWithTimestamps() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeFieldWithDateDimension(); + compositeField = getStarTreeFieldWithDateDimension(); StarTreeValues starTreeValues = getStarTreeValuesWithDates( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -4316,7 +4322,7 @@ public void testMergeFlowWithTimestamps() throws IOException { getSortedNumericMock(dimList5, docsWithField5), getSortedNumericMock(metricsList, metricsWithField), getSortedNumericMock(metricsList1, metricsWithField1), - sf, + compositeField, "6" ); @@ -4327,10 +4333,17 @@ public void testMergeFlowWithTimestamps() throws IOException { getSortedNumericMock(dimList6, docsWithField6), getSortedNumericMock(metricsList2, metricsWithField2), getSortedNumericMock(metricsList21, metricsWithField21), - sf, + compositeField, "4" ); - builder = getStarTreeBuilder(sf, getWriteState(4), mapperService); + this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( + writeState, + Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, + Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION, + Composite99DocValuesFormat.META_DOC_VALUES_CODEC, + Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION + ); + builder = getStarTreeBuilder(metaOut, dataOut, compositeField, getWriteState(4, writeState.segmentInfo.getId()), mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); /** [1655287972000, 1655287972000, 1655287972000, 3] | [30.0, 3] @@ -4352,12 +4365,40 @@ public void testMergeFlowWithTimestamps() throws IOException { while (starTreeDocumentIterator.hasNext()) { count++; StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); - assertEquals(starTreeDocument.dimensions[3] * 10.0, (double) starTreeDocument.metrics[0], 0); - assertEquals(starTreeDocument.dimensions[3], starTreeDocument.metrics[1]); + System.out.println(starTreeDocument); + //assertEquals(starTreeDocument.dimensions[3] * 10.0, (double) starTreeDocument.metrics[0], 0); + //assertEquals(starTreeDocument.dimensions[3], starTreeDocument.metrics[1]); } assertEquals(10, count); - builder.build(starTreeDocumentsList.iterator()); + builder.build(starTreeDocumentsList.iterator(), new AtomicInteger(), docValuesConsumer); validateStarTree(builder.getRootNode(), 4, 10, builder.getStarTreeDocuments()); + metaOut.close(); + dataOut.close(); + docValuesConsumer.close(); + + StarTreeMetadata starTreeMetadata = new StarTreeMetadata( + "sf", + STAR_TREE, + mock(IndexInput.class), + VERSION_CURRENT, + builder.numStarTreeNodes, + getStarTreeDimensionNames(compositeField.getDimensionsOrder()), + compositeField.getMetrics(), + 10, + builder.numStarTreeDocs, + compositeField.getStarTreeConfig().maxLeafDocs(), + Set.of(), + getBuildMode(), + 0, + 231 + ); + + validateStarTreeFileFormats( + builder.getRootNode(), + builder.getStarTreeDocuments().size(), + starTreeMetadata, + builder.getStarTreeDocuments() + ); } private StarTreeValues getStarTreeValuesWithDates( @@ -4370,35 +4411,51 @@ private StarTreeValues getStarTreeValuesWithDates( StarTreeField sf, String number ) { - Map dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1_minute", - dimList, + ()->dimList, "field1_half-hour", - dimList4, + ()->dimList4, "field1_hour", - dimList2, + ()->dimList2, "field3", - dimList3 + ()->dimList3 ); - Map metricDocIdSetIterators = Map.of("field2_COUNT", metricsList, "field2_SUM", metricsList1); - return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of("numSegmentDocs", number)); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); + + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(0).getTypeName() + ), + () -> metricsList + ); + metricDocIdSetIterators.put( + fullyQualifiedFieldNameForStarTreeMetricsDocValues( + sf.getName(), + "field2", + sf.getMetrics().get(0).getMetrics().get(1).getTypeName() + ), + () -> metricsList1 + ); + return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of(SEGMENT_DOCS_COUNT, number)); } - private static StarTreeField getStarTreeFieldWithDateDimension() { + private StarTreeField getStarTreeFieldWithDateDimension() { List intervals = new ArrayList<>(); intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR)); intervals.add(new DateTimeUnitAdapter(Rounding.DateTimeUnit.HOUR_OF_DAY)); intervals.add(ExtendedDateTimeUnit.HALF_HOUR_OF_DAY); Dimension d1 = new DateDimension("field1", intervals, DateFieldMapper.Resolution.MILLISECONDS); Dimension d2 = new NumericDimension("field3"); - Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); - Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); + Metric m1 = new Metric("field2", List.of(MetricStat.SUM, MetricStat.VALUE_COUNT)); List dims = List.of(d1, d2); - List metrics = List.of(m1, m2); + List metrics = List.of(m1); StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( 10, new HashSet<>(), - StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP + getBuildMode() ); StarTreeField sf = new StarTreeField("sf", dims, metrics, c); return sf; @@ -4590,7 +4647,7 @@ private StarTreeField getStarTreeField(MetricStat count) { Metric m1 = new Metric("field2", List.of(count)); List dims = List.of(d1, d2); List metrics = List.of(m1); - StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1, new HashSet<>(), getBuildMode()); + StarTreeFieldConfiguration c = new StarTreeFieldConfiguration(1000, new HashSet<>(), getBuildMode()); return new StarTreeField("sf", dims, metrics, c); } diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 1aaf8b081632c..61caf4b43084b 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -100,7 +100,9 @@ public void testMetricsWithJustSum() throws IOException { Rounding.DateTimeUnit.DAY_OF_MONTH, Rounding.DateTimeUnit.MONTH_OF_YEAR ); - assertEquals(expectedTimeUnits, dateDim.getIntervals()); + for(int i=0; i expectedMetrics = Arrays.asList( - MetricStat.AVG, MetricStat.VALUE_COUNT, MetricStat.SUM, - MetricStat.MAX, - MetricStat.MIN + MetricStat.AVG ); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); @@ -229,11 +231,8 @@ public void testValidStarTreeDateDims() throws IOException { assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( - MetricStat.AVG, MetricStat.VALUE_COUNT, - MetricStat.SUM, - MetricStat.MAX, - MetricStat.MIN + MetricStat.SUM,MetricStat.AVG ); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); @@ -294,7 +293,7 @@ public void testNoMetrics() { public void testInvalidParam() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getInvalidMapping(false, false, false, false, true, false)) + () -> createMapperService(getInvalidMapping(false, false, false, false, true, false, false)) ); assertEquals( "Failed to parse mapping [_doc]: Star tree mapping definition has unsupported parameters: [invalid : {invalid=invalid}]", @@ -351,7 +350,7 @@ public void testInvalidMetricType() { public void testInvalidMetricTypeWithDocCount() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getInvalidMapping(false, false, false, false, false, true)) + () -> createMapperService(getInvalidMapping(false, false, false, false, false, true, false)) ); assertEquals("Failed to parse mapping [_doc]: Invalid metric stat: _doc_count", ex.getMessage()); } @@ -370,7 +369,7 @@ public void testInvalidDimType() { public void testInvalidDateDimType() { MapperParsingException ex = expectThrows( MapperParsingException.class, - () -> createMapperService(getInvalidMapping(false, false, true, false, false, true)) + () -> createMapperService(getInvalidMapping(false, false, true, false, false, false, true)) ); assertEquals( "Failed to parse mapping [_doc]: date_dimension [@timestamp] should be of type date for star tree field [startree]", @@ -629,14 +628,14 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.value("status"); } b.endArray(); - b.startArray("ordered_dimensions"); - b.startObject(); + b.startObject("date_dimension"); b.field("name", "@timestamp"); b.startArray("calendar_intervals"); b.value("day"); b.value("month"); b.endArray(); b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", dim); b.endObject(); @@ -679,14 +678,14 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.value("status"); } b.endArray(); - b.startArray("ordered_dimensions"); - b.startObject(); + b.startObject("date_dimension"); b.field("name", "@timestamp"); b.startArray("calendar_intervals"); b.value("day"); b.value("month"); b.endArray(); b.endObject(); + b.startArray("ordered_dimensions"); b.startObject(); b.field("name", dim); b.endObject(); @@ -1064,7 +1063,7 @@ private XContentBuilder getInvalidMappingWithDv( private XContentBuilder getInvalidMapping(boolean singleDim, boolean invalidSkipDims, boolean invalidDimType, boolean invalidMetricType) throws IOException { - return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false, false); + return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false, false, false); } protected boolean supportsOrIgnoresBoost() { From 9a4e7f3125ba5b01e35259e3daf58fc11f18df6e Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 9 Sep 2024 14:40:47 +0530 Subject: [PATCH 5/6] Adding support for date dim and adding back tests resolving conflicts Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 45 ++++++------ .../startree/builder/BaseStarTreeBuilder.java | 2 +- .../startree/index/StarTreeValues.java | 5 -- .../index/mapper/DateFieldMapper.java | 6 ++ .../builder/AbstractStarTreeBuilderTests.java | 5 +- .../index/mapper/StarTreeMapperTests.java | 68 +++++++++++-------- 6 files changed, 73 insertions(+), 58 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 1f396c97fd8ed..af0317ba31562 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -12,6 +12,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeUnit; @@ -22,6 +23,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; @@ -62,10 +64,7 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "star_tree") .startObject("config") .startObject("date_dimension") - .field("name", "numeric_dv_1") - .endObject() - .startObject() - .field("name", "numeric_dv_2") + .field("name", "timestamp") .endObject() .startArray("ordered_dimensions") .startObject() @@ -88,14 +87,6 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "integer") .field("doc_values", true) .endObject() - .startObject("numeric_dv_1") - .field("type", "integer") - .field("doc_values", true) - .endObject() - .startObject("numeric_dv_2") - .field("type", "integer") - .field("doc_values", true) - .endObject() .startObject("numeric") .field("type", "integer") .field("doc_values", false) @@ -268,7 +259,7 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .field("type", "star_tree") .startObject("config") .startObject("date_dimension") - .field("name", "numeric_dv1") + .field("name", "timestamp") .endObject() .startArray("ordered_dimensions") .startObject() @@ -291,10 +282,6 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea .field("type", "integer") .field("doc_values", true) .endObject() - .startObject("numeric_dv1") - .field("type", "integer") - .field("doc_values", true) - .endObject() .startObject("numeric") .field("type", "integer") .field("doc_values", false) @@ -327,7 +314,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "star_tree") .startObject("config") .startObject("date_dimension") - .field("name", "numeric_dv2") + .field("name", "timestamp") .endObject() .startArray("ordered_dimensions") .startObject() @@ -356,10 +343,6 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "integer") .field("doc_values", true) .endObject() - .startObject("numeric_dv2") - .field("type", "integer") - .field("doc_values", true) - .endObject() .startObject("numeric_dv1") .field("type", "integer") .field("doc_values", true) @@ -727,6 +710,24 @@ public void testMaxMetricsCompositeIndex() { ); } + public void testMaxCalendarIntervalsCompositeIndex() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) + .setSettings( + Settings.builder() + .put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1) + .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) + ) + .get() + ); + assertEquals( + "Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]", + ex.getMessage() + ); + } + public void testUnsupportedDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index a02f273c26937..5e5944838b98c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -445,7 +445,7 @@ protected void setReadersAndNumSegmentDocs( } // get doc id set iterators for metrics for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { - for (MetricStat metricStat : metric.getMetrics()) { + for (MetricStat metricStat : metric.getBaseMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( starTreeValues.getStarTreeField().getName(), metric.getField(), diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index aa2781b4c944b..a34bbbe9ee738 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -30,7 +30,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -249,8 +248,6 @@ public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { if (dimensionDocValuesIteratorMap.containsKey(dimension)) { return dimensionDocValuesIteratorMap.get(dimension).get(); - } else { - throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension %s not present", dimension)); } throw new IllegalArgumentException("dimension [" + dimension + "] does not exist in the segment."); @@ -266,8 +263,6 @@ public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricNam if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); - } else { - throw new IllegalArgumentException(String.format(Locale.ROOT, "Metric %s not present", fullyQualifiedMetricName)); } throw new IllegalArgumentException("metric [" + fullyQualifiedMetricName + "] does not exist in the segment."); diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index bc50f507010c4..4698440f08036 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -54,6 +54,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.LocaleUtils; +import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; import org.opensearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -76,6 +77,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -350,6 +352,10 @@ public DateFieldMapper build(BuilderContext context) { return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); } + @Override + public Optional getSupportedDataCubeDimensionType() { + return Optional.of(DimensionType.DATE); + } } public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 01634cddfcdd3..17c2c9ea5b4ef 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -4186,7 +4186,8 @@ public void testMergeFlow() throws IOException { builder.appendDocumentsToStarTree(starTreeDocumentIterator); for (StarTreeDocument starTreeDocument : builder.getStarTreeDocuments()) { assertEquals(starTreeDocument.dimensions[0] * 20.0, starTreeDocument.metrics[0]); - assertEquals(2L, starTreeDocument.metrics[1]); + assertEquals(starTreeDocument.dimensions[0] * 2, starTreeDocument.metrics[1]); + assertEquals(2L, starTreeDocument.metrics[2]); } builder.build(starTreeDocumentIterator, new AtomicInteger(), docValuesConsumer); @@ -4443,7 +4444,7 @@ private StarTreeValues getStarTreeValuesWithDates( ), () -> metricsList1 ); - return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of(SEGMENT_DOCS_COUNT, number)); + return new StarTreeValues(sf, null, dimDocIdSetIterators, metricDocIdSetIterators, Map.of(SEGMENT_DOCS_COUNT, number), null); } private StarTreeField getStarTreeFieldWithDateDimension() { diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 1cee6ad42d18f..dd3fbde1bea60 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -98,7 +98,7 @@ public void testValidStarTree() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("node", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -162,7 +162,7 @@ public void testMetricsWithJustSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("node", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -197,7 +197,7 @@ public void testMetricsWithCountAndSum() throws IOException { assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); assertEquals( - new HashSet<>(Arrays.asList("node", "status")), + new HashSet<>(Arrays.asList("@timestamp", "status")), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() ); } @@ -612,7 +612,7 @@ public void testValidations() throws IOException { ) ); assertEquals( - "Aggregations not supported for the dimension field [node] with field type [integer] as part of star tree field", + "Aggregations not supported for the dimension field [@timestamp] with field type [date] as part of star tree field", ex.getMessage() ); @@ -635,12 +635,16 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("node"); + b.value("@timestamp"); b.value("status"); } b.endArray(); b.startObject("date_dimension"); - b.field("name", "node"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value("day"); + b.value("month"); + b.endArray(); b.endObject(); b.startArray("ordered_dimensions"); b.startObject(); @@ -659,8 +663,8 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); - b.field("type", "integer"); + b.startObject("@timestamp"); + b.field("type", "date"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -681,7 +685,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "star_tree") .startObject("config") .startObject("date_dimension") - .field("name", "node") + .field("name", "timestamp") .endObject() .startArray("ordered_dimensions") .startObject() @@ -703,8 +707,8 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .endObject() .endObject() .startObject("properties") - .startObject("node") - .field("type", "integer") + .startObject("timestamp") + .field("type", "date") .endObject() .startObject("numeric_dv") .field("type", "integer") @@ -731,12 +735,16 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("node"); + b.value("@timestamp"); b.value("status"); } b.endArray(); b.startObject("date_dimension"); - b.field("name", "node"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value("day"); + b.value("month"); + b.endArray(); b.endObject(); b.startArray("ordered_dimensions"); b.startObject(); @@ -755,8 +763,8 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); - b.field("type", "integer"); + b.startObject("@timestamp"); + b.field("type", "date"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -777,12 +785,16 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.field("max_leaf_docs", 100); b.startArray("skip_star_node_creation_for_dimensions"); { - b.value("node"); + b.value("@timestamp"); b.value("status"); } b.endArray(); b.startObject("date_dimension"); - b.field("name", "node"); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value("day"); + b.value("month"); + b.endArray(); b.endObject(); b.startArray("ordered_dimensions"); b.startObject(); @@ -802,8 +814,8 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); - b.field("type", "integer"); + b.startObject("@timestamp"); + b.field("type", "date"); b.endObject(); b.startObject("status"); b.field("type", "integer"); @@ -902,7 +914,7 @@ private XContentBuilder getMinMapping( b.startObject("config"); if (!isEmptyDims) { b.startObject("date_dimension"); - b.field("name", "node"); + b.field("name", "@timestamp"); b.endObject(); b.startArray("ordered_dimensions"); b.startObject(); @@ -925,8 +937,8 @@ private XContentBuilder getMinMapping( b.endObject(); b.startObject("properties"); if (!missingDateDim) { - b.startObject("node"); - b.field("type", "integer"); + b.startObject("@timestamp"); + b.field("type", "date"); b.endObject(); } if (!missingDim) { @@ -1066,9 +1078,9 @@ private XContentBuilder getInvalidMapping( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); + b.startObject("@timestamp"); if (!invalidDate) { - b.field("type", "integer"); + b.field("type", "date"); } else { b.field("type", "keyword"); } @@ -1113,7 +1125,7 @@ private XContentBuilder getInvalidMappingWithDv( } b.endArray(); b.startObject("date_dimension"); - b.field("name", "node"); + b.field("name", "@timestamp"); b.endObject(); b.startArray("ordered_dimensions"); if (!singleDim) { @@ -1134,12 +1146,12 @@ private XContentBuilder getInvalidMappingWithDv( b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("node"); + b.startObject("@timestamp"); if (!invalidDimType) { - b.field("type", "integer"); + b.field("type", "date"); b.field("doc_values", "true"); } else { - b.field("type", "integer"); + b.field("type", "date"); b.field("doc_values", "false"); } b.endObject(); From 49dd2f5966b35864fb8fab2c02dc522ccec16d73 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 25 Sep 2024 18:48:04 +0530 Subject: [PATCH 6/6] star tree keyword changes Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 32 +++++--- .../Composite99DocValuesWriter.java | 54 +++++++++---- .../datacube/DimensionFactory.java | 28 ++++--- .../datacube/DimensionType.java | 8 +- .../datacube/KeywordDimension.java | 75 +++++++++++++++++++ .../builder/StarTreeDocsFileManager.java | 3 + .../index/mapper/KeywordFieldMapper.java | 7 ++ .../builder/AbstractStarTreeBuilderTests.java | 12 ++- .../index/mapper/StarTreeMapperTests.java | 43 ++++++++++- 9 files changed, 224 insertions(+), 38 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index af0317ba31562..9be7849e9208c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -56,7 +56,7 @@ public class StarTreeMapperIT extends OpenSearchIntegTestCase { .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)) .build(); - private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { + private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean ipdim) { try { return jsonBuilder().startObject() .startObject("composite") @@ -68,12 +68,15 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .endObject() .startArray("ordered_dimensions") .startObject() - .field("name", getDim(invalidDim, keywordDim)) + .field("name", getDim(invalidDim, ipdim)) + .endObject() + .startObject() + .field("name", "keyword_dv") .endObject() .endArray() .startArray("metrics") .startObject() - .field("name", getDim(invalidMetric, false)) + .field("name", getMetric(invalidMetric, false)) .endObject() .endArray() .endObject() @@ -99,6 +102,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool .field("type", "keyword") .field("doc_values", false) .endObject() + .startObject("ip") + .field("type", "ip") + .field("doc_values", false) + .endObject() .endObject() .endObject(); } catch (IOException e) { @@ -356,10 +363,19 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo } private static String getDim(boolean hasDocValues, boolean isKeyword) { + if (hasDocValues) { + return random().nextBoolean() ? "numeric" : "keyword"; + } else if (isKeyword) { + return "ip"; + } + return "numeric_dv"; + } + + private static String getMetric(boolean hasDocValues, boolean isKeyword) { if (hasDocValues) { return "numeric"; } else if (isKeyword) { - return "keyword"; + return "ip"; } return "numeric_dv"; } @@ -398,6 +414,7 @@ public void testValidCompositeIndex() { assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName()); } assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("keyword_dv", starTreeFieldType.getDimensions().get(2).getField()); assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG); assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); @@ -665,10 +682,7 @@ public void testInvalidDimCompositeIndex() { IllegalArgumentException.class, () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(true, false, false)).get() ); - assertEquals( - "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of star tree field", - ex.getMessage() - ); + assertTrue(ex.getMessage().startsWith("Aggregations not supported for the dimension field ")); } public void testMaxDimsCompositeIndex() { @@ -734,7 +748,7 @@ public void testUnsupportedDim() { () -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, true)).get() ); assertEquals( - "Failed to parse mapping [_doc]: unsupported field type associated with dimension [keyword] as part of star tree field [startree-1]", + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [ip] as part of star tree field [startree-1]", ex.getMessage() ); } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java index 0d4e35f7c3ab8..d94d89ff0dfc2 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99DocValuesWriter.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; @@ -34,6 +35,7 @@ import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.DocCountFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.StarTreeMapper; @@ -82,14 +84,7 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes(); compositeFieldSet = new HashSet<>(); segmentFieldSet = new HashSet<>(); - // TODO : add integ test for this - for (FieldInfo fi : this.state.fieldInfos) { - if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) { - segmentFieldSet.add(fi.name); - } else if (fi.name.equals(DocCountFieldMapper.NAME)) { - segmentFieldSet.add(fi.name); - } - } + addStarTreeSupportedFieldsFromSegment(); for (CompositeMappedFieldType type : compositeMappedFieldTypes) { compositeFieldSet.addAll(type.fields()); } @@ -148,6 +143,19 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentHasCompositeFields = Collections.disjoint(segmentFieldSet, compositeFieldSet) == false; } + private void addStarTreeSupportedFieldsFromSegment() { + // TODO : add integ test for this + for (FieldInfo fi : this.state.fieldInfos) { + if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) { + segmentFieldSet.add(fi.name); + } else if (DocValuesType.SORTED_SET.equals(fi.getDocValuesType())) { + segmentFieldSet.add(fi.name); + } else if (fi.name.equals(DocCountFieldMapper.NAME)) { + segmentFieldSet.add(fi.name); + } + } + } + @Override public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegate.addNumericField(field, valuesProducer); @@ -179,6 +187,10 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu @Override public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegate.addSortedSetField(field, valuesProducer); + // Perform this only during flush flow + if (mergeState.get() == null && segmentHasCompositeFields) { + createCompositeIndicesIfPossible(valuesProducer, field); + } } @Override @@ -235,6 +247,7 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, * Add empty doc values for fields not present in segment */ private void addDocValuesForEmptyField(String compositeField) { + // special case for doc count if (compositeField.equals(DocCountFieldMapper.NAME)) { fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { @Override @@ -243,16 +256,29 @@ public NumericDocValues getNumeric(FieldInfo field) { } }); } else { - fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) { - return DocValues.emptySortedNumeric(); - } - }); + if(isSortedSetField(compositeField)) { + fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { + @Override + public SortedSetDocValues getSortedSet(FieldInfo field) { + return DocValues.emptySortedSet(); + } + }); + } else { + fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + return DocValues.emptySortedNumeric(); + } + }); + } } compositeFieldSet.remove(compositeField); } + private boolean isSortedSetField(String field) { + return mapperService.fieldType(field) instanceof KeywordFieldMapper.KeywordFieldType; + } + @Override public void merge(MergeState mergeState) throws IOException { this.mergeState.compareAndSet(null, mergeState); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java index 7e72a3f0d9de6..e834706e2fa9d 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java @@ -24,6 +24,7 @@ import java.util.stream.Collectors; import static org.opensearch.index.compositeindex.datacube.DateDimension.CALENDAR_INTERVALS; +import static org.opensearch.index.compositeindex.datacube.KeywordDimension.KEYWORD; /** * Dimension factory class mainly used to parse and create dimension from the mappings @@ -43,6 +44,8 @@ public static Dimension parseAndCreateDimension( return parseAndCreateDateDimension(name, dimensionMap, c); case NumericDimension.NUMERIC: return new NumericDimension(name); + case KEYWORD: + return new KeywordDimension(name); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "unsupported field type associated with dimension [%s] as part of star tree field", name) @@ -56,16 +59,23 @@ public static Dimension parseAndCreateDimension( Map dimensionMap, Mapper.TypeParser.ParserContext c ) { - if (builder.getSupportedDataCubeDimensionType().isPresent() - && builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.DATE)) { - return parseAndCreateDateDimension(name, dimensionMap, c); - } else if (builder.getSupportedDataCubeDimensionType().isPresent() - && builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.NUMERIC)) { + if (builder.getSupportedDataCubeDimensionType().isEmpty()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) + ); + } + switch (builder.getSupportedDataCubeDimensionType().get()) { + case DATE: + return parseAndCreateDateDimension(name, dimensionMap, c); + case NUMERIC: return new NumericDimension(name); - } - throw new IllegalArgumentException( - String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) - ); + case KEYWORD: + return new KeywordDimension(name); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) + ); + } } private static DateDimension parseAndCreateDateDimension( diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java index 4b9faea331752..d327f8ca1fa1e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionType.java @@ -27,5 +27,11 @@ public enum DimensionType { * Represents a date dimension type. * This is used for dimensions that contain date or timestamp values. */ - DATE + DATE, + + /** + * Represents a keyword dimension type. + * This is used for dimensions that contain keyword ordinals. + */ + KEYWORD } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java new file mode 100644 index 0000000000000..dd6cedade43ff --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/KeywordDimension.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Composite index keyword dimension class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class KeywordDimension implements Dimension { + public static final String KEYWORD = "keyword"; + private final String field; + + public KeywordDimension(String field) { + this.field = field; + } + + @Override + public String getField() { + return field; + } + + @Override + public int getNumSubDimensions() { + return 1; + } + + @Override + public int setDimensionValues(Long value, Long[] dims, int index) { + dims[index++] = value; + return index; + } + + @Override + public List getDimensionFieldsNames() { + return List.of(field); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CompositeDataCubeFieldType.NAME, field); + builder.field(CompositeDataCubeFieldType.TYPE, KEYWORD); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KeywordDimension dimension = (KeywordDimension) o; + return Objects.equals(field, dimension.getField()); + } + + @Override + public int hashCode() { + return Objects.hash(field); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 7e920b912731d..98c3e5c6d71e6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -14,6 +14,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RandomAccessInput; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; @@ -45,7 +46,9 @@ *

The set of 'star-tree.documents' files is maintained, and a tracker array is used to keep track of the start document ID for each file. * Once the number of files reaches a set threshold, the files are merged. * + * @opensearch.experimental */ +@ExperimentalApi public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implements Closeable { private static final Logger logger = LogManager.getLogger(StarTreeDocsFileManager.class); private static final String STAR_TREE_DOC_FILE_NAME = "star-tree.documents"; diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 11ff601b3fd6d..13577d3c3aa56 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -59,6 +59,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.analysis.IndexAnalyzers; import org.opensearch.index.analysis.NamedAnalyzer; +import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.opensearch.index.query.QueryShardContext; @@ -73,6 +74,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Supplier; import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -254,6 +256,11 @@ public KeywordFieldMapper build(BuilderContext context) { this ); } + + @Override + public Optional getSupportedDataCubeDimensionType() { + return Optional.of(DimensionType.KEYWORD); + } } public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers())); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 329487961315f..49ecdd60460c8 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -4264,9 +4264,15 @@ public void testFlushFlowWithTimestamps() throws IOException { compositeField = getStarTreeFieldWithDateDimension(); SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); - SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList2, docsWithField2)); - SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricsList, metricsWithField)); - SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricsList, metricsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index dd3fbde1bea60..abce94092330e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -672,6 +672,9 @@ private XContentBuilder getExpandedMappingWithJustAvg(String dim, String metric) b.startObject("size"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -718,6 +721,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo .field("type", "integer") .field("doc_values", true) .endObject() + .endObject() .endObject(); } catch (IOException e) { @@ -772,6 +776,9 @@ private XContentBuilder getExpandedMappingWithJustSum(String dim, String metric) b.startObject("size"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -823,6 +830,9 @@ private XContentBuilder getExpandedMappingWithSumAndCount(String dim, String met b.startObject("size"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -866,6 +876,9 @@ private XContentBuilder getMinMappingWithDateDims(boolean calendarIntervalsExcee b.startObject(); b.field("name", "metric_field"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); } b.endArray(); @@ -895,6 +908,9 @@ private XContentBuilder getMinMappingWithDateDims(boolean calendarIntervalsExcee b.startObject("metric_field"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); @@ -920,6 +936,9 @@ private XContentBuilder getMinMapping( b.startObject(); b.field("name", "status"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); b.endArray(); } if (!isEmptyMetrics) { @@ -951,6 +970,9 @@ private XContentBuilder getMinMapping( b.field("type", "integer"); b.endObject(); } + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1018,7 +1040,9 @@ private XContentBuilder getMinMappingWith2StarTrees() throws IOException { b.startObject("metric_field"); b.field("type", "integer"); b.endObject(); - + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1058,6 +1082,9 @@ private XContentBuilder getInvalidMapping( b.startObject(); b.field("name", "status"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); } b.endArray(); b.startArray("metrics"); @@ -1090,7 +1117,7 @@ private XContentBuilder getInvalidMapping( if (!invalidDimType) { b.field("type", "integer"); } else { - b.field("type", "keyword"); + b.field("type", "ip"); } b.endObject(); b.startObject("metric_field"); @@ -1100,6 +1127,9 @@ private XContentBuilder getInvalidMapping( b.field("type", "integer"); } b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1132,6 +1162,9 @@ private XContentBuilder getInvalidMappingWithDv( b.startObject(); b.field("name", "status"); b.endObject(); + b.startObject(); + b.field("name", "keyword1"); + b.endObject(); } b.endArray(); b.startArray("metrics"); @@ -1168,6 +1201,9 @@ private XContentBuilder getInvalidMappingWithDv( b.field("doc_values", "true"); } b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }); } @@ -1224,6 +1260,9 @@ public void testEmptyName() { b.startObject("status"); b.field("type", "integer"); b.endObject(); + b.startObject("keyword1"); + b.field("type", "keyword"); + b.endObject(); b.endObject(); }))); assertThat(e.getMessage(), containsString("name cannot be empty string"));