From 3de2906c8d0c27b5b8394adab477d5cfcb384cb4 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 22 Aug 2023 14:47:17 +0530 Subject: [PATCH] feature flagging of Zstandard Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 3 ++- distribution/src/config/opensearch.yml | 6 ++++++ .../common/settings/FeatureFlagSettings.java | 1 + .../org/opensearch/common/util/FeatureFlags.java | 7 +++++++ .../org/opensearch/index/codec/CodecService.java | 16 ++++++++++++---- .../opensearch/index/engine/EngineConfig.java | 10 +++++++++- .../org/opensearch/index/codec/CodecTests.java | 10 ++++++++++ .../opensearch/test/OpenSearchIntegTestCase.java | 1 + 8 files changed, 48 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ab292192e64..339f0029ccb2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Make SearchTemplateRequest implement IndicesRequest.Replaceable ([#9122]()https://github.com/opensearch-project/OpenSearch/pull/9122) - [BWC and API enforcement] Define the initial set of annotations, their meaning and relations between them ([#9223](https://github.com/opensearch-project/OpenSearch/pull/9223)) - [Segment Replication] Support realtime reads for GET requests ([#9212](https://github.com/opensearch-project/OpenSearch/pull/9212)) +- Feature flagging of the ZStandard ([#9476](https://github.com/opensearch-project/OpenSearch/pull/9476)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) @@ -162,4 +163,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 3c4fe822005e0..c5b8a01070f48 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -129,3 +129,9 @@ ${path.logs} # index searcher threadpool. # #opensearch.experimental.feature.concurrent_segment_search.enabled: false +# +# +# Gates the visibility of the ZStd compression algorithm features +# for indexing operations. +# +#opensearch.experimental.feature.compression.zstd.enabled: false diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index e109d2a871cef..53f855ceb635c 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -37,6 +37,7 @@ protected FeatureFlagSettings( Arrays.asList( FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL_SETTING, FeatureFlags.REMOTE_STORE_SETTING, + FeatureFlags.ZSTD_COMPRESSION_SETTING, FeatureFlags.EXTENSIONS_SETTING, FeatureFlags.IDENTITY_SETTING, FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING, diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index e2663b56c5cca..4d03e68603e77 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -21,6 +21,11 @@ */ public class FeatureFlags { + /** + * Gates the visibility of the index settings that allows the utilization of ZStd compression algorithm features for indexing operations. + */ + public static final String ZSTD_COMPRESSION = "opensearch.experimental.feature.compression.zstd.enabled"; + /** * Gates the visibility of the segment replication experimental features that allows users to test unreleased beta features. */ @@ -96,6 +101,8 @@ public static boolean isEnabled(String featureFlagName) { Property.NodeScope ); + public static final Setting ZSTD_COMPRESSION_SETTING = Setting.boolSetting(ZSTD_COMPRESSION, false, Property.NodeScope); + public static final Setting REMOTE_STORE_SETTING = Setting.boolSetting(REMOTE_STORE, false, Property.NodeScope); public static final Setting EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope); diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index a1913d5fa2061..3ce396da09b3c 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -37,7 +37,9 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec; import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.customcodecs.ZstdCodec; import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec; @@ -67,7 +69,9 @@ public class CodecService { * the raw unfiltered lucene default. useful for testing */ public static final String LUCENE_DEFAULT_CODEC = "lucene_default"; + @ExperimentalApi public static final String ZSTD_CODEC = "zstd"; + @ExperimentalApi public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) { @@ -79,15 +83,19 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe codecs.put(LZ4, new Lucene95Codec()); codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); codecs.put(ZLIB, new Lucene95Codec(Mode.BEST_COMPRESSION)); - codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); + if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); + } } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); + if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); + } } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 3351931a6b068..2fab14be6f0a3 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -45,6 +45,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.indices.breaker.CircuitBreakerService; @@ -133,9 +134,13 @@ public Supplier retentionLeasesSupplier() { case "lz4": case "best_compression": case "zlib": + case "lucene_default": + return s; case "zstd": case "zstd_no_dict": - case "lucene_default": + if (!FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + throw new IllegalArgumentException("ZStandard feature must be enabled to set " + s + " codec."); + } return s; default: if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones @@ -183,6 +188,9 @@ private static void doValidateCodecSettings(final String codec) { switch (codec) { case "zstd": case "zstd_no_dict": + if (!FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + throw new IllegalArgumentException("Compression level cannot be set before enabling the ZStandard feature."); + } return; case "best_compression": case "zlib": diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 3ca759eaf0781..ce8ca5e27355d 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -45,6 +45,7 @@ import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.IndexAnalyzers; @@ -55,6 +56,7 @@ import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.plugins.MapperPlugin; +import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; @@ -96,6 +98,7 @@ public void testZlib() throws Exception { } public void testZstd() throws Exception { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); Codec codec = createCodecService(false).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); @@ -103,6 +106,7 @@ public void testZstd() throws Exception { } public void testZstdNoDict() throws Exception { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); Codec codec = createCodecService(false).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); @@ -110,6 +114,7 @@ public void testZstdNoDict() throws Exception { } public void testZstdWithCompressionLevel() throws Exception { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); @@ -118,6 +123,7 @@ public void testZstdWithCompressionLevel() throws Exception { } public void testZstdNoDictWithCompressionLevel() throws Exception { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); @@ -126,6 +132,7 @@ public void testZstdNoDictWithCompressionLevel() throws Exception { } public void testBestCompressionWithCompressionLevel() { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); final Settings zstdSettings = Settings.builder() .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC)) @@ -179,6 +186,7 @@ public void testLuceneCodecsWithCompressionLevel() { } public void testZstandardCompressionLevelSupport() throws Exception { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); CodecService codecService = createCodecService(false); CodecSettings zstdCodec = (CodecSettings) codecService.codec("zstd"); CodecSettings zstdNoDictCodec = (CodecSettings) codecService.codec("zstd_no_dict"); @@ -197,6 +205,7 @@ public void testBestCompressionMapperServiceNull() throws Exception { } public void testZstdMapperServiceNull() throws Exception { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); Codec codec = createCodecService(true).codec("zstd"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); @@ -204,6 +213,7 @@ public void testZstdMapperServiceNull() throws Exception { } public void testZstdNoDictMapperServiceNull() throws Exception { + FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION); Codec codec = createCodecService(true).codec("zstd_no_dict"); assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 382152652cafe..c2cc438a79acb 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -787,6 +787,7 @@ protected Settings featureFlagSettings() { } // Enabling Telemetry setting by default featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true); + featureSettings.put(FeatureFlags.ZSTD_COMPRESSION_SETTING.getKey(), true); return featureSettings.build(); }