From 7cea476a94c11aacb76d438319ef80af5a985655 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 22 Aug 2023 14:47:17 +0530 Subject: [PATCH] feature flagging of Zstandard Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 3 ++- distribution/src/config/opensearch.yml | 6 ++++++ .../common/settings/FeatureFlagSettings.java | 1 + .../org/opensearch/common/util/FeatureFlags.java | 12 ++++++++++++ .../org/opensearch/index/codec/CodecService.java | 16 ++++++++++++---- .../opensearch/index/engine/EngineConfig.java | 10 +++++++++- .../opensearch/test/OpenSearchIntegTestCase.java | 1 + 7 files changed, 43 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ab292192e64..339f0029ccb2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Make SearchTemplateRequest implement IndicesRequest.Replaceable ([#9122]()https://github.com/opensearch-project/OpenSearch/pull/9122) - [BWC and API enforcement] Define the initial set of annotations, their meaning and relations between them ([#9223](https://github.com/opensearch-project/OpenSearch/pull/9223)) - [Segment Replication] Support realtime reads for GET requests ([#9212](https://github.com/opensearch-project/OpenSearch/pull/9212)) +- Feature flagging of the ZStandard ([#9476](https://github.com/opensearch-project/OpenSearch/pull/9476)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) @@ -162,4 +163,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x \ No newline at end of file +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 3c4fe822005e0..c5b8a01070f48 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -129,3 +129,9 @@ ${path.logs} # index searcher threadpool. # #opensearch.experimental.feature.concurrent_segment_search.enabled: false +# +# +# Gates the visibility of the ZStd compression algorithm features +# for indexing operations. +# +#opensearch.experimental.feature.compression.zstd.enabled: false diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index e109d2a871cef..53f855ceb635c 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -37,6 +37,7 @@ protected FeatureFlagSettings( Arrays.asList( FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL_SETTING, FeatureFlags.REMOTE_STORE_SETTING, + FeatureFlags.ZSTD_COMPRESSION_SETTING, FeatureFlags.EXTENSIONS_SETTING, FeatureFlags.IDENTITY_SETTING, FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING, diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index e2663b56c5cca..4db526593edb0 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -21,6 +21,11 @@ */ public class FeatureFlags { + /** + * Gates the visibility of the index settings that allows the utilization of ZStd compression algorithm features for indexing operations. + */ + public static final String ZSTD_COMPRESSION = "opensearch.experimental.feature.compression.zstd.enabled"; + /** * Gates the visibility of the segment replication experimental features that allows users to test unreleased beta features. */ @@ -96,6 +101,13 @@ public static boolean isEnabled(String featureFlagName) { Property.NodeScope ); + public static final Setting ZSTD_COMPRESSION_SETTING = Setting.boolSetting( + ZSTD_COMPRESSION, + false, + Property.NodeScope, + Property.Dynamic + ); + public static final Setting REMOTE_STORE_SETTING = Setting.boolSetting(REMOTE_STORE, false, Property.NodeScope); public static final Setting EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope); diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index a1913d5fa2061..3ce396da09b3c 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -37,7 +37,9 @@ import org.apache.lucene.codecs.lucene95.Lucene95Codec; import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode; import org.opensearch.common.Nullable; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.customcodecs.ZstdCodec; import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec; @@ -67,7 +69,9 @@ public class CodecService { * the raw unfiltered lucene default. useful for testing */ public static final String LUCENE_DEFAULT_CODEC = "lucene_default"; + @ExperimentalApi public static final String ZSTD_CODEC = "zstd"; + @ExperimentalApi public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) { @@ -79,15 +83,19 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe codecs.put(LZ4, new Lucene95Codec()); codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION)); codecs.put(ZLIB, new Lucene95Codec(Mode.BEST_COMPRESSION)); - codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); + if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); + } } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); + if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); + } } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 3351931a6b068..2fab14be6f0a3 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -45,6 +45,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.indices.breaker.CircuitBreakerService; @@ -133,9 +134,13 @@ public Supplier retentionLeasesSupplier() { case "lz4": case "best_compression": case "zlib": + case "lucene_default": + return s; case "zstd": case "zstd_no_dict": - case "lucene_default": + if (!FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + throw new IllegalArgumentException("ZStandard feature must be enabled to set " + s + " codec."); + } return s; default: if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones @@ -183,6 +188,9 @@ private static void doValidateCodecSettings(final String codec) { switch (codec) { case "zstd": case "zstd_no_dict": + if (!FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) { + throw new IllegalArgumentException("Compression level cannot be set before enabling the ZStandard feature."); + } return; case "best_compression": case "zlib": diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 382152652cafe..c2cc438a79acb 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -787,6 +787,7 @@ protected Settings featureFlagSettings() { } // Enabling Telemetry setting by default featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true); + featureSettings.put(FeatureFlags.ZSTD_COMPRESSION_SETTING.getKey(), true); return featureSettings.build(); }