feature flagging of Zstandard

Signed-off-by: Sarthak Aggarwal <[email protected]>

CHANGELOG.md

@@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Make SearchTemplateRequest implement IndicesRequest.Replaceable ([#9122]()https://github.com/opensearch-project/OpenSearch/pull/9122)
 - [BWC and API enforcement] Define the initial set of annotations, their meaning and relations between them ([#9223](https://github.com/opensearch-project/OpenSearch/pull/9223))
 - [Segment Replication] Support realtime reads for GET requests ([#9212](https://github.com/opensearch-project/OpenSearch/pull/9212))
+- Feature flagging of the ZStandard ([#9476](https://github.com/opensearch-project/OpenSearch/pull/9476))
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))
@@ -162,4 +163,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x

distribution/src/config/opensearch.yml

@@ -129,3 +129,9 @@ ${path.logs}
 # index searcher threadpool.
 #
 #opensearch.experimental.feature.concurrent_segment_search.enabled: false
+#
+#
+# Gates the visibility of the ZStd compression algorithm features
+#  for indexing operations.
+#
+#opensearch.experimental.feature.compression.zstd.enabled: false

server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java

@@ -37,6 +37,7 @@ protected FeatureFlagSettings(
             Arrays.asList(
                 FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL_SETTING,
                 FeatureFlags.REMOTE_STORE_SETTING,
+                FeatureFlags.ZSTD_COMPRESSION_SETTING,
                 FeatureFlags.EXTENSIONS_SETTING,
                 FeatureFlags.IDENTITY_SETTING,
                 FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING,

server/src/main/java/org/opensearch/common/util/FeatureFlags.java

@@ -21,6 +21,11 @@
  */
 public class FeatureFlags {
 
+    /**
+     * Gates the visibility of the index settings that allows the utilization of ZStd compression algorithm features for indexing operations.
+     */
+    public static final String ZSTD_COMPRESSION = "opensearch.experimental.feature.compression.zstd.enabled";
+
     /**
      * Gates the visibility of the segment replication experimental features that allows users to test unreleased beta features.
      */
@@ -96,6 +101,8 @@ public static boolean isEnabled(String featureFlagName) {
         Property.NodeScope
     );
 
+    public static final Setting<Boolean> ZSTD_COMPRESSION_SETTING = Setting.boolSetting(ZSTD_COMPRESSION, false, Property.NodeScope);
+
     public static final Setting<Boolean> REMOTE_STORE_SETTING = Setting.boolSetting(REMOTE_STORE, false, Property.NodeScope);
 
     public static final Setting<Boolean> EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope);

server/src/main/java/org/opensearch/index/codec/CodecService.java

@@ -37,7 +37,9 @@
 import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.collect.MapBuilder;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.customcodecs.ZstdCodec;
 import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec;
@@ -67,7 +69,9 @@ public class CodecService {
      * the raw unfiltered lucene default. useful for testing
      */
     public static final String LUCENE_DEFAULT_CODEC = "lucene_default";
+    @ExperimentalApi
     public static final String ZSTD_CODEC = "zstd";
+    @ExperimentalApi
     public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict";
 
     public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) {
@@ -79,15 +83,19 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
             codecs.put(LZ4, new Lucene95Codec());
             codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
             codecs.put(ZLIB, new Lucene95Codec(Mode.BEST_COMPRESSION));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+            if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) {
+                codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
+                codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+            }
         } else {
             codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
             codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) {
+                codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
+                codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            }
         }
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {

server/src/main/java/org/opensearch/index/engine/EngineConfig.java

@@ -45,6 +45,7 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.unit.MemorySizeValue;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
@@ -133,9 +134,13 @@ public Supplier<RetentionLeases> retentionLeasesSupplier() {
             case "lz4":
             case "best_compression":
             case "zlib":
+            case "lucene_default":
+                return s;
             case "zstd":
             case "zstd_no_dict":
-            case "lucene_default":
+                if (!FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) {
+                    throw new IllegalArgumentException("ZStandard feature must be enabled to set " + s + " codec.");
+                }
                 return s;
             default:
                 if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones
@@ -183,6 +188,9 @@ private static void doValidateCodecSettings(final String codec) {
         switch (codec) {
             case "zstd":
             case "zstd_no_dict":
+                if (!FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) {
+                    throw new IllegalArgumentException("Compression level cannot be set before enabling the ZStandard feature.");
+                }
                 return;
             case "best_compression":
             case "zlib":

server/src/test/java/org/opensearch/index/codec/CodecTests.java

@@ -45,6 +45,7 @@
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.env.Environment;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.IndexAnalyzers;
@@ -55,6 +56,7 @@
 import org.opensearch.index.similarity.SimilarityService;
 import org.opensearch.indices.mapper.MapperRegistry;
 import org.opensearch.plugins.MapperPlugin;
+import org.opensearch.test.FeatureFlagSetter;
 import org.opensearch.test.IndexSettingsModule;
 import org.opensearch.test.OpenSearchTestCase;
 
@@ -96,20 +98,23 @@ public void testZlib() throws Exception {
     }
 
     public void testZstd() throws Exception {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         Codec codec = createCodecService(false).codec("zstd");
         assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec);
         Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat();
         assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
     }
 
     public void testZstdNoDict() throws Exception {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         Codec codec = createCodecService(false).codec("zstd_no_dict");
         assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec);
         Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat();
         assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
     }
 
     public void testZstdWithCompressionLevel() throws Exception {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         int randomCompressionLevel = randomIntBetween(1, 6);
         Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd");
         assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec);
@@ -118,6 +123,7 @@ public void testZstdWithCompressionLevel() throws Exception {
     }
 
     public void testZstdNoDictWithCompressionLevel() throws Exception {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         int randomCompressionLevel = randomIntBetween(1, 6);
         Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict");
         assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec);
@@ -126,6 +132,7 @@ public void testZstdNoDictWithCompressionLevel() throws Exception {
     }
 
     public void testBestCompressionWithCompressionLevel() {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         final Settings zstdSettings = Settings.builder()
             .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6))
             .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC))
@@ -179,6 +186,7 @@ public void testLuceneCodecsWithCompressionLevel() {
     }
 
     public void testZstandardCompressionLevelSupport() throws Exception {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         CodecService codecService = createCodecService(false);
         CodecSettings zstdCodec = (CodecSettings) codecService.codec("zstd");
         CodecSettings zstdNoDictCodec = (CodecSettings) codecService.codec("zstd_no_dict");
@@ -197,13 +205,15 @@ public void testBestCompressionMapperServiceNull() throws Exception {
     }
 
     public void testZstdMapperServiceNull() throws Exception {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         Codec codec = createCodecService(true).codec("zstd");
         assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec);
         Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat();
         assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel());
     }
 
     public void testZstdNoDictMapperServiceNull() throws Exception {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
         Codec codec = createCodecService(true).codec("zstd_no_dict");
         assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec);
         Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat();

test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java

@@ -787,6 +787,7 @@ protected Settings featureFlagSettings() {
         }
         // Enabling Telemetry setting by default
         featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true);
+        featureSettings.put(FeatureFlags.ZSTD_COMPRESSION_SETTING.getKey(), true);
         return featureSettings.build();
     }