Feature flag for ZStandard

CHANGELOG.md

@@ -162,6 +162,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Remote Store] Rate limiter integration for remote store uploads and downloads([#9448](https://github.com/opensearch-project/OpenSearch/pull/9448/))
 - [Remote Store] Implicitly use replication type SEGMENT for remote store clusters ([#9264](https://github.com/opensearch-project/OpenSearch/pull/9264))
 - Use non-concurrent path for sort request on timeseries index and field([#9562](https://github.com/opensearch-project/OpenSearch/pull/9562))
+- Move support for ZStd compression behind a feature flag ([#9476](https://github.com/opensearch-project/OpenSearch/pull/9476))
 
 ### Deprecated
 
@@ -180,4 +181,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x

distribution/src/config/opensearch.yml

@@ -129,3 +129,9 @@ ${path.logs}
 # index searcher threadpool.
 #
 #opensearch.experimental.feature.concurrent_segment_search.enabled: false
+#
+#
+# Gates the visibility of the ZStd compression algorithm features
+#  for indexing operations.
+#
+#opensearch.experimental.feature.compression.zstd.enabled: false

server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java

@@ -37,6 +37,7 @@ protected FeatureFlagSettings(
             Arrays.asList(
                 FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL_SETTING,
                 FeatureFlags.REMOTE_STORE_SETTING,
+                FeatureFlags.ZSTD_COMPRESSION_SETTING,
                 FeatureFlags.EXTENSIONS_SETTING,
                 FeatureFlags.IDENTITY_SETTING,
                 FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING,

server/src/main/java/org/opensearch/common/util/FeatureFlags.java

@@ -21,6 +21,11 @@
  */
 public class FeatureFlags {
 
+    /**
+     * Gates the visibility of the index settings that allows the utilization of ZStd compression algorithm features for indexing operations.
+     */
+    public static final String ZSTD_COMPRESSION = "opensearch.experimental.feature.compression.zstd.enabled";
+
     /**
      * Gates the visibility of the segment replication experimental features that allows users to test unreleased beta features.
      */
@@ -96,6 +101,8 @@ public static boolean isEnabled(String featureFlagName) {
         Property.NodeScope
     );
 
+    public static final Setting<Boolean> ZSTD_COMPRESSION_SETTING = Setting.boolSetting(ZSTD_COMPRESSION, false, Property.NodeScope);
+
     public static final Setting<Boolean> REMOTE_STORE_SETTING = Setting.boolSetting(REMOTE_STORE, false, Property.NodeScope);
 
     public static final Setting<Boolean> EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope);

server/src/main/java/org/opensearch/index/codec/CodecService.java

@@ -37,7 +37,9 @@
 import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.collect.MapBuilder;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.customcodecs.ZstdCodec;
 import org.opensearch.index.codec.customcodecs.ZstdNoDictCodec;
@@ -67,7 +69,9 @@ public class CodecService {
      * the raw unfiltered lucene default. useful for testing
      */
     public static final String LUCENE_DEFAULT_CODEC = "lucene_default";
+    @ExperimentalApi
     public static final String ZSTD_CODEC = "zstd";
+    @ExperimentalApi
     public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict";
 
     public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) {
@@ -79,15 +83,19 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
             codecs.put(LZ4, new Lucene95Codec());
             codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
             codecs.put(ZLIB, new Lucene95Codec(Mode.BEST_COMPRESSION));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+            if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) {
+                codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
+                codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+            }
         } else {
             codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
             codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION)) {
+                codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
+                codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            }
         }
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {

server/src/main/java/org/opensearch/index/engine/EngineConfig.java

@@ -45,6 +45,7 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.unit.MemorySizeValue;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
@@ -133,9 +134,17 @@ public Supplier<RetentionLeases> retentionLeasesSupplier() {
             case "lz4":
             case "best_compression":
             case "zlib":
+            case "lucene_default":
+                return s;
             case "zstd":
             case "zstd_no_dict":
-            case "lucene_default":
+                if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION) == false) {
+                    throw new IllegalArgumentException(
+                        "ZStandard must be enabled via [opensearch.experimental.feature.compression.zstd.enabled] feature flag to set "
+                            + s
+                            + " codec."
+                    );
+                }
                 return s;
             default:
                 if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones
@@ -183,6 +192,11 @@ private static void doValidateCodecSettings(final String codec) {
         switch (codec) {
             case "zstd":
             case "zstd_no_dict":
+                if (FeatureFlags.isEnabled(FeatureFlags.ZSTD_COMPRESSION) == false) {
+                    throw new IllegalArgumentException(
+                        "ZStandard must be enabled via [opensearch.experimental.feature.compression.zstd.enabled] feature flag to set compression level."
+                    );
+                }
                 return;
             case "best_compression":
             case "zlib":

server/src/test/java/org/opensearch/index/codec/CodecTests.java

@@ -45,6 +45,7 @@
 import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.env.Environment;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.IndexAnalyzers;
@@ -55,8 +56,10 @@
 import org.opensearch.index.similarity.SimilarityService;
 import org.opensearch.indices.mapper.MapperRegistry;
 import org.opensearch.plugins.MapperPlugin;
+import org.opensearch.test.FeatureFlagSetter;
 import org.opensearch.test.IndexSettingsModule;
 import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -67,6 +70,11 @@
 @SuppressCodecs("*") // we test against default codec so never get a random one here!
 public class CodecTests extends OpenSearchTestCase {
 
+    @Before
+    public void enableZstd() {
+        FeatureFlagSetter.set(FeatureFlags.ZSTD_COMPRESSION);
+    }
+
     public void testResolveDefaultCodecs() throws Exception {
         CodecService codecService = createCodecService(false);
         assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class));
@@ -148,6 +156,20 @@ public void testBestCompressionWithCompressionLevel() {
         assertTrue(e.getMessage().startsWith("Compression level cannot be set"));
     }
 
+    public void testZstdFeatureFlag() {
+        FeatureFlagSetter.clear();
+        final Settings zstdSettings = Settings.builder()
+            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(CodecService.ZSTD_CODEC, CodecService.ZSTD_NO_DICT_CODEC))
+            .build();
+
+        IndexScopedSettings zstdIndexScopedSettings = new IndexScopedSettings(zstdSettings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> zstdIndexScopedSettings.validate(zstdSettings, true)
+        );
+        assertTrue(e.getMessage().startsWith("ZStandard must be enabled via"));
+    }
+
     public void testLuceneCodecsWithCompressionLevel() {
         String codecName = randomFrom(Codec.availableCodecs());
         Codec codec = Codec.forName(codecName);

test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java

@@ -787,6 +787,8 @@ protected Settings featureFlagSettings() {
         }
         // Enabling Telemetry setting by default
         featureSettings.put(FeatureFlags.TELEMETRY_SETTING.getKey(), true);
+        // ZSTD is one of the randomly selected codeces, so the feature flag must be enabled
+        featureSettings.put(FeatureFlags.ZSTD_COMPRESSION, true);
         return featureSettings.build();
     }