diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 31b1cb5efe3a1..8cfd636b9fd2e 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -21,4 +21,5 @@ BWC_VERSION:
- "2.8.0"
- "2.8.1"
- "2.9.0"
+ - "2.9.1"
- "2.10.0"
diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml
index fdf42a9a2731e..764a365e7411c 100644
--- a/.github/workflows/version.yml
+++ b/.github/workflows/version.yml
@@ -54,8 +54,8 @@ jobs:
echo " - \"$CURRENT_VERSION\"" >> .ci/bwcVersions
sed -i "s/opensearch = $CURRENT_VERSION/opensearch = $NEXT_VERSION/g" buildSrc/version.properties
echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
- sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
- sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" server/src/main/java/org/opensearch/Version.java
+ sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
+ sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
@@ -82,7 +82,7 @@ jobs:
echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
sed -i "s/- \"$CURRENT_VERSION\"/\0\n - \"$NEXT_VERSION\"/g" .ci/bwcVersions
echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
- sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
+ sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ab07011aefee..9e4a26caa2b10 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -46,6 +46,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
- Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
- Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719))
+- [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792))
+- Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792))
### Deprecated
@@ -76,26 +78,42 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Added
- Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583)
- Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221)
+- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195)
+- Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606))
### Dependencies
- Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307))
- Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726))
- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724))
- Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725))
+- Bump `org.apache.zookeeper:zookeeper` from 3.8.1 to 3.8.2 ([#8844](https://github.com/opensearch-project/OpenSearch/pull/8844))
+- Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842))
+- Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838))
+- Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840))
+- OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868)
+- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995))
+- Bump `com.gradle.enterprise` from 3.13.3 to 3.14.1 ([#8996](https://github.com/opensearch-project/OpenSearch/pull/8996))
+- Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 ([#8995](https://github.com/opensearch-project/OpenSearch/pull/8995))
+- Bump `com.google.cloud:google-cloud-core-http` from 2.21.0 to 2.21.1 ([#8999](https://github.com/opensearch-project/OpenSearch/pull/8999))
### Changed
- Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303))
- Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620))
- Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735))
- Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805))
+- [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636))
+- Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807))
+- Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801))
+- [Remove] Deprecated Fractional ByteSizeValue support #9005 ([#9005](https://github.com/opensearch-project/OpenSearch/pull/9005))
### Deprecated
### Removed
### Fixed
+- Fix flaky ResourceAwareTasksTests.testBasicTaskResourceTracking test ([#8993](https://github.com/opensearch-project/OpenSearch/pull/8993))
### Security
[Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
index 0baf626142238..1dce1f8a75035 100644
--- a/DEVELOPER_GUIDE.md
+++ b/DEVELOPER_GUIDE.md
@@ -264,7 +264,10 @@ This repository is split into many top level directories. The most important one
### `distribution`
-Builds our tar and zip archives and our rpm and deb packages.
+Builds our tar and zip archives and our rpm and deb packages. There are several flavors of the distributions, with the classifier included in the name of the final deliverable (archive or package):
+ - default (no classifier), the distribution with bundled JDK
+ - `-no-jdk-` - the distribution without bundled JDK/JRE, assumes the JDK/JRE is going to be pre-installed on the target systems
+ - `-jre-` - the distribution bundled with JRE (smaller footprint), supported as experimental feature for some platforms
### `libs`
diff --git a/README.md b/README.md
index 55ff067e4d779..b5fc45509b002 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
[![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
[![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
[![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.8 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.8.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.8.0")
+[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0")
[![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
[![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
[![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
diff --git a/TESTING.md b/TESTING.md
index 93b4615da6f0b..a76ee07e1faac 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran
- [Test groups](#test-groups)
- [Load balancing and caches](#load-balancing-and-caches)
- [Test compatibility](#test-compatibility)
+ - [Retries](#retries)
- [Miscellaneous](#miscellaneous)
- [Running verification tasks](#running-verification-tasks)
- [Testing the REST layer](#testing-the-rest-layer)
@@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to
./gradlew test -Dtests.compatibility=1.0.0
+## Retries
+
+The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label.
+
## Miscellaneous
Run all tests without stopping on errors (inspect log files).
diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java
index 76851881730a3..b18ea4327cbc2 100644
--- a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java
+++ b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java
@@ -40,14 +40,14 @@
import org.opensearch.action.search.SearchPhaseController;
import org.opensearch.action.search.SearchProgressListener;
import org.opensearch.action.search.SearchRequest;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.index.Index;
import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.SearchModule;
import org.opensearch.search.SearchShardTarget;
diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java
index d3bfc9348cdb3..4f6b431cc56fd 100644
--- a/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java
+++ b/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java
@@ -27,8 +27,8 @@
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.index.store.remote.filecache.CachedIndexInput;
import org.opensearch.index.store.remote.filecache.FileCache;
import org.opensearch.index.store.remote.filecache.FileCacheFactory;
diff --git a/build.gradle b/build.gradle
index 6a14ab231894b..4d85638270eca 100644
--- a/build.gradle
+++ b/build.gradle
@@ -55,7 +55,7 @@ plugins {
id 'opensearch.docker-support'
id 'opensearch.global-build-info'
id "com.diffplug.spotless" version "6.19.0" apply false
- id "org.gradle.test-retry" version "1.5.3" apply false
+ id "org.gradle.test-retry" version "1.5.4" apply false
id "test-report-aggregation"
id 'jacoco-report-aggregation'
}
@@ -470,6 +470,81 @@ subprojects {
maxFailures = 10
}
failOnPassedAfterRetry = false
+ filter {
+ includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests")
+ includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT")
+ includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT")
+ includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT")
+ includeClasses.add("org.opensearch.aliases.IndexAliasesIT")
+ includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT")
+ includeClasses.add("org.opensearch.blocks.SimpleBlocksIT")
+ includeClasses.add("org.opensearch.client.PitIT")
+ includeClasses.add("org.opensearch.client.ReindexIT")
+ includeClasses.add("org.opensearch.cluster.ClusterHealthIT")
+ includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT")
+ includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT")
+ includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT")
+ includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests")
+ includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests")
+ includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT")
+ includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests")
+ includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT")
+ includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT")
+ includeClasses.add("org.opensearch.http.SearchRestCancellationIT")
+ includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests")
+ includeClasses.add("org.opensearch.index.IndexServiceTests")
+ includeClasses.add("org.opensearch.index.IndexSettingsTests")
+ includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT")
+ includeClasses.add("org.opensearch.index.ShardIndexingPressureIT")
+ includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT")
+ includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests")
+ includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests")
+ includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests")
+ includeClasses.add("org.opensearch.index.shard.IndexShardIT")
+ includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests")
+ includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests")
+ includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT")
+ includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests")
+ includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT")
+ includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT")
+ includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT")
+ includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests")
+ includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT")
+ includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests")
+ includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests")
+ includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep")
+ includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT")
+ includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT")
+ includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT")
+ includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT")
+ includeClasses.add("org.opensearch.remotestore.RemoteStoreIT")
+ includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT")
+ includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT")
+ includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT")
+ includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT")
+ includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT")
+ includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests")
+ includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests")
+ includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests")
+ includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests")
+ includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests")
+ includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT")
+ includeClasses.add("org.opensearch.search.SearchTimeoutIT")
+ includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT")
+ includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT")
+ includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT")
+ includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT")
+ includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT")
+ includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT")
+ includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT")
+ includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT")
+ includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT")
+ includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT")
+ includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT")
+ includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT")
+ includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT")
+ includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests")
+ }
}
}
}
diff --git a/server/src/main/java/org/opensearch/common/component/package-info.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java
similarity index 69%
rename from server/src/main/java/org/opensearch/common/component/package-info.java
rename to buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java
index 34d034b5a3ffb..2acc335d80df0 100644
--- a/server/src/main/java/org/opensearch/common/component/package-info.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java
@@ -6,5 +6,10 @@
* compatible open source license.
*/
-/** Base Lifecycle Component package. */
-package org.opensearch.common.component;
+package org.opensearch.gradle;
+
+public enum JavaPackageType {
+ NONE,
+ JRE,
+ JDK
+}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java
new file mode 100644
index 0000000000000..5f576984627a8
--- /dev/null
+++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java
@@ -0,0 +1,197 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle;
+
+import org.gradle.api.Buildable;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.TaskDependency;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+abstract class JavaVariant implements Buildable, Iterable {
+
+ private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList(
+ Arrays.asList("aarch64", "x64", "s390x", "ppc64le")
+ );
+ private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk"));
+ private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList(
+ Arrays.asList("darwin", "freebsd", "linux", "mac", "windows")
+ );
+ private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
+ private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?");
+
+ private final String name;
+ private final Configuration configuration;
+
+ private final Property vendor;
+ private final Property version;
+ private final Property platform;
+ private final Property architecture;
+ private String baseVersion;
+ private String major;
+ private String build;
+ private String hash;
+
+ JavaVariant(String name, Configuration configuration, ObjectFactory objectFactory) {
+ this.name = name;
+ this.configuration = configuration;
+ this.vendor = objectFactory.property(String.class);
+ this.version = objectFactory.property(String.class);
+ this.platform = objectFactory.property(String.class);
+ this.architecture = objectFactory.property(String.class);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getVendor() {
+ return vendor.get();
+ }
+
+ public void setVendor(final String vendor) {
+ if (ALLOWED_VENDORS.contains(vendor) == false) {
+ throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS);
+ }
+ this.vendor.set(vendor);
+ }
+
+ public String getVersion() {
+ return version.get();
+ }
+
+ public void setVersion(String version) {
+ if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
+ throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
+ }
+ parseVersion(version);
+ this.version.set(version);
+ }
+
+ public String getPlatform() {
+ return platform.get();
+ }
+
+ public void setPlatform(String platform) {
+ if (ALLOWED_PLATFORMS.contains(platform) == false) {
+ throw new IllegalArgumentException(
+ "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS
+ );
+ }
+ this.platform.set(platform);
+ }
+
+ public String getArchitecture() {
+ return architecture.get();
+ }
+
+ public void setArchitecture(final String architecture) {
+ String jdkArchitecture = translateJdkArchitecture(architecture);
+ if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) {
+ throw new IllegalArgumentException(
+ "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
+ );
+ }
+ this.architecture.set(jdkArchitecture);
+ }
+
+ public String getBaseVersion() {
+ return baseVersion;
+ }
+
+ public String getMajor() {
+ return major;
+ }
+
+ public String getBuild() {
+ return build;
+ }
+
+ public String getHash() {
+ return hash;
+ }
+
+ public String getPath() {
+ return configuration.getSingleFile().toString();
+ }
+
+ public String getConfigurationName() {
+ return configuration.getName();
+ }
+
+ @Override
+ public String toString() {
+ return getPath();
+ }
+
+ @Override
+ public TaskDependency getBuildDependencies() {
+ return configuration.getBuildDependencies();
+ }
+
+ // internal, make this jdks configuration unmodifiable
+ void finalizeValues() {
+ if (version.isPresent() == false) {
+ throw new IllegalArgumentException("version not specified for jdk [" + name + "]");
+ }
+ if (platform.isPresent() == false) {
+ throw new IllegalArgumentException("platform not specified for jdk [" + name + "]");
+ }
+ if (vendor.isPresent() == false) {
+ throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
+ }
+ if (architecture.isPresent() == false) {
+ throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
+ }
+ version.finalizeValue();
+ platform.finalizeValue();
+ vendor.finalizeValue();
+ architecture.finalizeValue();
+ }
+
+ @Override
+ public Iterator iterator() {
+ return configuration.iterator();
+ }
+
+ private void parseVersion(String version) {
+ // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build]
+ // Note the "patch" version is not yet handled here, as it has not yet been used by java.
+ Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version);
+ if (jdkVersionMatcher.matches() == false) {
+ // Try again with the pre-Java9 version format
+ jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);
+
+ if (jdkVersionMatcher.matches() == false) {
+ throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+ }
+ }
+
+ baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
+ major = jdkVersionMatcher.group(1);
+ build = jdkVersionMatcher.group(3);
+ hash = jdkVersionMatcher.group(5);
+ }
+
+ private String translateJdkArchitecture(String architecture) {
+ /*
+ * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands.
+ */
+ return "arm64".equals(architecture) ? "aarch64" : architecture;
+ }
+
+}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java
index 06e857744be2d..3218abe726639 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java
@@ -32,140 +32,13 @@
package org.opensearch.gradle;
-import org.gradle.api.Buildable;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.provider.Property;
-import org.gradle.api.tasks.TaskDependency;
import org.gradle.internal.os.OperatingSystem;
-import java.io.File;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class Jdk implements Buildable, Iterable {
-
- private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList(
- Arrays.asList("aarch64", "x64", "s390x", "ppc64le")
- );
- private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk"));
- private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList(
- Arrays.asList("darwin", "freebsd", "linux", "mac", "windows")
- );
- private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
- private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?");
-
- private final String name;
- private final Configuration configuration;
-
- private final Property vendor;
- private final Property version;
- private final Property platform;
- private final Property architecture;
- private String baseVersion;
- private String major;
- private String build;
- private String hash;
-
+public class Jdk extends JavaVariant {
Jdk(String name, Configuration configuration, ObjectFactory objectFactory) {
- this.name = name;
- this.configuration = configuration;
- this.vendor = objectFactory.property(String.class);
- this.version = objectFactory.property(String.class);
- this.platform = objectFactory.property(String.class);
- this.architecture = objectFactory.property(String.class);
- }
-
- public String getName() {
- return name;
- }
-
- public String getVendor() {
- return vendor.get();
- }
-
- public void setVendor(final String vendor) {
- if (ALLOWED_VENDORS.contains(vendor) == false) {
- throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS);
- }
- this.vendor.set(vendor);
- }
-
- public String getVersion() {
- return version.get();
- }
-
- public void setVersion(String version) {
- if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
- throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
- }
- parseVersion(version);
- this.version.set(version);
- }
-
- public String getPlatform() {
- return platform.get();
- }
-
- public void setPlatform(String platform) {
- if (ALLOWED_PLATFORMS.contains(platform) == false) {
- throw new IllegalArgumentException(
- "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS
- );
- }
- this.platform.set(platform);
- }
-
- public String getArchitecture() {
- return architecture.get();
- }
-
- public void setArchitecture(final String architecture) {
- String jdkArchitecture = translateJdkArchitecture(architecture);
- if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) {
- throw new IllegalArgumentException(
- "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
- );
- }
- this.architecture.set(jdkArchitecture);
- }
-
- public String getBaseVersion() {
- return baseVersion;
- }
-
- public String getMajor() {
- return major;
- }
-
- public String getBuild() {
- return build;
- }
-
- public String getHash() {
- return hash;
- }
-
- public String getPath() {
- return configuration.getSingleFile().toString();
- }
-
- public String getConfigurationName() {
- return configuration.getName();
- }
-
- @Override
- public String toString() {
- return getPath();
- }
-
- @Override
- public TaskDependency getBuildDependencies() {
- return configuration.getBuildDependencies();
+ super(name, configuration, objectFactory);
}
public Object getBinJavaPath() {
@@ -190,56 +63,4 @@ private String getHomeRoot() {
boolean isOSX = "mac".equals(getPlatform()) || "darwin".equals(getPlatform());
return getPath() + (isOSX ? "/Contents/Home" : "");
}
-
- // internal, make this jdks configuration unmodifiable
- void finalizeValues() {
- if (version.isPresent() == false) {
- throw new IllegalArgumentException("version not specified for jdk [" + name + "]");
- }
- if (platform.isPresent() == false) {
- throw new IllegalArgumentException("platform not specified for jdk [" + name + "]");
- }
- if (vendor.isPresent() == false) {
- throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]");
- }
- if (architecture.isPresent() == false) {
- throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]");
- }
- version.finalizeValue();
- platform.finalizeValue();
- vendor.finalizeValue();
- architecture.finalizeValue();
- }
-
- @Override
- public Iterator iterator() {
- return configuration.iterator();
- }
-
- private void parseVersion(String version) {
- // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build]
- // Note the "patch" version is not yet handled here, as it has not yet been used by java.
- Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version);
- if (jdkVersionMatcher.matches() == false) {
- // Try again with the pre-Java9 version format
- jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);
-
- if (jdkVersionMatcher.matches() == false) {
- throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
- }
- }
-
- baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
- major = jdkVersionMatcher.group(1);
- build = jdkVersionMatcher.group(3);
- hash = jdkVersionMatcher.group(5);
- }
-
- private String translateJdkArchitecture(String architecture) {
- /*
- * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands.
- */
- return "arm64".equals(architecture) ? "aarch64" : architecture;
- }
-
}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jre.java b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java
new file mode 100644
index 0000000000000..473bfc4860b80
--- /dev/null
+++ b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java
@@ -0,0 +1,18 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle;
+
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.model.ObjectFactory;
+
+public class Jre extends JavaVariant {
+ Jre(String name, Configuration configuration, ObjectFactory objectFactory) {
+ super(name, configuration, objectFactory);
+ }
+}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java
new file mode 100644
index 0000000000000..5a00f41f07a60
--- /dev/null
+++ b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java
@@ -0,0 +1,147 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle;
+
+import org.opensearch.gradle.transform.SymbolicLinkPreservingUntarTransform;
+import org.opensearch.gradle.transform.UnzipTransform;
+import org.gradle.api.GradleException;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.dsl.RepositoryHandler;
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.attributes.Attribute;
+import org.gradle.api.internal.artifacts.ArtifactAttributes;
+
+public class JreDownloadPlugin implements Plugin {
+ public static final String VENDOR_ADOPTIUM = "adoptium";
+
+ private static final String REPO_NAME_PREFIX = "jre_repo_";
+ private static final String EXTENSION_NAME = "jres";
+ public static final String JRE_TRIMMED_PREFIX = "jdk-?\\d.*-jre";
+
+ @Override
+ public void apply(Project project) {
+ Attribute jreAttribute = Attribute.of("jre", Boolean.class);
+ project.getDependencies().getAttributesSchema().attribute(jreAttribute);
+ project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE);
+ project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
+ transformSpec.getFrom()
+ .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE)
+ .attribute(jreAttribute, true);
+ transformSpec.getTo()
+ .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE)
+ .attribute(jreAttribute, true);
+ transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX));
+ });
+
+ ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
+ project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> {
+ transformSpec.getFrom()
+ .attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName())
+ .attribute(jreAttribute, true);
+ transformSpec.getTo()
+ .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE)
+ .attribute(jreAttribute, true);
+ transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX));
+ });
+
+ NamedDomainObjectContainer jresContainer = project.container(Jre.class, name -> {
+ Configuration configuration = project.getConfigurations().create("jre_" + name);
+ configuration.setCanBeConsumed(false);
+ configuration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
+ configuration.getAttributes().attribute(jreAttribute, true);
+ Jre jre = new Jre(name, configuration, project.getObjects());
+ configuration.defaultDependencies(dependencies -> {
+ jre.finalizeValues();
+ setupRepository(project, jre);
+ dependencies.add(project.getDependencies().create(dependencyNotation(jre)));
+ });
+ return jre;
+ });
+ project.getExtensions().add(EXTENSION_NAME, jresContainer);
+ }
+
+ private void setupRepository(Project project, Jre jre) {
+ RepositoryHandler repositories = project.getRepositories();
+
+ /*
+ * Define the appropriate repository for the given JRE vendor and version
+ *
+ * For Oracle/OpenJDK/AdoptOpenJDK we define a repository per-version.
+ */
+ String repoName = REPO_NAME_PREFIX + jre.getVendor() + "_" + jre.getVersion();
+ String repoUrl;
+ String artifactPattern;
+
+ if (jre.getVendor().equals(VENDOR_ADOPTIUM)) {
+ repoUrl = "https://github.com/adoptium/temurin" + jre.getMajor() + "-binaries/releases/download/";
+
+ if (jre.getMajor().equals("8")) {
+ // JDK-8 updates are always suffixed with 'U' (fe OpenJDK8U).
+ artifactPattern = "jdk"
+ + jre.getBaseVersion()
+ + "-"
+ + jre.getBuild()
+ + "/OpenJDK"
+ + jre.getMajor()
+ + "U"
+ + "-jre_[classifier]_[module]_hotspot_"
+ + jre.getBaseVersion()
+ + jre.getBuild()
+ + ".[ext]";
+ } else {
+ // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17).
+ // To distinguish between those, the GA releases have only major version component (fe 17+32),
+ // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
+ // version separator '.' should be enough.
+ artifactPattern = "jdk-" + jre.getBaseVersion() + "+" + jre.getBuild() + "/OpenJDK" + jre.getMajor()
+ // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release
+ + (jre.getBaseVersion().contains(".") || jre.getBaseVersion().matches("^2\\d+$") ? "U" : "")
+ + "-jre_[classifier]_[module]_hotspot_"
+ + jre.getBaseVersion()
+ + "_"
+ + jre.getBuild()
+ + ".[ext]";
+ }
+ } else {
+ throw new GradleException("Unknown JDK vendor [" + jre.getVendor() + "]");
+ }
+
+ // Define the repository if we haven't already
+ if (repositories.findByName(repoName) == null) {
+ repositories.ivy(repo -> {
+ repo.setName(repoName);
+ repo.setUrl(repoUrl);
+ repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+ repo.patternLayout(layout -> layout.artifact(artifactPattern));
+ repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeGroup(groupName(jre)));
+ });
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static NamedDomainObjectContainer getContainer(Project project) {
+ return (NamedDomainObjectContainer) project.getExtensions().getByName(EXTENSION_NAME);
+ }
+
+ private static String dependencyNotation(Jre jre) {
+ String platformDep = jre.getPlatform().equals("darwin") || jre.getPlatform().equals("mac") ? "mac" : jre.getPlatform();
+ String extension = jre.getPlatform().equals("windows") ? "zip" : "tar.gz";
+
+ return groupName(jre) + ":" + platformDep + ":" + jre.getBaseVersion() + ":" + jre.getArchitecture() + "@" + extension;
+ }
+
+ private static String groupName(Jre jre) {
+ return jre.getVendor() + "_" + jre.getMajor() + "_jre";
+ }
+
+}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java
index 968bd13bd4011..0575c23fee9f6 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java
@@ -101,7 +101,7 @@ public boolean shouldExtract() {
private final Property version;
private final Property type;
private final Property platform;
- private final Property bundledJdk;
+ private final Property bundledJdk;
private final Property failIfUnavailable;
private final Configuration extracted;
@@ -120,7 +120,7 @@ public boolean shouldExtract() {
this.type = objectFactory.property(Type.class);
this.type.convention(Type.ARCHIVE);
this.platform = objectFactory.property(Platform.class);
- this.bundledJdk = objectFactory.property(Boolean.class);
+ this.bundledJdk = objectFactory.property(JavaPackageType.class);
this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true);
this.extracted = extractedConfiguration;
}
@@ -154,8 +154,8 @@ public void setType(Type type) {
this.type.set(type);
}
- public boolean getBundledJdk() {
- return bundledJdk.getOrElse(true);
+ public JavaPackageType getBundledJdk() {
+ return bundledJdk.getOrElse(JavaPackageType.JDK);
}
public boolean isDocker() {
@@ -163,7 +163,7 @@ public boolean isDocker() {
return type == Type.DOCKER;
}
- public void setBundledJdk(Boolean bundledJdk) {
+ public void setBundledJdk(JavaPackageType bundledJdk) {
this.bundledJdk.set(bundledJdk);
}
@@ -266,7 +266,7 @@ void finalizeValues() {
}
if (bundledJdk.isPresent() == false) {
- bundledJdk.set(true);
+ bundledJdk.set(JavaPackageType.JDK);
}
version.finalizeValue();
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java
index 7c942358e12c3..4d8b62d95dff1 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java
@@ -76,6 +76,10 @@ public static String getBundledJdk(final String platform) {
return getBundledJdk(platform, null);
}
+ public static String getBundledJre(final String platform, final String arch) {
+ return getBundledJdk(platform, arch);
+ }
+
public static String getBundledJdkVendor() {
return bundledJdkVendor;
}
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java
index f4368b1cecc59..6a54612320c6c 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java
@@ -37,6 +37,7 @@
import org.opensearch.gradle.DistributionDependency;
import org.opensearch.gradle.DistributionDownloadPlugin;
import org.opensearch.gradle.DistributionResolution;
+import org.opensearch.gradle.JavaPackageType;
import org.opensearch.gradle.OpenSearchDistribution;
import org.opensearch.gradle.Version;
import org.opensearch.gradle.VersionProperties;
@@ -99,7 +100,7 @@ private void registerInternalDistributionResolutions(NamedDomainObjectContainer<
resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion()));
if (unreleasedInfo != null) {
- if (!distribution.getBundledJdk()) {
+ if (distribution.getBundledJdk() == JavaPackageType.NONE) {
throw new GradleException(
"Configuring a snapshot bwc distribution ('"
+ distribution.getName()
@@ -167,8 +168,10 @@ private static String distributionProjectName(OpenSearchDistribution distributio
? ""
: "-" + architecture.toString().toLowerCase();
- if (distribution.getBundledJdk() == false) {
+ if (distribution.getBundledJdk() == JavaPackageType.NONE) {
projectName += "no-jdk-";
+ } else if (distribution.getBundledJdk() == JavaPackageType.JRE) {
+ projectName += "jre-";
}
switch (distribution.getType()) {
case ARCHIVE:
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
index 7e0dc72cb9108..a311d89583acb 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -34,6 +34,7 @@
import org.opensearch.gradle.Architecture;
import org.opensearch.gradle.DistributionDownloadPlugin;
+import org.opensearch.gradle.JavaPackageType;
import org.opensearch.gradle.OpenSearchDistribution;
import org.opensearch.gradle.Jdk;
import org.opensearch.gradle.JdkDownloadPlugin;
@@ -71,13 +72,14 @@
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Stream;
public class DistroTestPlugin implements Plugin {
- private static final String SYSTEM_JDK_VERSION = "11.0.19+7";
+ private static final String SYSTEM_JDK_VERSION = "11.0.20+8";
private static final String SYSTEM_JDK_VENDOR = "adoptium";
- private static final String GRADLE_JDK_VERSION = "17.0.7+7";
+ private static final String GRADLE_JDK_VERSION = "17.0.8+7";
private static final String GRADLE_JDK_VENDOR = "adoptium";
// all distributions used by distro tests. this is temporary until tests are per distribution
@@ -137,7 +139,7 @@ public void apply(Project project) {
}
if ((distribution.getType() == OpenSearchDistribution.Type.DEB || distribution.getType() == OpenSearchDistribution.Type.RPM)
- && distribution.getBundledJdk()) {
+ && distribution.getBundledJdk() != JavaPackageType.NONE) {
for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) {
if (version.before("6.3.0")) {
continue; // before opening xpack
@@ -379,8 +381,8 @@ private List configureDistributions(Project project) {
OpenSearchDistribution.Type.RPM,
OpenSearchDistribution.Type.DOCKER
)) {
- for (boolean bundledJdk : Arrays.asList(true, false)) {
- if (bundledJdk == false) {
+ for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) {
+ if (bundledJdk == JavaPackageType.NONE) {
// We'll never publish an ARM (arm64) build without a bundled JDK.
if (architecture == Architecture.ARM64) {
continue;
@@ -403,8 +405,8 @@ private List configureDistributions(Project project) {
OpenSearchDistribution.Platform.LINUX,
OpenSearchDistribution.Platform.WINDOWS
)) {
- for (boolean bundledJdk : Arrays.asList(true, false)) {
- if (bundledJdk == false && architecture != Architecture.X64) {
+ for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) {
+ if (bundledJdk == JavaPackageType.NONE && architecture != Architecture.X64) {
// We will never publish distributions for non-x86 (amd64) platforms
// without a bundled JDK
continue;
@@ -432,7 +434,7 @@ private static OpenSearchDistribution createDistro(
Architecture architecture,
OpenSearchDistribution.Type type,
OpenSearchDistribution.Platform platform,
- boolean bundledJdk,
+ JavaPackageType bundledJdk,
String version
) {
String name = distroId(type, platform, bundledJdk, architecture) + "-" + version;
@@ -466,11 +468,12 @@ private static boolean isWindows(Project project) {
private static String distroId(
OpenSearchDistribution.Type type,
OpenSearchDistribution.Platform platform,
- boolean bundledJdk,
+ JavaPackageType bundledJdk,
Architecture architecture
) {
- return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk")
- + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase());
+ return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk != JavaPackageType.NONE
+ ? (bundledJdk == JavaPackageType.JDK ? "" : "-jre")
+ : "-no-jdk") + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase());
}
private static String destructiveDistroTestTaskName(OpenSearchDistribution distro) {
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties
new file mode 100644
index 0000000000000..e9253488ffbeb
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties
@@ -0,0 +1,12 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+
+implementation-class=org.opensearch.gradle.JreDownloadPlugin
diff --git a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java
index 1a9647573f948..e439e8a047f34 100644
--- a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java
+++ b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java
@@ -45,6 +45,7 @@
import java.util.Arrays;
import java.util.TreeSet;
+import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.core.StringContains.containsString;
public class DistributionDownloadPluginTests extends GradleUnitTestCase {
@@ -76,7 +77,14 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase {
);
public void testVersionDefault() {
- OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", null, Type.ARCHIVE, Platform.LINUX, true);
+ OpenSearchDistribution distro = checkDistro(
+ createProject(null, false),
+ "testdistro",
+ null,
+ Type.ARCHIVE,
+ Platform.LINUX,
+ JavaPackageType.JDK
+ );
assertEquals(distro.getVersion(), VersionProperties.getOpenSearch());
}
@@ -123,18 +131,32 @@ public void testBadVersionFormat() {
"badversion",
Type.ARCHIVE,
Platform.LINUX,
- true,
+ JavaPackageType.JDK,
"Invalid version format: 'badversion'"
);
}
public void testTypeDefault() {
- OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", null, Platform.LINUX, true);
+ OpenSearchDistribution distro = checkDistro(
+ createProject(null, false),
+ "testdistro",
+ "5.0.0",
+ null,
+ Platform.LINUX,
+ JavaPackageType.JDK
+ );
assertEquals(distro.getType(), Type.ARCHIVE);
}
public void testPlatformDefault() {
- OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, null, true);
+ OpenSearchDistribution distro = checkDistro(
+ createProject(null, false),
+ "testdistro",
+ "5.0.0",
+ Type.ARCHIVE,
+ null,
+ JavaPackageType.JDK
+ );
assertEquals(distro.getPlatform(), OpenSearchDistribution.CURRENT_PLATFORM);
}
@@ -151,8 +173,15 @@ public void testPlatformForIntegTest() {
}
public void testBundledJdkDefault() {
- OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, true);
- assertTrue(distro.getBundledJdk());
+ OpenSearchDistribution distro = checkDistro(
+ createProject(null, false),
+ "testdistro",
+ "5.0.0",
+ Type.ARCHIVE,
+ Platform.LINUX,
+ JavaPackageType.JDK
+ );
+ assertThat(distro.getBundledJdk(), equalTo(JavaPackageType.JDK));
}
public void testBundledJdkForIntegTest() {
@@ -162,7 +191,7 @@ public void testBundledJdkForIntegTest() {
"5.0.0",
Type.INTEG_TEST_ZIP,
null,
- true,
+ JavaPackageType.JDK,
"bundledJdk cannot be set on opensearch distribution [testdistro]"
);
}
@@ -178,7 +207,7 @@ public void testLocalCurrentVersionIntegTestZip() {
public void testLocalCurrentVersionArchives() {
for (Platform platform : Platform.values()) {
- for (boolean bundledJdk : new boolean[] { true, false }) {
+ for (JavaPackageType bundledJdk : JavaPackageType.values()) {
for (Architecture architecture : Architecture.values()) {
// create a new project in each iteration, so that we know we are resolving the only additional project being created
Project project = createProject(BWC_MINOR, true);
@@ -204,7 +233,7 @@ public void testLocalCurrentVersionArchives() {
public void testLocalCurrentVersionPackages() {
for (Type packageType : new Type[] { Type.RPM, Type.DEB }) {
- for (boolean bundledJdk : new boolean[] { true, false }) {
+ for (JavaPackageType bundledJdk : JavaPackageType.values()) {
Project project = createProject(BWC_MINOR, true);
String projectName = projectName(packageType.toString(), bundledJdk);
Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build();
@@ -219,7 +248,7 @@ public void testLocalCurrentVersionPackages() {
public void testLocalBwcArchives() {
for (Platform platform : Platform.values()) {
// note: no non bundled jdk for bwc
- String configName = projectName(platform.toString(), true);
+ String configName = projectName(platform.toString(), JavaPackageType.JDK);
configName += (platform == Platform.WINDOWS ? "-zip" : "-tar");
checkBwc("minor", configName, BWC_MINOR_VERSION, Type.ARCHIVE, platform, BWC_MINOR, true);
@@ -232,7 +261,7 @@ public void testLocalBwcArchives() {
public void testLocalBwcPackages() {
for (Type packageType : new Type[] { Type.RPM, Type.DEB }) {
// note: no non bundled jdk for bwc
- String configName = projectName(packageType.toString(), true);
+ String configName = projectName(packageType.toString(), JavaPackageType.JDK);
checkBwc("minor", configName, BWC_MINOR_VERSION, packageType, null, BWC_MINOR, true);
checkBwc("staged", configName, BWC_STAGED_VERSION, packageType, null, BWC_STAGED, true);
@@ -247,7 +276,7 @@ private void assertDistroError(
String version,
Type type,
Platform platform,
- Boolean bundledJdk,
+ JavaPackageType bundledJdk,
String message
) {
IllegalArgumentException e = expectThrows(
@@ -263,7 +292,7 @@ private OpenSearchDistribution createDistro(
String version,
Type type,
Platform platform,
- Boolean bundledJdk
+ JavaPackageType bundledJdk
) {
NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project);
return distros.create(name, distro -> {
@@ -289,7 +318,7 @@ private OpenSearchDistribution checkDistro(
String version,
Type type,
Platform platform,
- Boolean bundledJdk
+ JavaPackageType bundledJdk
) {
OpenSearchDistribution distribution = createDistro(project, name, version, type, platform, bundledJdk);
distribution.finalizeValues();
@@ -315,7 +344,7 @@ private void checkBwc(
Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build();
archiveProject.getConfigurations().create(config);
archiveProject.getArtifacts().add(config, new File("doesnotmatter"));
- final OpenSearchDistribution distro = createDistro(project, "distro", version.toString(), type, platform, true);
+ final OpenSearchDistribution distro = createDistro(project, "distro", version.toString(), type, platform, JavaPackageType.JDK);
distro.setArchitecture(Architecture.current());
checkPlugin(project);
}
@@ -335,7 +364,7 @@ private Project createProject(BwcVersions bwcVersions, boolean isInternal) {
return project;
}
- private static String projectName(String base, boolean bundledJdk) {
- return bundledJdk ? base : ("no-jdk-" + base);
+ private static String projectName(String base, JavaPackageType bundledJdk) {
+ return (bundledJdk == JavaPackageType.JDK) ? base : ((bundledJdk == JavaPackageType.NONE) ? ("no-jdk-" + base) : "jre-" + base);
}
}
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 246f6e3444224..e4a9293c59b8f 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,8 +1,8 @@
opensearch = 3.0.0
-lucene = 9.7.0
+lucene = 9.8.0-snapshot-4373c3b
bundled_jdk_vendor = adoptium
-bundled_jdk = 20.0.1+9
+bundled_jdk = 20.0.2+9
# See please https://github.com/adoptium/temurin-build/issues/3371
bundled_jdk_linux_ppc64le = 20+36
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java
index 6a75837f8a0fd..332e089ad9e76 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java
@@ -99,7 +99,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
defaultPipeline,
defaultRequireAlias,
true,
- request.getXContentType()
+ request.getMediaType()
);
// short circuit the call to the transport layer
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
index f3fbadf07d304..d23a5976fada6 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java
@@ -157,7 +157,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
// Bulk API only supports newline delimited JSON or Smile. Before executing
// the bulk, we need to check that all requests have the same content-type
// and this content-type is supported by the Bulk API.
- XContentType bulkContentType = null;
+ MediaType bulkContentType = null;
for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
DocWriteRequest> action = bulkRequest.requests().get(i);
@@ -245,7 +245,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
IndexRequest indexRequest = (IndexRequest) action;
BytesReference indexSource = indexRequest.source();
- XContentType indexXContentType = indexRequest.getContentType();
+ MediaType mediaType = indexRequest.getContentType();
try (
XContentParser parser = XContentHelper.createParser(
@@ -257,7 +257,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
indexSource,
- indexXContentType
+ mediaType
)
) {
try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
@@ -266,7 +266,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
}
}
} else if (opType == DocWriteRequest.OpType.UPDATE) {
- source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef();
+ source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, ToXContent.EMPTY_PARAMS, false).toBytesRef();
}
if (source != null) {
@@ -392,30 +392,30 @@ static Request update(UpdateRequest updateRequest) throws IOException {
// set for the partial document and the upsert document. This client
// only accepts update requests that have the same content types set
// for both doc and upsert.
- XContentType xContentType = null;
+ MediaType mediaType = null;
if (updateRequest.doc() != null) {
- xContentType = updateRequest.doc().getContentType();
+ mediaType = updateRequest.doc().getContentType();
}
if (updateRequest.upsertRequest() != null) {
- XContentType upsertContentType = updateRequest.upsertRequest().getContentType();
- if ((xContentType != null) && (xContentType != upsertContentType)) {
+ MediaType upsertContentType = updateRequest.upsertRequest().getContentType();
+ if ((mediaType != null) && (mediaType != upsertContentType)) {
throw new IllegalStateException(
"Update request cannot have different content types for doc ["
- + xContentType
+ + mediaType
+ "]"
+ " and upsert ["
+ upsertContentType
+ "] documents"
);
} else {
- xContentType = upsertContentType;
+ mediaType = upsertContentType;
}
}
- if (xContentType == null) {
- xContentType = Requests.INDEX_CONTENT_TYPE;
+ if (mediaType == null) {
+ mediaType = Requests.INDEX_CONTENT_TYPE;
}
request.addParameters(parameters.asMap());
- request.setEntity(createEntity(updateRequest, xContentType));
+ request.setEntity(createEntity(updateRequest, mediaType));
return request;
}
@@ -816,14 +816,13 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest)
return request;
}
- static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
- return createEntity(toXContent, xContentType, ToXContent.EMPTY_PARAMS);
+ static HttpEntity createEntity(ToXContent toXContent, MediaType mediaType) throws IOException {
+ return createEntity(toXContent, mediaType, ToXContent.EMPTY_PARAMS);
}
- static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType, ToXContent.Params toXContentParams)
- throws IOException {
- BytesRef source = XContentHelper.toXContent(toXContent, xContentType, toXContentParams, false).toBytesRef();
- return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
+ static HttpEntity createEntity(ToXContent toXContent, MediaType mediaType, ToXContent.Params toXContentParams) throws IOException {
+ BytesRef source = XContentHelper.toXContent(toXContent, mediaType, toXContentParams, false).toBytesRef();
+ return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(mediaType));
}
static String endpoint(String index, String id) {
@@ -868,20 +867,6 @@ static String endpoint(String[] indices, String endpoint, String type) {
return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build();
}
- /**
- * Returns a {@link ContentType} from a given {@link XContentType}.
- *
- * @param xContentType the {@link XContentType}
- * @return the {@link ContentType}
- *
- * @deprecated use {@link #createContentType(MediaType)} instead
- */
- @Deprecated
- @SuppressForbidden(reason = "Only allowed place to convert a XContentType to a ContentType")
- public static ContentType createContentType(final XContentType xContentType) {
- return ContentType.create(xContentType.mediaTypeWithoutParameters(), (Charset) null);
- }
-
/**
* Returns a {@link ContentType} from a given {@link XContentType}.
*
@@ -1265,8 +1250,8 @@ Params withWaitForEvents(Priority waitForEvents) {
*
* @return the {@link IndexRequest}'s content type
*/
- static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) {
- XContentType requestContentType = indexRequest.getContentType();
+ static MediaType enforceSameContentType(IndexRequest indexRequest, @Nullable MediaType mediaType) {
+ MediaType requestContentType = indexRequest.getContentType();
if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE) {
throw new IllegalArgumentException(
"Unsupported content-type found for request with content-type ["
@@ -1274,19 +1259,19 @@ static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable
+ "], only JSON and SMILE are supported"
);
}
- if (xContentType == null) {
+ if (mediaType == null) {
return requestContentType;
}
- if (requestContentType != xContentType) {
+ if (requestContentType != mediaType) {
throw new IllegalArgumentException(
"Mismatching content-type found for request with content-type ["
+ requestContentType
+ "], previous requests have content-type ["
- + xContentType
+ + mediaType
+ "]"
);
}
- return xContentType;
+ return mediaType;
}
/**
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java
index 9574d4b17819d..1ff5c81c4a0e3 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java
@@ -88,9 +88,9 @@
import org.opensearch.core.ParseField;
import org.opensearch.core.xcontent.ContextParser;
import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.rankeval.RankEvalRequest;
import org.opensearch.index.rankeval.RankEvalResponse;
import org.opensearch.index.reindex.BulkByScrollResponse;
@@ -2227,11 +2227,11 @@ protected final Resp parseEntity(final HttpEntity entity, final CheckedFu
if (entity.getContentType() == null) {
throw new IllegalStateException("OpenSearch didn't return the [Content-Type] header, unable to parse response body");
}
- XContentType xContentType = XContentType.fromMediaType(entity.getContentType());
- if (xContentType == null) {
+ MediaType medaiType = MediaType.fromMediaType(entity.getContentType());
+ if (medaiType == null) {
throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType());
}
- try (XContentParser parser = xContentType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) {
+ try (XContentParser parser = medaiType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) {
return entityParser.apply(parser);
}
}
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java
index 3ed41684b090e..cffed98fce3aa 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java
@@ -44,10 +44,10 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.ParseField;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -77,7 +77,7 @@ public class CreateIndexRequest extends TimedRequest implements Validatable, ToX
private Settings settings = EMPTY_SETTINGS;
private BytesReference mappings;
- private XContentType mappingsXContentType;
+ private MediaType mappingsMediaType;
private final Set aliases = new HashSet<>();
@@ -123,17 +123,6 @@ public CreateIndexRequest settings(Settings settings) {
return this;
}
- /**
- * The settings to create the index with (either json or yaml format)
- *
- * @deprecated use {@link #settings(String source, MediaType mediaType)} instead
- */
- @Deprecated
- public CreateIndexRequest settings(String source, XContentType xContentType) {
- this.settings = Settings.builder().loadFromSource(source, xContentType).build();
- return this;
- }
-
/**
* The settings to create the index with (either json or yaml format)
*/
@@ -162,23 +151,8 @@ public BytesReference mappings() {
return mappings;
}
- public XContentType mappingsXContentType() {
- return mappingsXContentType;
- }
-
- /**
- * Adds mapping that will be added when the index gets created.
- *
- * Note that the definition should *not* be nested under a type name.
- *
- * @param source The mapping source
- * @param xContentType The content type of the source
- *
- * @deprecated use {@link #mapping(String source, MediaType mediaType)} instead
- */
- @Deprecated
- public CreateIndexRequest mapping(String source, XContentType xContentType) {
- return mapping(new BytesArray(source), xContentType);
+ public MediaType mappingsMediaType() {
+ return mappingsMediaType;
}
/**
@@ -213,7 +187,7 @@ public CreateIndexRequest mapping(XContentBuilder source) {
*/
public CreateIndexRequest mapping(Map source) {
try {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.getDefaultMediaType());
builder.map(source);
return mapping(BytesReference.bytes(builder), builder.contentType());
} catch (IOException e) {
@@ -221,24 +195,6 @@ public CreateIndexRequest mapping(Map source) {
}
}
- /**
- * Adds mapping that will be added when the index gets created.
- *
- * Note that the definition should *not* be nested under a type name.
- *
- * @param source The mapping source
- * @param xContentType the content type of the mapping source
- *
- * @deprecated use {@link #mapping(BytesReference source, MediaType mediaType)} instead
- */
- @Deprecated
- public CreateIndexRequest mapping(BytesReference source, XContentType xContentType) {
- Objects.requireNonNull(xContentType);
- mappings = source;
- mappingsXContentType = xContentType;
- return this;
- }
-
/**
* Adds mapping that will be added when the index gets created.
*
@@ -250,7 +206,7 @@ public CreateIndexRequest mapping(BytesReference source, XContentType xContentTy
public CreateIndexRequest mapping(BytesReference source, MediaType mediaType) {
Objects.requireNonNull(mediaType);
mappings = source;
- mappingsXContentType = XContentType.fromMediaType(mediaType);
+ mappingsMediaType = mediaType;
return this;
}
@@ -278,16 +234,6 @@ public CreateIndexRequest aliases(XContentBuilder source) {
return aliases(BytesReference.bytes(source), source.contentType());
}
- /**
- * Sets the aliases that will be associated with the index when it gets created
- *
- * @deprecated use {@link #aliases(String, MediaType)} instead
- */
- @Deprecated
- public CreateIndexRequest aliases(String source, XContentType contentType) {
- return aliases(new BytesArray(source), contentType);
- }
-
/**
* Sets the aliases that will be associated with the index when it gets created
*/
@@ -295,16 +241,6 @@ public CreateIndexRequest aliases(String source, MediaType mediaType) {
return aliases(new BytesArray(source), mediaType);
}
- /**
- * Sets the aliases that will be associated with the index when it gets created
- *
- * @deprecated use {@link #aliases(BytesReference source, MediaType contentType)} instead
- */
- @Deprecated
- public CreateIndexRequest aliases(BytesReference source, XContentType contentType) {
- return aliases(source, (MediaType) contentType);
- }
-
/**
* Sets the aliases that will be associated with the index when it gets created
*/
@@ -345,18 +281,6 @@ public CreateIndexRequest aliases(Collection aliases) {
return this;
}
- /**
- * Sets the settings and mappings as a single source.
- *
- * Note that the mapping definition should *not* be nested under a type name.
- *
- * @deprecated use {@link #source(String, MediaType)} instead
- */
- @Deprecated
- public CreateIndexRequest source(String source, XContentType xContentType) {
- return source(new BytesArray(source), xContentType);
- }
-
/**
* Sets the settings and mappings as a single source.
*
@@ -375,20 +299,6 @@ public CreateIndexRequest source(XContentBuilder source) {
return source(BytesReference.bytes(source), source.contentType());
}
- /**
- * Sets the settings and mappings as a single source.
- *
- * Note that the mapping definition should *not* be nested under a type name.
- *
- * @deprecated use {@link #source(BytesReference, MediaType)} instead
- */
- @Deprecated
- public CreateIndexRequest source(BytesReference source, XContentType xContentType) {
- Objects.requireNonNull(xContentType);
- source(XContentHelper.convertToMap(source, false, xContentType).v2());
- return this;
- }
-
/**
* Sets the settings and mappings as a single source.
*
@@ -458,7 +368,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
if (mappings != null) {
try (InputStream stream = mappings.streamInput()) {
- builder.rawField(MAPPINGS.getPreferredName(), stream, mappingsXContentType);
+ builder.rawField(MAPPINGS.getPreferredName(), stream, mappingsMediaType);
}
}
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java
index 2c90d5a734aa0..327836160cceb 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java
@@ -34,7 +34,7 @@
import org.opensearch.client.core.BroadcastResponse;
import org.opensearch.core.ParseField;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ConstructingObjectParser;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.XContentParser;
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java
index fed958955af99..09dbbd63b9479 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java
@@ -48,6 +48,7 @@
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.common.xcontent.support.XContentMapValues;
import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -267,7 +268,7 @@ public PutIndexTemplateRequest mapping(Map source) {
private PutIndexTemplateRequest internalMapping(Map source) {
try {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.map(source);
MediaType mediaType = builder.contentType();
Objects.requireNonNull(mediaType);
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java
index a2237efaed25f..6d7e95d191ba6 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java
@@ -38,9 +38,8 @@
import org.opensearch.client.TimedRequest;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -98,17 +97,7 @@ public BytesReference source() {
}
/**
- * The {@link XContentType} of the mapping source.
- *
- * @deprecated use {@link #mediaType()} instead
- */
- @Deprecated
- public XContentType xContentType() {
- return XContentType.fromMediaType(mediaType);
- }
-
- /**
- * The {@link XContentType} of the mapping source.
+ * The {@link MediaType} of the mapping source.
*/
public MediaType mediaType() {
return mediaType;
@@ -121,7 +110,7 @@ public MediaType mediaType() {
*/
public PutMappingRequest source(Map mappingSource) {
try {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.getDefaultMediaType());
builder.map(mappingSource);
return source(builder);
} catch (IOException e) {
@@ -129,20 +118,6 @@ public PutMappingRequest source(Map mappingSource) {
}
}
- /**
- * The mapping source definition.
- *
- * Note that the definition should *not* be nested under a type name.
- *
- * @deprecated use {@link #source(String, MediaType)} instead
- */
- @Deprecated
- public PutMappingRequest source(String mappingSource, XContentType xContentType) {
- this.source = new BytesArray(mappingSource);
- this.mediaType = xContentType;
- return this;
- }
-
/**
* The mapping source definition.
*
@@ -165,20 +140,6 @@ public PutMappingRequest source(XContentBuilder builder) {
return this;
}
- /**
- * The mapping source definition.
- *
- * Note that the definition should *not* be nested under a type name.
- *
- * @deprecated use {@link #source(BytesReference, MediaType)} instead
- */
- @Deprecated
- public PutMappingRequest source(BytesReference source, XContentType xContentType) {
- this.source = source;
- this.mediaType = xContentType;
- return this;
- }
-
/**
* The mapping source definition.
*
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java
index 61799a83e5df5..2ffb5c605c27f 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java
@@ -39,7 +39,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import java.io.IOException;
import java.util.Collections;
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java
index 443cef45e646b..327fccfb35951 100644
--- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java
+++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java
@@ -37,7 +37,7 @@
import org.opensearch.action.admin.indices.rollover.MaxSizeCondition;
import org.opensearch.client.TimedRequest;
import org.opensearch.client.indices.CreateIndexRequest;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java
index 707f4246009aa..e9b297d21bcba 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java
@@ -36,7 +36,6 @@
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContent;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.test.OpenSearchTestCase;
@@ -60,7 +59,7 @@ public final void testFromXContent() throws IOException {
final XContentType xContentType = randomFrom(XContentType.values());
final BytesReference bytes = toShuffledXContent(clientTestInstance, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
- final XContent xContent = XContentFactory.xContent(xContentType);
+ final XContent xContent = xContentType.xContent();
final XContentParser parser = xContent.createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, bytes.streamInput());
final S serverInstance = doParseToServerInstance(parser);
assertInstances(serverInstance, clientTestInstance);
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java
index 5f7df0cd5860c..b5ee31c87b8d4 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java
@@ -36,7 +36,6 @@
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContent;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.test.OpenSearchTestCase;
@@ -59,7 +58,7 @@ public final void testFromXContent() throws IOException {
final S serverTestInstance = createServerTestInstance(xContentType);
final BytesReference bytes = toShuffledXContent(serverTestInstance, xContentType, getParams(), randomBoolean());
- final XContent xContent = XContentFactory.xContent(xContentType);
+ final XContent xContent = xContentType.xContent();
final XContentParser parser = xContent.createParser(
NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE,
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java
index 49acbe8cd2bc1..dd793fdbb7ff6 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java
@@ -44,8 +44,8 @@
import org.opensearch.action.search.SearchRequest;
import org.opensearch.common.Strings;
import org.opensearch.core.common.bytes.BytesArray;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentType;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java
index 2ac0eee407b95..3c7d988e3f01d 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java
@@ -61,7 +61,7 @@
import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.support.XContentMapValues;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java
index eaf8f4f8efff7..cefe992b58c64 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java
@@ -60,8 +60,8 @@
import org.opensearch.client.indices.GetIndexRequest;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java
index 8ca5c5fa58742..ea57f1857fbfb 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java
@@ -112,8 +112,8 @@
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java
index e21619ff15ef6..c672ed6be110d 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java
@@ -79,7 +79,7 @@
import org.opensearch.core.common.Strings;
import org.opensearch.test.OpenSearchTestCase;
import org.junit.Assert;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import java.io.IOException;
import java.util.Arrays;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java
index 84cd2fb8b20dd..e3c8197dc2c90 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java
@@ -161,10 +161,7 @@ private void testSimulatePipeline(boolean isVerbose, boolean isFailure) throws I
}
builder.endObject();
- SimulatePipelineRequest request = new SimulatePipelineRequest(
- BytesReference.bytes(builder),
- XContentType.fromMediaType(builder.contentType())
- );
+ SimulatePipelineRequest request = new SimulatePipelineRequest(BytesReference.bytes(builder), builder.contentType());
request.setVerbose(isVerbose);
SimulatePipelineResponse response = execute(
request,
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java
index ee6aeeeacf633..15a99b3e91685 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java
@@ -76,6 +76,7 @@
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
@@ -809,7 +810,7 @@ public void testUpdate() throws IOException {
UpdateRequest parsedUpdateRequest = new UpdateRequest();
- XContentType entityContentType = XContentType.fromMediaType(entity.getContentType());
+ MediaType entityContentType = MediaType.fromMediaType(entity.getContentType());
try (XContentParser parser = createParser(entityContentType.xContent(), entity.getContent())) {
parsedUpdateRequest.fromXContent(parser);
}
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java
index e86de6ba718f9..af178ad2a5d47 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java
@@ -48,7 +48,7 @@
import org.opensearch.action.support.master.AcknowledgedRequest;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.repositories.fs.FsRepository;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java
index 03d6e6720ffed..376eb3c5e407c 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java
@@ -35,9 +35,9 @@
import org.opensearch.client.AbstractResponseTestCase;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -61,7 +61,7 @@ static class SourceOnlyResponse implements ToXContentObject {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
// this implementation copied from RestGetSourceAction.RestGetSourceResponseListener::buildResponse
try (InputStream stream = source.streamInput()) {
- builder.rawValue(stream, XContentHelper.xContentType(source));
+ builder.rawValue(stream, MediaTypeRegistry.xContentType(source));
}
return builder;
}
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java
index a973753aa2032..178d9296bd242 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java
@@ -77,8 +77,8 @@
import org.opensearch.common.Strings;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java
index f85fcae7af365..ccbb64b13b2d2 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java
@@ -63,7 +63,7 @@
import org.opensearch.common.Priority;
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.indices.recovery.RecoverySettings;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java
index 73346bc57646e..de2eb91c6660b 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java
@@ -104,8 +104,8 @@
import org.opensearch.cluster.metadata.Template;
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java
index b83bdab899a41..7babb339e176a 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java
@@ -39,7 +39,6 @@
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContent;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.index.Index;
@@ -194,7 +193,7 @@ public final void testBwcFromXContent() throws IOException {
final XContentType xContentType = randomFrom(XContentType.values());
final BytesReference bytes = toShuffledXContent(expected, xContentType, getParams(), randomBoolean());
- final XContent xContent = XContentFactory.xContent(xContentType);
+ final XContent xContent = xContentType.xContent();
final XContentParser parser = xContent.createParser(
NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE,
@@ -215,7 +214,7 @@ public final void testBwcFromXContent() throws IOException {
final XContentType xContentType = randomFrom(XContentType.values());
final BytesReference bytes = toShuffledXContent(expected, xContentType, getParams(), randomBoolean());
- final XContent xContent = XContentFactory.xContent(xContentType);
+ final XContent xContent = xContentType.xContent();
final XContentParser parser = xContent.createParser(
NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE,
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java
index 1ac5ff06d5624..0cfbd8c29fe43 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java
@@ -69,8 +69,8 @@ private void assertMappingsEqual(CreateIndexRequest expected, CreateIndexRequest
} else {
assertNotNull(actual.mappings());
try (
- XContentParser expectedJson = createParser(expected.mappingsXContentType().xContent(), expected.mappings());
- XContentParser actualJson = createParser(actual.mappingsXContentType().xContent(), actual.mappings())
+ XContentParser expectedJson = createParser(expected.mappingsMediaType().xContent(), expected.mappings());
+ XContentParser actualJson = createParser(actual.mappingsMediaType().xContent(), actual.mappings())
) {
assertEquals(expectedJson.map(), actualJson.map());
} catch (IOException e) {
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java
index 96db2fb43c161..ef8655b0226dd 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java
@@ -36,7 +36,7 @@
import org.opensearch.action.admin.indices.datastream.DataStreamsStatsAction;
import org.opensearch.core.action.support.DefaultShardOperationFailedException;
import org.opensearch.client.AbstractResponseTestCase;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java
index 6f924898096a0..8c83791d94d42 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java
@@ -39,10 +39,10 @@
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -262,7 +262,7 @@ private static AliasMetadata randomAliasMetadata(String name) {
}
static XContentBuilder randomMapping(String type, XContentType xContentType) throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType);
builder.startObject().startObject(type);
randomMappingFields(builder, true);
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java
index 9d51c09ba2e44..c8372d5001fac 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java
@@ -36,7 +36,7 @@
import org.opensearch.action.admin.indices.rollover.MaxAgeCondition;
import org.opensearch.action.admin.indices.rollover.MaxDocsCondition;
import org.opensearch.action.admin.indices.rollover.MaxSizeCondition;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java
index ff2418c5caee2..0fb00f6510147 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java
@@ -36,7 +36,7 @@
import org.opensearch.action.admin.indices.rollover.MaxAgeCondition;
import org.opensearch.action.admin.indices.rollover.MaxDocsCondition;
import org.opensearch.action.admin.indices.rollover.MaxSizeCondition;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index e7137127053e2..161b8008525b4 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -28,9 +28,11 @@
* under the License.
*/
+import org.opensearch.gradle.JavaPackageType
+
apply plugin: 'opensearch.internal-distribution-archive-setup'
-CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean jdk) {
+CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, JavaPackageType java) {
return copySpec {
into("opensearch-${version}") {
into('lib') {
@@ -39,19 +41,23 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
into('config') {
dirMode 0750
fileMode 0660
- with configFiles(distributionType, jdk)
+ with configFiles(distributionType, java)
from {
dirMode 0750
jvmOptionsDir.getParent()
}
}
into('bin') {
- with binFiles(distributionType, jdk)
+ with binFiles(distributionType, java)
}
- if (jdk) {
+ if (java == JavaPackageType.JDK) {
into("darwin".equals(platform) ? 'jdk.app' : 'jdk') {
with jdkFiles(project, platform, architecture)
}
+ } else if (java == JavaPackageType.JRE) {
+ into("darwin".equals(platform) ? 'jre.app' : 'jre') {
+ with jreFiles(project, platform, architecture)
+ }
}
into('') {
from {
@@ -73,7 +79,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
rename { 'LICENSE.txt' }
}
- with noticeFile(jdk)
+ with noticeFile(java)
into('modules') {
with modulesFiles
}
@@ -84,77 +90,84 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
distribution_archives {
integTestZip {
content {
- archiveFiles(transportModulesFiles, 'zip', null, 'x64', false)
+ archiveFiles(transportModulesFiles, 'zip', null, 'x64', JavaPackageType.NONE)
}
}
darwinTar {
archiveClassifier = 'darwin-x64'
content {
- archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', true)
+ archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.JDK)
}
}
darwinArm64Tar {
archiveClassifier = 'darwin-arm64'
content {
- archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', true)
+ archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.JDK)
}
}
noJdkDarwinTar {
archiveClassifier = 'no-jdk-darwin-x64'
content {
- archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', false)
+ archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.NONE)
}
}
noJdkDarwinArm64Tar {
archiveClassifier = 'no-jdk-darwin-arm64'
content {
- archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', false)
+ archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.NONE)
}
}
freebsdTar {
archiveClassifier = 'freebsd-x64'
content {
- archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false)
+ archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE)
}
}
noJdkFreebsdTar {
archiveClassifier = 'no-jdk-freebsd-x64'
content {
- archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false)
+ archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE)
}
}
linuxArm64Tar {
archiveClassifier = 'linux-arm64'
content {
- archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', true)
+ archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.JDK)
}
}
noJdkLinuxArm64Tar {
archiveClassifier = 'no-jdk-linux-arm64'
content {
- archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', false)
+ archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.NONE)
}
}
linuxTar {
archiveClassifier = 'linux-x64'
content {
- archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', true)
+ archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JDK)
}
}
noJdkLinuxTar {
archiveClassifier = 'no-jdk-linux-x64'
content {
- archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', false)
+ archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.NONE)
+ }
+ }
+
+ jreLinuxTar {
+ archiveClassifier = 'jre-linux-x64'
+ content {
+ archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JRE)
}
}
@@ -163,7 +176,7 @@ distribution_archives {
linuxS390xTar {
archiveClassifier = 'linux-s390x'
content {
- archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', false)
+ archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', JavaPackageType.NONE)
}
}
@@ -171,28 +184,28 @@ distribution_archives {
linuxPpc64leTar {
archiveClassifier = 'linux-ppc64le'
content {
- archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', true)
+ archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.JDK)
}
}
noJdkLinuxPpc64leTar {
archiveClassifier = 'no-jdk-linux-ppc64le'
content {
- archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', false)
+ archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.NONE)
}
}
windowsZip {
archiveClassifier = 'windows-x64'
content {
- archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', true)
+ archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.JDK)
}
}
noJdkWindowsZip {
archiveClassifier = 'no-jdk-windows-x64'
content {
- archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', false)
+ archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.NONE)
}
}
}
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 2378ed833ead7..35ca84ca66dba 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -36,6 +36,7 @@ import org.opensearch.gradle.MavenFilteringHack
import org.opensearch.gradle.NoticeTask
import org.opensearch.gradle.VersionProperties
import org.opensearch.gradle.info.BuildParams
+import org.opensearch.gradle.JavaPackageType
import java.nio.file.Files
import java.nio.file.Path
@@ -275,13 +276,14 @@ project(':test:external-modules').subprojects.each { Project testModule ->
configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
apply plugin: 'opensearch.jdk-download'
+ apply plugin: 'opensearch.jre-download'
apply plugin: 'opensearch.repositories'
// Setup all required JDKs
project.jdks {
['darwin', 'linux', 'windows'].each { platform ->
(platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture ->
- "bundled_${platform}_${architecture}" {
+ "bundled_jdk_${platform}_${architecture}" {
it.platform = platform
it.version = VersionProperties.getBundledJdk(platform, architecture)
it.vendor = VersionProperties.bundledJdkVendor
@@ -291,6 +293,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
}
+ // Setup all required JREs
+ project.jres {
+ ['darwin', 'linux', 'windows'].each { platform ->
+ (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture ->
+ "bundled_jre_${platform}_${architecture}" {
+ it.platform = platform
+ it.version = VersionProperties.getBundledJre(platform, architecture)
+ it.vendor = VersionProperties.bundledJdkVendor
+ it.architecture = architecture
+ }
+ }
+ }
+ }
+
// TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
/*****************************************************************************
* Properties to expand when copying packaging files *
@@ -377,20 +393,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
from buildTransportModulesTaskProvider
}
- configFiles = { distributionType, jdk ->
+ configFiles = { distributionType, java ->
copySpec {
with copySpec {
// main config files, processed with distribution specific substitutions
from '../src/config'
exclude 'log4j2.properties' // this is handled separately below
- MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk))
+ MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java))
}
from project(':distribution').buildLog4jConfig
from project(':distribution').buildConfig
}
}
- binFiles = { distributionType, jdk ->
+ binFiles = { distributionType, java ->
copySpec {
// non-windows files, for all distributions
with copySpec {
@@ -398,7 +414,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
exclude '*.exe'
exclude '*.bat'
eachFile { it.setMode(0755) }
- MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk))
+ MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java))
}
// windows files, only for zip
if (distributionType == 'zip') {
@@ -406,7 +422,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
from '../src/bin'
include '*.bat'
filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
- MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk))
+ MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java))
}
with copySpec {
from '../src/bin'
@@ -424,12 +440,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
}
- noticeFile = { jdk ->
+ noticeFile = { java ->
copySpec {
if (project.name == 'integ-test-zip') {
from buildServerNoticeTaskProvider
} else {
- if (jdk) {
+ if (java != JavaPackageType.NONE) {
from buildNoticeTaskProvider
} else {
from buildNoJdkNoticeTaskProvider
@@ -446,7 +462,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
if ("arm64".equals(architecture)) {
architecture = "aarch64"
}
- from project.jdks."bundled_${platform}_${architecture}"
+ from project.jdks."bundled_jdk_${platform}_${architecture}"
exclude "demo/**"
/*
* The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude
@@ -465,6 +481,31 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
}
}
+
+ jreFiles = { Project project, String platform, String architecture ->
+ return copySpec {
+ /*
+ * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands.
+ */
+ if ("arm64".equals(architecture)) {
+ architecture = "aarch64"
+ }
+ from project.jres."bundled_jre_${platform}_${architecture}"
+ exclude "demo/**"
+ /*
+ * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude
+ * it from the build.
+ */
+ if ("darwin".equals(platform)) {
+ exclude "Contents/MacOS"
+ }
+ eachFile { FileCopyDetails details ->
+ if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
+ details.mode = 0755
+ }
+ }
+ }
+ }
}
}
@@ -583,7 +624,7 @@ subprojects {
],
'opensearch.bundled_jdk': [
- 'def': jdk ? 'true' : 'false'
+ 'def': jdk != JavaPackageType.NONE ? true : false
],
'license.name': [
diff --git a/distribution/docker/docker-arm64-export/build.gradle b/distribution/docker/docker-arm64-export/build.gradle
new file mode 100644
index 0000000000000..3506c4e39c234
--- /dev/null
+++ b/distribution/docker/docker-arm64-export/build.gradle
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.
diff --git a/distribution/docker/docker-s390x-export/build.gradle b/distribution/docker/docker-s390x-export/build.gradle
new file mode 100644
index 0000000000000..3506c4e39c234
--- /dev/null
+++ b/distribution/docker/docker-s390x-export/build.gradle
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+// This file is intentionally blank. All configuration of the
+// export is done in the parent project.
diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index a0ae4c5c2472a..7914fcc172ef4 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -63,7 +63,7 @@ import java.util.regex.Pattern
*/
plugins {
- id "com.netflix.nebula.ospackage-base" version "11.3.0"
+ id "com.netflix.nebula.ospackage-base" version "11.4.0"
}
void addProcessFilesTask(String type, boolean jdk) {
diff --git a/distribution/src/bin/opensearch-env b/distribution/src/bin/opensearch-env
index f94824576b122..cd7a0b2b4520b 100644
--- a/distribution/src/bin/opensearch-env
+++ b/distribution/src/bin/opensearch-env
@@ -46,7 +46,7 @@ fi
# now set the classpath
OPENSEARCH_CLASSPATH="$OPENSEARCH_HOME/lib/*"
-# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JDK
+# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JRE -> bundled JDK
if [ ! -z "$OPENSEARCH_JAVA_HOME" ]; then
JAVA="$OPENSEARCH_JAVA_HOME/bin/java"
JAVA_TYPE="OPENSEARCH_JAVA_HOME"
@@ -57,13 +57,18 @@ else
if [ $OS = "darwin" ]; then
# macOS bundled Java
JAVA="$OPENSEARCH_HOME/jdk.app/Contents/Home/bin/java"
+ JAVA_TYPE="bundled jdk"
elif [ $OS = "freebsd" ]; then
# using FreeBSD default java from ports if JAVA_HOME is not set
JAVA="/usr/local/bin/java"
+ JAVA_TYPE="bundled jdk"
+ elif [ -d "$OPENSEARCH_HOME/jre" ]; then
+ JAVA="$OPENSEARCH_HOME/jre/bin/java"
+ JAVA_TYPE="bundled jre"
else
JAVA="$OPENSEARCH_HOME/jdk/bin/java"
+ JAVA_TYPE="bundled jdk"
fi
- JAVA_TYPE="bundled jdk"
fi
if [ ! -x "$JAVA" ]; then
diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index e15afc0f677c3..952110c6c0289 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -82,3 +82,6 @@ ${error.file}
# JDK 20+ Incubating Vector Module for SIMD optimizations;
# disabling may reduce performance on vector optimized lucene
20:--add-modules=jdk.incubator.vector
+
+# HDFS ForkJoinPool.common() support by SecurityManager
+-Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory
diff --git a/server/src/main/java/org/opensearch/common/component/AbstractLifecycleComponent.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/AbstractLifecycleComponent.java
similarity index 98%
rename from server/src/main/java/org/opensearch/common/component/AbstractLifecycleComponent.java
rename to libs/common/src/main/java/org/opensearch/common/lifecycle/AbstractLifecycleComponent.java
index 837f8af44bf13..111556fbe43cf 100644
--- a/server/src/main/java/org/opensearch/common/component/AbstractLifecycleComponent.java
+++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/AbstractLifecycleComponent.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.component;
+package org.opensearch.common.lifecycle;
import java.io.IOException;
import java.io.UncheckedIOException;
diff --git a/server/src/main/java/org/opensearch/common/component/Lifecycle.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java
similarity index 99%
rename from server/src/main/java/org/opensearch/common/component/Lifecycle.java
rename to libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java
index fb12c1fc9ac4b..e76d49cbf49e8 100644
--- a/server/src/main/java/org/opensearch/common/component/Lifecycle.java
+++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.component;
+package org.opensearch.common.lifecycle;
/**
* Lifecycle state. Allows the following transitions:
diff --git a/server/src/main/java/org/opensearch/common/component/LifecycleComponent.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java
similarity index 97%
rename from server/src/main/java/org/opensearch/common/component/LifecycleComponent.java
rename to libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java
index 984d55df1bdfa..f343f9ada01ef 100644
--- a/server/src/main/java/org/opensearch/common/component/LifecycleComponent.java
+++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.component;
+package org.opensearch.common.lifecycle;
import org.opensearch.common.lease.Releasable;
diff --git a/server/src/main/java/org/opensearch/common/component/LifecycleListener.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleListener.java
similarity index 97%
rename from server/src/main/java/org/opensearch/common/component/LifecycleListener.java
rename to libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleListener.java
index 89c344b955bc9..7ac41a5eb0df0 100644
--- a/server/src/main/java/org/opensearch/common/component/LifecycleListener.java
+++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleListener.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.component;
+package org.opensearch.common.lifecycle;
/**
* Base lifecycle listener.
diff --git a/libs/common/src/main/java/org/opensearch/common/lifecycle/package-info.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/package-info.java
new file mode 100644
index 0000000000000..1bedde5585e36
--- /dev/null
+++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/package-info.java
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Foundation implementation for a object lifecycle.
+ *
+ * See {@link org.opensearch.common.lifecycle.Lifecycle} for example usage
+ *
+ * @opensearch.internal
+ */
+package org.opensearch.common.lifecycle;
diff --git a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0.jar.sha1
deleted file mode 100644
index 2b0f77275c0ab..0000000000000
--- a/libs/core/licenses/lucene-core-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ad391210ffd806931334be9670a35af00c56f959
\ No newline at end of file
diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..dc363f2776429
--- /dev/null
+++ b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+30c3afcf058532d3d2b8820375043000e7f34a9b
\ No newline at end of file
diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchException.java b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
index f75a225af1b4d..1a1273ad8963c 100644
--- a/libs/core/src/main/java/org/opensearch/OpenSearchException.java
+++ b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
@@ -118,6 +118,14 @@ public class OpenSearchException extends RuntimeException implements Writeable,
UNKNOWN_VERSION_ADDED
)
);
+ registerExceptionHandle(
+ new OpenSearchExceptionHandle(
+ org.opensearch.OpenSearchParseException.class,
+ org.opensearch.OpenSearchParseException::new,
+ 35,
+ UNKNOWN_VERSION_ADDED
+ )
+ );
registerExceptionHandle(
new OpenSearchExceptionHandle(
org.opensearch.core.common.ParsingException.class,
@@ -134,6 +142,14 @@ public class OpenSearchException extends RuntimeException implements Writeable,
UNKNOWN_VERSION_ADDED
)
);
+ registerExceptionHandle(
+ new OpenSearchExceptionHandle(
+ org.opensearch.core.common.breaker.CircuitBreakingException.class,
+ org.opensearch.core.common.breaker.CircuitBreakingException::new,
+ 133,
+ UNKNOWN_VERSION_ADDED
+ )
+ );
}
/**
diff --git a/server/src/main/java/org/opensearch/OpenSearchParseException.java b/libs/core/src/main/java/org/opensearch/OpenSearchParseException.java
similarity index 100%
rename from server/src/main/java/org/opensearch/OpenSearchParseException.java
rename to libs/core/src/main/java/org/opensearch/OpenSearchParseException.java
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index f85da63bdbb1f..3f83282245fd8 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -55,7 +55,7 @@ public class Version implements Comparable, ToXContentFragment {
* The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
* values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99
* indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id
- *
+ *>>
* IMPORTANT: Unreleased vs. Released Versions
*
* All listed versions MUST be released versions, except the last major, the last minor and the last revison. ONLY those are required
@@ -90,8 +90,9 @@ public class Version implements Comparable, ToXContentFragment {
public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0);
public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0);
public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0);
+ public static final Version V_2_9_1 = new Version(2090199, org.apache.lucene.util.Version.LUCENE_9_7_0);
public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0);
- public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0);
+ public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0);
public static final Version CURRENT = V_3_0_0;
public static Version fromId(int id) {
diff --git a/libs/core/src/main/java/org/opensearch/core/common/Strings.java b/libs/core/src/main/java/org/opensearch/core/common/Strings.java
index d6c484e5e4746..ffb7711f84492 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/Strings.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/Strings.java
@@ -515,4 +515,27 @@ private static String changeFirstCharacterCase(String str, boolean capitalize) {
sb.append(str.substring(1));
return sb.toString();
}
+
+ /**
+ * Format the double value with a single decimal points, trimming trailing '.0'.
+ */
+ public static String format1Decimals(double value, String suffix) {
+ String p = String.valueOf(value);
+ int ix = p.indexOf('.') + 1;
+ int ex = p.indexOf('E');
+ char fraction = p.charAt(ix);
+ if (fraction == '0') {
+ if (ex != -1) {
+ return p.substring(0, ix - 1) + p.substring(ex) + suffix;
+ } else {
+ return p.substring(0, ix - 1) + suffix;
+ }
+ } else {
+ if (ex != -1) {
+ return p.substring(0, ix) + fraction + p.substring(ex) + suffix;
+ } else {
+ return p.substring(0, ix) + fraction + suffix;
+ }
+ }
+ }
}
diff --git a/server/src/main/java/org/opensearch/common/breaker/CircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java
similarity index 99%
rename from server/src/main/java/org/opensearch/common/breaker/CircuitBreaker.java
rename to libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java
index 4cbd375e8c1ff..0f75f763d21c1 100644
--- a/server/src/main/java/org/opensearch/common/breaker/CircuitBreaker.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.breaker;
+package org.opensearch.core.common.breaker;
import java.util.Locale;
diff --git a/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java
similarity index 98%
rename from server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java
rename to libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java
index 4cab014912970..4869127ee473d 100644
--- a/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java
@@ -29,7 +29,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.breaker;
+package org.opensearch.core.common.breaker;
import org.opensearch.OpenSearchException;
import org.opensearch.core.common.io.stream.StreamInput;
diff --git a/server/src/main/java/org/opensearch/common/breaker/NoopCircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java
similarity index 98%
rename from server/src/main/java/org/opensearch/common/breaker/NoopCircuitBreaker.java
rename to libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java
index ddd72280faa4f..86a0a7ccb96fd 100644
--- a/server/src/main/java/org/opensearch/common/breaker/NoopCircuitBreaker.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.breaker;
+package org.opensearch.core.common.breaker;
/**
* A CircuitBreaker that doesn't increment or adjust, and all operations are
diff --git a/libs/core/src/main/java/org/opensearch/core/common/breaker/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/package-info.java
new file mode 100644
index 0000000000000..f9fb83d2207e1
--- /dev/null
+++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Foundation classes for the Circuit Breaker
+ */
+package org.opensearch.core.common.breaker;
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
index 24795204c6dc8..d9040da569345 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
@@ -53,6 +53,8 @@
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
@@ -344,6 +346,10 @@ public BigInteger readBigInteger() throws IOException {
return new BigInteger(readString());
}
+ public MediaType readMediaType() throws IOException {
+ return MediaTypeRegistry.fromMediaType(readString());
+ }
+
@Nullable
public Text readOptionalText() throws IOException {
int length = readInt();
diff --git a/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java
similarity index 99%
rename from server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java
rename to libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java
index b95e39feb8fac..68486dd7c975f 100644
--- a/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.unit;
+package org.opensearch.core.common.unit;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
diff --git a/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeValue.java
similarity index 88%
rename from server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java
rename to libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeValue.java
index a123c79464727..529501226f5e3 100644
--- a/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeValue.java
@@ -30,16 +30,13 @@
* GitHub history for details.
*/
-package org.opensearch.common.unit;
+package org.opensearch.core.common.unit;
import org.opensearch.OpenSearchParseException;
-import org.opensearch.common.Strings;
+import org.opensearch.core.common.Strings;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.logging.DeprecationLogger;
-import org.opensearch.common.logging.LogConfigurator;
-import org.opensearch.common.network.NetworkService;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -54,17 +51,6 @@
*/
public class ByteSizeValue implements Writeable, Comparable, ToXContentFragment {
- /**
- * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured
- * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any
- * {@link ByteSizeValue} object constructed in, for example, settings in {@link NetworkService}.
- *
- * @opensearch.internal
- */
- static class DeprecationLoggerHolder {
- static DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ByteSizeValue.class);
- }
-
public static final ByteSizeValue ZERO = new ByteSizeValue(0, ByteSizeUnit.BYTES);
private final long size;
@@ -262,14 +248,14 @@ private static ByteSizeValue parse(
return new ByteSizeValue(Long.parseLong(s), unit);
} catch (final NumberFormatException e) {
try {
- final double doubleValue = Double.parseDouble(s);
- DeprecationLoggerHolder.deprecationLogger.deprecate(
- "fractional_byte_values",
- "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for setting [{}]",
+ Double.parseDouble(s);
+ throw new OpenSearchParseException(
+ "Failed to parse bytes value [{}]. Fractional bytes values have been "
+ + "deprecated since Legacy 6.2. Use non-fractional bytes values instead: found for setting [{}]",
+ e,
initialInput,
settingName
);
- return new ByteSizeValue((long) (doubleValue * unit.toBytes(1)));
} catch (final NumberFormatException ignored) {
throw new OpenSearchParseException("failed to parse [{}]", e, initialInput);
}
diff --git a/libs/core/src/main/java/org/opensearch/core/common/unit/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/unit/package-info.java
new file mode 100644
index 0000000000000..79b5dcdcba3b6
--- /dev/null
+++ b/libs/core/src/main/java/org/opensearch/core/common/unit/package-info.java
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Common units of measurement used by the core library. These units of measurement classes exist
+ * in the core because they depend on core functionality beyond the common library (e.g., serializable).
+ *
+ * @opensearch.api
+ * @opensearch.experimental
+ */
+package org.opensearch.core.common.unit;
diff --git a/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java
similarity index 98%
rename from server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java
rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java
index 83f3f9532948f..ab887acb85a87 100644
--- a/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.indices.breaker;
+package org.opensearch.core.indices.breaker;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
diff --git a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerService.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java
similarity index 92%
rename from server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerService.java
rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java
index b5cc1a6b1c6c5..ee9c94f432a36 100644
--- a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerService.java
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java
@@ -30,12 +30,12 @@
* GitHub history for details.
*/
-package org.opensearch.indices.breaker;
+package org.opensearch.core.indices.breaker;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
+import org.opensearch.core.common.breaker.CircuitBreaker;
/**
* Interface for Circuit Breaker services, which provide breakers to classes
diff --git a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java
similarity index 97%
rename from server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java
rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java
index 94e63acd10648..0e53a38908a96 100644
--- a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java
@@ -30,12 +30,12 @@
* GitHub history for details.
*/
-package org.opensearch.indices.breaker;
+package org.opensearch.core.indices.breaker;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/indices/breaker/NoneCircuitBreakerService.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java
similarity index 91%
rename from server/src/main/java/org/opensearch/indices/breaker/NoneCircuitBreakerService.java
rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java
index bcb47b48a5f14..4095fd32b6d3c 100644
--- a/server/src/main/java/org/opensearch/indices/breaker/NoneCircuitBreakerService.java
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java
@@ -30,10 +30,10 @@
* GitHub history for details.
*/
-package org.opensearch.indices.breaker;
+package org.opensearch.core.indices.breaker;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
/**
* Class that returns a breaker that never breaks
diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/package-info.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/package-info.java
new file mode 100644
index 0000000000000..a98f9ab1d9f1e
--- /dev/null
+++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/package-info.java
@@ -0,0 +1,15 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Top Level core circuit breaker implementation
+ *
+ * @opensearch.internal
+ * @opensearch.experimental
+ */
+package org.opensearch.core.indices.breaker;
diff --git a/libs/core/src/main/java/org/opensearch/core/indices/package-info.java b/libs/core/src/main/java/org/opensearch/core/indices/package-info.java
new file mode 100644
index 0000000000000..c80edf3d2f01a
--- /dev/null
+++ b/libs/core/src/main/java/org/opensearch/core/indices/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Top Level Package for implementations used across indexes
+ */
+package org.opensearch.core.indices;
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java
index 82d16315f6527..8e3c115c7ba58 100644
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java
@@ -32,12 +32,18 @@
package org.opensearch.core.xcontent;
+import org.opensearch.core.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Locale;
+
/**
* Abstracts a Media Type and a format parameter.
* Media types are used as values on Content-Type and Accept headers
* format is an URL parameter, specifies response media type.
*/
-public interface MediaType {
+public interface MediaType extends Writeable {
/**
* Returns a type part of a MediaType
* i.e. application for application/json
@@ -65,9 +71,54 @@ default String typeWithSubtype() {
XContent xContent();
+ boolean detectedXContent(final byte[] bytes, int offset, int length);
+
+ boolean detectedXContent(final CharSequence content, final int length);
+
default String mediaType() {
return mediaTypeWithoutParameters();
}
String mediaTypeWithoutParameters();
+
+ XContentBuilder contentBuilder() throws IOException;
+
+ XContentBuilder contentBuilder(final OutputStream os) throws IOException;
+
+ /**
+ * Accepts a format string, which is most of the time is equivalent to {@link MediaType#subtype()}
+ * and attempts to match the value to an {@link MediaType}.
+ * The comparisons are done in lower case format.
+ * This method will return {@code null} if no match is found
+ */
+ static MediaType fromFormat(String mediaType) {
+ return MediaTypeRegistry.fromFormat(mediaType);
+ }
+
+ /**
+ * Attempts to match the given media type with the known {@link MediaType} values. This match is done in a case-insensitive manner.
+ * The provided media type can optionally has parameters.
+ * This method is suitable for parsing of the {@code Content-Type} and {@code Accept} HTTP headers.
+ * This method will return {@code null} if no match is found
+ */
+ static MediaType fromMediaType(String mediaTypeHeaderValue) {
+ mediaTypeHeaderValue = removeVersionInMediaType(mediaTypeHeaderValue);
+ return MediaTypeRegistry.fromMediaType(mediaTypeHeaderValue);
+ }
+
+ /**
+ * Clients compatible with ES 7.x might start sending media types with versioned media type
+ * in a form of application/vnd.elasticsearch+json;compatible-with=7.
+ * This has to be removed in order to be used in 7.x server.
+ * The same client connecting using that media type will be able to communicate with ES 8 thanks to compatible API.
+ * @param mediaType - a media type used on Content-Type header, might contain versioned media type.
+ *
+ * @return a media type string without
+ */
+ private static String removeVersionInMediaType(String mediaType) {
+ if (mediaType != null && (mediaType = mediaType.toLowerCase(Locale.ROOT)).contains("vnd.opensearch")) {
+ return mediaType.replaceAll("vnd.opensearch\\+", "").replaceAll("\\s*;\\s*compatible-with=\\d+", "");
+ }
+ return mediaType;
+ }
}
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java
deleted file mode 100644
index cce95e7b4d6b1..0000000000000
--- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.core.xcontent;
-
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * Parses supported internet media types
- *
- * @opensearch.internal
- */
-public class MediaTypeParser {
- private final Map formatToMediaType;
- private final Map typeWithSubtypeToMediaType;
-
- public MediaTypeParser(T[] acceptedMediaTypes) {
- this(acceptedMediaTypes, Map.of());
- }
-
- public MediaTypeParser(T[] acceptedMediaTypes, Map additionalMediaTypes) {
- final int size = acceptedMediaTypes.length + additionalMediaTypes.size();
- Map formatMap = new HashMap<>(size);
- Map typeMap = new HashMap<>(size);
- for (T mediaType : acceptedMediaTypes) {
- typeMap.put(mediaType.typeWithSubtype(), mediaType);
- formatMap.put(mediaType.format(), mediaType);
- }
- for (Map.Entry entry : additionalMediaTypes.entrySet()) {
- String typeWithSubtype = entry.getKey();
- T mediaType = entry.getValue();
-
- typeMap.put(typeWithSubtype.toLowerCase(Locale.ROOT), mediaType);
- formatMap.put(mediaType.format(), mediaType);
- }
-
- this.formatToMediaType = Map.copyOf(formatMap);
- this.typeWithSubtypeToMediaType = Map.copyOf(typeMap);
- }
-
- public T fromMediaType(String mediaType) {
- ParsedMediaType parsedMediaType = parseMediaType(mediaType);
- return parsedMediaType != null ? parsedMediaType.getMediaType() : null;
- }
-
- public T fromFormat(String format) {
- if (format == null) {
- return null;
- }
- return formatToMediaType.get(format.toLowerCase(Locale.ROOT));
- }
-
- /**
- * parsing media type that follows https://tools.ietf.org/html/rfc7231#section-3.1.1.1
- * @param headerValue a header value from Accept or Content-Type
- * @return a parsed media-type
- */
- public ParsedMediaType parseMediaType(String headerValue) {
- if (headerValue != null) {
- String[] split = headerValue.toLowerCase(Locale.ROOT).split(";");
-
- String[] typeSubtype = split[0].trim().split("/");
- if (typeSubtype.length == 2) {
- String type = typeSubtype[0];
- String subtype = typeSubtype[1];
- T xContentType = typeWithSubtypeToMediaType.get(type + "/" + subtype);
- if (xContentType != null) {
- Map parameters = new HashMap<>();
- for (int i = 1; i < split.length; i++) {
- // spaces are allowed between parameters, but not between '=' sign
- String[] keyValueParam = split[i].trim().split("=");
- if (keyValueParam.length != 2 || hasSpaces(keyValueParam[0]) || hasSpaces(keyValueParam[1])) {
- return null;
- }
- parameters.put(keyValueParam[0], keyValueParam[1]);
- }
- return new ParsedMediaType(xContentType, parameters);
- }
- }
-
- }
- return null;
- }
-
- private boolean hasSpaces(String s) {
- return s.trim().equals(s) == false;
- }
-
- /**
- * A media type object that contains all the information provided on a Content-Type or Accept header
- */
- public class ParsedMediaType {
- private final Map parameters;
- private final T mediaType;
-
- public ParsedMediaType(T mediaType, Map parameters) {
- this.parameters = parameters;
- this.mediaType = mediaType;
- }
-
- public T getMediaType() {
- return mediaType;
- }
-
- public Map getParameters() {
- return parameters;
- }
- }
-}
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java
new file mode 100644
index 0000000000000..064f154602caf
--- /dev/null
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java
@@ -0,0 +1,406 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.core.xcontent;
+
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.spi.MediaTypeProvider;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Parses supported internet media types
+ *
+ * @opensearch.internal
+ */
+public final class MediaTypeRegistry {
+ private static Map formatToMediaType = Map.of();
+ private static Map typeWithSubtypeToMediaType = Map.of();
+
+ // Default mediaType singleton
+ private static MediaType DEFAULT_MEDIA_TYPE;
+ public static final int GUESS_HEADER_LENGTH = 20;
+
+ // JSON is a core type, so we create a static instance for implementations that require JSON format (e.g., tests)
+ // todo we should explore moving the concrete JSON implementation from the xcontent library to core
+ public static final MediaType JSON;
+
+ static {
+ List mediaTypes = new ArrayList<>();
+ Map amt = new HashMap<>();
+ for (MediaTypeProvider provider : ServiceLoader.load(MediaTypeProvider.class, MediaTypeProvider.class.getClassLoader())) {
+ mediaTypes.addAll(provider.getMediaTypes());
+ amt = Stream.of(amt, provider.getAdditionalMediaTypes())
+ .flatMap(map -> map.entrySet().stream())
+ .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+ register(mediaTypes.toArray(new MediaType[0]), amt);
+ JSON = fromMediaType("application/json");
+ setDefaultMediaType(JSON);
+ }
+
+ private static void register(MediaType[] acceptedMediaTypes, Map additionalMediaTypes) {
+ // ensures the map is not overwritten:
+ Map typeMap = new HashMap<>(typeWithSubtypeToMediaType);
+ Map formatMap = new HashMap<>(formatToMediaType);
+ for (MediaType mediaType : acceptedMediaTypes) {
+ if (formatMap.containsKey(mediaType.format())) {
+ throw new IllegalArgumentException("unable to register mediaType: [" + mediaType.format() + "]. Type already exists.");
+ }
+ typeMap.put(mediaType.typeWithSubtype(), mediaType);
+ formatMap.put(mediaType.format(), mediaType);
+ }
+ for (Map.Entry entry : additionalMediaTypes.entrySet()) {
+ String typeWithSubtype = entry.getKey().toLowerCase(Locale.ROOT);
+ if (typeMap.containsKey(typeWithSubtype)) {
+ throw new IllegalArgumentException(
+ "unable to register mediaType: ["
+ + entry.getKey()
+ + "]. "
+ + "Type already exists and is mapped to: [."
+ + entry.getValue().format()
+ + "]"
+ );
+ }
+
+ MediaType mediaType = entry.getValue();
+ typeMap.put(typeWithSubtype, mediaType);
+ formatMap.putIfAbsent(mediaType.format(), mediaType); // ignore if the additional type mapping already exists
+ }
+
+ formatToMediaType = Map.copyOf(formatMap);
+ typeWithSubtypeToMediaType = Map.copyOf(typeMap);
+ }
+
+ public static MediaType fromMediaType(String mediaType) {
+ ParsedMediaType parsedMediaType = parseMediaType(mediaType);
+ return parsedMediaType != null ? parsedMediaType.getMediaType() : null;
+ }
+
+ public static MediaType fromFormat(String format) {
+ if (format == null) {
+ return null;
+ }
+ return formatToMediaType.get(format.toLowerCase(Locale.ROOT));
+ }
+
+ /**
+ * Returns a binary content builder for the provided content type.
+ */
+ public static XContentBuilder contentBuilder(MediaType type) throws IOException {
+ for (var mediaType : formatToMediaType.values()) {
+ if (type == mediaType) {
+ return type.contentBuilder();
+ }
+ }
+ throw new IllegalArgumentException("No matching content type for " + type);
+ }
+
+ public static XContentBuilder contentBuilder(MediaType type, OutputStream outputStream) throws IOException {
+ for (var mediaType : formatToMediaType.values()) {
+ if (type == mediaType) {
+ return type.contentBuilder(outputStream);
+ }
+ }
+ throw new IllegalArgumentException("No matching content type for " + type);
+ }
+
+ /**
+ * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent}
+ *
+ * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
+ * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
+ * This method is deprecated to prevent usages of it from spreading further without specific reasons.
+ */
+ @Deprecated
+ public static MediaType xContent(final byte[] data, int offset, int length) {
+ MediaType type = mediaTypeFromBytes(data, offset, length);
+ if (type == null) {
+ throw new XContentParseException("Failed to derive xcontent");
+ }
+ return type;
+ }
+
+ /**
+ * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent}
+ *
+ * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
+ * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
+ * This method is deprecated to prevent usages of it from spreading further without specific reasons.
+ */
+ @Deprecated
+ public static MediaType xContent(byte[] data) {
+ return xContent(data, 0, data.length);
+ }
+
+ /**
+ * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent}
+ *
+ * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
+ * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
+ * This method is deprecated to prevent usages of it from spreading further without specific reasons.
+ */
+ @Deprecated
+ public static MediaType xContent(CharSequence content) {
+ MediaType type = xContentType(content);
+ if (type == null) {
+ throw new XContentParseException("Failed to derive xcontent");
+ }
+ return type;
+ }
+
+ /**
+ * Guesses the content type based on the provided char sequence.
+ *
+ * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
+ * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
+ * This method is deprecated to prevent usages of it from spreading further without specific reasons.
+ */
+ @Deprecated
+ public static MediaType xContentType(CharSequence content) {
+ int length = content.length() < GUESS_HEADER_LENGTH ? content.length() : GUESS_HEADER_LENGTH;
+ if (length == 0) {
+ return null;
+ }
+ for (var mediaType : formatToMediaType.values()) {
+ if (mediaType.detectedXContent(content, length)) {
+ return mediaType;
+ }
+ }
+
+ // fallback for json
+ for (int i = 0; i < length; i++) {
+ char c = content.charAt(i);
+ if (c == '{') {
+ return MediaType.fromMediaType("application/json");
+ }
+ if (Character.isWhitespace(c) == false) {
+ break;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Guesses the content type based on the provided input stream without consuming it.
+ *
+ * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
+ * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
+ * This method is deprecated to prevent usages of it from spreading further without specific reasons.
+ */
+ @Deprecated
+ public static MediaType xContentType(InputStream si) throws IOException {
+ /*
+ * We need to guess the content type. To do this, we look for the first non-whitespace character and then try to guess the content
+ * type on the GUESS_HEADER_LENGTH bytes that follow. We do this in a way that does not modify the initial read position in the
+ * underlying input stream. This is why the input stream must support mark/reset and why we repeatedly mark the read position and
+ * reset.
+ */
+ if (si.markSupported() == false) {
+ throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass());
+ }
+ si.mark(Integer.MAX_VALUE);
+ try {
+ // scan until we find the first non-whitespace character or the end of the stream
+ int current;
+ do {
+ current = si.read();
+ if (current == -1) {
+ return null;
+ }
+ } while (Character.isWhitespace((char) current));
+ // now guess the content type off the next GUESS_HEADER_LENGTH bytes including the current byte
+ final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH];
+ firstBytes[0] = (byte) current;
+ int read = 1;
+ while (read < GUESS_HEADER_LENGTH) {
+ final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read);
+ if (r == -1) {
+ break;
+ }
+ read += r;
+ }
+ return mediaTypeFromBytes(firstBytes, 0, read);
+ } finally {
+ si.reset();
+ }
+
+ }
+
+ /**
+ * Guesses the content type based on the provided bytes.
+ *
+ * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
+ * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
+ * This method is deprecated to prevent usages of it from spreading further without specific reasons.
+ */
+ @Deprecated
+ public static MediaType xContentType(BytesReference bytes) {
+ if (bytes instanceof BytesArray) {
+ final BytesArray array = (BytesArray) bytes;
+ return mediaTypeFromBytes(array.array(), array.offset(), array.length());
+ }
+ try {
+ final InputStream inputStream = bytes.streamInput();
+ assert inputStream.markSupported();
+ return xContentType(inputStream);
+ } catch (IOException e) {
+ assert false : "Should not happen, we're just reading bytes from memory";
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /**
+ * Guesses the content type based on the provided bytes.
+ *
+ * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
+ * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
+ * This method is deprecated to prevent usages of it from spreading further without specific reasons.
+ */
+ @Deprecated
+ public static MediaType mediaTypeFromBytes(final byte[] data, int offset, int length) {
+ int totalLength = data.length;
+ if (totalLength == 0 || length == 0) {
+ return null;
+ } else if ((offset + length) > totalLength) {
+ return null;
+ }
+ for (var mediaType : formatToMediaType.values()) {
+ if (mediaType.detectedXContent(data, offset, length)) {
+ return mediaType;
+ }
+ }
+
+ // a last chance for JSON
+ int jsonStart = 0;
+ // JSON may be preceded by UTF-8 BOM
+ if (length > 3 && data[offset] == (byte) 0xEF && data[offset + 1] == (byte) 0xBB && data[offset + 2] == (byte) 0xBF) {
+ jsonStart = 3;
+ }
+
+ for (int i = jsonStart; i < length; i++) {
+ byte b = data[offset + i];
+ if (b == '{') {
+ return fromMediaType("application/json");
+ }
+ if (Character.isWhitespace(b) == false) {
+ break;
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * parsing media type that follows https://tools.ietf.org/html/rfc7231#section-3.1.1.1
+ * @param headerValue a header value from Accept or Content-Type
+ * @return a parsed media-type
+ */
+ public static ParsedMediaType parseMediaType(String headerValue) {
+ if (headerValue != null) {
+ String[] split = headerValue.toLowerCase(Locale.ROOT).split(";");
+
+ String[] typeSubtype = split[0].trim().split("/");
+ if (typeSubtype.length == 2) {
+ String type = typeSubtype[0];
+ String subtype = typeSubtype[1];
+ MediaType mediaType = typeWithSubtypeToMediaType.get(type + "/" + subtype);
+ if (mediaType != null) {
+ Map parameters = new HashMap<>();
+ for (int i = 1; i < split.length; i++) {
+ // spaces are allowed between parameters, but not between '=' sign
+ String[] keyValueParam = split[i].trim().split("=");
+ if (keyValueParam.length != 2 || hasSpaces(keyValueParam[0]) || hasSpaces(keyValueParam[1])) {
+ return null;
+ }
+ parameters.put(keyValueParam[0], keyValueParam[1]);
+ }
+ return new ParsedMediaType(mediaType, parameters);
+ }
+ }
+
+ }
+ return null;
+ }
+
+ private static boolean hasSpaces(String s) {
+ return s.trim().equals(s) == false;
+ }
+
+ /**
+ * A media type object that contains all the information provided on a Content-Type or Accept header
+ */
+ public static class ParsedMediaType {
+ private final Map parameters;
+ private final MediaType mediaType;
+
+ public ParsedMediaType(MediaType mediaType, Map parameters) {
+ this.parameters = parameters;
+ this.mediaType = mediaType;
+ }
+
+ public MediaType getMediaType() {
+ return mediaType;
+ }
+
+ public Map getParameters() {
+ return parameters;
+ }
+ }
+
+ private static void setDefaultMediaType(final MediaType mediaType) {
+ if (DEFAULT_MEDIA_TYPE != null) {
+ throw new RuntimeException(
+ "unable to reset the default media type from current default [" + DEFAULT_MEDIA_TYPE + "] to [" + mediaType + "]"
+ );
+ } else {
+ DEFAULT_MEDIA_TYPE = mediaType;
+ }
+ }
+
+ public static MediaType getDefaultMediaType() {
+ return DEFAULT_MEDIA_TYPE;
+ }
+}
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java
new file mode 100644
index 0000000000000..eeaadc1698df6
--- /dev/null
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java
@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.core.xcontent.spi;
+
+import org.opensearch.core.xcontent.MediaType;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Service Provider Interface for plugins, modules, extensions providing
+ * their own Media Types
+ *
+ * @opensearch.experimental
+ * @opensearch.api
+ */
+public interface MediaTypeProvider {
+ /** Extensions that implement their own concrete {@link MediaType}s provide them through this interface method */
+ List getMediaTypes();
+
+ /** Extensions that implement additional {@link MediaType} aliases provide them through this interface method */
+ Map getAdditionalMediaTypes();
+}
diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java
new file mode 100644
index 0000000000000..67ccd981dafa8
--- /dev/null
+++ b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java
@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Service Provider Interface for extensible media types */
+package org.opensearch.core.xcontent.spi;
diff --git a/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeUnitTests.java
similarity index 91%
rename from server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java
rename to libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeUnitTests.java
index c6bcceec99fbd..07b9131602ac3 100644
--- a/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java
+++ b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeUnitTests.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.unit;
+package org.opensearch.core.common.unit;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
@@ -38,12 +38,12 @@
import java.io.IOException;
-import static org.opensearch.common.unit.ByteSizeUnit.BYTES;
-import static org.opensearch.common.unit.ByteSizeUnit.GB;
-import static org.opensearch.common.unit.ByteSizeUnit.KB;
-import static org.opensearch.common.unit.ByteSizeUnit.MB;
-import static org.opensearch.common.unit.ByteSizeUnit.PB;
-import static org.opensearch.common.unit.ByteSizeUnit.TB;
+import static org.opensearch.core.common.unit.ByteSizeUnit.BYTES;
+import static org.opensearch.core.common.unit.ByteSizeUnit.GB;
+import static org.opensearch.core.common.unit.ByteSizeUnit.KB;
+import static org.opensearch.core.common.unit.ByteSizeUnit.MB;
+import static org.opensearch.core.common.unit.ByteSizeUnit.PB;
+import static org.opensearch.core.common.unit.ByteSizeUnit.TB;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
diff --git a/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeValueTests.java
similarity index 97%
rename from server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java
rename to libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeValueTests.java
index 99c1feb78527f..def1694a72ba4 100644
--- a/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java
+++ b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeValueTests.java
@@ -30,7 +30,7 @@
* GitHub history for details.
*/
-package org.opensearch.common.unit;
+package org.opensearch.core.common.unit;
import org.opensearch.OpenSearchParseException;
import org.opensearch.core.common.io.stream.Writeable.Reader;
@@ -336,12 +336,10 @@ public void testParseInvalidNumber() throws IOException {
public void testParseFractionalNumber() throws IOException {
ByteSizeUnit unit = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values()));
String fractionalValue = "23.5" + unit.getSuffix();
- ByteSizeValue instance = ByteSizeValue.parseBytesSizeValue(fractionalValue, "test");
- assertEquals(fractionalValue, instance.toString());
- assertWarnings(
- "Fractional bytes values are deprecated. Use non-fractional bytes values instead: ["
- + fractionalValue
- + "] found for setting [test]"
+ // test exception is thrown: fractional byte size values has been deprecated since Legacy 6.2
+ OpenSearchParseException e = expectThrows(
+ OpenSearchParseException.class,
+ () -> ByteSizeValue.parseBytesSizeValue(fractionalValue, "test")
);
}
diff --git a/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java
new file mode 100644
index 0000000000000..fe239fea8129e
--- /dev/null
+++ b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.secure_sm;
+
+import java.security.AccessControlContext;
+import java.security.AccessController;
+import java.security.Permission;
+import java.security.Permissions;
+import java.security.PrivilegedAction;
+import java.security.ProtectionDomain;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory;
+import java.util.concurrent.ForkJoinWorkerThread;
+
+public class SecuredForkJoinWorkerThreadFactory implements ForkJoinWorkerThreadFactory {
+ static AccessControlContext contextWithPermissions(Permission... perms) {
+ Permissions permissions = new Permissions();
+ for (Permission perm : perms)
+ permissions.add(perm);
+ return new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, permissions) });
+ }
+
+ // ACC for access to the factory
+ private static final AccessControlContext ACC = contextWithPermissions(
+ new RuntimePermission("getClassLoader"),
+ new RuntimePermission("setContextClassLoader"),
+ new RuntimePermission("modifyThreadGroup"),
+ new RuntimePermission("modifyThread")
+ );
+
+ public final ForkJoinWorkerThread newThread(ForkJoinPool pool) {
+ return AccessController.doPrivileged(new PrivilegedAction<>() {
+ public ForkJoinWorkerThread run() {
+ return new ForkJoinWorkerThread(pool) {
+
+ };
+ }
+ }, ACC);
+ }
+}
diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java
index 94ac264546dbb..9f423bc9abad3 100644
--- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java
+++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java
@@ -32,19 +32,14 @@
package org.opensearch.common.xcontent;
-import com.fasterxml.jackson.dataformat.cbor.CBORConstants;
-import com.fasterxml.jackson.dataformat.smile.SmileConstants;
-import org.opensearch.common.xcontent.cbor.CborXContent;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.common.xcontent.smile.SmileXContent;
import org.opensearch.common.xcontent.yaml.YamlXContent;
-import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.core.xcontent.XContentParseException;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
/**
@@ -52,13 +47,11 @@
*/
public class XContentFactory {
- static final int GUESS_HEADER_LENGTH = 20;
-
/**
* Returns a content builder using JSON format ({@link org.opensearch.common.xcontent.XContentType#JSON}.
*/
public static XContentBuilder jsonBuilder() throws IOException {
- return contentBuilder(XContentType.JSON);
+ return MediaTypeRegistry.contentBuilder(XContentType.JSON);
}
/**
@@ -72,7 +65,7 @@ public static XContentBuilder jsonBuilder(OutputStream os) throws IOException {
* Returns a content builder using SMILE format ({@link org.opensearch.common.xcontent.XContentType#SMILE}.
*/
public static XContentBuilder smileBuilder() throws IOException {
- return contentBuilder(XContentType.SMILE);
+ return MediaTypeRegistry.contentBuilder(XContentType.SMILE);
}
/**
@@ -86,7 +79,7 @@ public static XContentBuilder smileBuilder(OutputStream os) throws IOException {
* Returns a content builder using YAML format ({@link org.opensearch.common.xcontent.XContentType#YAML}.
*/
public static XContentBuilder yamlBuilder() throws IOException {
- return contentBuilder(XContentType.YAML);
+ return MediaTypeRegistry.contentBuilder(XContentType.YAML);
}
/**
@@ -100,271 +93,6 @@ public static XContentBuilder yamlBuilder(OutputStream os) throws IOException {
* Returns a content builder using CBOR format ({@link org.opensearch.common.xcontent.XContentType#CBOR}.
*/
public static XContentBuilder cborBuilder() throws IOException {
- return contentBuilder(XContentType.CBOR);
- }
-
- /**
- * Constructs a new cbor builder that will output the result into the provided output stream.
- */
- public static XContentBuilder cborBuilder(OutputStream os) throws IOException {
- return new XContentBuilder(CborXContent.cborXContent, os);
- }
-
- /**
- * Constructs a xcontent builder that will output the result into the provided output stream.
- */
- public static XContentBuilder contentBuilder(XContentType type, OutputStream outputStream) throws IOException {
- if (type == XContentType.JSON) {
- return jsonBuilder(outputStream);
- } else if (type == XContentType.SMILE) {
- return smileBuilder(outputStream);
- } else if (type == XContentType.YAML) {
- return yamlBuilder(outputStream);
- } else if (type == XContentType.CBOR) {
- return cborBuilder(outputStream);
- }
- throw new IllegalArgumentException("No matching content type for " + type);
- }
-
- /**
- * Returns a binary content builder for the provided media type.
- */
- public static XContentBuilder contentBuilder(MediaType type) throws IOException {
- if (type instanceof XContentType) {
- return contentBuilder(XContentType.fromMediaType(type));
- }
- throw new IllegalArgumentException("Content type [" + type.getClass().getName() + "] not supported");
- }
-
- /**
- * Returns a binary content builder for the provided content type.
- */
- public static XContentBuilder contentBuilder(XContentType type) throws IOException {
- if (type == XContentType.JSON) {
- return JsonXContent.contentBuilder();
- } else if (type == XContentType.SMILE) {
- return SmileXContent.contentBuilder();
- } else if (type == XContentType.YAML) {
- return YamlXContent.contentBuilder();
- } else if (type == XContentType.CBOR) {
- return CborXContent.contentBuilder();
- }
- throw new IllegalArgumentException("No matching content type for " + type);
- }
-
- /**
- * Returns the {@link XContent} for the provided content type.
- */
- public static XContent xContent(MediaType type) {
- if (type == null) {
- throw new IllegalArgumentException("Cannot get xcontent for unknown type");
- }
- return type.xContent();
- }
-
- /**
- * Guesses the content type based on the provided char sequence.
- *
- * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
- * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
- * This method is deprecated to prevent usages of it from spreading further without specific reasons.
- */
- @Deprecated
- public static XContentType xContentType(CharSequence content) {
- int length = content.length() < GUESS_HEADER_LENGTH ? content.length() : GUESS_HEADER_LENGTH;
- if (length == 0) {
- return null;
- }
- char first = content.charAt(0);
- if (first == '{') {
- return XContentType.JSON;
- }
- // Should we throw a failure here? Smile idea is to use it in bytes....
- if (length > 2
- && first == SmileConstants.HEADER_BYTE_1
- && content.charAt(1) == SmileConstants.HEADER_BYTE_2
- && content.charAt(2) == SmileConstants.HEADER_BYTE_3) {
- return XContentType.SMILE;
- }
- if (length > 2 && first == '-' && content.charAt(1) == '-' && content.charAt(2) == '-') {
- return XContentType.YAML;
- }
-
- // CBOR is not supported
-
- for (int i = 0; i < length; i++) {
- char c = content.charAt(i);
- if (c == '{') {
- return XContentType.JSON;
- }
- if (Character.isWhitespace(c) == false) {
- break;
- }
- }
- return null;
- }
-
- /**
- * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent}
- *
- * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
- * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
- * This method is deprecated to prevent usages of it from spreading further without specific reasons.
- */
- @Deprecated
- public static XContent xContent(CharSequence content) {
- XContentType type = xContentType(content);
- if (type == null) {
- throw new XContentParseException("Failed to derive xcontent");
- }
- return xContent(type);
- }
-
- /**
- * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent}
- *
- * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
- * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
- * This method is deprecated to prevent usages of it from spreading further without specific reasons.
- */
- @Deprecated
- public static XContent xContent(byte[] data) {
- return xContent(data, 0, data.length);
- }
-
- /**
- * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent}
- *
- * @deprecated guessing the content type should not be needed ideally. We should rather know the content type upfront or read it
- * from headers. Till we fixed the REST layer to read the Content-Type header, that should be the only place where guessing is needed.
- */
- @Deprecated
- public static XContent xContent(byte[] data, int offset, int length) {
- XContentType type = xContentType(data, offset, length);
- if (type == null) {
- throw new XContentParseException("Failed to derive xcontent");
- }
- return xContent(type);
- }
-
- /**
- * Guesses the content type based on the provided input stream without consuming it.
- *
- * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
- * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
- * This method is deprecated to prevent usages of it from spreading further without specific reasons.
- */
- @Deprecated
- public static XContentType xContentType(InputStream si) throws IOException {
- /*
- * We need to guess the content type. To do this, we look for the first non-whitespace character and then try to guess the content
- * type on the GUESS_HEADER_LENGTH bytes that follow. We do this in a way that does not modify the initial read position in the
- * underlying input stream. This is why the input stream must support mark/reset and why we repeatedly mark the read position and
- * reset.
- */
- if (si.markSupported() == false) {
- throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass());
- }
- si.mark(Integer.MAX_VALUE);
- try {
- // scan until we find the first non-whitespace character or the end of the stream
- int current;
- do {
- current = si.read();
- if (current == -1) {
- return null;
- }
- } while (Character.isWhitespace((char) current));
- // now guess the content type off the next GUESS_HEADER_LENGTH bytes including the current byte
- final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH];
- firstBytes[0] = (byte) current;
- int read = 1;
- while (read < GUESS_HEADER_LENGTH) {
- final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read);
- if (r == -1) {
- break;
- }
- read += r;
- }
- return xContentType(firstBytes, 0, read);
- } finally {
- si.reset();
- }
-
- }
-
- /**
- * Guesses the content type based on the provided bytes.
- *
- * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
- * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
- * This method is deprecated to prevent usages of it from spreading further without specific reasons.
- */
- @Deprecated
- public static XContentType xContentType(byte[] bytes) {
- return xContentType(bytes, 0, bytes.length);
- }
-
- /**
- * Guesses the content type based on the provided bytes.
- *
- * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
- * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
- * This method is deprecated to prevent usages of it from spreading further without specific reasons.
- */
- @Deprecated
- public static XContentType xContentType(byte[] bytes, int offset, int length) {
- int totalLength = bytes.length;
- if (totalLength == 0 || length == 0) {
- return null;
- } else if ((offset + length) > totalLength) {
- return null;
- }
- byte first = bytes[offset];
- if (first == '{') {
- return XContentType.JSON;
- }
- if (length > 2
- && first == SmileConstants.HEADER_BYTE_1
- && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2
- && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3) {
- return XContentType.SMILE;
- }
- if (length > 2 && first == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-') {
- return XContentType.YAML;
- }
- // CBOR logic similar to CBORFactory#hasCBORFormat
- if (first == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1) {
- return XContentType.CBOR;
- }
- if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first) && length > 2) {
- // Actually, specific "self-describe tag" is a very good indicator
- if (first == (byte) 0xD9 && bytes[offset + 1] == (byte) 0xD9 && bytes[offset + 2] == (byte) 0xF7) {
- return XContentType.CBOR;
- }
- }
- // for small objects, some encoders just encode as major type object, we can safely
- // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort
- if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, first)) {
- return XContentType.CBOR;
- }
-
- int jsonStart = 0;
- // JSON may be preceded by UTF-8 BOM
- if (length > 3 && first == (byte) 0xEF && bytes[offset + 1] == (byte) 0xBB && bytes[offset + 2] == (byte) 0xBF) {
- jsonStart = 3;
- }
-
- // a last chance for JSON
- for (int i = jsonStart; i < length; i++) {
- byte b = bytes[offset + i];
- if (b == '{') {
- return XContentType.JSON;
- }
- if (Character.isWhitespace(b) == false) {
- break;
- }
- }
- return null;
+ return MediaTypeRegistry.contentBuilder(XContentType.CBOR);
}
}
diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java
index b3a8f5e27635d..7026dbd5e78c3 100644
--- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java
+++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java
@@ -32,16 +32,19 @@
package org.opensearch.common.xcontent;
+import com.fasterxml.jackson.dataformat.cbor.CBORConstants;
+import com.fasterxml.jackson.dataformat.smile.SmileConstants;
import org.opensearch.common.xcontent.cbor.CborXContent;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.common.xcontent.smile.SmileXContent;
import org.opensearch.common.xcontent.yaml.YamlXContent;
+import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.MediaType;
-import org.opensearch.core.xcontent.MediaTypeParser;
import org.opensearch.core.xcontent.XContent;
+import org.opensearch.core.xcontent.XContentBuilder;
-import java.util.Locale;
-import java.util.Map;
+import java.io.IOException;
+import java.io.OutputStream;
/**
* The content type of {@link XContent}.
@@ -71,6 +74,26 @@ public String subtype() {
public XContent xContent() {
return JsonXContent.jsonXContent;
}
+
+ @Override
+ public boolean detectedXContent(final byte[] bytes, int offset, int length) {
+ return bytes[offset] == '{';
+ }
+
+ @Override
+ public boolean detectedXContent(final CharSequence content, final int length) {
+ return content.charAt(0) == '{';
+ }
+
+ @Override
+ public XContentBuilder contentBuilder() throws IOException {
+ return JsonXContent.contentBuilder();
+ }
+
+ @Override
+ public XContentBuilder contentBuilder(final OutputStream os) throws IOException {
+ return new XContentBuilder(JsonXContent.jsonXContent, os);
+ }
},
/**
* The jackson based smile binary format. Fast and compact binary format.
@@ -90,6 +113,32 @@ public String subtype() {
public XContent xContent() {
return SmileXContent.smileXContent;
}
+
+ @Override
+ public boolean detectedXContent(final byte[] bytes, int offset, int length) {
+ return length > 2
+ && bytes[offset] == SmileConstants.HEADER_BYTE_1
+ && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2
+ && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3;
+ }
+
+ @Override
+ public boolean detectedXContent(final CharSequence content, final int length) {
+ return length > 2
+ && content.charAt(0) == SmileConstants.HEADER_BYTE_1
+ && content.charAt(1) == SmileConstants.HEADER_BYTE_2
+ && content.charAt(2) == SmileConstants.HEADER_BYTE_3;
+ }
+
+ @Override
+ public XContentBuilder contentBuilder() throws IOException {
+ return SmileXContent.contentBuilder();
+ }
+
+ @Override
+ public XContentBuilder contentBuilder(final OutputStream os) throws IOException {
+ return new XContentBuilder(SmileXContent.smileXContent, os);
+ }
},
/**
* A YAML based content type.
@@ -109,6 +158,26 @@ public String subtype() {
public XContent xContent() {
return YamlXContent.yamlXContent;
}
+
+ @Override
+ public boolean detectedXContent(final byte[] bytes, int offset, int length) {
+ return length > 2 && bytes[offset] == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-';
+ }
+
+ @Override
+ public boolean detectedXContent(final CharSequence content, final int length) {
+ return length > 2 && content.charAt(0) == '-' && content.charAt(1) == '-' && content.charAt(2) == '-';
+ }
+
+ @Override
+ public XContentBuilder contentBuilder() throws IOException {
+ return YamlXContent.contentBuilder();
+ }
+
+ @Override
+ public XContentBuilder contentBuilder(final OutputStream os) throws IOException {
+ return new XContentBuilder(YamlXContent.yamlXContent, os);
+ }
},
/**
* A CBOR based content type.
@@ -128,56 +197,42 @@ public String subtype() {
public XContent xContent() {
return CborXContent.cborXContent;
}
- };
- /** a parser of media types */
- private static final MediaTypeParser MEDIA_TYPE_PARSER = new MediaTypeParser<>(
- XContentType.values(),
- Map.of("application/*", JSON, "application/x-ndjson", JSON)
- );
-
- /** gets the {@link MediaTypeParser} singleton for use outside class */
- @SuppressWarnings("rawtypes")
- public static MediaTypeParser getMediaTypeParser() {
- return MEDIA_TYPE_PARSER;
- }
+ @Override
+ public boolean detectedXContent(final byte[] bytes, int offset, int length) {
+ // CBOR logic similar to CBORFactory#hasCBORFormat
+ if (bytes[offset] == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1) {
+ return true;
+ }
+ if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, bytes[offset]) && length > 2) {
+ // Actually, specific "self-describe tag" is a very good indicator
+ if (bytes[offset] == (byte) 0xD9 && bytes[offset + 1] == (byte) 0xD9 && bytes[offset + 2] == (byte) 0xF7) {
+ return true;
+ }
+ }
+ // for small objects, some encoders just encode as major type object, we can safely
+ // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort
+ if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, bytes[offset])) {
+ return true;
+ }
+ return false;
+ }
- /**
- * Accepts a format string, which is most of the time is equivalent to {@link XContentType#subtype()}
- * and attempts to match the value to an {@link XContentType}.
- * The comparisons are done in lower case format.
- * This method will return {@code null} if no match is found
- */
- public static XContentType fromFormat(String mediaType) {
- return MEDIA_TYPE_PARSER.fromFormat(mediaType);
- }
+ @Override
+ public boolean detectedXContent(final CharSequence content, final int length) {
+ return false;
+ }
- /**
- * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner.
- * The provided media type can optionally has parameters.
- * This method is suitable for parsing of the {@code Content-Type} and {@code Accept} HTTP headers.
- * This method will return {@code null} if no match is found
- */
- public static XContentType fromMediaType(String mediaTypeHeaderValue) {
- mediaTypeHeaderValue = removeVersionInMediaType(mediaTypeHeaderValue);
- return MEDIA_TYPE_PARSER.fromMediaType(mediaTypeHeaderValue);
- }
+ @Override
+ public XContentBuilder contentBuilder() throws IOException {
+ return CborXContent.contentBuilder();
+ }
- /**
- * Clients compatible with ES 7.x might start sending media types with versioned media type
- * in a form of application/vnd.elasticsearch+json;compatible-with=7.
- * This has to be removed in order to be used in 7.x server.
- * The same client connecting using that media type will be able to communicate with ES 8 thanks to compatible API.
- * @param mediaType - a media type used on Content-Type header, might contain versioned media type.
- *
- * @return a media type string without
- */
- private static String removeVersionInMediaType(String mediaType) {
- if (mediaType != null && (mediaType = mediaType.toLowerCase(Locale.ROOT)).contains("vnd.opensearch")) {
- return mediaType.replaceAll("vnd.opensearch\\+", "").replaceAll("\\s*;\\s*compatible-with=\\d+", "");
+ @Override
+ public XContentBuilder contentBuilder(final OutputStream os) throws IOException {
+ return new XContentBuilder(CborXContent.cborXContent, os);
}
- return mediaType;
- }
+ };
private int index;
@@ -199,12 +254,8 @@ public String format() {
return subtype();
}
- /** Converts from a {@link MediaType} to an explicit {@link XContentType} */
- public static XContentType fromMediaType(MediaType mediaType) {
- if (mediaType instanceof XContentType) {
- return (XContentType) mediaType;
- } else {
- return mediaType != null ? MEDIA_TYPE_PARSER.fromMediaType(mediaType.mediaTypeWithoutParameters()) : null;
- }
+ @Override
+ public void writeTo(StreamOutput output) throws IOException {
+ output.writeString(this.mediaType());
}
}
diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java
index 9164b5216f9a5..3d4bbba9cc50c 100644
--- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java
@@ -43,9 +43,9 @@
import com.fasterxml.jackson.core.util.JsonGeneratorDelegate;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContent;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentGenerator;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -339,7 +339,7 @@ public void writeRawField(String name, InputStream content) throws IOException {
// needed for the XContentFactory.xContentType call
content = new BufferedInputStream(content);
}
- XContentType contentType = XContentFactory.xContentType(content);
+ MediaType contentType = MediaTypeRegistry.xContentType(content);
if (contentType == null) {
throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
}
@@ -354,7 +354,7 @@ public void writeRawField(String name, InputStream content, MediaType mediaType)
if (mayWriteRawData(mediaType) == false) {
// EMPTY is safe here because we never call namedObject when writing raw data
try (
- XContentParser parser = XContentFactory.xContent(mediaType)
+ XContentParser parser = mediaType.xContent()
// It's okay to pass the throwing deprecation handler
// because we should not be writing raw fields when
// generating JSON
diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java
new file mode 100644
index 0000000000000..af5ab67507b81
--- /dev/null
+++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.xcontent.spi;
+
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.spi.MediaTypeProvider;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Media Type implementations provided by xcontent library
+ *
+ * @opensearch.internal
+ */
+public class XContentProvider implements MediaTypeProvider {
+ /** Returns the concrete {@link MediaType} provided by the xcontent library */
+ @Override
+ public List getMediaTypes() {
+ return List.of(XContentType.values());
+ }
+
+ /** Returns the additional {@link MediaType} aliases provided by the xcontent library */
+ @Override
+ public Map getAdditionalMediaTypes() {
+ return Map.of("application/*", XContentType.JSON, "application/x-ndjson", XContentType.JSON);
+ }
+}
diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java
new file mode 100644
index 0000000000000..c265021f12763
--- /dev/null
+++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java
@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** SPI implementation for the xcontent library */
+package org.opensearch.common.xcontent.spi;
diff --git a/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider b/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider
new file mode 100644
index 0000000000000..ce3fab93087dd
--- /dev/null
+++ b/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider
@@ -0,0 +1,9 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+org.opensearch.common.xcontent.spi.XContentProvider
diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java
index f5616f2159f77..64d36f0a8b78f 100644
--- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java
+++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java
@@ -32,7 +32,7 @@
package org.opensearch.common.xcontent;
-import org.opensearch.core.xcontent.MediaTypeParser;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.test.OpenSearchTestCase;
import java.util.Collections;
@@ -44,42 +44,39 @@
public class MediaTypeParserTests extends OpenSearchTestCase {
- @SuppressWarnings("unchecked")
- MediaTypeParser mediaTypeParser = XContentType.getMediaTypeParser();
-
public void testJsonWithParameters() throws Exception {
String mediaType = "application/json";
- assertThat(mediaTypeParser.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap()));
- assertThat(mediaTypeParser.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap()));
- assertThat(mediaTypeParser.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8")));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap()));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap()));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8")));
assertThat(
- mediaTypeParser.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(),
+ MediaTypeRegistry.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(),
equalTo(Map.of("charset", "utf-8", "custom", "123"))
);
}
public void testWhiteSpaceInTypeSubtype() {
String mediaType = " application/json ";
- assertThat(mediaTypeParser.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON));
assertThat(
- mediaTypeParser.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(),
+ MediaTypeRegistry.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(),
equalTo(Map.of("charset", "utf-8", "custom", "123"))
);
assertThat(
- mediaTypeParser.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(),
+ MediaTypeRegistry.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(),
equalTo(Map.of("charset", "utf-8", "custom", "123"))
);
mediaType = " application / json ";
- assertThat(mediaTypeParser.parseMediaType(mediaType), is(nullValue()));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType), is(nullValue()));
}
public void testInvalidParameters() {
String mediaType = "application/json";
- assertThat(mediaTypeParser.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue()));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue()));
- assertThat(mediaTypeParser.parseMediaType(mediaType + "; key = value"), is(nullValue()));
- assertThat(mediaTypeParser.parseMediaType(mediaType + "; key="), is(nullValue()));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; key = value"), is(nullValue()));
+ assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; key="), is(nullValue()));
}
}
diff --git a/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
index c5dc68ff4c800..0384615c2f2f5 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
@@ -37,7 +37,7 @@
import org.opensearch.common.util.MockPageCacheRecycler;
import org.opensearch.core.xcontent.ContextParser;
import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.plugins.SearchPlugin;
import org.opensearch.script.ScriptService;
import org.opensearch.search.aggregations.Aggregation;
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java
index 3bd1137975800..b76fe41c8e67d 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java
@@ -32,7 +32,7 @@
package org.opensearch.ingest.common;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import java.util.Map;
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java
index bbd9ff4c8b912..ce8c182b60a61 100644
--- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java
@@ -33,15 +33,14 @@
package org.opensearch.ingest.common;
import org.opensearch.OpenSearchException;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.OpenSearchParseException;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.ingest.IngestDocument;
import org.opensearch.ingest.Processor;
import org.opensearch.ingest.RandomDocumentPicks;
import org.hamcrest.CoreMatchers;
-import static org.hamcrest.Matchers.equalTo;
-
public class BytesProcessorTests extends AbstractStringProcessorTestCase {
private String modifiedInput;
@@ -101,14 +100,16 @@ public void testMissingUnits() {
assertThat(exception.getMessage(), CoreMatchers.containsString("unit is missing or unrecognized"));
}
- public void testFractional() throws Exception {
+ public void testFractional() {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "1.1kb");
Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
- processor.execute(ingestDocument);
- assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(1126L));
- assertWarnings(
- "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for setting " + "[Ingest Field]"
+ OpenSearchParseException e = expectThrows(OpenSearchParseException.class, () -> processor.execute(ingestDocument));
+ assertThat(
+ e.getMessage(),
+ CoreMatchers.containsString(
+ "Fractional bytes values have been deprecated since Legacy 6.2. " + "Use non-fractional bytes values instead:"
+ )
);
}
}
diff --git a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java
index b189c8ed8905e..87598aa28ce8f 100644
--- a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java
+++ b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java
@@ -35,7 +35,6 @@
import org.opensearch.OpenSearchParseException;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import java.io.IOException;
@@ -68,7 +67,7 @@ final class UserAgentParser {
private void init(InputStream regexStream) throws IOException {
// EMPTY is safe here because we don't use namedObject
- XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML)
+ XContentParser yamlParser = XContentType.YAML.xContent()
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, regexStream);
XContentParser.Token token = yamlParser.nextToken();
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1
deleted file mode 100644
index ecf696b4b3b83..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-297e1cfade4ef71466cc9d4f361d81807c8dc4c8
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..6eaa40708e4ae
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+9f8a34fc3d450343ab05ccb5af318a836a6a5fb3
\ No newline at end of file
diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java
index 6e56ecf3950bb..da67a0d2dd13a 100644
--- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java
@@ -40,8 +40,8 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.xcontent.StatusToXContentObject;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.rest.RestStatus;
@@ -104,11 +104,11 @@ public static SearchTemplateResponse fromXContent(XContentParser parser) throws
if (contentAsMap.containsKey(TEMPLATE_OUTPUT_FIELD.getPreferredName())) {
Object source = contentAsMap.get(TEMPLATE_OUTPUT_FIELD.getPreferredName());
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).value(source);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).value(source);
searchTemplateResponse.setSource(BytesReference.bytes(builder));
} else {
MediaType contentType = parser.contentType();
- XContentBuilder builder = XContentFactory.contentBuilder(contentType).map(contentAsMap);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType).map(contentAsMap);
XContentParser searchResponseParser = contentType.xContent()
.createParser(parser.getXContentRegistry(), parser.getDeprecationHandler(), BytesReference.bytes(builder).streamInput());
diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java
index bbda8d15d9d41..f6b0cc4eecf9a 100644
--- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java
@@ -42,7 +42,6 @@
import org.opensearch.common.inject.Inject;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.rest.action.search.RestSearchAction;
@@ -132,8 +131,7 @@ static SearchRequest convert(
}
try (
- XContentParser parser = XContentFactory.xContent(XContentType.JSON)
- .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, source)
+ XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, source)
) {
SearchSourceBuilder builder = SearchSourceBuilder.searchSource();
builder.parseXContent(parser, false);
diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java
index 664e87da0a7d8..e3669ef67ff36 100644
--- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java
+++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java
@@ -33,9 +33,9 @@
package org.opensearch.script.mustache;
import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -101,7 +101,7 @@ public void testToXContentWithInlineTemplate() throws IOException {
request.setScriptParams(scriptParams);
XContentType contentType = randomFrom(XContentType.values());
- XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType)
+ XContentBuilder expectedRequest = MediaTypeRegistry.contentBuilder(contentType)
.startObject()
.field("source", "{\"query\": { \"match\" : { \"{{my_field}}\" : \"{{my_value}}\" } } }")
.startObject("params")
@@ -112,7 +112,7 @@ public void testToXContentWithInlineTemplate() throws IOException {
.field("profile", true)
.endObject();
- XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType);
+ XContentBuilder actualRequest = MediaTypeRegistry.contentBuilder(contentType);
request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType);
@@ -131,7 +131,7 @@ public void testToXContentWithStoredTemplate() throws IOException {
request.setScriptParams(params);
XContentType contentType = randomFrom(XContentType.values());
- XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType)
+ XContentBuilder expectedRequest = MediaTypeRegistry.contentBuilder(contentType)
.startObject()
.field("id", "match_template")
.startObject("params")
@@ -142,7 +142,7 @@ public void testToXContentWithStoredTemplate() throws IOException {
.field("profile", false)
.endObject();
- XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType);
+ XContentBuilder actualRequest = MediaTypeRegistry.contentBuilder(contentType);
request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType);
diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java
index 804a72561c10e..fd0a4e9612a8f 100644
--- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java
+++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java
@@ -36,6 +36,7 @@
import org.opensearch.action.search.SearchResponse;
import org.opensearch.action.search.ShardSearchFailure;
import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
@@ -164,7 +165,7 @@ public void testSourceToXContent() throws IOException {
response.setSource(BytesReference.bytes(source));
XContentType contentType = randomFrom(XContentType.values());
- XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType)
+ XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType)
.startObject()
.startObject("template_output")
.startObject("query")
@@ -175,7 +176,7 @@ public void testSourceToXContent() throws IOException {
.endObject()
.endObject();
- XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType);
+ XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType);
response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS);
assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType);
@@ -210,7 +211,7 @@ public void testSearchResponseToXContent() throws IOException {
response.setResponse(searchResponse);
XContentType contentType = randomFrom(XContentType.values());
- XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType)
+ XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType)
.startObject()
.field("took", 0)
.field("timed_out", false)
@@ -235,7 +236,7 @@ public void testSearchResponseToXContent() throws IOException {
.endObject()
.endObject();
- XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType);
+ XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType);
response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS);
assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType);
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index 3b3f2a7f9ca38..d7af8621c478a 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -84,8 +84,7 @@ shadowJar {
tasks.validateNebulaPom.dependsOn tasks.generatePomFileForShadowPublication
tasks.validateShadowPom.dependsOn tasks.generatePomFileForNebulaPublication
-tasks.publishNebulaPublicationToMavenLocal.dependsOn tasks.generatePomFileForShadowPublication
-tasks.publishShadowPublicationToMavenLocal.dependsOn tasks.generatePomFileForNebulaPublication
+tasks.withType(AbstractPublishToMaven)*.dependsOn "generatePomFileForShadowPublication", "generatePomFileForNebulaPublication"
tasks.named("dependencyLicenses").configure {
mapping from: /asm-.*/, to: 'asm'
diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java
index faae24083eeca..9a2c8c1f0aa55 100644
--- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java
+++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java
@@ -65,9 +65,9 @@
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.ParseField;
import org.opensearch.core.xcontent.ConstructingObjectParser;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -178,11 +178,11 @@ static class ContextSetup implements Writeable, ToXContentObject {
private final BytesReference document;
private final QueryBuilder query;
- private XContentType xContentType;
+ private MediaType mediaType;
static ContextSetup parse(XContentParser parser, Void context) throws IOException {
ContextSetup contextSetup = PARSER.parse(parser, null);
- contextSetup.setXContentType(XContentType.fromMediaType(parser.contentType()));
+ contextSetup.setXContentType(parser.contentType());
return contextSetup;
}
@@ -195,9 +195,9 @@ static ContextSetup parse(XContentParser parser, Void context) throws IOExceptio
ContextSetup(StreamInput in) throws IOException {
index = in.readOptionalString();
document = in.readOptionalBytesReference();
- String xContentType = in.readOptionalString();
- if (xContentType != null) {
- this.xContentType = XContentType.fromMediaType(xContentType);
+ String mediaType = in.readOptionalString();
+ if (mediaType != null) {
+ this.mediaType = MediaType.fromMediaType(mediaType);
}
query = in.readOptionalNamedWriteable(QueryBuilder.class);
}
@@ -214,12 +214,12 @@ public QueryBuilder getQuery() {
return query;
}
- public XContentType getXContentType() {
- return xContentType;
+ public MediaType getXContentType() {
+ return mediaType;
}
- public void setXContentType(XContentType xContentType) {
- this.xContentType = xContentType;
+ public void setXContentType(MediaType mediaType) {
+ this.mediaType = mediaType;
}
@Override
@@ -230,19 +230,19 @@ public boolean equals(Object o) {
return Objects.equals(index, that.index)
&& Objects.equals(document, that.document)
&& Objects.equals(query, that.query)
- && Objects.equals(xContentType, that.xContentType);
+ && Objects.equals(mediaType, that.mediaType);
}
@Override
public int hashCode() {
- return Objects.hash(index, document, query, xContentType);
+ return Objects.hash(index, document, query, mediaType);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(index);
out.writeOptionalBytesReference(document);
- out.writeOptionalString(xContentType != null ? xContentType.mediaTypeWithoutParameters() : null);
+ out.writeOptionalString(mediaType != null ? mediaType.mediaTypeWithoutParameters() : null);
out.writeOptionalNamedWriteable(query);
}
@@ -257,7 +257,7 @@ public String toString() {
+ ", query="
+ query
+ ", xContentType="
- + xContentType
+ + mediaType
+ '}';
}
@@ -275,7 +275,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE,
document,
- xContentType
+ mediaType
)
) {
builder.generator().copyCurrentStructure(parser);
@@ -594,8 +594,8 @@ private static Response prepareRamIndex(
try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(defaultAnalyzer))) {
String index = indexService.index().getName();
BytesReference document = request.contextSetup.document;
- XContentType xContentType = request.contextSetup.xContentType;
- SourceToParse sourceToParse = new SourceToParse(index, "_id", document, xContentType);
+ MediaType mediaType = request.contextSetup.mediaType;
+ SourceToParse sourceToParse = new SourceToParse(index, "_id", document, mediaType);
ParsedDocument parsedDocument = indexService.mapperService().documentMapper().parse(sourceToParse);
indexWriter.addDocuments(parsedDocument.docs());
try (IndexReader indexReader = DirectoryReader.open(indexWriter)) {
diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java
index 8084420295280..c8a28158ad8db 100644
--- a/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java
+++ b/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java
@@ -32,8 +32,8 @@
package org.opensearch.painless.api;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.painless.CompilerSettings;
import java.util.regex.Pattern;
diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java
index c3233bc0d924a..26bebfdee2fd0 100644
--- a/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java
+++ b/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java
@@ -34,7 +34,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.settings.Settings;
import java.util.Collections;
diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java
index a7787f4bc3c29..4117eb331197f 100644
--- a/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java
+++ b/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java
@@ -32,7 +32,7 @@
package org.opensearch.painless.api;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.test.OpenSearchTestCase;
import java.util.regex.Pattern;
diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java
index 4f26a15031dd4..b38bc5d7b45fc 100644
--- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java
+++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java
@@ -416,11 +416,12 @@ public void visit(QueryVisitor visitor) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = super.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = super.rewrite(searcher);
if (rewritten != this) {
return rewritten;
}
+ IndexReader reader = searcher.getIndexReader();
if (reader instanceof DirectoryReader) {
IndexSearcher indexSearcher = new IndexSearcher(reader);
indexSearcher.setQueryCache(null);
diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java
index 86d30f009e709..2702a02a7a951 100644
--- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java
@@ -32,7 +32,6 @@
package org.opensearch.percolator;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
@@ -89,8 +88,8 @@ final class PercolateQuery extends Query implements Accountable {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = candidateMatchesQuery.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = candidateMatchesQuery.rewrite(searcher);
if (rewritten != candidateMatchesQuery) {
return new PercolateQuery(
name,
diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java
index d5b61d5c5a517..08d9a4855c473 100644
--- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java
@@ -71,6 +71,8 @@
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.ParseField;
import org.opensearch.core.xcontent.ConstructingObjectParser;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
@@ -90,8 +92,8 @@
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.index.query.QueryShardException;
import org.opensearch.index.query.Rewriteable;
-import org.opensearch.indices.breaker.CircuitBreakerService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -123,7 +125,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder documents;
- private final XContentType documentXContentType;
+ private final MediaType documentXContentType;
private final String indexedDocumentIndex;
private final String indexedDocumentId;
@@ -150,7 +152,7 @@ public PercolateQueryBuilder(String field, BytesReference document, XContentType
* @param documents The binary blob containing document to percolate
* @param documentXContentType The content type of the binary blob containing the document to percolate
*/
- public PercolateQueryBuilder(String field, List documents, XContentType documentXContentType) {
+ public PercolateQueryBuilder(String field, List documents, MediaType documentXContentType) {
if (field == null) {
throw new IllegalArgumentException("[field] is a required argument");
}
@@ -252,7 +254,11 @@ protected PercolateQueryBuilder(String field, Supplier documentS
}
documents = in.readList(StreamInput::readBytesReference);
if (documents.isEmpty() == false) {
- documentXContentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ documentXContentType = in.readMediaType();
+ } else {
+ documentXContentType = in.readEnum(XContentType.class);
+ }
} else {
documentXContentType = null;
}
@@ -298,7 +304,11 @@ protected void doWriteTo(StreamOutput out) throws IOException {
out.writeBytesReference(document);
}
if (documents.isEmpty() == false) {
- out.writeEnum(documentXContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ documentXContentType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) documentXContentType);
+ }
}
}
@@ -432,7 +442,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) {
PercolateQueryBuilder rewritten = new PercolateQueryBuilder(
field,
Collections.singletonList(source),
- XContentHelper.xContentType(source)
+ MediaTypeRegistry.xContentType(source)
);
if (name != null) {
rewritten.setName(name);
@@ -560,7 +570,7 @@ public List getDocuments() {
}
// pkg-private for testing
- XContentType getXContentType() {
+ MediaType getXContentType() {
return documentXContentType;
}
diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java
index 3b953fcfe65e1..1040d014483e1 100644
--- a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java
+++ b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java
@@ -1275,7 +1275,7 @@ private CustomQuery(Term term) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
+ public Query rewrite(IndexSearcher searcher) throws IOException {
return new TermQuery(term);
}
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java
index ea001de0ee7c6..d97d5a3a7dcd5 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -36,9 +36,9 @@
import org.opensearch.common.Strings;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -274,7 +274,7 @@ public static DiscountedCumulativeGain createTestItem() {
public void testXContentRoundtrip() throws IOException {
DiscountedCumulativeGain testItem = createTestItem();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java
index 014f52faa9d57..32c7edc845e32 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java
@@ -35,9 +35,9 @@
import org.opensearch.action.OriginalIndices;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -166,7 +166,7 @@ public static ExpectedReciprocalRank createTestItem() {
public void testXContentRoundtrip() throws IOException {
ExpectedReciprocalRank testItem = createTestItem();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java
index 3df79acfa6ce1..d34e6ba8bd72e 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java
@@ -35,9 +35,9 @@
import org.opensearch.action.OriginalIndices;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -183,7 +183,7 @@ public void testNoResults() throws Exception {
public void testXContentRoundtrip() throws IOException {
MeanReciprocalRank testItem = createTestItem();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java
index 3317a2d2f00f1..41340cfd1003d 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java
@@ -35,9 +35,9 @@
import org.opensearch.action.OriginalIndices;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -199,7 +199,7 @@ public static PrecisionAtK createTestItem() {
public void testXContentRoundtrip() throws IOException {
PrecisionAtK testItem = createTestItem();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java
index 7c0590566bba9..b021d739cc6a6 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java
@@ -39,14 +39,14 @@
import org.opensearch.cluster.block.ClusterBlockException;
import org.opensearch.cluster.coordination.NoClusterManagerBlockService;
import org.opensearch.core.common.ParsingException;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentLocation;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -177,7 +177,7 @@ public void testToXContent() throws IOException {
Collections.singletonMap("coffee_query", coffeeQueryQuality),
Collections.singletonMap("beer_query", new ParsingException(new XContentLocation(0, 0), "someMsg"))
);
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
String xContent = BytesReference.bytes(response.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString();
assertEquals(
("{"
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java
index 988784b6e57a3..01f5a3a12da01 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java
@@ -34,9 +34,9 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -57,7 +57,7 @@ public static RatedDocument createRatedDocument() {
public void testXContentParsing() throws IOException {
RatedDocument testItem = createRatedDocument();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
RatedDocument parsedItem = RatedDocument.fromXContent(itemParser);
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java
index ac2401f30e6f0..6e99e31a2b819 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java
@@ -35,10 +35,10 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.settings.Settings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -134,7 +134,7 @@ public static RatedRequest createTestItem(boolean forceRequest) {
public void testXContentRoundtrip() throws IOException {
RatedRequest testItem = createTestItem(randomBoolean());
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java
index 85e024f6bb1e9..37f778fbc5059 100644
--- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java
+++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java
@@ -35,9 +35,9 @@
import org.opensearch.action.OriginalIndices;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -185,7 +185,7 @@ public static RecallAtK createTestItem() {
public void testXContentRoundtrip() throws IOException {
RecallAtK testItem = createTestItem();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 6170c1adabbea..a0ad02899ea27 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -53,7 +53,7 @@
import org.opensearch.action.support.TransportAction;
import org.opensearch.client.ParentTaskAssigningClient;
import org.opensearch.common.Nullable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.index.VersionType;
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java
index 6f36fafd852ef..4b1e416ac63ea 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java
@@ -37,8 +37,8 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.action.search.RestSearchAction;
@@ -106,7 +106,7 @@ private XContentParser extractRequestSpecificFields(RestRequest restRequest, Map
}
try (
XContentParser parser = restRequest.contentOrSourceParamParser();
- XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType())
) {
Map body = parser.map();
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java
index ba974d248ac9b..cb07e593f8155 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java
@@ -61,10 +61,10 @@
import org.opensearch.common.lucene.uid.Versions;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.VersionType;
import org.opensearch.index.mapper.VersionFieldMapper;
import org.opensearch.index.reindex.remote.RemoteScrollableHitSource;
@@ -378,27 +378,24 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc)
index.id(doc.getId());
// the source xcontent type and destination could be different
- final XContentType sourceXContentType = doc.getXContentType();
- final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType();
- if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) {
+ final MediaType sourceMediaType = doc.getMediaType();
+ final MediaType mainRequestMediaType = mainRequest.getDestination().getContentType();
+ if (mainRequestMediaType != null && doc.getMediaType() != mainRequestMediaType) {
// we need to convert
try (
InputStream stream = doc.getSource().streamInput();
- XContentParser parser = sourceXContentType.xContent()
+ XContentParser parser = sourceMediaType.xContent()
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream);
- XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent())
+ XContentBuilder builder = XContentBuilder.builder(mainRequestMediaType.xContent())
) {
parser.nextToken();
builder.copyCurrentStructure(parser);
index.source(BytesReference.bytes(builder), builder.contentType());
} catch (IOException e) {
- throw new UncheckedIOException(
- "failed to convert hit from " + sourceXContentType + " to " + mainRequestXContentType,
- e
- );
+ throw new UncheckedIOException("failed to convert hit from " + sourceMediaType + " to " + mainRequestMediaType, e);
}
} else {
- index.source(doc.getSource(), doc.getXContentType());
+ index.source(doc.getSource(), doc.getMediaType());
}
/*
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java
index 5297723ab3687..1ea9ec5fb7beb 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java
@@ -138,7 +138,7 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc)
IndexRequest index = new IndexRequest();
index.index(doc.getIndex());
index.id(doc.getId());
- index.source(doc.getSource(), doc.getXContentType());
+ index.source(doc.getSource(), doc.getMediaType());
index.setIfSeqNo(doc.getSeqNo());
index.setIfPrimaryTerm(doc.getPrimaryTerm());
index.setPipeline(mainRequest.getPipeline());
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java
index 3d606b6b77c44..aaca04641d76a 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java
@@ -41,12 +41,12 @@
import org.opensearch.common.collect.Tuple;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
import org.opensearch.core.xcontent.ConstructingObjectParser;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.ObjectParser.ValueType;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentLocation;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.reindex.ScrollableHitSource.BasicHit;
import org.opensearch.index.reindex.ScrollableHitSource.Hit;
import org.opensearch.index.reindex.ScrollableHitSource.Response;
@@ -72,7 +72,7 @@ private RemoteResponseParsers() {}
/**
* Parser for an individual {@code hit} element.
*/
- public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> {
+ public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> {
int i = 0;
String index = (String) a[i++];
String id = (String) a[i++];
@@ -106,7 +106,7 @@ private RemoteResponseParsers() {}
class Fields {
String routing;
}
- ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new);
+ ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new);
HIT_PARSER.declareObject((hit, fields) -> { hit.setRouting(fields.routing); }, fieldsParser, new ParseField("fields"));
fieldsParser.declareString((fields, routing) -> fields.routing = routing, routingField);
fieldsParser.declareLong((fields, ttl) -> {}, ttlField); // ignore ttls since they have been removed
@@ -116,7 +116,7 @@ class Fields {
/**
* Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List)]}.
*/
- public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a);
+ public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a);
static {
HITS_PARSER.declareField(constructorArg(), (p, c) -> {
if (p.currentToken() == XContentParser.Token.START_OBJECT) {
@@ -184,7 +184,7 @@ class Fields {
SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures"));
}
- public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>(
+ public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>(
"search_response",
true,
a -> {
@@ -296,13 +296,13 @@ public void setCausedBy(Throwable causedBy) {
/**
* Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out.
*/
- public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>(
+ public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>(
"/",
true,
a -> (Version) a[0]
);
static {
- ConstructingObjectParser versionParser = new ConstructingObjectParser<>(
+ ConstructingObjectParser versionParser = new ConstructingObjectParser<>(
"version",
true,
a -> a[0] == null
diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java
index 39d3adcd82ad5..3c305c2a8239d 100644
--- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java
+++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java
@@ -55,10 +55,10 @@
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.reindex.RejectAwareActionListener;
import org.opensearch.index.reindex.ScrollableHitSource;
import org.opensearch.core.rest.RestStatus;
@@ -182,7 +182,7 @@ protected void cleanup(Runnable onCompletion) {
private void execute(
Request request,
- BiFunction parser,
+ BiFunction parser,
RejectAwareActionListener super T> listener
) {
// Preserve the thread context so headers survive after the call
@@ -198,12 +198,12 @@ public void onSuccess(org.opensearch.client.Response response) {
try {
HttpEntity responseEntity = response.getEntity();
InputStream content = responseEntity.getContent();
- XContentType xContentType = null;
+ MediaType mediaType = null;
if (responseEntity.getContentType() != null) {
final String mimeType = ContentType.parse(responseEntity.getContentType()).getMimeType();
- xContentType = XContentType.fromMediaType(mimeType);
+ mediaType = MediaType.fromMediaType(mimeType);
}
- if (xContentType == null) {
+ if (mediaType == null) {
try {
logger.debug("Response didn't include Content-Type: " + bodyMessage(response.getEntity()));
throw new OpenSearchException(
@@ -217,10 +217,10 @@ public void onSuccess(org.opensearch.client.Response response) {
}
// EMPTY is safe here because we don't call namedObject
try (
- XContentParser xContentParser = xContentType.xContent()
+ XContentParser xContentParser = mediaType.xContent()
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, content)
) {
- parsedResponse = parser.apply(xContentParser, xContentType);
+ parsedResponse = parser.apply(xContentParser, mediaType);
} catch (XContentParseException e) {
/* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it
* is totally wrong and we're probably not talking to Elasticsearch. */
diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java
index 2946544a8f901..651ccd47df7bd 100644
--- a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java
+++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java
@@ -74,7 +74,7 @@ public void testPipelineQueryParameterIsError() throws IOException {
body.endObject();
}
body.endObject();
- request.withContent(BytesReference.bytes(body), XContentType.fromMediaType(body.contentType()));
+ request.withContent(BytesReference.bytes(body), body.contentType());
}
request.withParams(singletonMap("pipeline", "doesn't matter"));
Exception e = expectThrows(
diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
index ebbd2da776ace..e671fec1fedee 100644
--- a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
+++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
@@ -44,8 +44,8 @@
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.util.FileSystemUtils;
import org.opensearch.common.io.Streams;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
diff --git a/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java b/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java
index 1bf461d67862b..9c61bca316a56 100644
--- a/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java
+++ b/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java
@@ -38,7 +38,7 @@
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.client.Client;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.plugin.repository.url.URLRepositoryModulePlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.repositories.fs.FsRepository;
diff --git a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java
index fbfbf5e006fee..0fad0cbe21033 100644
--- a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java
+++ b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java
@@ -37,8 +37,8 @@
import org.opensearch.common.blobstore.BlobStore;
import org.opensearch.common.blobstore.BlobStoreException;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import java.net.MalformedURLException;
import java.net.URL;
diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
index db76c0b145840..95440e28686e7 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
@@ -39,8 +39,8 @@
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.http.HttpServerTransport;
import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
diff --git a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java
index 2584b768707cd..f7e1c6106cf5a 100644
--- a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java
+++ b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java
@@ -38,7 +38,7 @@
import org.opensearch.client.ResponseException;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.http.HttpTransportSettings;
import org.opensearch.test.rest.OpenSearchRestTestCase;
import org.opensearch.test.rest.yaml.ObjectPath;
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java
index 124bc02527bd1..998c89590c53c 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java
@@ -78,8 +78,8 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
import org.opensearch.core.xcontent.NamedXContentRegistry;
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java
index 9a5459a5ab572..a30cf00ce4047 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java
@@ -52,7 +52,7 @@
import io.netty.channel.RecvByteBufAllocator;
import io.netty.channel.socket.nio.NioSocketChannel;
import org.opensearch.common.SuppressForbidden;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import java.io.IOException;
import java.nio.ByteBuffer;
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java
index d7f2f6eb6acbb..ef60797bca067 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java
@@ -45,7 +45,7 @@
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.netty4.Netty4HttpServerTransport;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.plugins.NetworkPlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.threadpool.ThreadPool;
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java
index f2f6538d305d9..d32a43e3eb9eb 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java
@@ -42,7 +42,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.common.Booleans;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.monitor.jvm.JvmInfo;
import java.util.concurrent.atomic.AtomicBoolean;
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java
index 561cac2facbff..637cb10c383bd 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java
@@ -57,13 +57,13 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
import org.opensearch.common.util.net.NetUtils;
import org.opensearch.common.lease.Releasables;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.Netty4NioSocketChannel;
import org.opensearch.transport.NettyAllocator;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java
index ef014aa39367b..5fcfc4ee5e151 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java
@@ -44,7 +44,7 @@
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.HttpTransportSettings;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.rest.BytesRestResponse;
import org.opensearch.rest.RestChannel;
import org.opensearch.rest.RestRequest;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
index cad2e50327023..9f359002e441a 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
@@ -72,8 +72,8 @@
import io.netty.util.AttributeKey;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.tasks.Task;
import org.opensearch.transport.NettyAllocator;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java
index adf4d59a0c139..6c8cf69afb148 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -52,7 +52,7 @@
import org.opensearch.http.HttpResponse;
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.NullDispatcher;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.TestThreadPool;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java
index 5a43057b1b7d1..d23edfda829f9 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -65,7 +65,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.MockBigArrays;
import org.opensearch.common.util.MockPageCacheRecycler;
@@ -75,7 +75,7 @@
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.HttpTransportSettings;
import org.opensearch.http.NullDispatcher;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.rest.BytesRestResponse;
import org.opensearch.rest.RestChannel;
import org.opensearch.rest.RestRequest;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
index f80d7f41b5f55..5c1c5970a7cfb 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
@@ -39,7 +39,7 @@
import org.opensearch.common.transport.TransportAddress;
import org.opensearch.common.util.MockPageCacheRecycler;
import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.SharedGroupFactory;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java
index d3fa8ea56ffe7..5ee194b7bb513 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java
@@ -36,14 +36,14 @@
import io.netty.buffer.CompositeByteBuf;
import io.netty.buffer.Unpooled;
import org.apache.lucene.util.BytesRef;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.bytes.AbstractBytesReferenceTestCase;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.io.stream.ReleasableBytesStreamOutput;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.io.IOException;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java
index 5d7841df4bf33..c10a92ba1900b 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java
@@ -32,14 +32,14 @@
package org.opensearch.transport.netty4;
import org.opensearch.Version;
-import org.opensearch.common.component.Lifecycle;
+import org.opensearch.common.lifecycle.Lifecycle;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.network.NetworkService;
import org.opensearch.common.network.NetworkUtils;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.MockPageCacheRecycler;
import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.TestThreadPool;
import org.opensearch.threadpool.ThreadPool;
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java
index 619f473b8bef2..ee2bf12a4246f 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java
@@ -43,7 +43,7 @@
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.common.util.net.NetUtils;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.transport.MockTransportService;
import org.opensearch.test.transport.StubbableTransport;
import org.opensearch.transport.AbstractSimpleTransportTestCase;
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1
deleted file mode 100644
index 0ed030926ab93..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-94293b169fb8572f440a5a4a523320ecf9778ffe
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..8a3332c950b6d
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+fde64e3b23bc9a0849b9897febfe9f13c5113143
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1
deleted file mode 100644
index ddd67276606a5..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2df800a38b64867b8dcd61fc2cd986114e4a80cb
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..33c2afacf2395
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+b01a791705fa01fce48dd02ea79fa8045de8dd5e
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1
deleted file mode 100644
index 0cd68af98e724..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a01e8153f34d72e8c8c0180c1dea5b10f677dd3a
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..1e7986dafa11e
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+43d19320b1b9cd18638b1602fa87d5f21ee043bc
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1
deleted file mode 100644
index c7b4d2dc6da75..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b7d47d54683b0b1e09b271c32d1b7d3eb1990f49
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..14880d9c2d243
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+9244dc232f175010b480d4d88e13945c17a0b28b
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1
deleted file mode 100644
index 8df7245044171..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e68b9816e6cff8ee15f5b350cf2ffa54f9828b7
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..edc4de3fffe28
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+3101a4f79820c1ca3dfb8f49b74c5fb5b32940e1
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1
deleted file mode 100644
index 974e4202f5ffb..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d23b1f05b471e05d0d6068b3ece7c8c65672eae7
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..54c310277b09b
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+f12b2a22cd5ebcd84f40a40e78fdd4e268b3b26d
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1
deleted file mode 100644
index dce408a7d40ef..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dfb4313f3c68d337310522840d7144c1605d084a
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..358db9ea3f0f5
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+7dbf5cc3dff93cc1ffe45d79b129859590d001dd
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
index 9dbf08a3e1a01..23db0a74dc3fc 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
@@ -51,7 +51,7 @@
import org.opensearch.OpenSearchException;
import org.opensearch.SpecialPermission;
import org.opensearch.cloud.azure.classic.AzureServiceRemoteException;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.common.Strings;
diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java
index 861926a9e67c9..afa35f63ae4dc 100644
--- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -46,7 +46,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.transport.TransportAddress;
import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.transport.MockTransportService;
import org.opensearch.transport.Transport;
import org.opensearch.transport.TransportService;
diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java
index cc65e92b1485a..ea10d03576d94 100644
--- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java
@@ -45,7 +45,7 @@
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.discovery.SeedHostsProvider;
import org.opensearch.discovery.SeedHostsResolver;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.transport.MockTransportService;
import org.opensearch.transport.TransportService;
import org.opensearch.transport.nio.MockNioTransport;
diff --git a/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java b/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java
index 4873cb6dcbf7a..b7ae7f8b404be 100644
--- a/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java
+++ b/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java
@@ -46,7 +46,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.cloud.gce.util.Access;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index 62651216c8144..17213b96a25f3 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -89,7 +89,7 @@ dependencies {
api "org.apache.james:apache-mime4j-core:${versions.mime4j}"
api "org.apache.james:apache-mime4j-dom:${versions.mime4j}"
// EPUB books
- api 'org.apache.commons:commons-lang3:3.12.0'
+ api 'org.apache.commons:commons-lang3:3.13.0'
// Microsoft Word files with visio diagrams
api 'org.apache.commons:commons-math3:3.6.1'
// POIs dependency
diff --git a/plugins/ingest-attachment/licenses/commons-lang3-3.12.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.12.0.jar.sha1
deleted file mode 100644
index 9273d8c01aaba..0000000000000
--- a/plugins/ingest-attachment/licenses/commons-lang3-3.12.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c6842c86792ff03b9f1d1fe2aab8dc23aa6c6f0e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1
new file mode 100644
index 0000000000000..d0c2f2486ee1f
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1
@@ -0,0 +1 @@
+b7263237aa89c1f99b327197c41d0669707a462e
\ No newline at end of file
diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java
index 67e79addfedc5..48285f80150be 100644
--- a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java
+++ b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java
@@ -46,7 +46,7 @@
import org.opensearch.common.regex.Regex;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.plugins.Plugin;
import org.opensearch.repositories.blobstore.OpenSearchMockAPIBasedRepositoryIntegTestCase;
import org.opensearch.core.rest.RestStatus;
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java
index 2677604ecb622..3846dd14559b5 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java
@@ -41,7 +41,7 @@
import org.opensearch.common.blobstore.BlobStore;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.indices.recovery.RecoverySettings;
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
index c518cc2716db6..88e9a63384f7a 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
@@ -61,8 +61,8 @@
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsException;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import java.net.Authenticator;
diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java
index 8b68ccebf8c53..ab16edc7b1a2d 100644
--- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java
+++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java
@@ -53,7 +53,7 @@
import org.opensearch.common.network.InetAddresses;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.CountDown;
import org.opensearch.core.rest.RestStatus;
diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java
index 24a226290985a..bfae0a3c4438c 100644
--- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java
+++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java
@@ -38,8 +38,8 @@
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.env.Environment;
import org.opensearch.indices.recovery.RecoverySettings;
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
index 1e8e358833cc9..0fb98c8b9abff 100644
--- a/plugins/repository-gcs/build.gradle
+++ b/plugins/repository-gcs/build.gradle
@@ -67,7 +67,7 @@ dependencies {
api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}"
api 'com.google.cloud:google-cloud-core:2.5.10'
- api 'com.google.cloud:google-cloud-core-http:2.21.0'
+ api 'com.google.cloud:google-cloud-core-http:2.21.1'
api 'com.google.cloud:google-cloud-storage:1.113.1'
api 'com.google.code.gson:gson:2.9.0'
@@ -77,7 +77,7 @@ dependencies {
api 'com.google.http-client:google-http-client:1.43.2'
api 'com.google.http-client:google-http-client-appengine:1.43.2'
- api 'com.google.http-client:google-http-client-gson:1.43.2'
+ api 'com.google.http-client:google-http-client-gson:1.43.3'
api 'com.google.http-client:google-http-client-jackson2:1.43.2'
api 'com.google.oauth-client:google-oauth-client:1.34.1'
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1
deleted file mode 100644
index 2ef0a9bf9b33e..0000000000000
--- a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-07da4710ccdbcfee253672c0b9e00e7370626c26
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1
new file mode 100644
index 0000000000000..cc5e7a53098ac
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1
@@ -0,0 +1 @@
+88dd2b413dd06826c611e39e6e3259e069f02f66
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1
deleted file mode 100644
index df0374aa27c70..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b1c2e3e89804c113dba7b342aa8e0fc2cf3d9378
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1
new file mode 100644
index 0000000000000..43f4fe4a127e1
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1
@@ -0,0 +1 @@
+252e267acf720ef6333488740a696a1d5e204639
\ No newline at end of file
diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index f0f0fb7681c1a..92a2f99f4a441 100644
--- a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -55,8 +55,8 @@
import org.opensearch.common.regex.Regex;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.env.Environment;
import org.opensearch.indices.recovery.RecoverySettings;
diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java
index 226aec437fc0f..f5c20003ea7b6 100644
--- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java
+++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java
@@ -55,8 +55,8 @@
import org.opensearch.common.blobstore.support.PlainBlobMetadata;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.common.io.Streams;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import java.io.ByteArrayInputStream;
import java.io.IOException;
diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java
index a743ac72bdb8b..05d9739c00d50 100644
--- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -38,8 +38,8 @@
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.settings.Setting;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.indices.recovery.RecoverySettings;
diff --git a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
index 488376d36cdc4..466344668d966 100644
--- a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
+++ b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
@@ -53,7 +53,7 @@
import org.opensearch.common.network.InetAddresses;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.CountDown;
import org.opensearch.core.common.Strings;
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 0b13e18ba3f4d..6626bfccc6662 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -48,7 +48,7 @@ opensearchplugin {
}
versions << [
- 'hadoop3': '3.3.4'
+ 'hadoop3': '3.3.6'
]
testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"
@@ -440,3 +440,7 @@ thirdPartyAudit {
'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField',
)
}
+
+tasks.withType(JavaForkOptions) {
+ systemProperty "java.util.concurrent.ForkJoinPool.common.threadFactory", "org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory"
+}
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1
deleted file mode 100644
index dd79b8a10cebc..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6339a8f7279310c8b1f7ef314b592d8c71ca72ef
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1
new file mode 100644
index 0000000000000..d99793bc56522
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1
@@ -0,0 +1 @@
+12ac6f103a0ff29fce17a078c7c64d25320b6165
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1
deleted file mode 100644
index 32d58d1dc501a..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1
new file mode 100644
index 0000000000000..ea22d763b7bfa
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1
@@ -0,0 +1 @@
+81065531e63fccbe85fb04a3274709593fb00d3c
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1
deleted file mode 100644
index 532d25a44531f..0000000000000
--- a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-036ef2f86dc44410d2bb5d54ce40435d2484d9a5
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1
new file mode 100644
index 0000000000000..fe60968056eb7
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1
@@ -0,0 +1 @@
+ba40aca60f39599d5b1f1d32b35295bfde1f3c8b
\ No newline at end of file
diff --git a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java
index 88c58942e9bbf..10c3bc2d0364b 100644
--- a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java
@@ -48,7 +48,7 @@
import org.opensearch.common.SuppressForbidden;
import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.env.Environment;
diff --git a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
index b9b0e9e87dd0c..2758bd020e979 100644
--- a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
+++ b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java
@@ -43,6 +43,9 @@
* to ignore the offending thread until a version of Hadoop is released that addresses the incorrect
* interrupt handling.
*
+ * In Hadoop 3.3.6, the org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap uses ForkJoinPool
+ * to perform statistics calculation, leaving dangling workers.
+ *
* @see https://issues.apache.org/jira/browse/HADOOP-12829
* @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner"
* @see "org.apache.hadoop.fs.FileSystem.Statistics"
@@ -53,6 +56,6 @@ public final class HdfsClientThreadLeakFilter implements ThreadFilter {
@Override
public boolean reject(Thread t) {
- return t.getName().equals(OFFENDING_THREAD_NAME);
+ return t.getName().equals(OFFENDING_THREAD_NAME) || t.getName().startsWith("ForkJoinPool.commonPool-");
}
}
diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index 3070c654a96ee..805af9874b552 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -45,7 +45,7 @@
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.indices.recovery.RecoverySettings;
import org.opensearch.plugins.Plugin;
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
index 81a902a6992d8..1a644934245cb 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
@@ -51,8 +51,8 @@
import org.opensearch.common.blobstore.support.AbstractBlobContainer;
import org.opensearch.common.blobstore.support.PlainBlobMetadata;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
index 30040e182cbc9..ed1ebf1f531f8 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
@@ -39,7 +39,7 @@
import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.blobstore.BlobStore;
import org.opensearch.common.blobstore.BlobStoreException;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
import software.amazon.awssdk.services.s3.model.StorageClass;
import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
index d42bfc0be7e4f..f98b775d9ce4b 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
@@ -46,8 +46,8 @@
import org.opensearch.common.settings.SecureSetting;
import org.opensearch.core.common.settings.SecureString;
import org.opensearch.common.settings.Setting;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.indices.recovery.RecoverySettings;
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
index 5b43ae84c51dc..cb6851652f208 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
@@ -17,7 +17,7 @@
import org.opensearch.common.blobstore.exception.CorruptFileException;
import org.opensearch.common.blobstore.stream.write.WritePriority;
import org.opensearch.common.io.InputStreamContainer;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.util.ByteUtils;
import org.opensearch.repositories.s3.io.CheckedContainer;
import org.opensearch.repositories.s3.SocketAccess;
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
index 10137f0475177..8bb446fff0b61 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
@@ -24,7 +24,7 @@
import org.opensearch.common.io.InputStreamContainer;
import org.opensearch.common.lucene.store.ByteArrayIndexInput;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
import org.opensearch.repositories.s3.async.AsyncTransferManager;
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
index 1a1fb123aa5ea..016be07e5d533 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -57,8 +57,8 @@
import org.opensearch.common.network.InetAddresses;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.CountDown;
import org.opensearch.common.util.io.IOUtils;
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
index a2a7ca8d8bdd5..f88b3616d2f0a 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
@@ -40,7 +40,7 @@
import org.opensearch.common.blobstore.BlobStoreException;
import org.opensearch.common.blobstore.DeleteResult;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.test.OpenSearchTestCase;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
index 84d56c7ae2854..e5fd9e5caab9c 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
@@ -36,8 +36,8 @@
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.indices.recovery.RecoverySettings;
import org.opensearch.repositories.RepositoryException;
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
index 596291a1d94fb..2fc10c65eaa65 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
@@ -14,7 +14,7 @@
import org.opensearch.common.blobstore.exception.CorruptFileException;
import org.opensearch.common.blobstore.stream.write.WritePriority;
import org.opensearch.common.io.InputStreamContainer;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.repositories.blobstore.ZeroInputStream;
import org.opensearch.test.OpenSearchTestCase;
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java
index 1befc110eb6a5..9dd13144bc454 100644
--- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java
+++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java
@@ -41,7 +41,7 @@
import org.opensearch.common.network.NetworkService;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java
index 1509e0b179bfe..0990f97124e53 100644
--- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java
+++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java
@@ -44,7 +44,7 @@
import org.opensearch.common.network.NetworkService;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.nio.BytesChannelContext;
import org.opensearch.nio.ChannelFactory;
import org.opensearch.nio.Config;
diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java
index 67598aec154fa..bd8aec534f608 100644
--- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java
+++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java
@@ -47,7 +47,7 @@
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.nio.NioHttpServerTransport;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.plugins.NetworkPlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.threadpool.ThreadPool;
diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java
index 29ef19a2aec87..ee96e9d85c03b 100644
--- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java
+++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java
@@ -32,7 +32,7 @@
package org.opensearch.transport.nio;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.bytes.CompositeBytesReference;
import org.opensearch.common.bytes.ReleasableBytesReference;
diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java
index c606a4818a324..ca62c0dedd452 100644
--- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java
+++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java
@@ -49,7 +49,7 @@
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.http.CorsHandler;
import org.opensearch.http.HttpChannel;
diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java
index edaee15507df9..2922f28e3be18 100644
--- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java
+++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java
@@ -51,8 +51,8 @@
import org.opensearch.action.support.PlainActionFuture;
import org.opensearch.common.network.NetworkService;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.nio.BytesChannelContext;
import org.opensearch.nio.ChannelFactory;
diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java
index c69fe23002dfe..4d0db18d433ec 100644
--- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java
+++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java
@@ -54,7 +54,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.MockBigArrays;
import org.opensearch.common.util.MockPageCacheRecycler;
@@ -64,7 +64,7 @@
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.HttpTransportSettings;
import org.opensearch.http.NullDispatcher;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.nio.NioSocketChannel;
import org.opensearch.rest.BytesRestResponse;
import org.opensearch.rest.RestChannel;
diff --git a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java
index 4b06c4e15bce7..d0e779edded7d 100644
--- a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java
+++ b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java
@@ -43,7 +43,7 @@
import org.opensearch.common.util.MockPageCacheRecycler;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.common.util.net.NetUtils;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.transport.MockTransportService;
import org.opensearch.test.transport.StubbableTransport;
import org.opensearch.transport.AbstractSimpleTransportTestCase;
diff --git a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1
deleted file mode 100644
index 45d8f459573b1..0000000000000
--- a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-27ba6caaa4587a982cd451f7217b5a982bcfc44a
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..e7c7dc2bbc046
--- /dev/null
+++ b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+1446b7641743a1082b566179d1bf2960f5a0724b
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1
deleted file mode 100644
index 3981ea4fa226e..0000000000000
--- a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6389463bfbfcf902c8d31d12e9513a6818ac9d5e
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..d0f64519cd6ff
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+127032ea137d2501b24f0e35e5f9a2e1c7864633
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.7.0.jar.sha1 b/server/licenses/lucene-core-9.7.0.jar.sha1
deleted file mode 100644
index 2b0f77275c0ab..0000000000000
--- a/server/licenses/lucene-core-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ad391210ffd806931334be9670a35af00c56f959
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..dc363f2776429
--- /dev/null
+++ b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+30c3afcf058532d3d2b8820375043000e7f34a9b
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.7.0.jar.sha1 b/server/licenses/lucene-grouping-9.7.0.jar.sha1
deleted file mode 100644
index 90acbf6dcee8d..0000000000000
--- a/server/licenses/lucene-grouping-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8e6f0c229f4861be641047c33b05067176e4279c
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..294beba43f62a
--- /dev/null
+++ b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+e6f742efe0ef3b383468fe38f88ab2dd69ed3d2c
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0.jar.sha1
deleted file mode 100644
index bfcca0bc6cb5b..0000000000000
--- a/server/licenses/lucene-highlighter-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-facb7c7ee0f75ed457a2d98f10d6430e25a53691
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..c2a2ef5b13946
--- /dev/null
+++ b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+3162856444777130dee2c4cabe1bf6d18710ff63
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.7.0.jar.sha1 b/server/licenses/lucene-join-9.7.0.jar.sha1
deleted file mode 100644
index 0dab3a7ddc41a..0000000000000
--- a/server/licenses/lucene-join-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d041bdc0947a14223cf68357407ee18b21027587
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..7c6adaaba9cf1
--- /dev/null
+++ b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+5fe8383516eca7300f978ce38042e327b0a57877
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.7.0.jar.sha1 b/server/licenses/lucene-memory-9.7.0.jar.sha1
deleted file mode 100644
index 357a9c4b2ea26..0000000000000
--- a/server/licenses/lucene-memory-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0fade51ee353e15ddbbc45262aafe6f99ed020f1
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..586702c968a77
--- /dev/null
+++ b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+b3e77970485be6d2dd59b999bbaa65a2cb993744
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.7.0.jar.sha1 b/server/licenses/lucene-misc-9.7.0.jar.sha1
deleted file mode 100644
index da5e1921626b2..0000000000000
--- a/server/licenses/lucene-misc-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7fcf451e2376526c3a027958812866cc5b0ff13f
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..493598eefff5e
--- /dev/null
+++ b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+86d667ea2f7fb2142d2acacf801dcea47d014a5e
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.7.0.jar.sha1 b/server/licenses/lucene-queries-9.7.0.jar.sha1
deleted file mode 100644
index fa82e95a7e19f..0000000000000
--- a/server/licenses/lucene-queries-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-126989d4622419aa06fcbf3a342e859cab8c8799
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..1bf937f10d795
--- /dev/null
+++ b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+930d004de698f374da8ac5530fd80e241edeba45
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0.jar.sha1
deleted file mode 100644
index 438db0aea66e1..0000000000000
--- a/server/licenses/lucene-queryparser-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6e77bde908ff698354e4a2149e6dd4658b56d7b0
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..74458bc93f90b
--- /dev/null
+++ b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+f62882823d5aa9ed4cf0081a8c18f35e21992080
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0.jar.sha1
deleted file mode 100644
index 38b0b1cccbc29..0000000000000
--- a/server/licenses/lucene-sandbox-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9f3e8e1947f2f1c5784132444af51a060ff0b4bf
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..3231d0e067940
--- /dev/null
+++ b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+f1ec1527e283b423b7ff5e12cd8d889e7247199d
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1
deleted file mode 100644
index 48679df469fd1..0000000000000
--- a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-01b0bc7a407d8c35a70a1adf7966bb3e7caae928
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..dd47faf91f206
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+de787c052879893e47d21fa161c93413665d55d7
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1
deleted file mode 100644
index 55d4d217fa6b9..0000000000000
--- a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7c6b1b6e0a70c9cd177371e648648c2f896742a2
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..2b378438bfb14
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+7e541ed960a571f5d9a0ecff5c26fd5ca857581e
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.7.0.jar.sha1 b/server/licenses/lucene-suggest-9.7.0.jar.sha1
deleted file mode 100644
index d4d7e6cd6bed9..0000000000000
--- a/server/licenses/lucene-suggest-9.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5c37fd9a5d71dc87fe1cd4c18ff295ec8cfac170
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1
new file mode 100644
index 0000000000000..1e3ed6561e3ef
--- /dev/null
+++ b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1
@@ -0,0 +1 @@
+4b222ef09a5f20896d031a8322f2e69304c16384
\ No newline at end of file
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java
index 51598d7775623..a75448dadf427 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java
@@ -65,7 +65,7 @@
import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.opensearch.common.Priority;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.index.Index;
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java
index 7f175289f3a88..d6a7dcf4b6152 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java
@@ -47,8 +47,8 @@
import org.opensearch.cluster.routing.allocation.AllocationService;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.time.DateFormatter;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.plugins.Plugin;
import org.opensearch.test.OpenSearchIntegTestCase;
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java
index 850034bc631b1..c77178a5165bc 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java
@@ -41,8 +41,8 @@
import org.opensearch.client.Requests;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.test.OpenSearchIntegTestCase;
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java
index afa5ac908c137..24a66083d85ff 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java
@@ -47,7 +47,7 @@
import org.opensearch.action.support.WriteRequest;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexMetadata;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java
index 61171b0a817b0..4683535a3a095 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java
@@ -50,7 +50,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java
index cafce1a194caa..2d2714723802f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java
@@ -59,8 +59,8 @@
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.io.PathUtilsForTesting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.index.IndexSettings;
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java
index 79b674b23fd48..f1c34aa4f4141 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java
@@ -42,7 +42,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsException;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.indices.recovery.RecoverySettings;
import org.opensearch.test.OpenSearchIntegTestCase;
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java
index 23335f6e82ef1..9433bebc24c20 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java
@@ -46,7 +46,7 @@
import org.opensearch.common.Priority;
import org.opensearch.common.network.NetworkModule;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.indices.ShardLimitValidator;
import org.opensearch.snapshots.SnapshotInfo;
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java
index 9a465c2f9121c..e48e13b471ba5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java
@@ -38,8 +38,8 @@
import org.opensearch.cluster.routing.UnassignedInfo;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Priority;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.index.Index;
import org.opensearch.index.IndexService;
diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java
index ba9f335cd24d4..4b95c2fc4bce5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java
@@ -58,8 +58,8 @@
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.common.lucene.uid.Versions;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.util.io.IOUtils;
@@ -84,7 +84,7 @@
import org.opensearch.index.translog.Translog;
import org.opensearch.index.translog.TranslogStats;
import org.opensearch.indices.IndicesService;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.recovery.RecoveryState;
import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher;
import org.opensearch.plugins.Plugin;
diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java
index b6124ff09d992..df2c8c62ca392 100644
--- a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java
@@ -65,8 +65,8 @@
import org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java
index d51e4bbff11b5..89031f68aba97 100644
--- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java
@@ -66,8 +66,8 @@
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.core.index.Index;
diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java
index 1dd0f6a3d664e..6cc1c51ed65d9 100644
--- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java
@@ -38,8 +38,8 @@
import org.opensearch.action.search.SearchPhaseExecutionException;
import org.opensearch.cluster.routing.UnassignedInfo;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.MockEngineFactoryPlugin;
import org.opensearch.index.translog.TestTranslog;
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index 2ab44f8318617..10bd179ddc5fd 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -46,17 +46,17 @@
import org.opensearch.client.Requests;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.xcontent.XContentType;
-import org.opensearch.indices.breaker.CircuitBreakerStats;
-import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.indices.breaker.CircuitBreakerStats;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.rest.RestStatus;
+import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
import org.opensearch.search.sort.SortOrder;
import org.opensearch.test.OpenSearchIntegTestCase;
import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
@@ -197,7 +197,7 @@ public void testRamAccountingTermsEnum() throws Exception {
prepareCreate("ramtest").setSource(
"{\"mappings\": {\"type\": {\"properties\": {\"test\": "
+ "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}",
- XContentType.JSON
+ MediaTypeRegistry.JSON
)
);
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
index 341c0a965f94e..5ce9de7d0eae0 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
@@ -44,7 +44,7 @@
import org.opensearch.action.search.SearchPhaseExecutionException;
import org.opensearch.action.search.SearchRequestBuilder;
import org.opensearch.common.Strings;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java
index c31b5e1f3bc5b..850f08b8136c1 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java
@@ -74,12 +74,12 @@
import org.opensearch.common.Priority;
import org.opensearch.common.SetOnce;
import org.opensearch.common.Strings;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.concurrent.GatedCloseable;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
import org.opensearch.common.xcontent.XContentType;
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
index d345578c7de6e..3ab1a2a8564c5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java
@@ -875,69 +875,84 @@ public void testPressureServiceStats() throws Exception {
waitForDocs(initialDocCount, indexer);
refresh(INDEX_NAME);
+ // get shard references.
+ final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME);
+ final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME);
+ logger.info("Replica aid {}", replicaShard.routingEntry().allocationId());
+ logger.info("former primary aid {}", primaryShard.routingEntry().allocationId());
+
+ // fetch pressure stats from the Primary's Node.
SegmentReplicationPressureService pressureService = internalCluster().getInstance(
SegmentReplicationPressureService.class,
primaryNode
);
- final Map shardStats = pressureService.nodeStats().getShardStats();
- assertEquals(1, shardStats.size());
- final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME);
- IndexShard replica = getIndexShard(replicaNode, INDEX_NAME);
- SegmentReplicationPerGroupStats groupStats = shardStats.get(primaryShard.shardId());
- Set replicaStats = groupStats.getReplicaStats();
- assertEquals(1, replicaStats.size());
-
- // assert replica node returns nothing.
+ // Fetch pressure stats from the Replica's Node we will assert replica node returns nothing until it is promoted.
SegmentReplicationPressureService replicaNode_service = internalCluster().getInstance(
SegmentReplicationPressureService.class,
replicaNode
);
+
+ final Map shardStats = pressureService.nodeStats().getShardStats();
+ assertEquals("We should have stats returned for the replication group", 1, shardStats.size());
+
+ SegmentReplicationPerGroupStats groupStats = shardStats.get(primaryShard.shardId());
+ Set replicaStats = groupStats.getReplicaStats();
+ assertAllocationIdsInReplicaShardStats(Set.of(replicaShard.routingEntry().allocationId().getId()), replicaStats);
+
assertTrue(replicaNode_service.nodeStats().getShardStats().isEmpty());
- // drop the primary, this won't hand off SR state.
+ // drop the primary, this won't hand off pressure stats between old/new primary.
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
ensureYellowAndNoInitializingShards(INDEX_NAME);
- replicaNode_service = internalCluster().getInstance(SegmentReplicationPressureService.class, replicaNode);
- replica = getIndexShard(replicaNode, INDEX_NAME);
- assertTrue("replica should be promoted as a primary", replica.routingEntry().primary());
- assertEquals(1, replicaNode_service.nodeStats().getShardStats().size());
- // we don't have a replica assigned yet, so this should be 0.
- assertEquals(0, replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats().size());
+
+ assertTrue("replica should be promoted as a primary", replicaShard.routingEntry().primary());
+ assertEquals(
+ "We should have stats returned for the replication group",
+ 1,
+ replicaNode_service.nodeStats().getShardStats().size()
+ );
+ // after the primary is dropped and replica is promoted we won't have a replica assigned yet, so stats per replica should return
+ // empty.
+ replicaStats = replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats();
+ assertTrue(replicaStats.isEmpty());
// start another replica.
String replicaNode_2 = internalCluster().startDataOnlyNode();
ensureGreen(INDEX_NAME);
- String docId = String.valueOf(initialDocCount + 1);
- client().prepareIndex(INDEX_NAME).setId(docId).setSource("foo", "bar").get();
- refresh(INDEX_NAME);
- waitForSearchableDocs(initialDocCount + 1, replicaNode_2);
+ final IndexShard secondReplicaShard = getIndexShard(replicaNode_2, INDEX_NAME);
+ final String second_replica_aid = secondReplicaShard.routingEntry().allocationId().getId();
+ waitForSearchableDocs(initialDocCount, replicaNode_2);
- replicaNode_service = internalCluster().getInstance(SegmentReplicationPressureService.class, replicaNode);
- replica = getIndexShard(replicaNode_2, INDEX_NAME);
- assertEquals(1, replicaNode_service.nodeStats().getShardStats().size());
- replicaStats = replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats();
- assertEquals(1, replicaStats.size());
+ assertEquals(
+ "We should have stats returned for the replication group",
+ 1,
+ replicaNode_service.nodeStats().getShardStats().size()
+ );
+ replicaStats = replicaNode_service.nodeStats().getShardStats().get(replicaShard.shardId()).getReplicaStats();
+ assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), replicaStats);
+ final SegmentReplicationShardStats replica_entry = replicaStats.stream().findFirst().get();
+ assertEquals(replica_entry.getCheckpointsBehindCount(), 0);
// test a checkpoint without any new segments
flush(INDEX_NAME);
assertBusy(() -> {
- final SegmentReplicationPressureService service = internalCluster().getInstance(
- SegmentReplicationPressureService.class,
- replicaNode
- );
- assertEquals(1, service.nodeStats().getShardStats().size());
- final Set shardStatsSet = service.nodeStats()
+ assertEquals(1, replicaNode_service.nodeStats().getShardStats().size());
+ final Set shardStatsSet = replicaNode_service.nodeStats()
.getShardStats()
- .get(primaryShard.shardId())
+ .get(replicaShard.shardId())
.getReplicaStats();
- assertEquals(1, shardStatsSet.size());
+ assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), shardStatsSet);
final SegmentReplicationShardStats stats = shardStatsSet.stream().findFirst().get();
assertEquals(0, stats.getCheckpointsBehindCount());
});
}
}
+ private void assertAllocationIdsInReplicaShardStats(Set expected, Set replicaStats) {
+ assertEquals(expected, replicaStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet()));
+ }
+
/**
* Tests a scroll query on the replica
* @throws Exception when issue is encountered
@@ -1162,6 +1177,10 @@ public void testScrollWithOngoingSegmentReplication() throws Exception {
}
public void testPitCreatedOnReplica() throws Exception {
+ assumeFalse(
+ "Skipping the test as it is flaky with remote store. Tracking issue https://github.com/opensearch-project/OpenSearch/issues/8850",
+ segmentReplicationWithRemoteEnabled()
+ );
final String primary = internalCluster().startDataOnlyNode();
createIndex(INDEX_NAME);
ensureYellowAndNoInitializingShards(INDEX_NAME);
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java
index 7426ca2f13f84..3024eeb798b48 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java
@@ -22,6 +22,8 @@
import org.opensearch.common.Priority;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.SegmentReplicationShardStats;
+import org.opensearch.index.shard.IndexShard;
import org.opensearch.indices.IndicesService;
import org.opensearch.test.OpenSearchIntegTestCase;
import org.opensearch.test.transport.MockTransportService;
@@ -29,8 +31,10 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -56,7 +60,7 @@ public void testPrimaryRelocation() throws Exception {
createIndex(1);
final String replica = internalCluster().startNode();
ensureGreen(INDEX_NAME);
- final int initialDocCount = scaledRandomIntBetween(100, 1000);
+ final int initialDocCount = scaledRandomIntBetween(10, 100);
final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
final List> pendingIndexResponses = new ArrayList<>();
for (int i = 0; i < initialDocCount; i++) {
@@ -133,7 +137,7 @@ public void testPrimaryRelocationWithSegRepFailure() throws Exception {
createIndex(1);
final String replica = internalCluster().startNode();
ensureGreen(INDEX_NAME);
- final int initialDocCount = scaledRandomIntBetween(100, 1000);
+ final int initialDocCount = scaledRandomIntBetween(10, 100);
final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
final List> pendingIndexResponses = new ArrayList<>();
for (int i = 0; i < initialDocCount; i++) {
@@ -528,13 +532,27 @@ public void testFlushAfterRelocation() throws Exception {
client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
}
- // Verify segment replication event never happened on replica shard
+ final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME);
+
+ // Verify segment replication event never happened on replica shard other than recovery.
+ assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setPreference("_only_local").setSize(0).get(), 0);
+ assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setPreference("_only_local").setSize(0).get(), 0);
+
SegmentReplicationStatsResponse segmentReplicationStatsResponse = client().admin()
.indices()
.prepareSegmentReplicationStats(INDEX_NAME)
.execute()
.actionGet();
- assertTrue(segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0).getReplicaStats().isEmpty());
+ final Set replicaStats = segmentReplicationStatsResponse.getReplicationStats()
+ .get(INDEX_NAME)
+ .get(0)
+ .getReplicaStats();
+ assertEquals(
+ Set.of(replicaShard.routingEntry().allocationId().getId()),
+ replicaStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet())
+ );
+ // the primary still has not refreshed to update its checkpoint, so our replica is not yet behind.
+ assertEquals(0, replicaStats.stream().findFirst().get().getCheckpointsBehindCount());
// Relocate primary to new primary. When new primary starts it does perform a flush.
logger.info("--> relocate the shard from primary to newPrimary");
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java
index 28bd5a6ae252d..e95f10bd7abdb 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java
@@ -46,8 +46,8 @@
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.set.Sets;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.IndexSettings;
diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java
index b5d7bd476059d..2454f6553951e 100644
--- a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java
@@ -39,8 +39,8 @@
import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.indices.recovery.PeerRecoveryTargetService;
import org.opensearch.indices.recovery.FileChunkRequest;
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java
index 3fe7f3d553a1b..608d7d9d02581 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java
@@ -12,7 +12,7 @@
import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse;
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
index c5d023bdd7a64..2887fbc56106c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
@@ -26,6 +26,7 @@
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -37,6 +38,13 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase {
protected static final int REPLICA_COUNT = 1;
protected Path absolutePath;
protected Path absolutePath2;
+ private final List documentKeys = List.of(
+ randomAlphaOfLength(5),
+ randomAlphaOfLength(5),
+ randomAlphaOfLength(5),
+ randomAlphaOfLength(5),
+ randomAlphaOfLength(5)
+ );
@Override
protected boolean addMockInternalEngine() {
@@ -59,7 +67,7 @@ public Settings indexSettings() {
IndexResponse indexSingleDoc(String indexName) {
return client().prepareIndex(indexName)
.setId(UUIDs.randomBase64UUID())
- .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5))
+ .setSource(documentKeys.get(randomIntBetween(0, documentKeys.size() - 1)), randomAlphaOfLength(5))
.get();
}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java
index b4456f887cbaa..4d5648c74ba5c 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java
@@ -104,9 +104,17 @@ private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlus
Map indexStats = indexData(numberOfIterations, invokeFlush, flushAfterMerge, deletedDocs);
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME)));
- assertAcked(client().admin().indices().prepareClose(INDEX_NAME));
- client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture());
+ boolean restoreAllShards = randomBoolean();
+ if (restoreAllShards) {
+ assertAcked(client().admin().indices().prepareClose(INDEX_NAME));
+ }
+ client().admin()
+ .cluster()
+ .restoreRemoteStore(
+ new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(restoreAllShards),
+ PlainActionFuture.newFuture()
+ );
ensureGreen(INDEX_NAME);
if (deletedDocs == -1) {
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
index 42bd4b5173fa3..693c4113f8f3b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
@@ -11,6 +11,7 @@
import org.hamcrest.MatcherAssert;
import org.junit.Before;
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
+import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse;
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
import org.opensearch.action.admin.indices.recovery.RecoveryResponse;
import org.opensearch.action.index.IndexResponse;
@@ -18,7 +19,6 @@
import org.opensearch.cluster.health.ClusterHealthStatus;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.routing.RecoverySource;
-import org.opensearch.common.UUIDs;
import org.opensearch.common.settings.Settings;
import org.opensearch.index.shard.RemoteStoreRefreshListener;
import org.opensearch.indices.recovery.RecoveryState;
@@ -34,15 +34,16 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import static org.hamcrest.Matchers.comparesEqualTo;
-import static org.hamcrest.Matchers.oneOf;
import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.oneOf;
+import static org.hamcrest.Matchers.comparesEqualTo;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0)
public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase {
private static final String INDEX_NAME = "remote-store-test-idx-1";
@@ -68,13 +69,6 @@ public Settings indexSettings() {
return remoteStoreIndexSettings(0);
}
- private IndexResponse indexSingleDoc() {
- return client().prepareIndex(INDEX_NAME)
- .setId(UUIDs.randomBase64UUID())
- .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5))
- .get();
- }
-
private Map indexData(int numberOfIterations, boolean invokeFlush, String index) {
long totalOperations = 0;
long refreshedOrFlushedOperations = 0;
@@ -93,7 +87,7 @@ private Map indexData(int numberOfIterations, boolean invokeFlush,
refreshedOrFlushedOperations = totalOperations;
int numberOfOperations = randomIntBetween(20, 50);
for (int j = 0; j < numberOfOperations; j++) {
- IndexResponse response = INDEX_NAME.equals(index) ? indexSingleDoc() : indexSingleDoc(index);
+ IndexResponse response = indexSingleDoc(index);
maxSeqNo = response.getSeqNo();
shardId = response.getShardId().id();
indexingStats.put(MAX_SEQ_NO_TOTAL + "-shard-" + shardId, maxSeqNo);
@@ -109,12 +103,14 @@ private Map indexData(int numberOfIterations, boolean invokeFlush,
}
private void verifyRestoredData(Map indexStats, boolean checkTotal, String indexName) {
+ // This is required to get updated number from already active shards which were not restored
+ refresh(indexName);
String statsGranularity = checkTotal ? TOTAL_OPERATIONS : REFRESHED_OR_FLUSHED_OPERATIONS;
String maxSeqNoGranularity = checkTotal ? MAX_SEQ_NO_TOTAL : MAX_SEQ_NO_REFRESHED_OR_FLUSHED;
ensureYellowAndNoInitializingShards(indexName);
ensureGreen(indexName);
assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity));
- IndexResponse response = INDEX_NAME.equals(indexName) ? indexSingleDoc() : indexSingleDoc(indexName);
+ IndexResponse response = indexSingleDoc(indexName);
assertEquals(indexStats.get(maxSeqNoGranularity + "-shard-" + response.getShardId().id()) + 1, response.getSeqNo());
refresh(indexName);
assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1);
@@ -130,6 +126,28 @@ private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, St
}
}
+ private void restore(String... indices) {
+ boolean restoreAllShards = randomBoolean();
+ if (restoreAllShards) {
+ assertAcked(client().admin().indices().prepareClose(indices));
+ }
+ client().admin()
+ .cluster()
+ .restoreRemoteStore(
+ new RestoreRemoteStoreRequest().indices(indices).restoreAllShards(restoreAllShards),
+ PlainActionFuture.newFuture()
+ );
+ }
+
+ private void restoreAndVerify(int shardCount, int replicaCount, Map indexStats) {
+ restore(INDEX_NAME);
+ ensureGreen(INDEX_NAME);
+ // This is required to get updated number from already active shards which were not restored
+ assertEquals(shardCount * (1 + replicaCount), getNumShards(INDEX_NAME).totalNumShards);
+ assertEquals(replicaCount, getNumShards(INDEX_NAME).numReplicas);
+ verifyRestoredData(indexStats, true, INDEX_NAME);
+ }
+
/**
* Helper function to test restoring an index with no replication from remote store. Only primary node is dropped.
* @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data.
@@ -144,23 +162,16 @@ private void testRestoreFlow(int numberOfIterations, boolean invokeFlush, int sh
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME)));
ensureRed(INDEX_NAME);
- assertAcked(client().admin().indices().prepareClose(INDEX_NAME));
- client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture());
-
- ensureGreen(INDEX_NAME);
- assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards);
- verifyRestoredData(indexStats, true, INDEX_NAME);
+ restoreAndVerify(shardCount, 0, indexStats);
}
/**
* Helper function to test restoring an index having replicas from remote store when all the nodes housing the primary/replica drop.
- * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store.
* @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data.
* @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked.
* @throws IOException IO Exception.
*/
- private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount)
- throws IOException {
+ private void testRestoreFlowBothPrimaryReplicasDown(int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException {
prepareCluster(1, 2, INDEX_NAME, 1, shardCount);
Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME);
assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards);
@@ -170,14 +181,7 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int
ensureRed(INDEX_NAME);
internalCluster().startDataOnlyNodes(2);
- assertAcked(client().admin().indices().prepareClose(INDEX_NAME));
- client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture());
-
- ensureGreen(INDEX_NAME);
-
- assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards);
- assertEquals(0, getNumShards(INDEX_NAME).numReplicas);
- verifyRestoredData(indexStats, true, INDEX_NAME);
+ restoreAndVerify(shardCount, 1, indexStats);
}
/**
@@ -212,10 +216,16 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo
ensureRed(indices);
internalCluster().startDataOnlyNodes(3);
- assertAcked(client().admin().indices().prepareClose(indices));
+ boolean restoreAllShards = randomBoolean();
+ if (restoreAllShards) {
+ assertAcked(client().admin().indices().prepareClose(indices));
+ }
client().admin()
.cluster()
- .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")), PlainActionFuture.newFuture());
+ .restoreRemoteStore(
+ new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")).restoreAllShards(restoreAllShards),
+ PlainActionFuture.newFuture()
+ );
ensureGreen(indices);
for (String index : indices) {
assertEquals(shardCount, getNumShards(index).totalNumShards);
@@ -223,6 +233,37 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo
}
}
+ public void testRestoreFlowAllShardsNoRedIndex() throws InterruptedException {
+ int shardCount = randomIntBetween(1, 5);
+ prepareCluster(0, 3, INDEX_NAME, 0, shardCount);
+ indexData(randomIntBetween(2, 5), true, INDEX_NAME);
+ assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards);
+
+ PlainActionFuture future = PlainActionFuture.newFuture();
+ client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(true), future);
+ try {
+ future.get();
+ } catch (ExecutionException e) {
+ // If the request goes to co-ordinator, e.getCause() can be RemoteTransportException
+ assertTrue(e.getCause() instanceof IllegalStateException || e.getCause().getCause() instanceof IllegalStateException);
+ }
+ }
+
+ public void testRestoreFlowNoRedIndex() {
+ int shardCount = randomIntBetween(1, 5);
+ prepareCluster(0, 3, INDEX_NAME, 0, shardCount);
+ Map indexStats = indexData(randomIntBetween(2, 5), true, INDEX_NAME);
+ assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards);
+
+ client().admin()
+ .cluster()
+ .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(false), PlainActionFuture.newFuture());
+
+ ensureGreen(INDEX_NAME);
+ assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards);
+ verifyRestoredData(indexStats, true, INDEX_NAME);
+ }
+
/**
* Simulates all data restored using Remote Translog Store.
* @throws IOException IO Exception.
@@ -265,7 +306,7 @@ public void testRemoteTranslogRestoreWithCommittedData() throws IOException {
// @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188")
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479")
public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException {
- testRestoreFlowBothPrimaryReplicasDown(true, 1, true, randomIntBetween(1, 5));
+ testRestoreFlowBothPrimaryReplicasDown(1, true, randomIntBetween(1, 5));
}
/**
@@ -274,7 +315,7 @@ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOExce
*/
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479")
public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException {
- testRestoreFlowBothPrimaryReplicasDown(true, 1, false, randomIntBetween(1, 5));
+ testRestoreFlowBothPrimaryReplicasDown(1, false, randomIntBetween(1, 5));
}
/**
@@ -284,7 +325,7 @@ public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOExc
*/
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479")
public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException {
- testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5));
+ testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), false, randomIntBetween(1, 5));
}
/**
@@ -294,7 +335,7 @@ public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOExcepti
*/
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479")
public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException {
- testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5));
+ testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), true, randomIntBetween(1, 5));
}
/**
@@ -341,10 +382,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio
ensureRed(indices);
internalCluster().startDataOnlyNodes(3);
- assertAcked(client().admin().indices().prepareClose(indices));
- client().admin()
- .cluster()
- .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(new String[] {}), PlainActionFuture.newFuture());
+ restore(indices);
ensureGreen(indices);
for (String index : indices) {
@@ -381,10 +419,16 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc
ensureRed(indices);
internalCluster().startDataOnlyNodes(3);
- assertAcked(client().admin().indices().prepareClose(indices[0], indices[1]));
+ boolean restoreAllShards = randomBoolean();
+ if (restoreAllShards) {
+ assertAcked(client().admin().indices().prepareClose(indices[0], indices[1]));
+ }
client().admin()
.cluster()
- .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indices[0], indices[1]), PlainActionFuture.newFuture());
+ .restoreRemoteStore(
+ new RestoreRemoteStoreRequest().indices(indices[0], indices[1]).restoreAllShards(restoreAllShards),
+ PlainActionFuture.newFuture()
+ );
ensureGreen(indices[0], indices[1]);
assertEquals(shardCount, getNumShards(indices[0]).totalNumShards);
verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]);
@@ -427,10 +471,16 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOExc
ensureRed(indices);
internalCluster().startDataOnlyNodes(3);
- assertAcked(client().admin().indices().prepareClose(indices[0], indices[1]));
+ boolean restoreAllShards = randomBoolean();
+ if (restoreAllShards) {
+ assertAcked(client().admin().indices().prepareClose(indices[0], indices[1]));
+ }
client().admin()
.cluster()
- .restoreRemoteStore(new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*"), PlainActionFuture.newFuture());
+ .restoreRemoteStore(
+ new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*").restoreAllShards(restoreAllShards),
+ PlainActionFuture.newFuture()
+ );
ensureGreen(indices[0], indices[1]);
assertEquals(shardCount, getNumShards(indices[0]).totalNumShards);
verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]);
@@ -490,7 +540,7 @@ private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throw
assertEquals(0, recoverySource.get().getIndex().recoveredFileCount());
}
- IndexResponse response = indexSingleDoc();
+ IndexResponse response = indexSingleDoc(INDEX_NAME);
assertEquals(indexStats.get(MAX_SEQ_NO_TOTAL) + 1, response.getSeqNo());
refresh(INDEX_NAME);
assertBusy(
diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java
index 4e7f2ae486c93..d5c744a49c62d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java
@@ -37,8 +37,8 @@
import org.opensearch.common.blobstore.fs.FsBlobStore;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.repositories.blobstore.OpenSearchBlobStoreRepositoryIntegTestCase;
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java
index c7392b260319a..aa8ef3f29c989 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java
@@ -79,7 +79,6 @@ public void testSimpleTimeout() throws Exception {
.get();
assertTrue(searchResponse.isTimedOut());
assertEquals(0, searchResponse.getFailedShards());
- assertTrue(numDocs > searchResponse.getHits().getTotalHits().value);
}
public void testSimpleDoesNotTimeout() throws Exception {
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java
index e554a3be20528..fe62f5902cbba 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java
@@ -35,8 +35,8 @@
import org.opensearch.action.search.SearchResponse;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.settings.Settings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.query.WrapperQueryBuilder;
import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder;
@@ -59,7 +59,7 @@ public void testWrapperQueryIsRewritten() throws IOException {
XContentType xContentType = randomFrom(XContentType.values());
BytesReference bytesReference;
- try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) {
builder.startObject();
{
builder.startObject("terms");
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
index be69428453952..a54ca880c3dd0 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
@@ -35,7 +35,7 @@
import org.opensearch.OpenSearchException;
import org.opensearch.ExceptionsHelper;
import org.opensearch.action.index.IndexRequestBuilder;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.settings.Settings;
import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
import org.opensearch.search.aggregations.Aggregator;
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java
index f67b913a75871..3651a7354e5de 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java
@@ -179,8 +179,18 @@ public void testExplainScript() throws InterruptedException, IOException, Execut
for (SearchHit hit : hits.getHits()) {
assertThat(hit.getId(), equalTo(Integer.toString(idCounter)));
assertThat(hit.getExplanation().toString(), containsString(Double.toString(idCounter)));
- assertThat(hit.getExplanation().toString(), containsString("1 = n"));
- assertThat(hit.getExplanation().toString(), containsString("1 = N"));
+
+ // Since Apache Lucene 9.8, the scores are not computed because script (see please ExplainableScriptPlugin)
+ // says "needs_score() == false"
+ // 19.0 = min of:
+ // 19.0 = This script returned 19.0
+ // 0.0 = _score:
+ // 0.0 = weight(text:text in 0) [PerFieldSimilarity], result of:
+ // 0.0 = score(freq=1.0), with freq of:
+ // 1.0 = freq, occurrences of term within document
+ // 3.4028235E38 = maxBoost
+
+ assertThat(hit.getExplanation().toString(), containsString("1.0 = freq, occurrences of term within document"));
assertThat(hit.getExplanation().getDetails().length, equalTo(2));
idCounter--;
}
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java
index 7f306b3ee550f..bb14ed1ea5578 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java
@@ -406,9 +406,7 @@ public void testBulk() throws Exception {
.endObject();
client().admin().indices().prepareCreate("countries").setSettings(settings).setMapping(xContentBuilder).get();
- BulkResponse bulk = client().prepareBulk()
- .add(bulkAction, 0, bulkAction.length, null, XContentType.fromMediaType(xContentBuilder.contentType()))
- .get();
+ BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, xContentBuilder.contentType()).get();
for (BulkItemResponse item : bulk.getItems()) {
assertFalse("unable to index data", item.isFailed());
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java
index 0f08c537d74d8..9d0c30c5a488f 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java
@@ -90,7 +90,52 @@ public class AggregationProfilerIT extends OpenSearchIntegTestCase {
COLLECT + "_count",
POST_COLLECTION + "_count",
BUILD_AGGREGATION + "_count",
- REDUCE + "_count"
+ REDUCE + "_count",
+ INITIALIZE + "_start_time",
+ BUILD_LEAF_COLLECTOR + "_start_time",
+ COLLECT + "_start_time",
+ POST_COLLECTION + "_start_time",
+ BUILD_AGGREGATION + "_start_time",
+ REDUCE + "_start_time"
+ );
+
+ private static final Set CONCURRENT_SEARCH_BREAKDOWN_KEYS = Set.of(
+ INITIALIZE,
+ BUILD_LEAF_COLLECTOR,
+ COLLECT,
+ POST_COLLECTION,
+ BUILD_AGGREGATION,
+ REDUCE,
+ INITIALIZE + "_count",
+ BUILD_LEAF_COLLECTOR + "_count",
+ COLLECT + "_count",
+ POST_COLLECTION + "_count",
+ BUILD_AGGREGATION + "_count",
+ REDUCE + "_count",
+ "max_" + INITIALIZE,
+ "max_" + BUILD_LEAF_COLLECTOR,
+ "max_" + COLLECT,
+ "max_" + POST_COLLECTION,
+ "max_" + BUILD_AGGREGATION,
+ "max_" + REDUCE,
+ "min_" + INITIALIZE,
+ "min_" + BUILD_LEAF_COLLECTOR,
+ "min_" + COLLECT,
+ "min_" + POST_COLLECTION,
+ "min_" + BUILD_AGGREGATION,
+ "min_" + REDUCE,
+ "avg_" + INITIALIZE,
+ "avg_" + BUILD_LEAF_COLLECTOR,
+ "avg_" + COLLECT,
+ "avg_" + POST_COLLECTION,
+ "avg_" + BUILD_AGGREGATION,
+ "avg_" + REDUCE,
+ "max_" + BUILD_LEAF_COLLECTOR + "_count",
+ "max_" + COLLECT + "_count",
+ "min_" + BUILD_LEAF_COLLECTOR + "_count",
+ "min_" + COLLECT + "_count",
+ "avg_" + BUILD_LEAF_COLLECTOR + "_count",
+ "avg_" + COLLECT + "_count"
);
private static final String TOTAL_BUCKETS = "total_buckets";
@@ -169,7 +214,12 @@ public void testSimpleProfile() {
assertThat(histoAggResult.getTime(), greaterThan(0L));
Map breakdown = histoAggResult.getTimeBreakdown();
assertThat(breakdown, notNullValue());
- assertThat(breakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (histoAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(breakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(breakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(breakdown.get(INITIALIZE), greaterThan(0L));
assertThat(breakdown.get(COLLECT), greaterThan(0L));
assertThat(breakdown.get(BUILD_AGGREGATION).longValue(), greaterThan(0L));
@@ -212,7 +262,12 @@ public void testMultiLevelProfile() {
assertThat(histoAggResult.getTime(), greaterThan(0L));
Map histoBreakdown = histoAggResult.getTimeBreakdown();
assertThat(histoBreakdown, notNullValue());
- assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (histoAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(histoBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(histoBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(histoBreakdown.get(COLLECT), greaterThan(0L));
assertThat(histoBreakdown.get(BUILD_AGGREGATION), greaterThan(0L));
@@ -230,7 +285,12 @@ public void testMultiLevelProfile() {
assertThat(termsAggResult.getTime(), greaterThan(0L));
Map termsBreakdown = termsAggResult.getTimeBreakdown();
assertThat(termsBreakdown, notNullValue());
- assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (termsAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(termsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(termsBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(termsBreakdown.get(COLLECT), greaterThan(0L));
assertThat(termsBreakdown.get(BUILD_AGGREGATION), greaterThan(0L));
@@ -245,7 +305,12 @@ public void testMultiLevelProfile() {
assertThat(avgAggResult.getTime(), greaterThan(0L));
Map avgBreakdown = termsAggResult.getTimeBreakdown();
assertThat(avgBreakdown, notNullValue());
- assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (avgAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(avgBreakdown.get(COLLECT), greaterThan(0L));
assertThat(avgBreakdown.get(BUILD_AGGREGATION), greaterThan(0L));
@@ -298,7 +363,12 @@ public void testMultiLevelProfileBreadthFirst() {
assertThat(histoAggResult.getTime(), greaterThan(0L));
Map histoBreakdown = histoAggResult.getTimeBreakdown();
assertThat(histoBreakdown, notNullValue());
- assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (histoAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(histoBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(histoBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(histoBreakdown.get(COLLECT), greaterThan(0L));
assertThat(histoBreakdown.get(BUILD_AGGREGATION), greaterThan(0L));
@@ -316,7 +386,12 @@ public void testMultiLevelProfileBreadthFirst() {
assertThat(termsAggResult.getTime(), greaterThan(0L));
Map termsBreakdown = termsAggResult.getTimeBreakdown();
assertThat(termsBreakdown, notNullValue());
- assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (termsAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(termsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(termsBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(termsBreakdown.get(COLLECT), greaterThan(0L));
assertThat(termsBreakdown.get(BUILD_AGGREGATION), greaterThan(0L));
@@ -331,7 +406,12 @@ public void testMultiLevelProfileBreadthFirst() {
assertThat(avgAggResult.getTime(), greaterThan(0L));
Map avgBreakdown = avgAggResult.getTimeBreakdown();
assertThat(avgBreakdown, notNullValue());
- assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (avgAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(avgBreakdown.get(COLLECT), greaterThan(0L));
assertThat(avgBreakdown.get(BUILD_AGGREGATION), greaterThan(0L));
@@ -369,7 +449,12 @@ public void testDiversifiedAggProfile() {
assertThat(diversifyAggResult.getTime(), greaterThan(0L));
Map diversifyBreakdown = diversifyAggResult.getTimeBreakdown();
assertThat(diversifyBreakdown, notNullValue());
- assertThat(diversifyBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (diversifyAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(diversifyBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(diversifyBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(diversifyBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(diversifyBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(diversifyBreakdown.get(COLLECT), greaterThan(0L));
@@ -386,7 +471,12 @@ public void testDiversifiedAggProfile() {
assertThat(maxAggResult.getTime(), greaterThan(0L));
Map maxBreakdown = maxAggResult.getTimeBreakdown();
assertThat(maxBreakdown, notNullValue());
- assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (maxAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(diversifyBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(diversifyBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(diversifyBreakdown.get(COLLECT), greaterThan(0L));
@@ -439,7 +529,12 @@ public void testComplexProfile() {
assertThat(histoAggResult.getTime(), greaterThan(0L));
Map histoBreakdown = histoAggResult.getTimeBreakdown();
assertThat(histoBreakdown, notNullValue());
- assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (histoAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(histoBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(histoBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(histoBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(histoBreakdown.get(COLLECT), greaterThan(0L));
@@ -462,7 +557,12 @@ public void testComplexProfile() {
assertThat(tagsAggResult.getTime(), greaterThan(0L));
Map tagsBreakdown = tagsAggResult.getTimeBreakdown();
assertThat(tagsBreakdown, notNullValue());
- assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (tagsAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(tagsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(tagsBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(tagsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(tagsBreakdown.get(COLLECT), greaterThan(0L));
@@ -482,7 +582,12 @@ public void testComplexProfile() {
assertThat(avgAggResult.getTime(), greaterThan(0L));
Map avgBreakdown = avgAggResult.getTimeBreakdown();
assertThat(avgBreakdown, notNullValue());
- assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (avgAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(avgBreakdown.get(COLLECT), greaterThan(0L));
@@ -498,7 +603,12 @@ public void testComplexProfile() {
assertThat(maxAggResult.getTime(), greaterThan(0L));
Map maxBreakdown = maxAggResult.getTimeBreakdown();
assertThat(maxBreakdown, notNullValue());
- assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (maxAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(maxBreakdown.get(COLLECT), greaterThan(0L));
@@ -514,7 +624,12 @@ public void testComplexProfile() {
assertThat(stringsAggResult.getTime(), greaterThan(0L));
Map stringsBreakdown = stringsAggResult.getTimeBreakdown();
assertThat(stringsBreakdown, notNullValue());
- assertThat(stringsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (stringsAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(stringsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(stringsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(stringsBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(stringsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(stringsBreakdown.get(COLLECT), greaterThan(0L));
@@ -534,7 +649,12 @@ public void testComplexProfile() {
assertThat(avgAggResult.getTime(), greaterThan(0L));
avgBreakdown = avgAggResult.getTimeBreakdown();
assertThat(avgBreakdown, notNullValue());
- assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (avgAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(avgBreakdown.get(COLLECT), greaterThan(0L));
@@ -550,7 +670,12 @@ public void testComplexProfile() {
assertThat(maxAggResult.getTime(), greaterThan(0L));
maxBreakdown = maxAggResult.getTimeBreakdown();
assertThat(maxBreakdown, notNullValue());
- assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (maxAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(maxBreakdown.get(COLLECT), greaterThan(0L));
@@ -567,7 +692,12 @@ public void testComplexProfile() {
assertThat(tagsAggResult.getTime(), greaterThan(0L));
tagsBreakdown = tagsAggResult.getTimeBreakdown();
assertThat(tagsBreakdown, notNullValue());
- assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (tagsAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(tagsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(tagsBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(tagsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(tagsBreakdown.get(COLLECT), greaterThan(0L));
@@ -587,7 +717,12 @@ public void testComplexProfile() {
assertThat(avgAggResult.getTime(), greaterThan(0L));
avgBreakdown = avgAggResult.getTimeBreakdown();
assertThat(avgBreakdown, notNullValue());
- assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (avgAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(avgBreakdown.get(COLLECT), greaterThan(0L));
@@ -603,7 +738,12 @@ public void testComplexProfile() {
assertThat(maxAggResult.getTime(), greaterThan(0L));
maxBreakdown = maxAggResult.getTimeBreakdown();
assertThat(maxBreakdown, notNullValue());
- assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ if (maxAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS));
+ } else {
+ assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS));
+ }
assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L));
assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L));
assertThat(maxBreakdown.get(COLLECT), greaterThan(0L));
@@ -700,7 +840,12 @@ public void testGlobalAggWithStatsSubAggregatorProfile() {
assertThat(globalAggResult.getTime(), greaterThan(0L));
Map breakdown = globalAggResult.getTimeBreakdown();
assertThat(breakdown, notNullValue());
- assertEquals(BREAKDOWN_KEYS, breakdown.keySet());
+ if (globalAggResult.getMaxSliceTime() != null) {
+ // concurrent segment search enabled
+ assertEquals(CONCURRENT_SEARCH_BREAKDOWN_KEYS, breakdown.keySet());
+ } else {
+ assertEquals(BREAKDOWN_KEYS, breakdown.keySet());
+ }
assertThat(breakdown.get(INITIALIZE), greaterThan(0L));
assertThat(breakdown.get(COLLECT), greaterThan(0L));
assertThat(breakdown.get(BUILD_AGGREGATION).longValue(), greaterThan(0L));
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java
index 483d698f3c9a4..861279c4000d8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java
@@ -41,7 +41,7 @@
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.cluster.metadata.RepositoriesMetadata;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.repositories.IndexId;
import org.opensearch.repositories.RepositoriesService;
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java
index f6df7cccf96f7..bbca3bdc417c7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java
@@ -44,7 +44,7 @@
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.core.util.FileSystemUtils;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.repositories.RepositoriesService;
import org.opensearch.repositories.RepositoryException;
import org.opensearch.repositories.RepositoryVerificationException;
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
index dbd96a7fd109f..8677e61efeb46 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
@@ -52,7 +52,7 @@
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.common.xcontent.XContentFactory;
@@ -276,7 +276,10 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
client.admin()
.cluster()
- .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture());
+ .restoreRemoteStore(
+ new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true),
+ PlainActionFuture.newFuture()
+ );
ensureYellowAndNoInitializingShards(restoredIndexName1);
ensureGreen(restoredIndexName1);
assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1);
@@ -434,7 +437,9 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
// Re-initialize client to make sure we are not using client from stopped node.
client = client(clusterManagerNode);
assertAcked(client.admin().indices().prepareClose(indexName1));
- client.admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture());
+ client.admin()
+ .cluster()
+ .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(true), PlainActionFuture.newFuture());
ensureYellowAndNoInitializingShards(indexName1);
ensureGreen(indexName1);
assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
@@ -515,7 +520,10 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException
assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
client.admin()
.cluster()
- .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture());
+ .restoreRemoteStore(
+ new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true),
+ PlainActionFuture.newFuture()
+ );
ensureYellowAndNoInitializingShards(restoredIndexName1);
ensureGreen(restoredIndexName1);
// indexing some new docs and validating
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
index 300e1db09b4c5..efd072fce511d 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
@@ -30,7 +30,7 @@
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.core.index.Index;
import org.opensearch.index.IndexModule;
import org.opensearch.index.IndexNotFoundException;
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 4bba25039d376..420df6b4c34c8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -66,7 +66,7 @@
import org.opensearch.cluster.routing.UnassignedInfo;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.util.BytesRefUtils;
import org.opensearch.core.index.Index;
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
index c22dd90cc930b..e47a2b94fc715 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
@@ -46,7 +46,7 @@
import org.opensearch.client.Client;
import org.opensearch.cluster.SnapshotsInProgress;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.common.util.io.IOUtils;
diff --git a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
index 3a36a6ff103e0..9d276cfe283eb 100644
--- a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
+++ b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
@@ -42,6 +42,7 @@
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.TermQuery;
@@ -93,11 +94,12 @@ public BlendedTermQuery(Term[] terms, float[] boosts) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = super.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = super.rewrite(searcher);
if (rewritten != this) {
return rewritten;
}
+ IndexReader reader = searcher.getIndexReader();
IndexReaderContext context = reader.getContext();
TermStates[] ctx = new TermStates[terms.length];
int[] docFreqs = new int[ctx.length];
diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java
index d53164427debf..b8c6d1e78d25a 100644
--- a/server/src/main/java/org/opensearch/OpenSearchServerException.java
+++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java
@@ -288,14 +288,6 @@ public static void registerExceptions() {
UNKNOWN_VERSION_ADDED
)
);
- registerExceptionHandle(
- new OpenSearchExceptionHandle(
- org.opensearch.OpenSearchParseException.class,
- org.opensearch.OpenSearchParseException::new,
- 35,
- UNKNOWN_VERSION_ADDED
- )
- );
registerExceptionHandle(
new OpenSearchExceptionHandle(
org.opensearch.search.SearchException.class,
@@ -915,14 +907,6 @@ public static void registerExceptions() {
UNKNOWN_VERSION_ADDED
)
);
- registerExceptionHandle(
- new OpenSearchExceptionHandle(
- org.opensearch.common.breaker.CircuitBreakingException.class,
- org.opensearch.common.breaker.CircuitBreakingException::new,
- 133,
- UNKNOWN_VERSION_ADDED
- )
- );
registerExceptionHandle(
new OpenSearchExceptionHandle(
org.opensearch.transport.NodeNotConnectedException.class,
diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java
index 2ce1d4bcd4b02..b775095861150 100644
--- a/server/src/main/java/org/opensearch/action/ActionModule.java
+++ b/server/src/main/java/org/opensearch/action/ActionModule.java
@@ -301,7 +301,7 @@
import org.opensearch.index.seqno.RetentionLeaseActions;
import org.opensearch.identity.IdentityService;
import org.opensearch.indices.SystemIndices;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.persistent.CompletionPersistentTaskAction;
import org.opensearch.persistent.RemovePersistentTaskAction;
import org.opensearch.persistent.StartPersistentTaskAction;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java
index acf40e3a9de3c..1d2dbf66920f8 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java
@@ -40,7 +40,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.http.HttpInfo;
import org.opensearch.ingest.IngestInfo;
import org.opensearch.monitor.jvm.JvmInfo;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index 3b6c85ee6e091..82e03d1fd79ac 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -49,7 +49,7 @@
import org.opensearch.index.stats.ShardIndexingPressureStats;
import org.opensearch.index.store.remote.filecache.FileCacheStats;
import org.opensearch.indices.NodeIndicesStats;
-import org.opensearch.indices.breaker.AllCircuitBreakerStats;
+import org.opensearch.core.indices.breaker.AllCircuitBreakerStats;
import org.opensearch.ingest.IngestStats;
import org.opensearch.monitor.fs.FsInfo;
import org.opensearch.monitor.jvm.JvmStats;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java
index 703b9575a88ad..eb1935158c231 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java
@@ -35,7 +35,8 @@
public class RestoreRemoteStoreRequest extends ClusterManagerNodeRequest implements ToXContentObject {
private String[] indices = Strings.EMPTY_ARRAY;
- private Boolean waitForCompletion;
+ private Boolean waitForCompletion = false;
+ private Boolean restoreAllShards = false;
public RestoreRemoteStoreRequest() {}
@@ -43,6 +44,7 @@ public RestoreRemoteStoreRequest(StreamInput in) throws IOException {
super(in);
indices = in.readStringArray();
waitForCompletion = in.readOptionalBoolean();
+ restoreAllShards = in.readOptionalBoolean();
}
@Override
@@ -50,6 +52,7 @@ public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeStringArray(indices);
out.writeOptionalBoolean(waitForCompletion);
+ out.writeOptionalBoolean(restoreAllShards);
}
@Override
@@ -118,6 +121,27 @@ public boolean waitForCompletion() {
return waitForCompletion;
}
+ /**
+ * Set the value for restoreAllShards, denoting whether to restore all shards or only unassigned shards
+ *
+ * @param restoreAllShards If true, the operation will restore all the shards of the given indices.
+ * If false, the operation will restore only the unassigned shards of the given indices.
+ * @return this request
+ */
+ public RestoreRemoteStoreRequest restoreAllShards(boolean restoreAllShards) {
+ this.restoreAllShards = restoreAllShards;
+ return this;
+ }
+
+ /**
+ * Returns restoreAllShards setting
+ *
+ * @return true if the operation will restore all the shards of the given indices
+ */
+ public boolean restoreAllShards() {
+ return restoreAllShards;
+ }
+
/**
* Parses restore definition
*
@@ -167,12 +191,14 @@ public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
RestoreRemoteStoreRequest that = (RestoreRemoteStoreRequest) o;
- return waitForCompletion == that.waitForCompletion && Arrays.equals(indices, that.indices);
+ return waitForCompletion == that.waitForCompletion
+ && restoreAllShards == that.restoreAllShards
+ && Arrays.equals(indices, that.indices);
}
@Override
public int hashCode() {
- int result = Objects.hash(waitForCompletion);
+ int result = Objects.hash(waitForCompletion, restoreAllShards);
result = 31 * result + Arrays.hashCode(indices);
return result;
}
@@ -181,4 +207,5 @@ public int hashCode() {
public String toString() {
return org.opensearch.common.Strings.toString(XContentType.JSON, this);
}
+
}
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 5da3f2eb01260..3bb552b80084d 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -44,6 +44,7 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
@@ -387,7 +388,7 @@ public CreateSnapshotRequest settings(String source, MediaType mediaType) {
*/
public CreateSnapshotRequest settings(Map source) {
try {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.map(source);
settings(Strings.toString(builder), builder.contentType());
} catch (IOException e) {
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 840564a4bd7a2..7a142e70305ae 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -517,7 +517,7 @@ public String snapshotUuid() {
/**
* Sets the storage type for this request.
*/
- RestoreSnapshotRequest storageType(StorageType storageType) {
+ public RestoreSnapshotRequest storageType(StorageType storageType) {
this.storageType = storageType;
return this;
}
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java
index 282585a43183a..8b718aeaf70c7 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java
@@ -36,7 +36,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentObject;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java
index 699884ca0eab3..bddb3fb746eb1 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java
@@ -42,7 +42,7 @@
import org.opensearch.common.network.NetworkModule;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.ToXContentFragment;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
index ecfcd6310dd0a..761373a001ffe 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
@@ -32,12 +32,14 @@
package org.opensearch.action.admin.cluster.storedscripts;
+import org.opensearch.Version;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.action.support.master.AcknowledgedRequest;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentHelper;
@@ -59,14 +61,18 @@ public class PutStoredScriptRequest extends AcknowledgedRequest filter) {
return this;
}
try {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.map(filter);
this.filter = Strings.toString(builder);
return this;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 142cbe6a0ab0b..6ed9648502ee3 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -48,6 +48,7 @@
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.ConstructingObjectParser;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.ObjectParser.ValueType;
import org.opensearch.core.xcontent.ToXContent;
@@ -428,7 +429,7 @@ public AliasActions filter(Map filter) {
return this;
}
try {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.map(filter);
this.filter = org.opensearch.common.Strings.toString(builder);
return this;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
index a716959614065..27f20f028ea74 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java
@@ -39,6 +39,7 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentType;
@@ -200,8 +201,8 @@ public CreateIndexRequestBuilder addAlias(Alias alias) {
/**
* Sets the settings and mappings as a single source.
*/
- public CreateIndexRequestBuilder setSource(String source, XContentType xContentType) {
- request.source(source, xContentType);
+ public CreateIndexRequestBuilder setSource(String source, MediaType mediaType) {
+ request.source(source, mediaType);
return this;
}
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java
index 0b2375850f1fc..2b536d24c946f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java
@@ -55,7 +55,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.IndexService;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 465a44556c081..2588dfd718e71 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -49,6 +49,7 @@
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.index.Index;
@@ -298,7 +299,7 @@ public PutMappingRequest source(XContentBuilder mappingBuilder) {
*/
public PutMappingRequest source(Map mappingSource) {
try {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.map(mappingSource);
return source(BytesReference.bytes(builder), builder.contentType());
} catch (IOException e) {
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java
index 280dc307447b7..e014d6d703500 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java
@@ -34,7 +34,7 @@
import org.opensearch.Version;
import org.opensearch.core.common.io.stream.NamedWriteable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import java.util.Objects;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java
index f0ab571ea9f75..faa3558420a5c 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java
@@ -34,8 +34,8 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
index 95a4b6573611d..11a7555d15d3f 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java
@@ -40,7 +40,7 @@
import org.opensearch.core.ParseField;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.XContentParser;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index ed598c14acec3..1603f95f4d512 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -36,7 +36,7 @@
import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder;
import org.opensearch.client.OpenSearchClient;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
/**
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java
index 4ddff1563885a..ca2921ceb70c8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -55,7 +55,7 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.inject.Inject;
import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.index.shard.DocsStats;
import org.opensearch.tasks.Task;
import org.opensearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java
index b2e7ed92e608a..aef2dc8f2c7c8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java
@@ -42,7 +42,7 @@
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.ParseField;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.ToXContentObject;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java
index eb05c0a69b78b..855e678c77b9b 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java
@@ -37,7 +37,7 @@
import org.opensearch.action.support.master.AcknowledgedRequestBuilder;
import org.opensearch.client.OpenSearchClient;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
/**
* Transport request builder for resizing an index
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java
index 0d31c90a98f56..328768bc9ae0e 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java
@@ -57,7 +57,7 @@
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.TransportService;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.index.store.StoreStats;
import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java
index 5a3a34e9a2ebe..e4abaef4ddfa8 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java
@@ -37,7 +37,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java
index 7fe663a347ee3..28215475416ba 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java
@@ -36,7 +36,7 @@
import org.opensearch.action.support.broadcast.BroadcastResponse;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java
index 4695b44c4986b..a01fc82fffd01 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java
@@ -41,8 +41,8 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.threadpool.Scheduler;
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
index 07d29413667cc..a3294e160659b 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
@@ -50,8 +50,8 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.search.fetch.subphase.FetchSourceContext;
import java.io.IOException;
@@ -236,30 +236,30 @@ public long estimatedSizeInBytes() {
/**
* Adds a framed data in binary format
*/
- public BulkRequest add(byte[] data, int from, int length, XContentType xContentType) throws IOException {
- return add(data, from, length, null, xContentType);
+ public BulkRequest add(byte[] data, int from, int length, MediaType mediaType) throws IOException {
+ return add(data, from, length, null, mediaType);
}
/**
* Adds a framed data in binary format
*/
- public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, XContentType xContentType) throws IOException {
- return add(new BytesArray(data, from, length), defaultIndex, xContentType);
+ public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, MediaType mediaType) throws IOException {
+ return add(new BytesArray(data, from, length), defaultIndex, mediaType);
}
/**
* Adds a framed data in binary format
*/
- public BulkRequest add(BytesReference data, @Nullable String defaultIndex, XContentType xContentType) throws IOException {
- return add(data, defaultIndex, null, null, null, null, true, xContentType);
+ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, MediaType mediaType) throws IOException {
+ return add(data, defaultIndex, null, null, null, null, true, mediaType);
}
/**
* Adds a framed data in binary format
*/
- public BulkRequest add(BytesReference data, @Nullable String defaultIndex, boolean allowExplicitIndex, XContentType xContentType)
+ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, boolean allowExplicitIndex, MediaType mediaType)
throws IOException {
- return add(data, defaultIndex, null, null, null, null, allowExplicitIndex, xContentType);
+ return add(data, defaultIndex, null, null, null, null, allowExplicitIndex, mediaType);
}
public BulkRequest add(
@@ -269,9 +269,9 @@ public BulkRequest add(
@Nullable FetchSourceContext defaultFetchSourceContext,
@Nullable String defaultPipeline,
boolean allowExplicitIndex,
- XContentType xContentType
+ MediaType mediaType
) throws IOException {
- return add(data, defaultIndex, defaultRouting, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, xContentType);
+ return add(data, defaultIndex, defaultRouting, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, mediaType);
}
public BulkRequest add(
@@ -282,7 +282,7 @@ public BulkRequest add(
@Nullable String defaultPipeline,
@Nullable Boolean defaultRequireAlias,
boolean allowExplicitIndex,
- XContentType xContentType
+ MediaType mediaType
) throws IOException {
String routing = valueOrDefault(defaultRouting, globalRouting);
String pipeline = valueOrDefault(defaultPipeline, globalPipeline);
@@ -295,7 +295,7 @@ public BulkRequest add(
pipeline,
requireAlias,
allowExplicitIndex,
- xContentType,
+ mediaType,
this::internalAdd,
this::internalAdd,
this::add
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java
index d2e0d558fcc1b..08eee82a53cf9 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java
@@ -46,6 +46,7 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
/**
* A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes
@@ -124,9 +125,8 @@ public BulkRequestBuilder add(byte[] data, int from, int length, XContentType xC
/**
* Adds a framed data in binary format
*/
- public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, XContentType xContentType)
- throws Exception {
- request.add(data, from, length, defaultIndex, xContentType);
+ public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, MediaType mediaType) throws Exception {
+ request.add(data, from, length, defaultIndex, mediaType);
return this;
}
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java
index 569aa8f9e4f34..3a86ccd578991 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java
@@ -42,6 +42,7 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.lucene.uid.Versions;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContent;
import org.opensearch.core.xcontent.XContentParser;
@@ -101,10 +102,10 @@ private static BytesReference sliceTrimmingCarriageReturn(
BytesReference bytesReference,
int from,
int nextMarker,
- XContentType xContentType
+ MediaType mediaType
) {
final int length;
- if (XContentType.JSON == xContentType && bytesReference.get(nextMarker - 1) == (byte) '\r') {
+ if (XContentType.JSON == mediaType && bytesReference.get(nextMarker - 1) == (byte) '\r') {
length = nextMarker - from - 1;
} else {
length = nextMarker - from;
@@ -125,12 +126,12 @@ public void parse(
@Nullable String defaultPipeline,
@Nullable Boolean defaultRequireAlias,
boolean allowExplicitIndex,
- XContentType xContentType,
+ MediaType mediaType,
Consumer indexRequestConsumer,
Consumer updateRequestConsumer,
Consumer deleteRequestConsumer
) throws IOException {
- XContent xContent = xContentType.xContent();
+ XContent xContent = mediaType.xContent();
int line = 0;
int from = 0;
byte marker = xContent.streamSeparator();
@@ -312,7 +313,7 @@ public void parse(
.setPipeline(pipeline)
.setIfSeqNo(ifSeqNo)
.setIfPrimaryTerm(ifPrimaryTerm)
- .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType)
+ .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType)
.setRequireAlias(requireAlias)
);
} else {
@@ -325,7 +326,7 @@ public void parse(
.setPipeline(pipeline)
.setIfSeqNo(ifSeqNo)
.setIfPrimaryTerm(ifPrimaryTerm)
- .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType)
+ .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType)
.setRequireAlias(requireAlias)
);
}
@@ -339,7 +340,7 @@ public void parse(
.setPipeline(pipeline)
.setIfSeqNo(ifSeqNo)
.setIfPrimaryTerm(ifPrimaryTerm)
- .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType)
+ .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType)
.setRequireAlias(requireAlias)
);
} else if ("update".equals(action)) {
@@ -358,7 +359,7 @@ public void parse(
.routing(routing);
try (
XContentParser sliceParser = createParser(
- sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType),
+ sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType),
xContent
)
) {
diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
index 0b675fd9125fa..140c1320daa91 100644
--- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
@@ -78,8 +78,8 @@
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.lease.Releasable;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.index.IndexingPressureService;
import org.opensearch.index.SegmentReplicationPressureService;
@@ -752,7 +752,7 @@ static BulkItemResponse processUpdateResponse(
if (updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) {
final BytesReference indexSourceAsBytes = updateIndexRequest.source();
- final Tuple> sourceAndContent = XContentHelper.convertToMap(
+ final Tuple extends MediaType, Map> sourceAndContent = XContentHelper.convertToMap(
indexSourceAsBytes,
true,
updateIndexRequest.getContentType()
diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequest.java b/server/src/main/java/org/opensearch/action/index/IndexRequest.java
index 86ba13dedfa06..ac4c8436aab5a 100644
--- a/server/src/main/java/org/opensearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/opensearch/action/index/IndexRequest.java
@@ -53,12 +53,12 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.lucene.uid.Versions;
-import org.opensearch.common.unit.ByteSizeValue;
-import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.VersionType;
import org.opensearch.index.mapper.MapperService;
@@ -78,10 +78,10 @@
* created using {@link org.opensearch.client.Requests#indexRequest(String)}.
*
* The index requires the {@link #index()}, {@link #id(String)} and
- * {@link #source(byte[], XContentType)} to be set.
+ * {@link #source(byte[], MediaType)} to be set.
*
- * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}),
- * its string form ({@link #source(String, XContentType)}) or using a {@link XContentBuilder}
+ * The source (content to index) can be set in its bytes form using ({@link #source(byte[], MediaType)}),
+ * its string form ({@link #source(String, MediaType)}) or using a {@link XContentBuilder}
* ({@link #source(XContentBuilder)}).
*
* If the {@link #id(String)} is not set, it will be automatically generated.
@@ -116,7 +116,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement
private long version = Versions.MATCH_ANY;
private VersionType versionType = VersionType.INTERNAL;
- private XContentType contentType;
+ private MediaType contentType;
private String pipeline;
private String finalPipeline;
@@ -159,7 +159,11 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio
isRetry = in.readBoolean();
autoGeneratedTimestamp = in.readLong();
if (in.readBoolean()) {
- contentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ contentType = in.readMediaType();
+ } else {
+ contentType = in.readEnum(XContentType.class);
+ }
} else {
contentType = null;
}
@@ -174,7 +178,7 @@ public IndexRequest() {
/**
* Constructs a new index request against the specific index. The
- * {@link #source(byte[], XContentType)} must be set.
+ * {@link #source(byte[], MediaType)} must be set.
*/
public IndexRequest(String index) {
super(NO_SHARD_ID);
@@ -249,7 +253,7 @@ public IndicesOptions indicesOptions() {
* The content type. This will be used when generating a document from user provided objects like Maps and when parsing the
* source at index time
*/
- public XContentType getContentType() {
+ public MediaType getContentType() {
return contentType;
}
@@ -372,9 +376,9 @@ public IndexRequest source(Map source) throws OpenSearchGenerationExc
*
* @param source The map to index
*/
- public IndexRequest source(Map source, XContentType contentType) throws OpenSearchGenerationException {
+ public IndexRequest source(Map source, MediaType contentType) throws OpenSearchGenerationException {
try {
- XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType);
builder.map(source);
return source(builder);
} catch (IOException e) {
@@ -386,10 +390,10 @@ public IndexRequest source(Map source, XContentType contentType) thro
* Sets the document source to index.
*
* Note, its preferable to either set it using {@link #source(XContentBuilder)}
- * or using the {@link #source(byte[], XContentType)}.
+ * or using the {@link #source(byte[], MediaType)}.
*/
- public IndexRequest source(String source, XContentType xContentType) {
- return source(new BytesArray(source), xContentType);
+ public IndexRequest source(String source, MediaType mediaType) {
+ return source(new BytesArray(source), mediaType);
}
/**
@@ -419,7 +423,7 @@ public IndexRequest source(Object... source) {
* valid String representation.
*
*/
- public IndexRequest source(XContentType xContentType, Object... source) {
+ public IndexRequest source(MediaType mediaType, Object... source) {
if (source.length % 2 != 0) {
throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]");
}
@@ -430,7 +434,7 @@ public IndexRequest source(XContentType xContentType, Object... source) {
);
}
try {
- XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(mediaType);
builder.startObject();
for (int i = 0; i < source.length; i++) {
builder.field(source[i++].toString(), source[i]);
@@ -447,27 +451,28 @@ public IndexRequest source(XContentType xContentType, Object... source) {
*/
public IndexRequest source(BytesReference source, MediaType mediaType) {
this.source = Objects.requireNonNull(source);
- this.contentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType));
+ this.contentType = Objects.requireNonNull(mediaType);
return this;
}
/**
* Sets the document to index in bytes form.
*/
- public IndexRequest source(byte[] source, XContentType xContentType) {
- return source(source, 0, source.length, xContentType);
+ public IndexRequest source(byte[] source, MediaType mediaType) {
+ return source(source, 0, source.length, mediaType);
}
/**
* Sets the document to index in bytes form (assumed to be safe to be used from different
* threads).
*
- * @param source The source to index
- * @param offset The offset in the byte array
- * @param length The length of the data
+ * @param source The source to index
+ * @param offset The offset in the byte array
+ * @param length The length of the data
+ * @param mediaType The data format over the wire
*/
- public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) {
- return source(new BytesArray(source, offset, length), xContentType);
+ public IndexRequest source(byte[] source, int offset, int length, MediaType mediaType) {
+ return source(new BytesArray(source, offset, length), mediaType);
}
/**
@@ -665,7 +670,11 @@ private void writeBody(StreamOutput out) throws IOException {
out.writeLong(autoGeneratedTimestamp);
if (contentType != null) {
out.writeBoolean(true);
- out.writeEnum(contentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ contentType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) contentType);
+ }
} else {
out.writeBoolean(false);
}
diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java
index a3067fe0ba160..7a88f817c70bf 100644
--- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java
@@ -32,12 +32,13 @@
package org.opensearch.action.ingest;
+import org.opensearch.Version;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.action.support.master.AcknowledgedRequest;
+import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -54,19 +55,7 @@ public class PutPipelineRequest extends AcknowledgedRequest
private String id;
private BytesReference source;
- private XContentType xContentType;
-
- /**
- * Create a new pipeline request with the id and source along with the content type of the source
- *
- * @deprecated use {@link #PutPipelineRequest(String, BytesReference, MediaType)} instead
- */
- @Deprecated
- public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) {
- this.id = Objects.requireNonNull(id);
- this.source = Objects.requireNonNull(source);
- this.xContentType = Objects.requireNonNull(xContentType);
- }
+ private MediaType mediaType;
/**
* Create a new pipeline request with the id and source along with the content type of the source
@@ -74,17 +63,18 @@ public PutPipelineRequest(String id, BytesReference source, XContentType xConten
public PutPipelineRequest(String id, BytesReference source, MediaType mediaType) {
this.id = Objects.requireNonNull(id);
this.source = Objects.requireNonNull(source);
- if (mediaType instanceof XContentType == false) {
- throw new IllegalArgumentException("PutPipelineRequest found unsupported media type [" + mediaType.getClass().getName() + "]");
- }
- this.xContentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType));
+ this.mediaType = Objects.requireNonNull(mediaType);
}
public PutPipelineRequest(StreamInput in) throws IOException {
super(in);
id = in.readString();
source = in.readBytesReference();
- xContentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType = in.readMediaType();
+ } else {
+ mediaType = in.readEnum(XContentType.class);
+ }
}
PutPipelineRequest() {}
@@ -102,8 +92,8 @@ public BytesReference getSource() {
return source;
}
- public XContentType getXContentType() {
- return xContentType;
+ public MediaType getMediaType() {
+ return mediaType;
}
@Override
@@ -111,13 +101,17 @@ public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(id);
out.writeBytesReference(source);
- out.writeEnum(xContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (source != null) {
- builder.rawValue(source.streamInput(), xContentType);
+ builder.rawValue(source.streamInput(), mediaType);
} else {
builder.startObject().endObject();
}
diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java
index 31f71e5720daa..1ac441a1afe64 100644
--- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java
@@ -32,15 +32,17 @@
package org.opensearch.action.ingest;
+import org.opensearch.Version;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.logging.DeprecationLogger;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.VersionType;
import org.opensearch.ingest.ConfigurationUtils;
import org.opensearch.ingest.IngestDocument;
@@ -66,14 +68,14 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent
private String id;
private boolean verbose;
private BytesReference source;
- private XContentType xContentType;
+ private MediaType mediaType;
/**
* Creates a new request with the given source and its content type
*/
- public SimulatePipelineRequest(BytesReference source, XContentType xContentType) {
+ public SimulatePipelineRequest(BytesReference source, MediaType mediaType) {
this.source = Objects.requireNonNull(source);
- this.xContentType = Objects.requireNonNull(xContentType);
+ this.mediaType = Objects.requireNonNull(mediaType);
}
SimulatePipelineRequest() {}
@@ -83,7 +85,11 @@ public SimulatePipelineRequest(BytesReference source, XContentType xContentType)
id = in.readOptionalString();
verbose = in.readBoolean();
source = in.readBytesReference();
- xContentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType = in.readMediaType();
+ } else {
+ mediaType = in.readEnum(XContentType.class);
+ }
}
@Override
@@ -111,8 +117,8 @@ public BytesReference getSource() {
return source;
}
- public XContentType getXContentType() {
- return xContentType;
+ public MediaType getXContentType() {
+ return mediaType;
}
@Override
@@ -121,12 +127,16 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(id);
out.writeBoolean(verbose);
out.writeBytesReference(source);
- out.writeEnum(xContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
- builder.rawValue(source.streamInput(), xContentType);
+ builder.rawValue(source.streamInput(), mediaType);
return builder;
}
diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java
index 822d1f2320c9b..d32aab0c8a561 100644
--- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java
@@ -8,6 +8,7 @@
package org.opensearch.action.search;
+import org.opensearch.Version;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.action.support.master.AcknowledgedRequest;
import org.opensearch.core.common.bytes.BytesReference;
@@ -29,7 +30,7 @@
public class PutSearchPipelineRequest extends AcknowledgedRequest implements ToXContentObject {
private String id;
private BytesReference source;
- private XContentType xContentType;
+ private MediaType mediaType;
public PutSearchPipelineRequest(String id, BytesReference source, MediaType mediaType) {
this.id = Objects.requireNonNull(id);
@@ -39,14 +40,18 @@ public PutSearchPipelineRequest(String id, BytesReference source, MediaType medi
PutSearchPipelineRequest.class.getSimpleName() + " found unsupported media type [" + mediaType.getClass().getName() + "]"
);
}
- this.xContentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType));
+ this.mediaType = Objects.requireNonNull(mediaType);
}
public PutSearchPipelineRequest(StreamInput in) throws IOException {
super(in);
id = in.readString();
source = in.readBytesReference();
- xContentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType = in.readMediaType();
+ } else {
+ mediaType = in.readEnum(XContentType.class);
+ }
}
@Override
@@ -62,8 +67,8 @@ public BytesReference getSource() {
return source;
}
- public XContentType getXContentType() {
- return xContentType;
+ public MediaType getMediaType() {
+ return mediaType;
}
@Override
@@ -71,13 +76,17 @@ public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(id);
out.writeBytesReference(source);
- out.writeEnum(xContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (source != null) {
- builder.rawValue(source.streamInput(), xContentType);
+ builder.rawValue(source.streamInput(), mediaType);
} else {
builder.startObject().endObject();
}
diff --git a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java
index 45c2dc4f29403..43c95133f12d6 100644
--- a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java
+++ b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java
@@ -35,8 +35,8 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.search.TopDocs;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.common.util.concurrent.AbstractRunnable;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java
index 512d3295c4cfc..2f002d21d9b68 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java
@@ -45,7 +45,7 @@
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.TotalHits.Relation;
import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.search.DocValueFormat;
diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
index c4af9ffa20194..1011f17c98dd6 100644
--- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
@@ -57,7 +57,7 @@
import org.opensearch.cluster.routing.ShardIterator;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Nullable;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.inject.Inject;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.Writeable;
@@ -70,7 +70,7 @@
import org.opensearch.core.index.Index;
import org.opensearch.index.query.Rewriteable;
import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.SearchPhaseResult;
import org.opensearch.search.SearchService;
import org.opensearch.search.SearchShardTarget;
diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java
index 1affc9202c32b..d3a617853b019 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java
@@ -47,7 +47,7 @@
import org.opensearch.cluster.routing.IndexShardRoutingTable;
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.common.Nullable;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java
index d33882fc1d53d..57cc4698cefce 100644
--- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java
+++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java
@@ -40,6 +40,7 @@
import org.opensearch.action.get.MultiGetRequest;
import org.opensearch.action.support.single.shard.SingleShardRequest;
import org.opensearch.common.Nullable;
+import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.ParseField;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
@@ -48,10 +49,9 @@
import org.opensearch.common.lucene.uid.Versions;
import org.opensearch.common.util.set.Sets;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.VersionType;
import org.opensearch.index.mapper.MapperService;
@@ -94,7 +94,7 @@ public class TermVectorsRequest extends SingleShardRequest i
private BytesReference doc;
- private XContentType xContentType;
+ private MediaType mediaType;
private String routing;
@@ -186,7 +186,11 @@ public TermVectorsRequest() {}
if (in.readBoolean()) {
doc = in.readBytesReference();
- xContentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType = in.readMediaType();
+ } else {
+ mediaType = in.readEnum(XContentType.class);
+ }
}
routing = in.readOptionalString();
preference = in.readOptionalString();
@@ -235,7 +239,7 @@ public TermVectorsRequest(TermVectorsRequest other) {
this.id = other.id();
if (other.doc != null) {
this.doc = new BytesArray(other.doc().toBytesRef(), true);
- this.xContentType = other.xContentType;
+ this.mediaType = other.mediaType;
}
this.flagsEnum = other.getFlags().clone();
this.preference = other.preference();
@@ -285,8 +289,8 @@ public BytesReference doc() {
return doc;
}
- public XContentType xContentType() {
- return xContentType;
+ public MediaType xContentType() {
+ return mediaType;
}
/**
@@ -302,19 +306,19 @@ public TermVectorsRequest doc(XContentBuilder documentBuilder) {
*/
@Deprecated
public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) {
- return this.doc(doc, generateRandomId, XContentHelper.xContentType(doc));
+ return this.doc(doc, generateRandomId, MediaTypeRegistry.xContentType(doc));
}
/**
* Sets an artificial document from which term vectors are requested for.
*/
- public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, MediaType xContentType) {
+ public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, MediaType mediaType) {
// assign a random id to this artificial document, for routing
if (generateRandomId) {
this.id(String.valueOf(randomInt.getAndAdd(1)));
}
this.doc = doc;
- this.xContentType = XContentType.fromMediaType(xContentType);
+ this.mediaType = mediaType;
return this;
}
@@ -534,7 +538,11 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(doc != null);
if (doc != null) {
out.writeBytesReference(doc);
- out.writeEnum(xContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
}
out.writeOptionalString(routing);
out.writeOptionalString(preference);
diff --git a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java
index d20d5e229a4c7..95433bc42c2d3 100644
--- a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java
+++ b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java
@@ -61,7 +61,7 @@
import org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.IndexService;
import org.opensearch.index.engine.VersionConflictEngineException;
@@ -238,7 +238,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener<
response.getResult()
);
if (request.fetchSource() != null && request.fetchSource().fetchSource()) {
- Tuple> sourceAndContent = XContentHelper.convertToMap(
+ Tuple extends MediaType, Map> sourceAndContent = XContentHelper.convertToMap(
upsertSourceBytes,
true,
upsertRequest.getContentType()
diff --git a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java
index e3aa4f1161ac7..a5568a838f21f 100644
--- a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java
+++ b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java
@@ -40,10 +40,11 @@
import org.opensearch.action.index.IndexRequest;
import org.opensearch.client.Requests;
import org.opensearch.common.Nullable;
-import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
@@ -352,7 +353,7 @@ public static GetResult extractGetResult(
long primaryTerm,
long version,
final Map source,
- XContentType sourceContentType,
+ MediaType sourceContentType,
@Nullable final BytesReference sourceAsBytes
) {
if (request.fetchSource() == null || request.fetchSource().fetchSource() == false) {
diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
index 8f529ec855d9b..86ebc0d9b69d6 100644
--- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
+++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java
@@ -48,11 +48,12 @@
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.ParseField;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -106,12 +107,12 @@ public class UpdateRequest extends InstanceShardOperationRequest
);
PARSER.declareBoolean(UpdateRequest::scriptedUpsert, SCRIPTED_UPSERT_FIELD);
PARSER.declareObject((request, builder) -> request.safeUpsertRequest().source(builder), (parser, context) -> {
- XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType());
builder.copyCurrentStructure(parser);
return builder;
}, UPSERT_FIELD);
PARSER.declareObject((request, builder) -> request.safeDoc().source(builder), (parser, context) -> {
- XContentBuilder docBuilder = XContentFactory.contentBuilder(parser.contentType());
+ XContentBuilder docBuilder = MediaTypeRegistry.contentBuilder(parser.contentType());
docBuilder.copyCurrentStructure(parser);
return docBuilder;
}, DOC_FIELD);
@@ -922,13 +923,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field("doc_as_upsert", docAsUpsert);
}
if (doc != null) {
- XContentType xContentType = doc.getContentType();
+ MediaType mediaType = doc.getContentType();
try (
XContentParser parser = XContentHelper.createParser(
NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE,
doc.source(),
- xContentType
+ mediaType
)
) {
builder.field("doc");
@@ -945,13 +946,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field("script", script);
}
if (upsertRequest != null) {
- XContentType xContentType = upsertRequest.getContentType();
+ MediaType mediaType = upsertRequest.getContentType();
try (
XContentParser parser = XContentHelper.createParser(
NamedXContentRegistry.EMPTY,
LoggingDeprecationHandler.INSTANCE,
upsertRequest.source(),
- xContentType
+ mediaType
)
) {
builder.field("upsert");
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java
index 3793b5094a4cb..7b1a2f0f12b69 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java
@@ -34,7 +34,7 @@
import org.opensearch.Version;
import org.opensearch.cluster.routing.ShardRouting;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
diff --git a/server/src/main/java/org/opensearch/cluster/DiskUsage.java b/server/src/main/java/org/opensearch/cluster/DiskUsage.java
index 961bfce053243..c472522baee51 100644
--- a/server/src/main/java/org/opensearch/cluster/DiskUsage.java
+++ b/server/src/main/java/org/opensearch/cluster/DiskUsage.java
@@ -32,11 +32,11 @@
package org.opensearch.cluster;
-import org.opensearch.common.Strings;
+import org.opensearch.core.common.Strings;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
index 0014d5c61fb2d..b5a9e4e7b30a0 100644
--- a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java
@@ -44,7 +44,7 @@
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.service.ClusterApplier;
import org.opensearch.common.Nullable;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
index de751d881bc0e..1c38e68c43466 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java
@@ -62,7 +62,7 @@
import org.opensearch.common.Priority;
import org.opensearch.common.SetOnce;
import org.opensearch.common.Strings;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java
index 44f6f2d7313a3..27ecca0358bd8 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java
@@ -45,6 +45,7 @@
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -307,7 +308,7 @@ public Builder filter(String filter) {
this.filter = null;
return this;
}
- return filter(XContentHelper.convertToMap(XContentFactory.xContent(filter), filter, true));
+ return filter(XContentHelper.convertToMap(MediaTypeRegistry.xContent(filter).xContent(), filter, true));
}
public Builder filter(Map filter) {
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java
index 5a019804f5eac..9b9b91a07a5cf 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java
@@ -36,10 +36,10 @@
import org.opensearch.common.Nullable;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryShardContext;
@@ -88,7 +88,7 @@ public void validateAliasStandalone(Alias alias) {
validateAliasStandalone(alias.name(), alias.indexRouting());
if (Strings.hasLength(alias.filter())) {
try {
- XContentHelper.convertToMap(XContentFactory.xContent(alias.filter()), alias.filter(), false);
+ XContentHelper.convertToMap(MediaTypeRegistry.xContent(alias.filter()).xContent(), alias.filter(), false);
} catch (Exception e) {
throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e);
}
@@ -134,7 +134,8 @@ public void validateAliasFilter(
) {
assert queryShardContext != null;
try (
- XContentParser parser = XContentFactory.xContent(filter)
+ XContentParser parser = MediaTypeRegistry.xContent(filter)
+ .xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter)
) {
validateAliasFilter(parser, queryShardContext);
@@ -158,7 +159,7 @@ public void validateAliasFilter(
try (
InputStream inputStream = filter.streamInput();
- XContentParser parser = XContentFactory.xContentType(inputStream)
+ XContentParser parser = MediaTypeRegistry.xContentType(inputStream)
.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter.streamInput())
) {
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
index 1ba38daa40566..bcf9cbd0efef8 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -53,9 +53,9 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -1296,7 +1296,7 @@ public Builder putMapping(String source) throws IOException {
putMapping(
new MappingMetadata(
MapperService.SINGLE_MAPPING_NAME,
- XContentHelper.convertToMap(XContentFactory.xContent(source), source, true)
+ XContentHelper.convertToMap(MediaTypeRegistry.xContent(source).xContent(), source, true)
)
);
return this;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java b/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java
index 844b78dccc59b..74a3342addc50 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java
@@ -42,7 +42,7 @@
import org.opensearch.cluster.routing.allocation.AllocationService;
import org.opensearch.cluster.routing.allocation.RoutingAllocation;
import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
index af348c1c98f2d..781ca5bb2255a 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
@@ -450,7 +450,11 @@ public Builder initializeAsRestore(IndexMetadata indexMetadata, SnapshotRecovery
/**
* Initializes an existing index, to be restored from remote store
*/
- public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, RemoteStoreRecoverySource recoverySource) {
+ public Builder initializeAsRemoteStoreRestore(
+ IndexMetadata indexMetadata,
+ RemoteStoreRecoverySource recoverySource,
+ Map activeInitializingShards
+ ) {
final UnassignedInfo unassignedInfo = new UnassignedInfo(
UnassignedInfo.Reason.EXISTING_INDEX_RESTORED,
"restore_source[remote_store]"
@@ -462,7 +466,11 @@ public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, Remot
for (int shardNumber = 0; shardNumber < indexMetadata.getNumberOfShards(); shardNumber++) {
ShardId shardId = new ShardId(index, shardNumber);
IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
- indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, true, recoverySource, unassignedInfo));
+ if (activeInitializingShards.containsKey(shardId)) {
+ indexShardRoutingBuilder.addShard(activeInitializingShards.get(shardId));
+ } else {
+ indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, true, recoverySource, unassignedInfo));
+ }
shards.put(shardNumber, indexShardRoutingBuilder.build());
}
return this;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java
index 1bee5d8176a0f..d6a67bc714689 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java
@@ -295,6 +295,16 @@ public ShardsIterator allShardsIncludingRelocationTargets(String[] indices) {
return allShardsSatisfyingPredicate(indices, shardRouting -> true, true);
}
+ /**
+ * All the shards on the node which match the predicate
+ * @param predicate condition to match
+ * @return iterator over shards matching the predicate
+ */
+ public ShardsIterator allShardsSatisfyingPredicate(Predicate predicate) {
+ String[] indices = indicesRouting.keySet().toArray(new String[0]);
+ return allShardsSatisfyingPredicate(indices, predicate, false);
+ }
+
private ShardsIterator allShardsSatisfyingPredicate(
String[] indices,
Predicate predicate,
@@ -562,9 +572,13 @@ public Builder addAsFromOpenToClose(IndexMetadata indexMetadata) {
return add(indexRoutingBuilder);
}
- public Builder addAsRemoteStoreRestore(IndexMetadata indexMetadata, RemoteStoreRecoverySource recoverySource) {
+ public Builder addAsRemoteStoreRestore(
+ IndexMetadata indexMetadata,
+ RemoteStoreRecoverySource recoverySource,
+ Map activeInitializingShards
+ ) {
IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex())
- .initializeAsRemoteStoreRestore(indexMetadata, recoverySource);
+ .initializeAsRemoteStoreRestore(indexMetadata, recoverySource, activeInitializingShards);
add(indexRoutingBuilder);
return this;
}
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java
index d54236ada6780..542aa9ca5becf 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java
@@ -34,13 +34,13 @@
import org.opensearch.OpenSearchParseException;
import org.opensearch.Version;
-import org.opensearch.common.Strings;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.RatioValue;
import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.Strings;
import java.util.Arrays;
import java.util.Iterator;
@@ -400,13 +400,13 @@ public boolean isCreateIndexBlockAutoReleaseEnabled() {
String describeLowThreshold() {
return freeBytesThresholdLow.equals(ByteSizeValue.ZERO)
- ? Strings.format1Decimals(100.0 - freeDiskThresholdLow, "%")
+ ? org.opensearch.core.common.Strings.format1Decimals(100.0 - freeDiskThresholdLow, "%")
: freeBytesThresholdLow.toString();
}
String describeHighThreshold() {
return freeBytesThresholdHigh.equals(ByteSizeValue.ZERO)
- ? Strings.format1Decimals(100.0 - freeDiskThresholdHigh, "%")
+ ? org.opensearch.core.common.Strings.format1Decimals(100.0 - freeDiskThresholdHigh, "%")
: freeBytesThresholdHigh.toString();
}
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java
index 946e2d5e5f3dc..ad568fc5ffa57 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java
@@ -39,7 +39,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index 4b69c05807ae4..3325b3475fd73 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -47,11 +47,11 @@
import org.opensearch.cluster.routing.ShardRoutingState;
import org.opensearch.cluster.routing.allocation.DiskThresholdSettings;
import org.opensearch.cluster.routing.allocation.RoutingAllocation;
-import org.opensearch.common.Strings;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.common.Strings;
import org.opensearch.core.index.Index;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.index.store.remote.filecache.FileCacheStats;
@@ -68,7 +68,7 @@
import static org.opensearch.cluster.routing.RoutingPool.getShardPool;
import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING;
import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING;
-import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO;
+import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING;
/**
* The {@link DiskThresholdDecider} checks that the node a shard is potentially
@@ -199,8 +199,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
final FileCacheStats fileCacheStats = clusterInfo.getNodeFileCacheStats().getOrDefault(node.nodeId(), null);
final long nodeCacheSize = fileCacheStats != null ? fileCacheStats.getTotal().getBytes() : 0;
final long totalNodeRemoteShardSize = currentNodeRemoteShardSize + shardSize;
-
- if (totalNodeRemoteShardSize > DATA_TO_FILE_CACHE_SIZE_RATIO * nodeCacheSize) {
+ final double dataToFileCacheSizeRatio = DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.get(allocation.metadata().settings());
+ if (dataToFileCacheSizeRatio > 0.0f && totalNodeRemoteShardSize > dataToFileCacheSizeRatio * nodeCacheSize) {
return allocation.decision(
Decision.NO,
NAME,
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
index 7f1c9f01f7e6f..c03f5ae619edf 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
@@ -51,7 +51,7 @@
import org.opensearch.common.Priority;
import org.opensearch.common.StopWatch;
import org.opensearch.common.StopWatch.TimingHandle;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
index a605c41bdeff8..e097803d86b48 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
@@ -45,7 +45,7 @@
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.routing.OperationRouting;
import org.opensearch.cluster.routing.RerouteService;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
index 790efaef95292..9f76460d2acdd 100644
--- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
@@ -55,7 +55,7 @@
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.common.Nullable;
import org.opensearch.common.Priority;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java
index 1f8a6aba0c883..86a2fe1397cec 100644
--- a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java
+++ b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/common/Strings.java b/server/src/main/java/org/opensearch/common/Strings.java
index 8e92c86836723..e9f4d32ed2664 100644
--- a/server/src/main/java/org/opensearch/common/Strings.java
+++ b/server/src/main/java/org/opensearch/common/Strings.java
@@ -155,29 +155,6 @@ public static String[] split(String toSplit, String delimiter) {
return new String[] { beforeDelimiter, afterDelimiter };
}
- /**
- * Format the double value with a single decimal points, trimming trailing '.0'.
- */
- public static String format1Decimals(double value, String suffix) {
- String p = String.valueOf(value);
- int ix = p.indexOf('.') + 1;
- int ex = p.indexOf('E');
- char fraction = p.charAt(ix);
- if (fraction == '0') {
- if (ex != -1) {
- return p.substring(0, ix - 1) + p.substring(ex) + suffix;
- } else {
- return p.substring(0, ix - 1) + suffix;
- }
- } else {
- if (ex != -1) {
- return p.substring(0, ix) + fraction + p.substring(ex) + suffix;
- } else {
- return p.substring(0, ix) + fraction + suffix;
- }
- }
- }
-
private Strings() {}
public static byte[] toUTF8Bytes(CharSequence charSequence) {
diff --git a/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java b/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java
index 923f592c6bc79..de4e6ad433c55 100644
--- a/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java
+++ b/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java
@@ -34,7 +34,9 @@
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.indices.breaker.BreakerSettings;
import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
diff --git a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java
index 62ec933fe5f37..ee2d5a650fc70 100644
--- a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java
+++ b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java
@@ -34,8 +34,8 @@
import org.opensearch.common.Nullable;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.common.compress.NotXContentException;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import java.io.IOException;
import java.util.Objects;
@@ -67,14 +67,14 @@ public static Compressor compressor(BytesReference bytes) {
// bytes should be either detected as compressed or as xcontent,
// if we have bytes that can be either detected as compressed or
// as a xcontent, we have a problem
- assert XContentHelper.xContentType(bytes) == null;
+ assert MediaTypeRegistry.xContentType(bytes) == null;
return DEFLATE_COMPRESSOR;
} else if (ZSTD_COMPRESSOR.isCompressed(bytes)) {
- assert XContentHelper.xContentType(bytes) == null;
+ assert MediaTypeRegistry.xContentType(bytes) == null;
return ZSTD_COMPRESSOR;
}
- if (XContentHelper.xContentType(bytes) == null) {
+ if (MediaTypeRegistry.xContentType(bytes) == null) {
throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes");
}
diff --git a/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java b/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java
index 80b5dd353703c..e853a6ddc34d2 100644
--- a/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java
+++ b/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java
@@ -32,7 +32,7 @@
package org.opensearch.common.io;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.threadpool.ThreadPool;
import java.nio.ByteBuffer;
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java
index b4440b85e0037..ef07f6ea8052c 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java
@@ -36,12 +36,12 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.similarities.ClassicSimilarity;
@@ -144,12 +144,12 @@ public boolean equals(Object obj) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = super.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = super.rewrite(searcher);
if (rewritten != this) {
return rewritten;
}
- XMoreLikeThis mlt = new XMoreLikeThis(reader, similarity == null ? new ClassicSimilarity() : similarity);
+ XMoreLikeThis mlt = new XMoreLikeThis(searcher.getIndexReader(), similarity == null ? new ClassicSimilarity() : similarity);
mlt.setFieldNames(moreLikeFields);
mlt.setAnalyzer(analyzer);
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
index 98ebb34fc040e..cc0468efb243e 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -39,6 +39,7 @@
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.Query;
@@ -159,8 +160,8 @@ public int[] getPositions() {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = super.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = super.rewrite(searcher);
if (rewritten != this) {
return rewritten;
}
@@ -177,7 +178,7 @@ public Query rewrite(IndexReader reader) throws IOException {
int position = positions.get(sizeMinus1);
Set terms = new HashSet<>();
for (Term term : suffixTerms) {
- getPrefixTerms(terms, term, reader);
+ getPrefixTerms(terms, term, searcher.getIndexReader());
if (terms.size() > maxExpansions) {
break;
}
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java b/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java
index d7ffa2df943b7..49148890abd55 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java
@@ -56,7 +56,9 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
@@ -808,8 +810,10 @@ public String describeParams() {
*/
private PriorityQueue retrieveTerms(int docNum) throws IOException {
Map termFreqMap = new HashMap<>();
+ final TermVectors termVectors = ir.termVectors();
+ final StoredFields storedFields = ir.storedFields();
for (String fieldName : fieldNames) {
- final Fields vectors = ir.getTermVectors(docNum);
+ final Fields vectors = termVectors.get(docNum);
final Terms vector;
if (vectors != null) {
vector = vectors.terms(fieldName);
@@ -819,7 +823,7 @@ private PriorityQueue retrieveTerms(int docNum) throws IOException {
// field does not store term vector info
if (vector == null) {
- Document d = ir.document(docNum);
+ Document d = storedFields.document(docNum);
IndexableField fields[] = d.getFields(fieldName);
for (IndexableField field : fields) {
final String stringValue = field.stringValue();
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
index ebd155b8a5679..db92d4f1cba96 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -128,7 +128,7 @@ protected int doHashCode() {
@Override
protected ScoreFunction rewrite(IndexReader reader) throws IOException {
- Query newFilter = filter.rewrite(reader);
+ Query newFilter = filter.rewrite(new IndexSearcher(reader));
if (newFilter == filter) {
return this;
}
@@ -322,16 +322,16 @@ public void visit(QueryVisitor visitor) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query rewritten = super.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query rewritten = super.rewrite(searcher);
if (rewritten != this) {
return rewritten;
}
- Query newQ = subQuery.rewrite(reader);
+ Query newQ = subQuery.rewrite(searcher);
ScoreFunction[] newFunctions = new ScoreFunction[functions.length];
boolean needsRewrite = (newQ != subQuery);
for (int i = 0; i < functions.length; i++) {
- newFunctions[i] = functions[i].rewrite(reader);
+ newFunctions[i] = functions[i].rewrite(searcher.getIndexReader());
needsRewrite |= (newFunctions[i] != functions[i]);
}
if (needsRewrite) {
diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
index 8bf5fc0f89d31..07084a6f690e4 100644
--- a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
+++ b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java
@@ -32,7 +32,6 @@
package org.opensearch.common.lucene.search.function;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.DocIdSetIterator;
@@ -105,12 +104,12 @@ public ScriptScoreQuery(
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query newQ = subQuery.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query newQ = subQuery.rewrite(searcher);
if (newQ != subQuery) {
return new ScriptScoreQuery(newQ, queryName, script, scriptBuilder, minScore, indexName, shardId, indexVersion);
}
- return super.rewrite(reader);
+ return super.rewrite(searcher);
}
@Override
diff --git a/server/src/main/java/org/opensearch/common/network/NetworkModule.java b/server/src/main/java/org/opensearch/common/network/NetworkModule.java
index d93f8d7c98b32..c052af41d66d9 100644
--- a/server/src/main/java/org/opensearch/common/network/NetworkModule.java
+++ b/server/src/main/java/org/opensearch/common/network/NetworkModule.java
@@ -53,7 +53,7 @@
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.http.HttpServerTransport;
import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.plugins.NetworkPlugin;
import org.opensearch.tasks.RawTaskStatus;
import org.opensearch.tasks.Task;
diff --git a/server/src/main/java/org/opensearch/common/network/NetworkService.java b/server/src/main/java/org/opensearch/common/network/NetworkService.java
index 0fb299ef66e70..1a893283eae21 100644
--- a/server/src/main/java/org/opensearch/common/network/NetworkService.java
+++ b/server/src/main/java/org/opensearch/common/network/NetworkService.java
@@ -34,7 +34,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 46a43842451d9..9da9e1b14d307 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -44,6 +44,7 @@
import org.opensearch.index.ShardIndexingPressureMemoryManager;
import org.opensearch.index.ShardIndexingPressureSettings;
import org.opensearch.index.ShardIndexingPressureStore;
+import org.opensearch.index.store.remote.filecache.FileCache;
import org.opensearch.search.backpressure.settings.NodeDuressSettings;
import org.opensearch.search.backpressure.settings.SearchBackpressureSettings;
import org.opensearch.search.backpressure.settings.SearchShardTaskSettings;
@@ -643,6 +644,7 @@ public void apply(Settings value, Settings current, Settings previous) {
// Settings related to Searchable Snapshots
Node.NODE_SEARCH_CACHE_SIZE_SETTING,
+ FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING,
// Settings related to Remote Refresh Segment Pressure
RemoteRefreshSegmentPressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED,
diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java
index 3bf2988e88e5a..ae390f1cc60ff 100644
--- a/server/src/main/java/org/opensearch/common/settings/Setting.java
+++ b/server/src/main/java/org/opensearch/common/settings/Setting.java
@@ -40,20 +40,21 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.Strings;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.common.regex.Regex;
-import org.opensearch.common.unit.ByteSizeValue;
import org.opensearch.common.unit.MemorySizeValue;
import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import java.io.IOException;
import java.util.ArrayList;
@@ -2047,7 +2048,7 @@ static boolean parseBoolean(String b, String key, boolean isFiltered) {
}
public static Setting byteSizeSetting(String key, ByteSizeValue value, Property... properties) {
- return byteSizeSetting(key, (s) -> value.toString(), properties);
+ return byteSizeSetting(key, (s) -> value.getBytes() + ByteSizeUnit.BYTES.getSuffix(), properties);
}
public static Setting byteSizeSetting(String key, Setting fallbackSetting, Property... properties) {
diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java
index a7649a1cd22c5..cd80e9727e0df 100644
--- a/server/src/main/java/org/opensearch/common/settings/Settings.java
+++ b/server/src/main/java/org/opensearch/common/settings/Settings.java
@@ -43,12 +43,12 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.logging.DeprecationLogger;
import org.opensearch.common.logging.LogConfigurator;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.MemorySizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
-import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.settings.SecureString;
@@ -88,7 +88,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue;
+import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
import static org.opensearch.common.unit.TimeValue.parseTimeValue;
/**
@@ -1081,7 +1081,7 @@ private void processLegacyLists(Map map) {
*/
public Builder loadFromMap(Map map) {
// TODO: do this without a serialization round-trip
- try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) {
builder.map(map);
return loadFromSource(Strings.toString(builder), builder.contentType());
} catch (IOException e) {
@@ -1094,7 +1094,7 @@ public Builder loadFromMap(Map map) {
*/
public Builder loadFromSource(String source, MediaType xContentType) {
try (
- XContentParser parser = XContentFactory.xContent(xContentType)
+ XContentParser parser = xContentType.xContent()
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, source)
) {
this.put(fromXContent(parser, true, true));
@@ -1127,7 +1127,7 @@ public Builder loadFromStream(String resourceName, InputStream is, boolean accep
}
// fromXContent doesn't use named xcontent or deprecation.
try (
- XContentParser parser = XContentFactory.xContent(xContentType)
+ XContentParser parser = xContentType.xContent()
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, is)
) {
if (parser.currentToken() == null) {
diff --git a/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java b/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java
index 8664b14119694..e197e88b3012e 100644
--- a/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java
+++ b/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java
@@ -23,7 +23,7 @@
import org.opensearch.common.settings.Setting.MinTimeValueParser;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Setting.RegexValidator;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import java.io.IOException;
import java.util.Arrays;
diff --git a/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java b/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java
index 18aae6277c379..96f128cb6dc38 100644
--- a/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java
+++ b/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java
@@ -33,11 +33,13 @@
package org.opensearch.common.unit;
import org.opensearch.OpenSearchParseException;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.monitor.jvm.JvmInfo;
import java.util.Objects;
-import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue;
+import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
/**
* Utility methods to get memory sizes.
diff --git a/server/src/main/java/org/opensearch/common/unit/SizeValue.java b/server/src/main/java/org/opensearch/common/unit/SizeValue.java
index 3b73955f8d046..766199ebbc8f8 100644
--- a/server/src/main/java/org/opensearch/common/unit/SizeValue.java
+++ b/server/src/main/java/org/opensearch/common/unit/SizeValue.java
@@ -33,7 +33,7 @@
package org.opensearch.common.unit;
import org.opensearch.OpenSearchParseException;
-import org.opensearch.common.Strings;
+import org.opensearch.core.common.Strings;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java
index 45c2092601a7b..3b5a057872bbc 100644
--- a/server/src/main/java/org/opensearch/common/util/BigArrays.java
+++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java
@@ -36,14 +36,14 @@
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.opensearch.common.Nullable;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
import org.opensearch.common.recycler.Recycler;
import org.opensearch.common.lease.Releasable;
import org.opensearch.common.lease.Releasables;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.core.common.util.BigArray;
import org.opensearch.core.common.util.ByteArray;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import java.util.Arrays;
diff --git a/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java b/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java
index f8c690471fb61..65be2a082f084 100644
--- a/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java
+++ b/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java
@@ -39,7 +39,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
import java.util.Arrays;
diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java b/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java
index b55280d43a473..12fe437b390bf 100644
--- a/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java
+++ b/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java
@@ -32,7 +32,7 @@
package org.opensearch.common.util.concurrent;
import org.apache.logging.log4j.Logger;
-import org.opensearch.common.component.Lifecycle;
+import org.opensearch.common.lifecycle.Lifecycle;
import java.util.Objects;
diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java
index cfe8008a4adca..a0903cbdd9f61 100644
--- a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java
+++ b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java
@@ -41,6 +41,7 @@
import org.opensearch.common.compress.CompressorFactory;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContent.Params;
@@ -52,7 +53,6 @@
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -86,14 +86,14 @@ public static XContentParser createParser(
if (compressedInput.markSupported() == false) {
compressedInput = new BufferedInputStream(compressedInput);
}
- final XContentType contentType = XContentFactory.xContentType(compressedInput);
- return XContentFactory.xContent(contentType).createParser(xContentRegistry, deprecationHandler, compressedInput);
+ final MediaType contentType = MediaTypeRegistry.xContentType(compressedInput);
+ return contentType.xContent().createParser(xContentRegistry, deprecationHandler, compressedInput);
} catch (Exception e) {
if (compressedInput != null) compressedInput.close();
throw e;
}
} else {
- return XContentFactory.xContent(xContentType(bytes)).createParser(xContentRegistry, deprecationHandler, bytes.streamInput());
+ return MediaTypeRegistry.xContentType(bytes).xContent().createParser(xContentRegistry, deprecationHandler, bytes.streamInput());
}
}
@@ -115,7 +115,7 @@ public static XContentParser createParser(
if (compressedInput.markSupported() == false) {
compressedInput = new BufferedInputStream(compressedInput);
}
- return XContentFactory.xContent(mediaType).createParser(xContentRegistry, deprecationHandler, compressedInput);
+ return mediaType.xContent().createParser(xContentRegistry, deprecationHandler, compressedInput);
} catch (Exception e) {
if (compressedInput != null) compressedInput.close();
throw e;
@@ -179,20 +179,14 @@ public static Tuple extends MediaType, Map> convertToMap(
final byte[] raw = arr.array();
final int offset = arr.offset();
final int length = arr.length();
- contentType = xContentType != null ? xContentType : XContentFactory.xContentType(raw, offset, length);
- return new Tuple<>(
- Objects.requireNonNull(contentType),
- convertToMap(XContentFactory.xContent(contentType), raw, offset, length, ordered)
- );
+ contentType = xContentType != null ? xContentType : MediaTypeRegistry.mediaTypeFromBytes(raw, offset, length);
+ return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(contentType.xContent(), raw, offset, length, ordered));
} else {
input = bytes.streamInput();
}
try (InputStream stream = input) {
- contentType = xContentType != null ? xContentType : XContentFactory.xContentType(stream);
- return new Tuple<>(
- Objects.requireNonNull(contentType),
- convertToMap(XContentFactory.xContent(contentType), stream, ordered)
- );
+ contentType = xContentType != null ? xContentType : MediaTypeRegistry.xContentType(stream);
+ return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(contentType.xContent(), stream, ordered));
}
} catch (IOException e) {
throw new OpenSearchParseException("Failed to parse content to map", e);
@@ -266,7 +260,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson) t
@Deprecated
public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException {
- return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes.toBytesRef().bytes));
+ return convertToJson(bytes, reformatJson, prettyPrint, MediaTypeRegistry.xContent(bytes.toBytesRef().bytes));
}
public static String convertToJson(BytesReference bytes, boolean reformatJson, MediaType xContentType) throws IOException {
@@ -311,7 +305,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson, b
if (bytes instanceof BytesArray) {
final BytesArray array = (BytesArray) bytes;
try (
- XContentParser parser = XContentFactory.xContent(mediaType)
+ XContentParser parser = mediaType.xContent()
.createParser(
NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
@@ -325,7 +319,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson, b
} else {
try (
InputStream stream = bytes.streamInput();
- XContentParser parser = XContentFactory.xContent(mediaType)
+ XContentParser parser = mediaType.xContent()
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)
) {
return toJsonString(prettyPrint, parser);
@@ -497,6 +491,7 @@ public static void writeRawField(String field, BytesReference source, XContentTy
* {@link XContentType}. Wraps the output into a new anonymous object according to the value returned
* by the {@link ToXContent#isFragment()} method returns.
*/
+ @Deprecated
public static BytesReference toXContent(ToXContent toXContent, XContentType xContentType, boolean humanReadable) throws IOException {
return toXContent(toXContent, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
}
@@ -544,29 +539,6 @@ public static BytesReference toXContent(ToXContent toXContent, MediaType mediaTy
}
}
- /**
- * Guesses the content type based on the provided bytes.
- *
- * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type.
- * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed.
- * This method is deprecated to prevent usages of it from spreading further without specific reasons.
- */
- @Deprecated
- public static XContentType xContentType(BytesReference bytes) {
- if (bytes instanceof BytesArray) {
- final BytesArray array = (BytesArray) bytes;
- return XContentFactory.xContentType(array.array(), array.offset(), array.length());
- }
- try {
- final InputStream inputStream = bytes.streamInput();
- assert inputStream.markSupported();
- return XContentFactory.xContentType(inputStream);
- } catch (IOException e) {
- assert false : "Should not happen, we're just reading bytes from memory";
- throw new UncheckedIOException(e);
- }
- }
-
/**
* Returns the contents of an object as an unparsed BytesReference
*
diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java b/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java
index 924db8bdea1dd..ab295b12302e1 100644
--- a/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java
+++ b/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java
@@ -35,7 +35,7 @@
import org.apache.lucene.util.BytesRef;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.time.DateFormatter;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentBuilderExtension;
diff --git a/server/src/main/java/org/opensearch/discovery/Discovery.java b/server/src/main/java/org/opensearch/discovery/Discovery.java
index 25b3cb6c2b90d..9d6807b6522c9 100644
--- a/server/src/main/java/org/opensearch/discovery/Discovery.java
+++ b/server/src/main/java/org/opensearch/discovery/Discovery.java
@@ -33,7 +33,7 @@
package org.opensearch.discovery;
import org.opensearch.cluster.coordination.ClusterStatePublisher;
-import org.opensearch.common.component.LifecycleComponent;
+import org.opensearch.common.lifecycle.LifecycleComponent;
/**
* A pluggable module allowing to implement discovery of other nodes, publishing of the cluster
diff --git a/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java b/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java
index 14805648c6771..cef7853011b82 100644
--- a/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java
+++ b/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java
@@ -35,7 +35,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.common.SetOnce;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.transport.TransportAddress;
diff --git a/server/src/main/java/org/opensearch/env/NodeEnvironment.java b/server/src/main/java/org/opensearch/env/NodeEnvironment.java
index f7d1f6e4343cc..59c5b43a7a904 100644
--- a/server/src/main/java/org/opensearch/env/NodeEnvironment.java
+++ b/server/src/main/java/org/opensearch/env/NodeEnvironment.java
@@ -57,7 +57,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.core.util.FileSystemUtils;
diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java
index 0992345735176..e6df6e964a31b 100644
--- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java
+++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java
@@ -9,13 +9,15 @@
package org.opensearch.extensions.rest;
import org.opensearch.OpenSearchParseException;
+import org.opensearch.Version;
+import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.RestRequest.Method;
import org.opensearch.transport.TransportRequest;
@@ -42,7 +44,7 @@ public class ExtensionRestRequest extends TransportRequest {
private String path;
private Map params;
private Map> headers;
- private XContentType xContentType = null;
+ private MediaType mediaType = null;
private BytesReference content;
// The owner of this request object
// Will be replaced with PrincipalIdentifierToken class from feature/identity
@@ -56,15 +58,15 @@ public class ExtensionRestRequest extends TransportRequest {
/**
* This object can be instantiated given method, uri, params, content and identifier
*
- * @param method of type {@link Method}
- * @param uri the REST uri string (excluding the query)
- * @param path the REST path
- * @param params the REST params
- * @param headers the REST headers
- * @param xContentType the content type, or null for plain text or no content
- * @param content the REST request content
+ * @param method of type {@link Method}
+ * @param uri the REST uri string (excluding the query)
+ * @param path the REST path
+ * @param params the REST params
+ * @param headers the REST headers
+ * @param mediaType the content type, or null for plain text or no content
+ * @param content the REST request content
* @param principalIdentifier the owner of this request
- * @param httpVersion the REST HTTP protocol version
+ * @param httpVersion the REST HTTP protocol version
*/
public ExtensionRestRequest(
Method method,
@@ -72,7 +74,7 @@ public ExtensionRestRequest(
String path,
Map params,
Map> headers,
- XContentType xContentType,
+ MediaType mediaType,
BytesReference content,
String principalIdentifier,
HttpRequest.HttpVersion httpVersion
@@ -82,7 +84,7 @@ public ExtensionRestRequest(
this.path = path;
this.params = params;
this.headers = headers;
- this.xContentType = xContentType;
+ this.mediaType = mediaType;
this.content = content;
this.principalIdentifierToken = principalIdentifier;
this.httpVersion = httpVersion;
@@ -102,7 +104,11 @@ public ExtensionRestRequest(StreamInput in) throws IOException {
params = in.readMap(StreamInput::readString, StreamInput::readString);
headers = in.readMap(StreamInput::readString, StreamInput::readStringList);
if (in.readBoolean()) {
- xContentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType = in.readMediaType();
+ } else {
+ mediaType = in.readEnum(XContentType.class);
+ }
}
content = in.readBytesReference();
principalIdentifierToken = in.readString();
@@ -117,9 +123,13 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeString(path);
out.writeMap(params, StreamOutput::writeString, StreamOutput::writeString);
out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeStringCollection);
- out.writeBoolean(xContentType != null);
- if (xContentType != null) {
- out.writeEnum(xContentType);
+ out.writeBoolean(mediaType != null);
+ if (mediaType != null) {
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
}
out.writeBytesReference(content);
out.writeString(principalIdentifierToken);
@@ -237,8 +247,8 @@ public Map> headers() {
*
* @return the content type of the {@link #content()}, or null if the context is plain text or if there is no content.
*/
- public XContentType getXContentType() {
- return xContentType;
+ public MediaType getXContentType() {
+ return mediaType;
}
/**
@@ -311,7 +321,7 @@ public String toString() {
+ ", headers="
+ headers.toString()
+ ", xContentType="
- + xContentType
+ + mediaType
+ ", contentLength="
+ content.length()
+ ", requester="
@@ -331,7 +341,7 @@ public boolean equals(Object obj) {
&& Objects.equals(path, that.path)
&& Objects.equals(params, that.params)
&& Objects.equals(headers, that.headers)
- && Objects.equals(xContentType, that.xContentType)
+ && Objects.equals(mediaType, that.mediaType)
&& Objects.equals(content, that.content)
&& Objects.equals(principalIdentifierToken, that.principalIdentifierToken)
&& Objects.equals(httpVersion, that.httpVersion);
@@ -339,6 +349,6 @@ public boolean equals(Object obj) {
@Override
public int hashCode() {
- return Objects.hash(method, uri, path, params, headers, xContentType, content, principalIdentifierToken, httpVersion);
+ return Objects.hash(method, uri, path, params, headers, mediaType, content, principalIdentifierToken, httpVersion);
}
}
diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java
index 2a3ad63379556..4b622b841a040 100644
--- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java
+++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java
@@ -80,7 +80,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
Tuple extends MediaType, Map> unreadExtensionTuple = XContentHelper.convertToMap(
request.content(),
false,
- request.getXContentType().xContent().mediaType()
+ request.getMediaType().xContent().mediaType()
);
Map extensionMap = unreadExtensionTuple.v2();
diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java
index e20a3c6b2814c..a96e87b0200cc 100644
--- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java
+++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java
@@ -14,7 +14,7 @@
import org.opensearch.client.node.NodeClient;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.extensions.DiscoveryExtensionNode;
import org.opensearch.extensions.ExtensionsManager;
import org.opensearch.rest.BaseRestHandler;
@@ -181,7 +181,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
String uri = httpRequest.uri();
Map params = request.params();
Map> headers = request.getHeaders();
- XContentType contentType = request.getXContentType();
+ MediaType contentType = request.getMediaType();
BytesReference content = request.content();
HttpRequest.HttpVersion httpVersion = httpRequest.protocolVersion();
diff --git a/server/src/main/java/org/opensearch/gateway/GatewayService.java b/server/src/main/java/org/opensearch/gateway/GatewayService.java
index cf105380e98ad..b8fd3d147523b 100644
--- a/server/src/main/java/org/opensearch/gateway/GatewayService.java
+++ b/server/src/main/java/org/opensearch/gateway/GatewayService.java
@@ -46,7 +46,7 @@
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.routing.allocation.AllocationService;
import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
diff --git a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java
index 9943f04c964e1..2a419031b2c5a 100644
--- a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java
+++ b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java
@@ -49,12 +49,12 @@
import org.opensearch.common.lucene.store.IndexOutputOutputStream;
import org.opensearch.common.lucene.store.InputStreamIndexInput;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
-import org.opensearch.common.util.io.IOUtils;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -288,7 +288,7 @@ private long write(final T state, boolean cleanup, final Path... locations) thro
}
protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {
- return XContentFactory.contentBuilder(type, stream);
+ return MediaTypeRegistry.contentBuilder(type, stream);
}
/**
@@ -321,7 +321,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro
long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
try (
- XContentParser parser = XContentFactory.xContent(FORMAT)
+ XContentParser parser = FORMAT.xContent()
.createParser(
namedXContentRegistry,
LoggingDeprecationHandler.INSTANCE,
diff --git a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java
index 8940b0ed25ed4..c01f51eb7188a 100644
--- a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java
@@ -45,6 +45,7 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
@@ -66,7 +67,6 @@
import org.opensearch.common.CheckedConsumer;
import org.opensearch.common.Nullable;
import org.opensearch.common.SetOnce;
-import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.bytes.RecyclingBytesStreamOutput;
import org.opensearch.common.io.Streams;
import org.opensearch.common.logging.Loggers;
@@ -75,14 +75,15 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.BigArrays;
-import org.opensearch.core.common.util.ByteArray;
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.common.lease.Releasable;
import org.opensearch.common.lease.Releasables;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.util.ByteArray;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -447,7 +448,7 @@ private OnDiskState loadOnDiskState(Path dataPath, DirectoryReader reader) throw
final SetOnce builderReference = new SetOnce<>();
consumeFromType(searcher, GLOBAL_TYPE_NAME, bytes -> {
final Metadata metadata = Metadata.Builder.fromXContent(
- XContentFactory.xContent(XContentType.SMILE)
+ XContentType.SMILE.xContent()
.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length)
);
logger.trace("found global metadata with last-accepted term [{}]", metadata.coordinationMetadata().term());
@@ -467,7 +468,7 @@ private OnDiskState loadOnDiskState(Path dataPath, DirectoryReader reader) throw
final Set indexUUIDs = new HashSet<>();
consumeFromType(searcher, INDEX_TYPE_NAME, bytes -> {
final IndexMetadata indexMetadata = IndexMetadata.fromXContent(
- XContentFactory.xContent(XContentType.SMILE)
+ XContentType.SMILE.xContent()
.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length)
);
logger.trace("found index metadata for {}", indexMetadata.getIndex());
@@ -507,12 +508,11 @@ private static void consumeFromType(IndexSearcher indexSearcher, String type, Ch
final Bits liveDocs = leafReaderContext.reader().getLiveDocs();
final IntPredicate isLiveDoc = liveDocs == null ? i -> true : liveDocs::get;
final DocIdSetIterator docIdSetIterator = scorer.iterator();
+ final StoredFields storedFields = leafReaderContext.reader().storedFields();
while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
if (isLiveDoc.test(docIdSetIterator.docID())) {
logger.trace("processing doc {}", docIdSetIterator.docID());
- bytesRefConsumer.accept(
- leafReaderContext.reader().document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME)
- );
+ bytesRefConsumer.accept(storedFields.document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME));
}
}
}
@@ -921,7 +921,7 @@ private Document makeDocument(String typeName, ToXContent metadata, DocumentBuff
try (RecyclingBytesStreamOutput streamOutput = documentBuffer.streamOutput()) {
try (
- XContentBuilder xContentBuilder = XContentFactory.contentBuilder(
+ XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(
XContentType.SMILE,
Streams.flushOnCloseStream(streamOutput)
)
diff --git a/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java b/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java
index 1b4e3d89bdc1e..44521ded150be 100644
--- a/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java
@@ -48,7 +48,7 @@
import org.opensearch.cluster.routing.allocation.decider.Decision;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.store.StoreFileMetadata;
import org.opensearch.indices.store.TransportNodesListShardStoreMetadata;
diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java
index 46f296f52ae01..0ba49be01d193 100644
--- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java
+++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java
@@ -37,7 +37,7 @@
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.opensearch.ExceptionsHelper;
import org.opensearch.action.ActionListener;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.network.CloseableChannel;
import org.opensearch.common.network.NetworkAddress;
import org.opensearch.common.network.NetworkService;
@@ -47,7 +47,7 @@
import org.opensearch.common.transport.NetworkExceptionHelper;
import org.opensearch.common.transport.PortsRange;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.core.common.Strings;
diff --git a/server/src/main/java/org/opensearch/http/HttpInfo.java b/server/src/main/java/org/opensearch/http/HttpInfo.java
index a5e981e98e3de..24c29b8dc7444 100644
--- a/server/src/main/java/org/opensearch/http/HttpInfo.java
+++ b/server/src/main/java/org/opensearch/http/HttpInfo.java
@@ -38,7 +38,7 @@
import org.opensearch.common.network.InetAddresses;
import org.opensearch.common.transport.BoundTransportAddress;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.node.ReportingService;
diff --git a/server/src/main/java/org/opensearch/http/HttpServerTransport.java b/server/src/main/java/org/opensearch/http/HttpServerTransport.java
index 425fa23047764..6549f0786fcda 100644
--- a/server/src/main/java/org/opensearch/http/HttpServerTransport.java
+++ b/server/src/main/java/org/opensearch/http/HttpServerTransport.java
@@ -32,7 +32,7 @@
package org.opensearch.http;
-import org.opensearch.common.component.LifecycleComponent;
+import org.opensearch.common.lifecycle.LifecycleComponent;
import org.opensearch.common.transport.BoundTransportAddress;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.node.ReportingService;
diff --git a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java
index 4522a59d67c05..acff87442d0a8 100644
--- a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java
+++ b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java
@@ -37,8 +37,8 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.transport.PortsRange;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import java.util.Collections;
diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
index e4e3a79c8e60c..8a0d563d51107 100644
--- a/server/src/main/java/org/opensearch/index/IndexModule.java
+++ b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -76,7 +76,7 @@
import org.opensearch.index.store.remote.filecache.FileCache;
import org.opensearch.index.translog.TranslogFactory;
import org.opensearch.indices.IndicesQueryCache;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.opensearch.indices.mapper.MapperRegistry;
import org.opensearch.indices.recovery.RecoveryState;
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
index 1ecc98b7f69f2..aed09710deec8 100644
--- a/server/src/main/java/org/opensearch/index/IndexService.java
+++ b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -91,7 +91,7 @@
import org.opensearch.index.store.Store;
import org.opensearch.index.translog.Translog;
import org.opensearch.index.translog.TranslogFactory;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.cluster.IndicesClusterStateService;
import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.opensearch.indices.mapper.MapperRegistry;
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 0749ad0876534..d0fdbd9ac4e03 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -41,8 +41,8 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.core.common.Strings;
diff --git a/server/src/main/java/org/opensearch/index/IndexingPressure.java b/server/src/main/java/org/opensearch/index/IndexingPressure.java
index 33be340feb335..9f57514a0751a 100644
--- a/server/src/main/java/org/opensearch/index/IndexingPressure.java
+++ b/server/src/main/java/org/opensearch/index/IndexingPressure.java
@@ -37,7 +37,7 @@
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.lease.Releasable;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
import org.opensearch.index.stats.IndexingPressureStats;
diff --git a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java
index 87605969feb6a..86c5d4542e5fa 100644
--- a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java
+++ b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java
@@ -244,7 +244,7 @@ private static Map prepareMap(
return map;
}
try {
- String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType());
+ String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getMediaType());
String trim = Strings.cleanTruncate(source, maxSourceCharsToLog).trim();
StringBuilder sb = new StringBuilder(trim);
StringBuilders.escapeJson(sb, 0);
@@ -279,7 +279,7 @@ private static String message(Index index, ParsedDocument doc, long tookInNanos,
return sb.toString();
}
try {
- String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType());
+ String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getMediaType());
sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog).trim()).append("]");
} catch (IOException e) {
sb.append(", source[_failed_to_convert_[").append(e.getMessage()).append("]]");
diff --git a/server/src/main/java/org/opensearch/index/MergePolicyConfig.java b/server/src/main/java/org/opensearch/index/MergePolicyConfig.java
index d0416aaf54a40..fe2af21dfe039 100644
--- a/server/src/main/java/org/opensearch/index/MergePolicyConfig.java
+++ b/server/src/main/java/org/opensearch/index/MergePolicyConfig.java
@@ -38,8 +38,8 @@
import org.apache.lucene.index.TieredMergePolicy;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
/**
* A shard in opensearch is a Lucene index, and a Lucene index is broken
diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java
index b0e6e5076d03c..3315a059ee783 100644
--- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java
+++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java
@@ -12,7 +12,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java
index a4f2628b5e5a3..88077aafb1495 100644
--- a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java
@@ -36,7 +36,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index ab8b4706e4ebe..24f68899c2ac7 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
index 04dd669d60883..234c67cc637f1 100644
--- a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
+++ b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java
@@ -10,7 +10,6 @@
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
@@ -87,12 +86,12 @@ public String toString(String field) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
+ public Query rewrite(IndexSearcher searcher) throws IOException {
if (Long.compareUnsigned(lowerValue, Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG) == 0
&& Long.compareUnsigned(upperValue, Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG) == 0) {
return new FieldExistsQuery(field);
}
- return super.rewrite(reader);
+ return super.rewrite(searcher);
}
abstract SortedNumericDocValues getValues(LeafReader reader, String field) throws IOException;
diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java
index 326e6aef45b08..74c9c25dc4c1f 100644
--- a/server/src/main/java/org/opensearch/index/engine/Engine.java
+++ b/server/src/main/java/org/opensearch/index/engine/Engine.java
@@ -69,7 +69,7 @@
import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver;
import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
import org.opensearch.common.metrics.CounterMetric;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ReleasableLock;
import org.opensearch.common.lease.Releasable;
diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
index 71bd64426161e..7900e63a95c39 100644
--- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
+++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
@@ -43,7 +43,7 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.MemorySizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.IndexSettings;
@@ -57,7 +57,7 @@
import org.opensearch.index.translog.TranslogDeletionPolicyFactory;
import org.opensearch.index.translog.TranslogFactory;
import org.opensearch.indices.IndexingMemoryController;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.threadpool.ThreadPool;
import java.util.Comparator;
diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java
index 744df13265250..c606c00228db9 100644
--- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java
+++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java
@@ -30,7 +30,7 @@
import org.opensearch.index.translog.TranslogConfig;
import org.opensearch.index.translog.TranslogDeletionPolicyFactory;
import org.opensearch.index.translog.TranslogFactory;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.plugins.EnginePlugin;
import org.opensearch.plugins.PluginsService;
import org.opensearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
index 77e2f5cbef7f9..77d63dfaade54 100644
--- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
@@ -49,6 +49,7 @@
import org.apache.lucene.index.ShuffleForcedMergePolicy;
import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
import org.apache.lucene.index.StandardDirectoryReader;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@@ -80,7 +81,7 @@
import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver;
import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo;
import org.opensearch.common.metrics.CounterMetric;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.util.concurrent.KeyedLock;
@@ -2889,6 +2890,7 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead
final CombinedDocValues dv = new CombinedDocValues(leaf.reader());
final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor();
final DocIdSetIterator iterator = scorer.iterator();
+ final StoredFields storedFields = leaf.reader().storedFields();
int docId;
while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
final long primaryTerm = dv.docPrimaryTerm(docId);
@@ -2896,7 +2898,7 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead
localCheckpointTracker.markSeqNoAsProcessed(seqNo);
localCheckpointTracker.markSeqNoAsPersisted(seqNo);
idFieldVisitor.reset();
- leaf.reader().document(docId, idFieldVisitor);
+ storedFields.document(docId, idFieldVisitor);
if (idFieldVisitor.getId() == null) {
assert dv.isTombstone(docId);
continue;
diff --git a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java
index abde2aff6e9e6..23fe59456887e 100644
--- a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java
+++ b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java
@@ -289,7 +289,7 @@ private Translog.Operation readDocAsOp(int docIndex) throws IOException {
? SourceFieldMapper.RECOVERY_SOURCE_NAME
: SourceFieldMapper.NAME;
final FieldsVisitor fields = new FieldsVisitor(true, sourceField);
- leaf.reader().document(segmentDocID, fields);
+ leaf.reader().storedFields().document(segmentDocID, fields);
final Translog.Operation op;
final boolean isTombstone = parallelArray.isTombStone[docIndex];
diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
index a9f7a2e70884c..e852658d7b3ba 100644
--- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
@@ -16,7 +16,7 @@
import org.opensearch.common.concurrent.GatedCloseable;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ReleasableLock;
import org.opensearch.common.util.io.IOUtils;
@@ -77,9 +77,10 @@ public NRTReplicationEngine(EngineConfig engineConfig) {
this.completionStatsCache = new CompletionStatsCache(() -> acquireSearcher("completion_stats"));
this.readerManager = readerManager;
this.readerManager.addListener(completionStatsCache);
- for (ReferenceManager.RefreshListener listener : engineConfig.getExternalRefreshListener()) {
- this.readerManager.addListener(listener);
- }
+ // NRT Replicas do not have a concept of Internal vs External reader managers.
+ // We also do not want to wire up refresh listeners for waitFor & pending refresh location.
+ // which are the current external listeners set from IndexShard.
+ // Only wire up the internal listeners.
for (ReferenceManager.RefreshListener listener : engineConfig.getInternalRefreshListener()) {
this.readerManager.addListener(listener);
}
@@ -322,22 +323,12 @@ public List segments(boolean verbose) {
@Override
public void refresh(String source) throws EngineException {
- maybeRefresh(source);
+ // Refresh on this engine should only ever happen in the reader after new segments arrive.
}
@Override
public boolean maybeRefresh(String source) throws EngineException {
- ensureOpen();
- try {
- return readerManager.maybeRefresh();
- } catch (IOException e) {
- try {
- failEngine("refresh failed source[" + source + "]", e);
- } catch (Exception inner) {
- e.addSuppressed(inner);
- }
- throw new RefreshFailedEngineException(shardId, e);
- }
+ return false;
}
@Override
diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java
index 35409437f605a..7b4c93c7235fe 100644
--- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java
+++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java
@@ -103,7 +103,7 @@ public void updateSegments(SegmentInfos infos) throws IOException {
// is always increased.
infos.updateGeneration(currentInfos);
currentInfos = infos;
- maybeRefreshBlocking();
+ maybeRefresh();
}
public SegmentInfos getSegmentInfos() {
diff --git a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java
index ce0f1e85e294d..6736354517ace 100644
--- a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java
+++ b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java
@@ -41,7 +41,7 @@
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.common.metrics.MeanMetric;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
diff --git a/server/src/main/java/org/opensearch/index/engine/Segment.java b/server/src/main/java/org/opensearch/index/engine/Segment.java
index 035d5bb293303..36b84723f0229 100644
--- a/server/src/main/java/org/opensearch/index/engine/Segment.java
+++ b/server/src/main/java/org/opensearch/index/engine/Segment.java
@@ -44,7 +44,7 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.common.lucene.Lucene;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import java.io.IOException;
import java.util.Map;
diff --git a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java
index dbadd50ebd1a1..1fd7f30237d0b 100644
--- a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java
+++ b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java
@@ -36,7 +36,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java
index 5efafb562df74..417cdd5a8f030 100644
--- a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java
+++ b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java
@@ -221,28 +221,33 @@ public int maxDoc() {
@Override
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
- if (docID != 0) {
- throw new IllegalArgumentException("no such doc ID " + docID);
- }
- if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) {
- assert operation.source().toBytesRef().offset == 0;
- assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length;
- visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes);
- }
- if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) {
- visitor.stringField(FAKE_ROUTING_FIELD, operation.routing());
- }
- if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) {
- BytesRef bytesRef = Uid.encodeId(operation.id());
- final byte[] id = new byte[bytesRef.length];
- System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length);
- visitor.binaryField(FAKE_ID_FIELD, id);
- }
+ storedFields().document(docID, visitor);
}
@Override
public StoredFields storedFields() throws IOException {
- throw new UnsupportedOperationException();
+ return new StoredFields() {
+ @Override
+ public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+ if (docID != 0) {
+ throw new IllegalArgumentException("no such doc ID " + docID);
+ }
+ if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) {
+ assert operation.source().toBytesRef().offset == 0;
+ assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length;
+ visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes);
+ }
+ if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) {
+ visitor.stringField(FAKE_ROUTING_FIELD, operation.routing());
+ }
+ if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) {
+ BytesRef bytesRef = Uid.encodeId(operation.id());
+ final byte[] id = new byte[bytesRef.length];
+ System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length);
+ visitor.binaryField(FAKE_ID_FIELD, id);
+ }
+ }
+ };
}
@Override
diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java
index 896039313ea1e..fc7db7d316214 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java
@@ -37,7 +37,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
index 36e6a242ecdec..05326b7027cac 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
@@ -50,7 +50,7 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.util.BigArrays;
import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java
index 0b370893cd90d..6914f5df31da8 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java
@@ -41,7 +41,7 @@
import org.opensearch.index.mapper.MappedFieldType;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.opensearch.search.lookup.SearchLookup;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java b/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java
index cd858278afaa3..517361e75d3ea 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java
@@ -34,7 +34,7 @@
import org.apache.lucene.index.FilteredTermsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.index.fielddata.plain.AbstractIndexOrdinalsFieldData;
import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
index e136e649d088a..90ee43d320743 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java
@@ -39,13 +39,13 @@
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.packed.PackedInts;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.fielddata.IndexOrdinalsFieldData;
import org.opensearch.index.fielddata.LeafOrdinalsFieldData;
import org.opensearch.index.fielddata.ScriptDocValues;
import org.opensearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import java.io.IOException;
import java.util.Collection;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java
index 2c6aabf04d4ee..d7b8179398920 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java
@@ -15,7 +15,7 @@
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.IndexFieldDataCache;
import org.opensearch.index.fielddata.LeafGeoShapeFieldData;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
index 0b3a5e09064ab..15eae77885686 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
@@ -48,7 +48,7 @@
import org.opensearch.index.fielddata.ScriptDocValues;
import org.opensearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
import org.opensearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.aggregations.support.ValuesSourceType;
import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java
index 6e9a13074a445..9cd824f570b05 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java
@@ -44,7 +44,7 @@
import org.opensearch.index.fielddata.IndexFieldDataCache;
import org.opensearch.index.fielddata.IndexGeoPointFieldData;
import org.opensearch.index.fielddata.LeafGeoPointFieldData;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java
index 3fbc605e35e1d..420864fbd7cb5 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java
@@ -40,7 +40,7 @@
import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.opensearch.index.fielddata.IndexFieldDataCache;
import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java
index e1a08344e68c7..5386d0eee2108 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java
@@ -40,7 +40,7 @@
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.opensearch.index.fielddata.IndexFieldDataCache;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java
index 2b08498f2b58b..25d199e9a79af 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java
@@ -49,7 +49,7 @@
import org.opensearch.index.fielddata.IndexOrdinalsFieldData;
import org.opensearch.index.fielddata.LeafOrdinalsFieldData;
import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java
index 8f6a1e46ef417..6f880b678a26a 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java
@@ -44,7 +44,7 @@
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
import org.opensearch.common.Nullable;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.util.BigArrays;
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -55,7 +55,7 @@
import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.opensearch.index.fielddata.ordinals.Ordinals;
import org.opensearch.index.fielddata.ordinals.OrdinalsBuilder;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java
index 812010b44b654..e7995f9188275 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java
@@ -53,7 +53,7 @@
import org.opensearch.index.fielddata.SortedNumericDoubleValues;
import org.opensearch.index.fielddata.fieldcomparator.LongValuesComparatorSource;
import org.opensearch.index.mapper.DocValueFetcher;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.apache.lucene.sandbox.document.HalfFloatPoint;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java
index 801186bbc4379..e52eaeb7dfc89 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java
@@ -46,7 +46,7 @@
import org.opensearch.index.fielddata.LeafOrdinalsFieldData;
import org.opensearch.index.fielddata.ScriptDocValues;
import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/get/ShardGetService.java b/server/src/main/java/org/opensearch/index/get/ShardGetService.java
index a815074119fb1..8f81e704ef2d4 100644
--- a/server/src/main/java/org/opensearch/index/get/ShardGetService.java
+++ b/server/src/main/java/org/opensearch/index/get/ShardGetService.java
@@ -51,10 +51,10 @@
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.common.metrics.MeanMetric;
import org.opensearch.common.util.set.Sets;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.support.XContentMapValues;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.VersionType;
import org.opensearch.index.engine.Engine;
@@ -276,7 +276,7 @@ private GetResult innerGetLoadFromStoredFields(
);
if (fieldVisitor != null) {
try {
- docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor);
+ docIdAndVersion.reader.storedFields().document(docIdAndVersion.docId, fieldVisitor);
} catch (IOException e) {
throw new OpenSearchException("Failed to get id [" + id + "]", e);
}
@@ -299,7 +299,7 @@ private GetResult innerGetLoadFromStoredFields(
shardId.getIndexName(),
id,
source,
- XContentHelper.xContentType(source),
+ MediaTypeRegistry.xContentType(source),
fieldVisitor.routing()
);
ParsedDocument doc = indexShard.mapperService().documentMapper().parse(sourceToParse);
@@ -378,7 +378,7 @@ private GetResult innerGetLoadFromStoredFields(
sourceAsMap = typeMapTuple.v2();
sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
try {
- source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
+ source = BytesReference.bytes(MediaTypeRegistry.contentBuilder(sourceContentType).map(sourceAsMap));
} catch (IOException e) {
throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
}
@@ -406,7 +406,7 @@ private GetResult innerGetLoadFromStoredFields(
sourceAsMap = typeMapTuple.v2();
sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
try {
- source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
+ source = BytesReference.bytes(MediaTypeRegistry.contentBuilder(sourceContentType).map(sourceAsMap));
} catch (IOException e) {
throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
index c5ee5be491095..f276d6ee2e579 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
@@ -43,8 +43,8 @@
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.mapper.DynamicTemplate.XContentFieldType;
@@ -77,14 +77,14 @@ final class DocumentParser {
ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadataFieldsMappers) throws MapperParsingException {
final Mapping mapping = docMapper.mapping();
final ParseContext.InternalParseContext context;
- final XContentType xContentType = source.getXContentType();
+ final MediaType mediaType = source.getMediaType();
try (
XContentParser parser = XContentHelper.createParser(
docMapperParser.getXContentRegistry(),
LoggingDeprecationHandler.INSTANCE,
source.source(),
- xContentType
+ mediaType
)
) {
context = new ParseContext.InternalParseContext(indexSettings, docMapperParser, docMapper, source, parser);
@@ -183,7 +183,7 @@ private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.
source.routing(),
context.docs(),
context.sourceToParse().source(),
- context.sourceToParse().getXContentType(),
+ context.sourceToParse().getMediaType(),
update
);
}
diff --git a/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java
index 4b1395ac02bf1..9ac226dcc0b70 100644
--- a/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java
@@ -54,7 +54,7 @@
import org.opensearch.index.fielddata.plain.PagedBytesIndexFieldData;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.indices.IndicesService;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.CoreValuesSourceType;
diff --git a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java
index 46f0352976408..1084c08cdf32c 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java
@@ -35,6 +35,7 @@
import org.apache.lucene.document.Field;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.index.mapper.ParseContext.Document;
import org.opensearch.index.mapper.MapperService.MergeReason;
@@ -57,7 +58,7 @@ public class ParsedDocument {
private final List documents;
private BytesReference source;
- private XContentType xContentType;
+ private MediaType mediaType;
private Mapping dynamicMappingsUpdate;
@@ -68,7 +69,7 @@ public ParsedDocument(
String routing,
List documents,
BytesReference source,
- XContentType xContentType,
+ MediaType mediaType,
Mapping dynamicMappingsUpdate
) {
this.version = version;
@@ -78,7 +79,7 @@ public ParsedDocument(
this.documents = documents;
this.source = source;
this.dynamicMappingsUpdate = dynamicMappingsUpdate;
- this.xContentType = xContentType;
+ this.mediaType = mediaType;
}
public String id() {
@@ -122,13 +123,13 @@ public BytesReference source() {
return this.source;
}
- public XContentType getXContentType() {
- return this.xContentType;
+ public MediaType getMediaType() {
+ return this.mediaType;
}
public void setSource(BytesReference source, XContentType xContentType) {
this.source = source;
- this.xContentType = xContentType;
+ this.mediaType = xContentType;
}
/**
diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java
index 95e71f168c2ca..deda9883bc07b 100644
--- a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java
@@ -39,15 +39,15 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.opensearch.common.Nullable;
-import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.util.CollectionUtils;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.support.XContentMapValues;
+import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.index.query.QueryShardException;
@@ -202,7 +202,7 @@ public boolean isComplete() {
@Override
public void preParse(ParseContext context) throws IOException {
BytesReference originalSource = context.sourceToParse().source();
- XContentType contentType = context.sourceToParse().getXContentType();
+ MediaType contentType = context.sourceToParse().getMediaType();
final BytesReference adaptedSource = applyFilters(originalSource, contentType);
if (adaptedSource != null) {
@@ -219,16 +219,16 @@ public void preParse(ParseContext context) throws IOException {
}
@Nullable
- public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable XContentType contentType) throws IOException {
+ public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable MediaType contentType) throws IOException {
if (enabled && originalSource != null) {
// Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data
if (filter != null) {
// we don't update the context source if we filter, we want to keep it as is...
- Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType);
+ Tuple extends MediaType, Map> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType);
Map filteredSource = filter.apply(mapTuple.v2());
BytesStreamOutput bStream = new BytesStreamOutput();
- XContentType actualContentType = mapTuple.v1();
- XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource);
+ MediaType actualContentType = mapTuple.v1();
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(actualContentType, bStream).map(filteredSource);
builder.close();
return bStream.bytes();
} else {
diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java
index 085e258c5501e..2b24a380c8bb6 100644
--- a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java
+++ b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java
@@ -37,7 +37,7 @@
import org.opensearch.common.Nullable;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
/**
* Stores the document source
@@ -54,20 +54,20 @@ public class SourceToParse {
private final @Nullable String routing;
- private final XContentType xContentType;
+ private final MediaType mediaType;
- public SourceToParse(String index, String id, BytesReference source, XContentType xContentType, @Nullable String routing) {
+ public SourceToParse(String index, String id, BytesReference source, MediaType mediaType, @Nullable String routing) {
this.index = Objects.requireNonNull(index);
this.id = Objects.requireNonNull(id);
// we always convert back to byte array, since we store it and Field only supports bytes..
// so, we might as well do it here, and improve the performance of working with direct byte arrays
this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef());
- this.xContentType = Objects.requireNonNull(xContentType);
+ this.mediaType = Objects.requireNonNull(mediaType);
this.routing = routing;
}
- public SourceToParse(String index, String id, BytesReference source, XContentType xContentType) {
- this(index, id, source, xContentType, null);
+ public SourceToParse(String index, String id, BytesReference source, MediaType mediaType) {
+ this(index, id, source, mediaType, null);
}
public BytesReference source() {
@@ -86,8 +86,8 @@ public String id() {
return this.routing;
}
- public XContentType getXContentType() {
- return this.xContentType;
+ public MediaType getMediaType() {
+ return this.mediaType;
}
/**
diff --git a/server/src/main/java/org/opensearch/index/merge/MergeStats.java b/server/src/main/java/org/opensearch/index/merge/MergeStats.java
index fee0ed904b68a..5bdbd0ab30613 100644
--- a/server/src/main/java/org/opensearch/index/merge/MergeStats.java
+++ b/server/src/main/java/org/opensearch/index/merge/MergeStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
index 2a7dbf314103d..001d77facb829 100644
--- a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
+++ b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java
@@ -32,8 +32,8 @@
package org.opensearch.index.query;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
@@ -60,7 +60,7 @@ public Query getQuery() {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
+ public Query rewrite(IndexSearcher searcher) throws IOException {
return in;
}
diff --git a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java
index 36b74c95c3200..7287634ecfacb 100644
--- a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java
@@ -58,6 +58,7 @@
import org.opensearch.common.lucene.search.MoreLikeThisQuery;
import org.opensearch.common.lucene.search.XMoreLikeThis;
import org.opensearch.common.lucene.uid.Versions;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
@@ -170,7 +171,7 @@ public static final class Item implements ToXContentObject, Writeable {
private String index;
private String id;
private BytesReference doc;
- private XContentType xContentType;
+ private MediaType mediaType;
private String[] fields;
private Map perFieldAnalyzer;
private String routing;
@@ -187,7 +188,7 @@ public Item() {}
this.id = copy.id;
this.routing = copy.routing;
this.doc = copy.doc;
- this.xContentType = copy.xContentType;
+ this.mediaType = copy.mediaType;
this.fields = copy.fields;
this.perFieldAnalyzer = copy.perFieldAnalyzer;
this.version = copy.version;
@@ -220,7 +221,7 @@ public Item(@Nullable String index, XContentBuilder doc) {
}
this.index = index;
this.doc = BytesReference.bytes(doc);
- this.xContentType = XContentType.fromMediaType(doc.contentType());
+ this.mediaType = doc.contentType();
}
/**
@@ -234,7 +235,11 @@ public Item(@Nullable String index, XContentBuilder doc) {
}
if (in.readBoolean()) {
doc = (BytesReference) in.readGenericValue();
- xContentType = in.readEnum(XContentType.class);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType = in.readMediaType();
+ } else {
+ mediaType = in.readEnum(XContentType.class);
+ }
} else {
id = in.readString();
}
@@ -255,7 +260,11 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(doc != null);
if (doc != null) {
out.writeGenericValue(doc);
- out.writeEnum(xContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
} else {
out.writeString(id);
}
@@ -331,8 +340,8 @@ public Item versionType(VersionType versionType) {
return this;
}
- XContentType xContentType() {
- return xContentType;
+ MediaType mediaType() {
+ return mediaType;
}
/**
@@ -351,7 +360,7 @@ TermVectorsRequest toTermVectorsRequest() {
.termStatistics(false);
// for artificial docs to make sure that the id has changed in the item too
if (doc != null) {
- termVectorsRequest.doc(doc, true, xContentType);
+ termVectorsRequest.doc(doc, true, mediaType);
this.id = termVectorsRequest.id();
}
return termVectorsRequest;
@@ -373,7 +382,7 @@ public static Item parse(XContentParser parser, Item item) throws IOException {
item.id = parser.text();
} else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) {
item.doc = BytesReference.bytes(jsonBuilder().copyCurrentStructure(parser));
- item.xContentType = XContentType.JSON;
+ item.mediaType = XContentType.JSON;
} else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
if (token == XContentParser.Token.START_ARRAY) {
List fields = new ArrayList<>();
@@ -419,7 +428,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
if (this.doc != null) {
try (InputStream stream = this.doc.streamInput()) {
- builder.rawField(DOC.getPreferredName(), stream, xContentType);
+ builder.rawField(DOC.getPreferredName(), stream, mediaType);
}
}
if (this.fields != null) {
diff --git a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java
index aab9adecb9a62..b0fc62ad8ec71 100644
--- a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java
@@ -34,15 +34,15 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.ParseField;
import org.opensearch.core.common.ParsingException;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import java.io.IOException;
@@ -175,7 +175,8 @@ protected boolean doEquals(WrapperQueryBuilder other) {
@Override
protected QueryBuilder doRewrite(QueryRewriteContext context) throws IOException {
try (
- XContentParser qSourceParser = XContentFactory.xContent(source)
+ XContentParser qSourceParser = MediaTypeRegistry.xContent(source)
+ .xContent()
.createParser(context.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, source)
) {
diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java
index 40b15eace2bad..a4c4aa9493288 100644
--- a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java
@@ -49,10 +49,10 @@
import org.opensearch.common.unit.DistanceUnit;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.fielddata.FieldData;
import org.opensearch.index.fielddata.IndexGeoPointFieldData;
@@ -230,7 +230,8 @@ protected ScoreFunction doToFunction(QueryShardContext context) throws IOExcepti
// EMPTY is safe because parseVariable doesn't use namedObject
try (
InputStream stream = functionBytes.streamInput();
- XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(functionBytes))
+ XContentParser parser = MediaTypeRegistry.xContentType(functionBytes)
+ .xContent()
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)
) {
scoreFunction = parseVariable(fieldName, parser, context, multiValueMode);
diff --git a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java
index 998427c27c1f6..2f03eb66bd71e 100644
--- a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java
+++ b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java
@@ -49,8 +49,8 @@
import org.opensearch.common.document.DocumentField;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
-import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.index.mapper.RoutingFieldMapper;
import org.opensearch.search.SearchHit;
import org.opensearch.threadpool.ThreadPool;
@@ -209,8 +209,8 @@ public BytesReference getSource() {
}
@Override
- public XContentType getXContentType() {
- return XContentHelper.xContentType(source);
+ public MediaType getMediaType() {
+ return MediaTypeRegistry.xContentType(source);
}
@Override
diff --git a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java
index f6b4793f3b87a..8a9a346a5344a 100644
--- a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java
+++ b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java
@@ -44,10 +44,10 @@
import org.opensearch.common.lucene.uid.Versions;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.VersionType;
import org.opensearch.index.query.QueryBuilder;
@@ -356,7 +356,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
request.getSearchRequest().indices(indices);
}
request.setRemoteInfo(buildRemoteInfo(source));
- XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType());
builder.map(source);
try (
InputStream stream = BytesReference.bytes(builder).streamInput();
diff --git a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java
index a4047507fcde1..47e882fc04127 100644
--- a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java
+++ b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java
@@ -46,6 +46,7 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentType;
@@ -296,7 +297,7 @@ public interface Hit {
* The content type of the hit source. Returns null if the source didn't come back from the search.
*/
@Nullable
- XContentType getXContentType();
+ MediaType getMediaType();
/**
* The routing on the hit if there is any or null if there isn't.
@@ -316,7 +317,7 @@ public static class BasicHit implements Hit {
private final long version;
private BytesReference source;
- private XContentType xContentType;
+ private MediaType mediaType;
private String routing;
private long seqNo;
private long primaryTerm;
@@ -358,13 +359,13 @@ public BytesReference getSource() {
}
@Override
- public XContentType getXContentType() {
- return xContentType;
+ public MediaType getMediaType() {
+ return mediaType;
}
- public BasicHit setSource(BytesReference source, XContentType xContentType) {
+ public BasicHit setSource(BytesReference source, MediaType mediaType) {
this.source = source;
- this.xContentType = xContentType;
+ this.mediaType = mediaType;
return this;
}
diff --git a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
index 770301629d964..cd28d8b6ef04e 100644
--- a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
+++ b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java
@@ -32,7 +32,6 @@
package org.opensearch.index.search;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -82,8 +81,8 @@ public ScoreMode getScoreMode() {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
- Query innerRewrite = query.rewrite(reader);
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ Query innerRewrite = query.rewrite(searcher);
if (innerRewrite != query) {
// Right now ToParentBlockJoinQuery always rewrites to a ToParentBlockJoinQuery
// so the else block will never be used. It is useful in the case that
@@ -97,7 +96,7 @@ public Query rewrite(IndexReader reader) throws IOException {
return innerRewrite;
}
}
- return super.rewrite(reader);
+ return super.rewrite(searcher);
}
@Override
diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
index 4d6cffa58510f..19fe9ee97cd2f 100644
--- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
@@ -247,7 +247,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
private final Consumer onReplicationGroupUpdated;
- private volatile ReplicationCheckpoint lastPublishedReplicationCheckpoint;
+ private volatile ReplicationCheckpoint latestReplicationCheckpoint;
/**
* Get all retention leases tracked on this shard.
@@ -1054,6 +1054,7 @@ public ReplicationTracker(
this.fileBasedRecoveryThreshold = IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.get(indexSettings.getSettings());
this.safeCommitInfoSupplier = safeCommitInfoSupplier;
this.onReplicationGroupUpdated = onReplicationGroupUpdated;
+ this.latestReplicationCheckpoint = indexSettings.isSegRepEnabled() ? ReplicationCheckpoint.empty(shardId) : null;
assert Version.V_EMPTY.equals(indexSettings.getIndexVersionCreated()) == false;
assert invariant();
}
@@ -1212,26 +1213,42 @@ public synchronized void updateVisibleCheckpointForShard(final String allocation
*/
public synchronized void setLatestReplicationCheckpoint(ReplicationCheckpoint checkpoint) {
assert indexSettings.isSegRepEnabled();
- assert handoffInProgress == false;
- if (checkpoint.equals(lastPublishedReplicationCheckpoint) == false) {
- this.lastPublishedReplicationCheckpoint = checkpoint;
- for (Map.Entry entry : checkpoints.entrySet()) {
- if (entry.getKey().equals(this.shardAllocationId) == false) {
- final CheckpointState cps = entry.getValue();
- if (cps.inSync) {
- cps.checkpointTimers.computeIfAbsent(checkpoint, ignored -> {
- final ReplicationTimer replicationTimer = new ReplicationTimer();
- replicationTimer.start();
- return replicationTimer;
- });
- logger.trace(
- () -> new ParameterizedMessage(
- "updated last published checkpoint to {} - timers [{}]",
- checkpoint,
- cps.checkpointTimers.keySet()
- )
- );
- }
+ if (checkpoint.equals(latestReplicationCheckpoint) == false) {
+ this.latestReplicationCheckpoint = checkpoint;
+ }
+ if (primaryMode) {
+ startReplicationLagTimers();
+ }
+ }
+
+ public ReplicationCheckpoint getLatestReplicationCheckpoint() {
+ return this.latestReplicationCheckpoint;
+ }
+
+ private void startReplicationLagTimers() {
+ for (Map.Entry entry : checkpoints.entrySet()) {
+ final String allocationId = entry.getKey();
+ if (allocationId.equals(this.shardAllocationId) == false) {
+ final CheckpointState cps = entry.getValue();
+ // if the shard is in checkpoints but is unavailable or out of sync we will not track its replication state.
+ // it is possible for a shard to be in-sync but not yet removed from the checkpoints collection after a failover event.
+ if (cps.inSync
+ && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false
+ && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint)) {
+ cps.checkpointTimers.computeIfAbsent(latestReplicationCheckpoint, ignored -> {
+ final ReplicationTimer replicationTimer = new ReplicationTimer();
+ replicationTimer.start();
+ return replicationTimer;
+ });
+ logger.trace(
+ () -> new ParameterizedMessage(
+ "updated last published checkpoint for {} at visible cp {} to {} - timers [{}]",
+ allocationId,
+ cps.visibleReplicationCheckpoint,
+ latestReplicationCheckpoint,
+ cps.checkpointTimers.keySet()
+ )
+ );
}
}
}
@@ -1244,12 +1261,17 @@ public synchronized void setLatestReplicationCheckpoint(ReplicationCheckpoint ch
*/
public synchronized Set getSegmentReplicationStats() {
assert indexSettings.isSegRepEnabled();
- final ReplicationCheckpoint lastPublishedCheckpoint = this.lastPublishedReplicationCheckpoint;
- if (primaryMode && lastPublishedCheckpoint != null) {
+ if (primaryMode) {
return this.checkpoints.entrySet()
.stream()
- .filter(entry -> entry.getKey().equals(this.shardAllocationId) == false && entry.getValue().inSync)
- .map(entry -> buildShardStats(lastPublishedCheckpoint.getLength(), entry.getKey(), entry.getValue()))
+ // filter out this shard's allocation id, any shards that are out of sync or unavailable (shard marked in-sync but has not
+ // been assigned to a node).
+ .filter(
+ entry -> entry.getKey().equals(this.shardAllocationId) == false
+ && entry.getValue().inSync
+ && replicationGroup.getUnavailableInSyncShards().contains(entry.getKey()) == false
+ )
+ .map(entry -> buildShardStats(latestReplicationCheckpoint.getLength(), entry.getKey(), entry.getValue()))
.collect(Collectors.toUnmodifiableSet());
}
return Collections.emptySet();
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index fbb2df7f9640a..bb5088866edb6 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -43,7 +43,6 @@
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryCachingPolicy;
import org.apache.lucene.search.ReferenceManager;
@@ -90,7 +89,7 @@
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.common.metrics.MeanMetric;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.concurrent.AbstractRunnable;
@@ -99,11 +98,11 @@
import org.opensearch.common.util.concurrent.RunOnce;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.common.util.set.Sets;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.common.lease.Releasable;
import org.opensearch.common.lease.Releasables;
import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.gateway.WriteStateException;
import org.opensearch.core.index.Index;
import org.opensearch.index.IndexModule;
@@ -175,7 +174,7 @@
import org.opensearch.index.warmer.WarmerStats;
import org.opensearch.indices.IndexingMemoryController;
import org.opensearch.indices.IndicesService;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.cluster.IndicesClusterStateService;
import org.opensearch.indices.recovery.PeerRecoveryTargetService;
import org.opensearch.indices.recovery.RecoveryFailedException;
@@ -948,7 +947,7 @@ public Engine.IndexResult applyIndexOperationOnReplica(
if (indexSettings.isSegRepEnabled()) {
Engine.Index index = new Engine.Index(
new Term(IdFieldMapper.NAME, Uid.encodeId(id)),
- new ParsedDocument(null, null, id, null, null, sourceToParse.source(), sourceToParse.getXContentType(), null),
+ new ParsedDocument(null, null, id, null, null, sourceToParse.source(), sourceToParse.getMediaType(), null),
seqNo,
opPrimaryTerm,
version,
@@ -2172,7 +2171,7 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o
shardId.getIndexName(),
index.id(),
index.source(),
- XContentHelper.xContentType(index.source()),
+ MediaTypeRegistry.xContentType(index.source()),
index.routing()
)
);
@@ -4424,7 +4423,8 @@ public void addRefreshListener(Translog.Location location, Consumer lis
readAllowed = isReadAllowed();
}
}
- if (readAllowed) {
+ // NRT Replicas will not accept refresh listeners.
+ if (readAllowed && isSegmentReplicationAllowed() == false) {
refreshListeners.addOrNotify(location, listener);
} else {
// we're not yet ready fo ready for reads, just ignore refresh cycles
@@ -4809,6 +4809,14 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo
return true;
} else {
logger.warn("Checksum mismatch between local and remote segment file: {}, will override local file", file);
+ // If there is a checksum mismatch and we are not serving reads it is safe to go ahead and delete the file now.
+ // Outside of engine resets this method will be invoked during recovery so this is safe.
+ if (isReadAllowed() == false) {
+ localDirectory.deleteFile(file);
+ } else {
+ // segment conflict with remote store while the shard is serving reads.
+ failShard("Local copy of segment " + file + " has a different checksum than the version in remote store", null);
+ }
}
} catch (NoSuchFileException | FileNotFoundException e) {
logger.debug("File {} does not exist in local FS, downloading from remote store", file);
diff --git a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java
index 4f0affb3035ec..5bc3f5f163352 100644
--- a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java
+++ b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java
@@ -44,8 +44,8 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.index.shard.ShardId;
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index 3ea8278038ac5..2385b906a7ae5 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -457,11 +457,10 @@ private void updateLocalSizeMapAndTracker(Collection segmentFiles) {
private void updateFinalStatusInSegmentTracker(boolean uploadStatus, long bytesBeforeUpload, long startTimeInNS) {
if (uploadStatus) {
long bytesUploaded = segmentTracker.getUploadBytesSucceeded() - bytesBeforeUpload;
- long timeTakenInMS = (System.nanoTime() - startTimeInNS) / 1_000_000L;
-
+ long timeTakenInMS = TimeValue.nsecToMSec(System.nanoTime() - startTimeInNS);
segmentTracker.incrementTotalUploadsSucceeded();
segmentTracker.addUploadBytes(bytesUploaded);
- segmentTracker.addUploadBytesPerSec((bytesUploaded * 1_000L) / timeTakenInMS);
+ segmentTracker.addUploadBytesPerSec((bytesUploaded * 1_000L) / Math.max(1, timeTakenInMS));
segmentTracker.addUploadTimeMs(timeTakenInMS);
} else {
segmentTracker.incrementTotalUploadsFailed();
diff --git a/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java
index 6d559a6cc0673..219ead931797a 100644
--- a/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java
+++ b/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java
@@ -286,7 +286,7 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
boolean matches(int doc) throws IOException {
routing = id = null;
leftToVisit = 2;
- leafReader.document(doc, this);
+ leafReader.storedFields().document(doc, this);
assert id != null : "docID must not be null - we might have hit a nested document";
int targetShardId = OperationRouting.generateShardId(indexMetadata, id, routing);
return targetShardId != shardId;
diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
index d4e779c83644f..5897fa7d513d7 100644
--- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
@@ -51,7 +51,7 @@
import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
import org.opensearch.common.UUIDs;
import org.opensearch.common.lucene.Lucene;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.core.index.Index;
diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
index 2d49b153c39f4..0b27afa5e9517 100644
--- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
+++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
@@ -38,7 +38,7 @@
import org.opensearch.core.ParseField;
import org.opensearch.common.Strings;
import org.opensearch.common.lucene.Lucene;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java b/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java
index 9e8c8d29c2058..d69ae02fa8e34 100644
--- a/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java
+++ b/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java
@@ -11,7 +11,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java
index 8f4f0b661ed33..dcdea8adbbcde 100644
--- a/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java
+++ b/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/store/StoreStats.java b/server/src/main/java/org/opensearch/index/store/StoreStats.java
index ba36e6b527031..aa73a2c629515 100644
--- a/server/src/main/java/org/opensearch/index/store/StoreStats.java
+++ b/server/src/main/java/org/opensearch/index/store/StoreStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java
index 3d23b4d22538c..45d60c8ad84db 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java
@@ -9,8 +9,9 @@
package org.opensearch.index.store.remote.filecache;
import org.apache.lucene.store.IndexInput;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.common.settings.Setting;
import org.opensearch.index.store.remote.utils.cache.CacheUsage;
import org.opensearch.index.store.remote.utils.cache.RefCountedCache;
import org.opensearch.index.store.remote.utils.cache.SegmentedCache;
@@ -49,8 +50,20 @@ public class FileCache implements RefCountedCache {
private final CircuitBreaker circuitBreaker;
- // TODO: Convert the constant into an integer setting
- public static final int DATA_TO_FILE_CACHE_SIZE_RATIO = 5;
+ /**
+ * Defines a limit of how much total remote data can be referenced as a ratio of the size of the disk reserved for
+ * the file cache. For example, if 100GB disk space is configured for use as a file cache and the
+ * remote_data_ratio of 5 is defined, then a total of 500GB of remote data can be loaded as searchable snapshots.
+ * This is designed to be a safeguard to prevent oversubscribing a cluster.
+ * Specify a value of zero for no limit, which is the default for compatibility reasons.
+ */
+ public static final Setting DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING = Setting.doubleSetting(
+ "cluster.filecache.remote_data_ratio",
+ 0.0,
+ 0.0,
+ Setting.Property.NodeScope,
+ Setting.Property.Dynamic
+ );
public FileCache(SegmentedCache cache, CircuitBreaker circuitBreaker) {
this.theCache = cache;
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java
index f23e057196096..9e8b2ee7d1938 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java
@@ -8,7 +8,7 @@
package org.opensearch.index.store.remote.filecache;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.cache.RemovalReason;
import org.opensearch.index.store.remote.utils.cache.SegmentedCache;
diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java
index 46a81adc1ab45..ebb6202c86ec9 100644
--- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java
@@ -11,7 +11,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java
index 3314b2e8d2c8c..4d515fadb5a13 100644
--- a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java
+++ b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java
@@ -39,6 +39,7 @@
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiTerms;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.memory.MemoryIndex;
import org.opensearch.OpenSearchException;
@@ -50,9 +51,9 @@
import org.opensearch.common.document.DocumentField;
import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.support.XContentMapValues;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.index.engine.Engine;
import org.opensearch.index.get.GetResult;
import org.opensearch.index.mapper.DocumentMapperForType;
@@ -127,7 +128,8 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ
/* or from an existing document */
else if (docIdAndVersion != null) {
// fields with stored term vectors
- termVectorsByField = docIdAndVersion.reader.getTermVectors(docIdAndVersion.docId);
+ TermVectors termVectors = docIdAndVersion.reader.termVectors();
+ termVectorsByField = termVectors.get(docIdAndVersion.docId);
Set selectedFields = request.selectedFields();
// generate tvs for fields where analyzer is overridden
if (selectedFields == null && request.perFieldAnalyzer() != null) {
@@ -322,7 +324,8 @@ private static Fields generateTermVectors(
}
}
/* and read vectors from it */
- return index.createSearcher().getIndexReader().getTermVectors(0);
+ TermVectors termVectors = index.createSearcher().getIndexReader().termVectors();
+ return termVectors.get(0);
}
private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {
@@ -390,13 +393,13 @@ private static ParsedDocument parseDocument(
IndexShard indexShard,
String index,
BytesReference doc,
- XContentType xContentType,
+ MediaType mediaType,
String routing
) {
MapperService mapperService = indexShard.mapperService();
DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate();
ParsedDocument parsedDocument = docMapper.getDocumentMapper()
- .parse(new SourceToParse(index, "_id_for_tv_api", doc, xContentType, routing));
+ .parse(new SourceToParse(index, "_id_for_tv_api", doc, mediaType, routing));
if (docMapper.getMapping() != null) {
parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping());
}
diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java
index 2862accfedc43..e02381c3f97f3 100644
--- a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java
+++ b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java
@@ -32,8 +32,8 @@
package org.opensearch.index.translog;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.index.IndexSettings;
import org.opensearch.core.index.shard.ShardId;
diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java
index c740e29963c4d..935f29a428877 100644
--- a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java
+++ b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java
index dd85f8f1f77ea..f3108444688f8 100644
--- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java
+++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java
@@ -44,7 +44,7 @@
import org.opensearch.common.io.Channels;
import org.opensearch.common.io.DiskIoBufferPool;
import org.opensearch.common.io.stream.ReleasableBytesStreamOutput;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.concurrent.ReleasableLock;
import org.opensearch.common.util.io.IOUtils;
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java
index 974e8af42b939..95424d86fba34 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java
@@ -213,17 +213,18 @@ public void listFoldersAsync(String threadpoolName, Iterable path, Actio
});
}
- public void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener) {
- blobStore.blobContainer((BlobPath) path).listBlobsByPrefixInSortedOrder("", limit, LEXICOGRAPHIC, listener);
+ public void listAllInSortedOrder(Iterable path, String filenamePrefix, int limit, ActionListener> listener) {
+ blobStore.blobContainer((BlobPath) path).listBlobsByPrefixInSortedOrder(filenamePrefix, limit, LEXICOGRAPHIC, listener);
}
public void listAllInSortedOrderAsync(
String threadpoolName,
Iterable path,
+ String filenamePrefix,
int limit,
ActionListener> listener
) {
- threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, limit, listener); });
+ threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, filenamePrefix, limit, listener); });
}
}
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java
index a240fd38cda11..885a2e32e915d 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java
@@ -125,8 +125,14 @@ void uploadBlobs(
*/
InputStream downloadBlob(Iterable path, String fileName) throws IOException;
- void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener);
-
- void listAllInSortedOrderAsync(String threadpoolName, Iterable path, int limit, ActionListener> listener);
+ void listAllInSortedOrder(Iterable path, String filenamePrefix, int limit, ActionListener> listener);
+
+ void listAllInSortedOrderAsync(
+ String threadpoolName,
+ Iterable path,
+ String filenamePrefix,
+ int limit,
+ ActionListener> listener
+ );
}
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
index e2bb5f74df234..850a0c1a6574c 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
@@ -210,7 +210,12 @@ public TranslogTransferMetadata readMetadata() throws IOException {
);
try {
- transferService.listAllInSortedOrder(remoteMetadataTransferPath, 1, latchedActionListener);
+ transferService.listAllInSortedOrder(
+ remoteMetadataTransferPath,
+ TranslogTransferMetadata.METADATA_PREFIX,
+ 1,
+ latchedActionListener
+ );
latch.await();
} catch (InterruptedException e) {
throw new IOException("Exception while reading/downloading metadafile", e);
@@ -367,6 +372,7 @@ public void deleteStaleTranslogMetadataFilesAsync(Runnable onCompletion) {
transferService.listAllInSortedOrderAsync(
ThreadPool.Names.REMOTE_PURGE,
remoteMetadataTransferPath,
+ TranslogTransferMetadata.METADATA_PREFIX,
Integer.MAX_VALUE,
new ActionListener<>() {
@Override
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java
index 75d6549b23f1e..a8b3404d3f2ce 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java
@@ -36,6 +36,8 @@ public class TranslogTransferMetadata {
public static final String METADATA_SEPARATOR = "__";
+ public static final String METADATA_PREFIX = "metadata";
+
static final int BUFFER_SIZE = 4096;
static final int CURRENT_VERSION = 1;
@@ -83,6 +85,7 @@ public String getFileName() {
return String.join(
METADATA_SEPARATOR,
Arrays.asList(
+ METADATA_PREFIX,
RemoteStoreUtils.invertLong(primaryTerm),
RemoteStoreUtils.invertLong(generation),
RemoteStoreUtils.invertLong(createdAt),
diff --git a/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java b/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java
index 9a87f1c26fa29..fc580d8517706 100644
--- a/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java
+++ b/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java
@@ -39,8 +39,8 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.index.engine.Engine;
diff --git a/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java b/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java
index 6ebed17437074..8f7df5157d673 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java
@@ -48,7 +48,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.index.cache.query.QueryCacheStats;
import org.opensearch.core.index.shard.ShardId;
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index c06880db42587..461747a21f3a7 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -49,7 +49,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index b574ffd1006c0..36b937e7df76a 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -60,9 +60,9 @@
import org.opensearch.common.CheckedFunction;
import org.opensearch.common.CheckedSupplier;
import org.opensearch.common.Nullable;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -72,7 +72,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.concurrent.AbstractRefCounted;
@@ -82,12 +82,12 @@
import org.opensearch.common.util.iterable.Iterables;
import org.opensearch.common.util.set.Sets;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.common.lease.Releasable;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
import org.opensearch.core.util.FileSystemUtils;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.env.NodeEnvironment;
@@ -137,7 +137,7 @@
import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory;
import org.opensearch.index.translog.TranslogFactory;
import org.opensearch.index.translog.TranslogStats;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.cluster.IndicesClusterStateService;
import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.opensearch.indices.mapper.MapperRegistry;
@@ -1727,7 +1727,7 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set filterParser = bytes -> {
try (
InputStream inputStream = bytes.streamInput();
- XContentParser parser = XContentFactory.xContentType(inputStream)
+ XContentParser parser = MediaTypeRegistry.xContentType(inputStream)
.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, inputStream)
) {
diff --git a/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java b/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java
index a421253227dbf..32dfb2c1c7529 100644
--- a/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java
+++ b/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java
@@ -32,10 +32,10 @@
package org.opensearch.indices.breaker;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
/**
* Settings for a {@link CircuitBreaker}
diff --git a/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java b/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java
index 40bb4894c7397..34cadd2c828d0 100644
--- a/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java
+++ b/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java
@@ -37,17 +37,20 @@
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.opensearch.common.Booleans;
import org.opensearch.common.breaker.ChildMemoryCircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ReleasableLock;
+import org.opensearch.core.indices.breaker.AllCircuitBreakerStats;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerStats;
import org.opensearch.monitor.jvm.GcNames;
import org.opensearch.monitor.jvm.JvmInfo;
diff --git a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java
index b3fc070d62e58..8f2b6cfde7a34 100644
--- a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java
+++ b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java
@@ -52,7 +52,7 @@
import org.opensearch.cluster.routing.RecoverySource.Type;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Nullable;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
diff --git a/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java b/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java
index dc18097419904..68364d2c2622a 100644
--- a/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java
+++ b/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java
@@ -51,7 +51,7 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.lease.Releasable;
import org.opensearch.core.index.Index;
import org.opensearch.index.fielddata.LeafFieldData;
diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java
index 66b5b3f8b7535..1ff1844d9e6ac 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java
@@ -46,7 +46,7 @@
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.concurrent.FutureUtils;
diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java
index a289c8f8a04b7..0ba57a9ee7f65 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java
@@ -49,7 +49,7 @@
import org.opensearch.common.Nullable;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.CancellableThreads;
import org.opensearch.common.util.concurrent.AbstractRunnable;
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java
index 8d60b44c2c6bd..12e628b9b7e7d 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java
@@ -36,7 +36,7 @@
import org.opensearch.OpenSearchWrapperException;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.index.shard.ShardId;
import java.io.IOException;
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
index 60076f1668af8..a4679a2dbae78 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
@@ -40,8 +40,8 @@
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
/**
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java
index 0b343fb0b0871..f8ba63def6239 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java
@@ -49,7 +49,7 @@
import org.opensearch.common.StopWatch;
import org.opensearch.common.concurrent.GatedCloseable;
import org.opensearch.common.logging.Loggers;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.CancellableThreads;
import org.opensearch.common.util.concurrent.FutureUtils;
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java
index 04bee5586cbab..4f1fb42a421f6 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java
@@ -14,7 +14,7 @@
import org.opensearch.action.ActionListenerResponseHandler;
import org.opensearch.action.support.RetryableAction;
import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.CancellableThreads;
diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
index e0e356f1531e1..4712ae6c18759 100644
--- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
+++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
@@ -139,13 +139,25 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener {
- if (segrepHandler != null) {
- logger.warn("Override handler for allocation id {}", request.getTargetAllocationId());
- cancelHandlers(handler -> handler.getAllocationId().equals(request.getTargetAllocationId()), "cancel due to retry");
- }
- return createTargetHandler(request.getTargetNode(), copyState, request.getTargetAllocationId(), fileChunkWriter);
- });
+ final SegmentReplicationSourceHandler newHandler = createTargetHandler(
+ request.getTargetNode(),
+ copyState,
+ request.getTargetAllocationId(),
+ fileChunkWriter
+ );
+ final SegmentReplicationSourceHandler existingHandler = allocationIdToHandlers.putIfAbsent(
+ request.getTargetAllocationId(),
+ newHandler
+ );
+ // If we are already replicating to this allocation Id, cancel the old and replace with a new execution.
+ // This will clear the old handler & referenced copy state holding an incref'd indexCommit.
+ if (existingHandler != null) {
+ logger.warn("Override handler for allocation id {}", request.getTargetAllocationId());
+ cancelHandlers(handler -> handler.getAllocationId().equals(request.getTargetAllocationId()), "cancel due to retry");
+ assert allocationIdToHandlers.containsKey(request.getTargetAllocationId()) == false;
+ allocationIdToHandlers.put(request.getTargetAllocationId(), newHandler);
+ }
+ assert allocationIdToHandlers.containsKey(request.getTargetAllocationId());
return copyState;
}
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java
index 9d8b30209ab80..8bb2a61e32e2d 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java
@@ -19,7 +19,7 @@
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Nullable;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.Settings;
import org.opensearch.index.IndexService;
import org.opensearch.index.shard.IndexEventListener;
diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java
index da01023ace47c..7549f3450e7f2 100644
--- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java
+++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java
@@ -33,6 +33,10 @@ public class ReplicationCheckpoint implements Writeable, Comparable();
}
- pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType()));
+ pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getMediaType()));
ClusterState.Builder newState = ClusterState.builder(currentState);
newState.metadata(
Metadata.builder(currentState.getMetadata()).putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines)).build()
@@ -487,7 +487,7 @@ void validatePipeline(Map ingestInfos, PutPipelineReq
throw new IllegalStateException("Ingest info is empty");
}
- Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
+ Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getMediaType()).v2();
Pipeline pipeline = Pipeline.create(request.getId(), pipelineConfig, processorFactories, scriptService);
List exceptions = new ArrayList<>();
for (Processor processor : pipeline.flattenAllProcessors()) {
diff --git a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java
index 08a3708ae60ae..04892e4653065 100644
--- a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java
+++ b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java
@@ -32,6 +32,7 @@
package org.opensearch.ingest;
+import org.opensearch.Version;
import org.opensearch.cluster.AbstractDiffable;
import org.opensearch.cluster.Diff;
import org.opensearch.core.ParseField;
@@ -77,7 +78,7 @@ private static class Builder {
private String id;
private BytesReference config;
- private XContentType xContentType;
+ private MediaType mediaType;
void setId(String id) {
this.id = id;
@@ -88,11 +89,11 @@ void setConfig(BytesReference config, MediaType mediaType) {
throw new IllegalArgumentException("PipelineConfiguration does not support media type [" + mediaType.getClass() + "]");
}
this.config = config;
- this.xContentType = XContentType.fromMediaType(mediaType);
+ this.mediaType = mediaType;
}
PipelineConfiguration build() {
- return new PipelineConfiguration(id, config, xContentType);
+ return new PipelineConfiguration(id, config, mediaType);
}
}
@@ -101,16 +102,12 @@ PipelineConfiguration build() {
// and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options)
// also the get pipeline api just directly returns this to the caller
private final BytesReference config;
- private final XContentType xContentType;
+ private final MediaType mediaType;
- public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) {
+ public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) {
this.id = Objects.requireNonNull(id);
this.config = Objects.requireNonNull(config);
- this.xContentType = Objects.requireNonNull(xContentType);
- }
-
- public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) {
- this(id, config, XContentType.fromMediaType(mediaType));
+ this.mediaType = Objects.requireNonNull(mediaType);
}
public String getId() {
@@ -118,12 +115,12 @@ public String getId() {
}
public Map getConfigAsMap() {
- return XContentHelper.convertToMap(config, true, xContentType).v2();
+ return XContentHelper.convertToMap(config, true, mediaType).v2();
}
// pkg-private for tests
- XContentType getXContentType() {
- return xContentType;
+ MediaType getMediaType() {
+ return mediaType;
}
// pkg-private for tests
@@ -141,7 +138,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
public static PipelineConfiguration readFrom(StreamInput in) throws IOException {
- return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class));
+ return new PipelineConfiguration(
+ in.readString(),
+ in.readBytesReference(),
+ in.getVersion().onOrAfter(Version.V_3_0_0) ? in.readMediaType() : in.readEnum(XContentType.class)
+ );
}
public static Diff readDiffFrom(StreamInput in) throws IOException {
@@ -157,7 +158,11 @@ public String toString() {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeBytesReference(config);
- out.writeEnum(xContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
}
@Override
diff --git a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
index 2af0237e800c0..5c904d8a7770d 100644
--- a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
+++ b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java
@@ -86,7 +86,8 @@ public boolean equals(Object obj) {
}
@Override
- public Query rewrite(IndexReader reader) throws IOException {
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ final IndexReader reader = searcher.getIndexReader();
if (Objects.equals(reader.getContext().id(), readerId) == false) {
return new MinDocQuery(minDoc, reader.getContext().id());
}
diff --git a/server/src/main/java/org/opensearch/monitor/MonitorService.java b/server/src/main/java/org/opensearch/monitor/MonitorService.java
index bed638484f7c0..ad02b18366b98 100644
--- a/server/src/main/java/org/opensearch/monitor/MonitorService.java
+++ b/server/src/main/java/org/opensearch/monitor/MonitorService.java
@@ -32,7 +32,7 @@
package org.opensearch.monitor;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.index.store.remote.filecache.FileCache;
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java
index 85a44aa905e08..d8090ada56b93 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java
@@ -38,7 +38,7 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.UUIDs;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
index cfa29436f24a5..114702ff0d351 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java
@@ -37,7 +37,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
index 50d1d981f3c98..e20d84cd9763e 100644
--- a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
+++ b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java
@@ -39,7 +39,7 @@
import org.opensearch.common.SuppressForbidden;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.io.PathUtils;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.env.NodeEnvironment.NodePath;
import org.opensearch.index.store.remote.filecache.FileCache;
diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java
index 7bdc74d9ae574..78379db6e2cff 100644
--- a/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java
+++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java
@@ -34,11 +34,11 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.monitor.jvm.JvmStats.GarbageCollector;
import org.opensearch.threadpool.Scheduler.Cancellable;
diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java
index 1acdf00ca6af0..9eafd80203a87 100644
--- a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java
+++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java
@@ -39,7 +39,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.node.ReportingService;
@@ -171,7 +171,7 @@ public class JvmInfo implements ReportingService.Info {
}
final boolean bundledJdk = Booleans.parseBoolean(System.getProperty("opensearch.bundled_jdk", Boolean.FALSE.toString()));
- final Boolean usingBundledJdk = bundledJdk ? usingBundledJdk() : null;
+ final Boolean usingBundledJdkOrJre = bundledJdk ? usingBundledJdkOrJre() : null;
INSTANCE = new JvmInfo(
JvmPid.getPid(),
@@ -180,7 +180,7 @@ public class JvmInfo implements ReportingService.Info {
runtimeMXBean.getVmVersion(),
runtimeMXBean.getVmVendor(),
bundledJdk,
- usingBundledJdk,
+ usingBundledJdkOrJre,
runtimeMXBean.getStartTime(),
configuredInitialHeapSize,
configuredMaxHeapSize,
@@ -201,7 +201,7 @@ public class JvmInfo implements ReportingService.Info {
}
@SuppressForbidden(reason = "PathUtils#get")
- private static boolean usingBundledJdk() {
+ private static boolean usingBundledJdkOrJre() {
/*
* We are using the bundled JDK if java.home is the jdk sub-directory of our working directory. This is because we always set
* the working directory of Elasticsearch to home, and the bundled JDK is in the jdk sub-directory there.
@@ -211,7 +211,8 @@ private static boolean usingBundledJdk() {
if (Constants.MAC_OS_X) {
return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk.app/Contents/Home").toAbsolutePath());
} else {
- return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath());
+ return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jre").toAbsolutePath())
+ || PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath());
}
}
diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java
index 5451821a565cd..b429de614f250 100644
--- a/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java
+++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/monitor/os/OsStats.java b/server/src/main/java/org/opensearch/monitor/os/OsStats.java
index c684bf10c4a03..697b86c6ba55c 100644
--- a/server/src/main/java/org/opensearch/monitor/os/OsStats.java
+++ b/server/src/main/java/org/opensearch/monitor/os/OsStats.java
@@ -37,7 +37,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java b/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java
index de49c07a5600f..9ab38b41a4648 100644
--- a/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java
+++ b/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index d768165451a5a..d8b51b0184a66 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -38,8 +38,8 @@
import org.opensearch.ExceptionsHelper;
import org.opensearch.common.SetOnce;
import org.opensearch.common.settings.SettingsException;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance;
import org.opensearch.index.IndexModule;
@@ -112,9 +112,9 @@
import org.opensearch.cluster.routing.allocation.DiskThresholdMonitor;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.StopWatch;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.component.Lifecycle;
-import org.opensearch.common.component.LifecycleComponent;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.common.lifecycle.Lifecycle;
+import org.opensearch.common.lifecycle.LifecycleComponent;
import org.opensearch.common.inject.Injector;
import org.opensearch.common.inject.Key;
import org.opensearch.common.inject.Module;
@@ -164,9 +164,9 @@
import org.opensearch.indices.SystemIndices;
import org.opensearch.indices.analysis.AnalysisModule;
import org.opensearch.indices.breaker.BreakerSettings;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.indices.cluster.IndicesClusterStateService;
import org.opensearch.indices.recovery.PeerRecoverySourceService;
import org.opensearch.indices.recovery.PeerRecoveryTargetService;
@@ -428,7 +428,7 @@ protected Node(
Constants.JVM_VERSION
);
if (jvmInfo.getBundledJdk()) {
- logger.info("JVM home [{}], using bundled JDK [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk());
+ logger.info("JVM home [{}], using bundled JDK/JRE [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk());
} else {
logger.info("JVM home [{}]", System.getProperty("java.home"));
deprecationLogger.deprecate(
@@ -941,8 +941,9 @@ protected Node(
clusterModule.getAllocationService(),
metadataCreateIndexService,
metadataIndexUpgradeService,
- clusterService.getClusterSettings(),
- shardLimitValidator
+ shardLimitValidator,
+ indicesService,
+ clusterInfoService::getClusterInfo
);
final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor(
diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java
index 6f4fe1e083ad7..95b3825cd992d 100644
--- a/server/src/main/java/org/opensearch/node/NodeService.java
+++ b/server/src/main/java/org/opensearch/node/NodeService.java
@@ -49,7 +49,7 @@
import org.opensearch.index.IndexingPressureService;
import org.opensearch.index.store.remote.filecache.FileCache;
import org.opensearch.indices.IndicesService;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.ingest.IngestService;
import org.opensearch.monitor.MonitorService;
import org.opensearch.plugins.PluginsService;
diff --git a/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java b/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java
index f52aab5b238b3..ab21d33700d54 100644
--- a/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java
@@ -32,10 +32,10 @@
package org.opensearch.plugins;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.settings.Settings;
import org.opensearch.indices.breaker.BreakerSettings;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
/**
* An extension point for {@link Plugin} implementations to add custom circuit breakers
diff --git a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java
index 8ea8333f4851a..fac29b39ec4c5 100644
--- a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java
@@ -45,7 +45,7 @@
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.http.HttpServerTransport;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.Transport;
import org.opensearch.transport.TransportInterceptor;
diff --git a/server/src/main/java/org/opensearch/plugins/Plugin.java b/server/src/main/java/org/opensearch/plugins/Plugin.java
index 02a46f44ef23b..e8c479a48d82b 100644
--- a/server/src/main/java/org/opensearch/plugins/Plugin.java
+++ b/server/src/main/java/org/opensearch/plugins/Plugin.java
@@ -41,7 +41,7 @@
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.cluster.node.DiscoveryNodeRole;
import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.component.LifecycleComponent;
+import org.opensearch.common.lifecycle.LifecycleComponent;
import org.opensearch.common.inject.Module;
import org.opensearch.core.common.io.stream.NamedWriteable;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
diff --git a/server/src/main/java/org/opensearch/plugins/PluginsService.java b/server/src/main/java/org/opensearch/plugins/PluginsService.java
index 3cd226c357367..67ce664c629ec 100644
--- a/server/src/main/java/org/opensearch/plugins/PluginsService.java
+++ b/server/src/main/java/org/opensearch/plugins/PluginsService.java
@@ -44,7 +44,7 @@
import org.opensearch.action.admin.cluster.node.info.PluginsAndModules;
import org.opensearch.bootstrap.JarHell;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.component.LifecycleComponent;
+import org.opensearch.common.lifecycle.LifecycleComponent;
import org.opensearch.common.inject.Module;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java
index 462d105dc0c68..52e7f374507d4 100644
--- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java
@@ -40,8 +40,8 @@
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.component.Lifecycle;
-import org.opensearch.common.component.LifecycleListener;
+import org.opensearch.common.lifecycle.Lifecycle;
+import org.opensearch.common.lifecycle.LifecycleListener;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.index.snapshots.IndexShardSnapshotStatus;
diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java
index e7f7a1d9c0554..e66f8ddee5678 100644
--- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java
@@ -58,7 +58,7 @@
import org.opensearch.cluster.service.ClusterManagerTaskThrottler;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Strings;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.regex.Regex;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java
index 8a712b0a0c098..683177b3c3639 100644
--- a/server/src/main/java/org/opensearch/repositories/Repository.java
+++ b/server/src/main/java/org/opensearch/repositories/Repository.java
@@ -42,7 +42,7 @@
import org.opensearch.cluster.metadata.RepositoryMetadata;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.common.Nullable;
-import org.opensearch.common.component.LifecycleComponent;
+import org.opensearch.common.lifecycle.LifecycleComponent;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.index.snapshots.IndexShardSnapshotStatus;
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index ae8080da73fa8..70db2e0c0a9bd 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -78,7 +78,7 @@
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.compress.Compressor;
import org.opensearch.common.compress.CompressorFactory;
import org.opensearch.common.compress.CompressorType;
@@ -89,8 +89,8 @@
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index cb3d779ece4a9..a138bfc8a7044 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -43,20 +43,20 @@
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.common.CheckedFunction;
import org.opensearch.common.blobstore.BlobContainer;
-import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.compress.Compressor;
import org.opensearch.common.io.Streams;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.lucene.store.ByteArrayIndexInput;
import org.opensearch.common.lucene.store.IndexOutputOutputStream;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.gateway.CorruptStateException;
import org.opensearch.snapshots.SnapshotInfo;
@@ -185,7 +185,7 @@ public void close() throws IOException {
// in order to write the footer we need to prevent closing the actual index input.
}
};
- XContentBuilder builder = XContentFactory.contentBuilder(
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(
XContentType.SMILE,
compressor.threadLocalOutputStream(indexOutputOutputStream)
)
diff --git a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java
index 0b9989ff64d9c..6971d8ec00ecf 100644
--- a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java
@@ -41,7 +41,7 @@
import org.opensearch.common.blobstore.fs.FsBlobStore;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.env.Environment;
diff --git a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java
index 2fc6ef86df603..11a116e8c858d 100644
--- a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java
+++ b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java
@@ -35,9 +35,9 @@
import org.opensearch.common.io.Streams;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentType;
import java.io.IOException;
import java.io.OutputStream;
@@ -86,43 +86,40 @@ protected AbstractRestChannel(RestRequest request, boolean detailedErrorsEnabled
@Override
public XContentBuilder newBuilder() throws IOException {
- return newBuilder(request.getXContentType(), true);
+ return newBuilder(request.getMediaType(), true);
}
@Override
public XContentBuilder newErrorBuilder() throws IOException {
// Disable filtering when building error responses
- return newBuilder(request.getXContentType(), false);
+ return newBuilder(request.getMediaType(), false);
}
/**
* Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following
- * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format
- * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map
+ * logic. If the request has a format parameter that will be used to attempt to map to an {@link MediaType}. If there is no format
+ * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link MediaType}. If this first attempt to map
* fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls
* back to JSON.
*/
@Override
- public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException {
+ public XContentBuilder newBuilder(@Nullable MediaType requestContentType, boolean useFiltering) throws IOException {
return newBuilder(requestContentType, null, useFiltering);
}
/**
* Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type can be sent as a parameter,
- * through {@code responseContentType} or it can fallback to {@link #newBuilder(XContentType, boolean)} logic if the sent type value
+ * through {@code responseContentType} or it can fallback to {@link #newBuilder(MediaType, boolean)} logic if the sent type value
* is {@code null}.
*/
@Override
- public XContentBuilder newBuilder(
- @Nullable XContentType requestContentType,
- @Nullable XContentType responseContentType,
- boolean useFiltering
- ) throws IOException {
+ public XContentBuilder newBuilder(@Nullable MediaType requestContentType, @Nullable MediaType responseContentType, boolean useFiltering)
+ throws IOException {
if (responseContentType == null) {
// TODO should format vs acceptHeader always be the same, do we allow overriding?
- responseContentType = XContentType.fromFormat(format);
+ responseContentType = MediaType.fromFormat(format);
if (responseContentType == null) {
- responseContentType = XContentType.fromMediaType(acceptHeader);
+ responseContentType = MediaType.fromMediaType(acceptHeader);
}
}
// try to determine the response content type from the media type or the format query string parameter, with the format parameter
@@ -134,7 +131,7 @@ public XContentBuilder newBuilder(
responseContentType = requestContentType;
} else {
// default to JSON output when all else fails
- responseContentType = XContentType.JSON;
+ responseContentType = MediaTypeRegistry.getDefaultMediaType();
}
}
@@ -147,12 +144,7 @@ public XContentBuilder newBuilder(
}
OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput());
- XContentBuilder builder = new XContentBuilder(
- XContentFactory.xContent(responseContentType),
- unclosableOutputStream,
- includes,
- excludes
- );
+ XContentBuilder builder = new XContentBuilder(responseContentType.xContent(), unclosableOutputStream, includes, excludes);
if (pretty) {
builder.prettyPrint().lfAtEnd();
}
diff --git a/server/src/main/java/org/opensearch/rest/RestChannel.java b/server/src/main/java/org/opensearch/rest/RestChannel.java
index 6e9e1559ce104..b8ce3e92e0098 100644
--- a/server/src/main/java/org/opensearch/rest/RestChannel.java
+++ b/server/src/main/java/org/opensearch/rest/RestChannel.java
@@ -34,8 +34,8 @@
import org.opensearch.common.Nullable;
import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentType;
import java.io.IOException;
@@ -50,9 +50,9 @@ public interface RestChannel {
XContentBuilder newErrorBuilder() throws IOException;
- XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException;
+ XContentBuilder newBuilder(@Nullable MediaType mediaType, boolean useFiltering) throws IOException;
- XContentBuilder newBuilder(@Nullable XContentType xContentType, @Nullable XContentType responseContentType, boolean useFiltering)
+ XContentBuilder newBuilder(@Nullable MediaType mediaType, @Nullable MediaType responseContentType, boolean useFiltering)
throws IOException;
BytesStreamOutput bytesOutput();
diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java
index 87d1542102f91..1a8bddc094d26 100644
--- a/server/src/main/java/org/opensearch/rest/RestController.java
+++ b/server/src/main/java/org/opensearch/rest/RestController.java
@@ -38,7 +38,7 @@
import org.opensearch.OpenSearchException;
import org.opensearch.client.node.NodeClient;
import org.opensearch.common.Nullable;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -47,6 +47,7 @@
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.core.common.Strings;
import org.opensearch.core.rest.RestStatus;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.common.xcontent.XContentType;
@@ -56,7 +57,7 @@
import org.opensearch.identity.Subject;
import org.opensearch.identity.tokens.AuthToken;
import org.opensearch.identity.tokens.RestTokenExtractor;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.usage.UsageService;
import java.io.ByteArrayOutputStream;
@@ -282,17 +283,17 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
private void dispatchRequest(RestRequest request, RestChannel channel, RestHandler handler) throws Exception {
final int contentLength = request.content().length();
if (contentLength > 0) {
- final XContentType xContentType = request.getXContentType();
- if (xContentType == null) {
+ final MediaType mediaType = request.getMediaType();
+ if (mediaType == null) {
sendContentTypeErrorMessage(request.getAllHeaderValues("Content-Type"), channel);
return;
}
- if (handler.supportsContentStream() && xContentType != XContentType.JSON && xContentType != XContentType.SMILE) {
+ if (handler.supportsContentStream() && mediaType != XContentType.JSON && mediaType != XContentType.SMILE) {
channel.sendResponse(
BytesRestResponse.createSimpleErrorResponse(
channel,
RestStatus.NOT_ACCEPTABLE,
- "Content-Type [" + xContentType + "] does not support stream parsing. Use JSON or SMILE instead"
+ "Content-Type [" + mediaType + "] does not support stream parsing. Use JSON or SMILE instead"
)
);
return;
@@ -592,14 +593,13 @@ public XContentBuilder newErrorBuilder() throws IOException {
}
@Override
- public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException {
- return delegate.newBuilder(xContentType, useFiltering);
+ public XContentBuilder newBuilder(@Nullable MediaType mediaType, boolean useFiltering) throws IOException {
+ return delegate.newBuilder(mediaType, useFiltering);
}
@Override
- public XContentBuilder newBuilder(XContentType xContentType, XContentType responseContentType, boolean useFiltering)
- throws IOException {
- return delegate.newBuilder(xContentType, responseContentType, useFiltering);
+ public XContentBuilder newBuilder(MediaType mediaType, MediaType responseContentType, boolean useFiltering) throws IOException {
+ return delegate.newBuilder(mediaType, responseContentType, useFiltering);
}
@Override
diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java
index 653720391d6a3..fb497a17cde72 100644
--- a/server/src/main/java/org/opensearch/rest/RestRequest.java
+++ b/server/src/main/java/org/opensearch/rest/RestRequest.java
@@ -41,9 +41,10 @@
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentParser;
@@ -64,7 +65,7 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue;
+import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
import static org.opensearch.common.unit.TimeValue.parseTimeValue;
/**
@@ -84,7 +85,7 @@ public class RestRequest implements ToXContent.Params {
private final Map> headers;
private final String rawPath;
private final Set consumedParams = new HashSet<>();
- private final SetOnce xContentType = new SetOnce<>();
+ private final SetOnce mediaType = new SetOnce<>();
private final HttpChannel httpChannel;
private HttpRequest httpRequest;
@@ -117,14 +118,14 @@ private RestRequest(
HttpChannel httpChannel,
long requestId
) {
- final XContentType xContentType;
+ final MediaType xContentType;
try {
xContentType = parseContentType(headers.get("Content-Type"));
} catch (final IllegalArgumentException e) {
throw new ContentTypeHeaderException(e);
}
if (xContentType != null) {
- this.xContentType.set(xContentType);
+ this.mediaType.set(xContentType);
}
this.xContentRegistry = xContentRegistry;
this.httpRequest = httpRequest;
@@ -295,7 +296,7 @@ protected BytesReference content(final boolean contentConsumed) {
public final BytesReference requiredContent() {
if (hasContent() == false) {
throw new OpenSearchParseException("request body is required");
- } else if (xContentType.get() == null) {
+ } else if (mediaType.get() == null) {
throw new IllegalStateException("unknown content type");
}
return content();
@@ -340,8 +341,8 @@ public final long getRequestId() {
* a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request
*/
@Nullable
- public final XContentType getXContentType() {
- return xContentType.get();
+ public final MediaType getMediaType() {
+ return mediaType.get();
}
public HttpChannel getHttpChannel() {
@@ -486,7 +487,7 @@ public NamedXContentRegistry getXContentRegistry() {
*/
public final XContentParser contentParser() throws IOException {
BytesReference content = requiredContent(); // will throw exception if body or content type missing
- return xContentType.get().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput());
+ return mediaType.get().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput());
}
/**
@@ -514,7 +515,7 @@ public final boolean hasContentOrSourceParam() {
* if you need to handle the absence request content gracefully.
*/
public final XContentParser contentOrSourceParamParser() throws IOException {
- Tuple tuple = contentOrSourceParam();
+ Tuple tuple = contentOrSourceParam();
return tuple.v1().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, tuple.v2().streamInput());
}
@@ -525,12 +526,12 @@ public final XContentParser contentOrSourceParamParser() throws IOException {
*/
public final void withContentOrSourceParamParserOrNull(CheckedConsumer withParser) throws IOException {
if (hasContentOrSourceParam()) {
- Tuple tuple = contentOrSourceParam();
+ Tuple tuple = contentOrSourceParam();
BytesReference content = tuple.v2();
- XContentType xContentType = tuple.v1();
+ MediaType mediaType = tuple.v1();
try (
InputStream stream = content.streamInput();
- XContentParser parser = xContentType.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream)
+ XContentParser parser = mediaType.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream)
) {
withParser.accept(parser);
}
@@ -543,11 +544,11 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer contentOrSourceParam() {
+ public final Tuple contentOrSourceParam() {
if (hasContentOrSourceParam() == false) {
throw new OpenSearchParseException("request body or source parameter is required");
} else if (hasContent()) {
- return new Tuple<>(xContentType.get(), requiredContent());
+ return new Tuple<>(mediaType.get(), requiredContent());
}
String source = param("source");
String typeParam = param("source_content_type");
@@ -555,18 +556,18 @@ public final Tuple contentOrSourceParam() {
throw new IllegalStateException("source and source_content_type parameters are required");
}
BytesArray bytes = new BytesArray(source);
- final XContentType xContentType = parseContentType(Collections.singletonList(typeParam));
- if (xContentType == null) {
+ final MediaType mediaType = parseContentType(Collections.singletonList(typeParam));
+ if (mediaType == null) {
throw new IllegalStateException("Unknown value for source_content_type [" + typeParam + "]");
}
- return new Tuple<>(xContentType, bytes);
+ return new Tuple<>(mediaType, bytes);
}
/**
* Parses the given content type string for the media type. This method currently ignores parameters.
*/
// TODO stop ignoring parameters such as charset...
- public static XContentType parseContentType(List header) {
+ public static MediaType parseContentType(List header) {
if (header == null || header.isEmpty()) {
return null;
} else if (header.size() > 1) {
@@ -580,7 +581,7 @@ public static XContentType parseContentType(List header) {
if (splitMediaType.length == 2
&& TCHAR_PATTERN.matcher(splitMediaType[0]).matches()
&& TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) {
- return XContentType.fromMediaType(elements[0]);
+ return MediaType.fromMediaType(elements[0]);
} else {
throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]");
}
diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
index 28381af769a51..47ab7ae585039 100644
--- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
@@ -35,7 +35,7 @@
import org.opensearch.client.node.NodeClient;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.logging.DeprecationLogger;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.rest.BaseRestHandler;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.action.RestToXContentListener;
@@ -80,10 +80,10 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
String id = request.param("id");
String context = request.param("context");
BytesReference content = request.requiredContent();
- XContentType xContentType = request.getXContentType();
- StoredScriptSource source = StoredScriptSource.parse(content, xContentType);
+ MediaType mediaType = request.getMediaType();
+ StoredScriptSource source = StoredScriptSource.parse(content, mediaType);
- PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, context, content, request.getXContentType(), source);
+ PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, context, content, request.getMediaType(), source);
putRequest.clusterManagerNodeTimeout(request.paramAsTime("cluster_manager_timeout", putRequest.clusterManagerNodeTimeout()));
parseDeprecatedMasterTimeoutParameter(putRequest, request, deprecationLogger, getName());
putRequest.timeout(request.paramAsTime("timeout", putRequest.timeout()));
diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java
index fca6745167bb4..414c82b4a470f 100644
--- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java
@@ -44,6 +44,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
request.paramAsTime("cluster_manager_timeout", restoreRemoteStoreRequest.masterNodeTimeout())
);
restoreRemoteStoreRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false));
+ restoreRemoteStoreRequest.restoreAllShards(request.paramAsBoolean("restore_all_shards", false));
request.applyContentParser(p -> restoreRemoteStoreRequest.source(p.mapOrdered()));
return channel -> client.admin().cluster().restoreRemoteStore(restoreRemoteStoreRequest, new RestToXContentListener<>(channel));
}
diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java
index e7335eee89c5a..d987e8fe63506 100644
--- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java
@@ -76,7 +76,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index"));
if (request.hasContent()) {
- Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2();
+ Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2();
sourceAsMap = prepareMappings(sourceAsMap);
createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE);
}
diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
index 86b0111e4b064..dde9f470161b9 100644
--- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
@@ -88,7 +88,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
putRequest.create(request.paramAsBoolean("create", false));
putRequest.cause(request.param("cause", ""));
- Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2();
+ Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2();
sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap);
putRequest.source(sourceAsMap);
diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java
index fd293dceb4cc1..8fdf000139d89 100644
--- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java
@@ -83,7 +83,7 @@ public String getName() {
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index")));
- Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2();
+ Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2();
if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) {
throw new IllegalArgumentException("Types cannot be provided in put mapping requests");
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java
index 702ffff1c9330..176e37abc0f65 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java
@@ -43,7 +43,7 @@
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.common.Table;
import org.opensearch.common.logging.DeprecationLogger;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.common.Strings;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.RestResponse;
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java
index 52890274d4198..2dfed426f35d6 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java
@@ -14,7 +14,7 @@
import org.opensearch.action.support.IndicesOptions;
import org.opensearch.client.node.NodeClient;
import org.opensearch.common.Table;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentOpenSearchExtension;
import org.opensearch.core.common.Strings;
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java
index a04bac0c30bc9..04bbdeeadc4c4 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java
@@ -37,7 +37,7 @@
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.opensearch.client.node.NodeClient;
import org.opensearch.common.Table;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.RestResponse;
import org.opensearch.rest.action.RestResponseListener;
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java
index 6346e5d23cd34..b54c8955283a2 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java
@@ -49,7 +49,7 @@
import org.opensearch.common.logging.DeprecationLogger;
import org.opensearch.common.network.NetworkAddress;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.http.HttpInfo;
import org.opensearch.index.cache.query.QueryCacheStats;
import org.opensearch.index.cache.request.RequestCacheStats;
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java
index 556928d4685a0..ec44b06318695 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java
@@ -38,12 +38,12 @@
import org.opensearch.common.io.UTF8StreamWriter;
import org.opensearch.core.common.io.stream.BytesStream;
import org.opensearch.common.regex.Regex;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.SizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.rest.BytesRestResponse;
import org.opensearch.rest.RestChannel;
import org.opensearch.rest.RestRequest;
@@ -69,18 +69,18 @@ public class RestTable {
public static RestResponse buildResponse(Table table, RestChannel channel) throws Exception {
RestRequest request = channel.request();
- XContentType xContentType = getXContentType(request);
+ MediaType xContentType = getXContentType(request);
if (xContentType != null) {
return buildXContentBuilder(table, channel);
}
return buildTextPlainResponse(table, channel);
}
- private static XContentType getXContentType(RestRequest request) {
+ private static MediaType getXContentType(RestRequest request) {
if (request.hasParam("format")) {
- return XContentType.fromFormat(request.param("format"));
+ return MediaType.fromFormat(request.param("format"));
}
- return XContentType.fromMediaType(request.header("Accept"));
+ return MediaType.fromMediaType(request.header("Accept"));
}
public static RestResponse buildXContentBuilder(Table table, RestChannel channel) throws Exception {
diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java
index 3fc02db0a8365..b046146707885 100644
--- a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java
@@ -105,7 +105,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
defaultPipeline,
defaultRequireAlias,
allowExplicitIndex,
- request.getXContentType()
+ request.getMediaType()
);
return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel));
diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java
index 7247067755721..ce5e12d6fd163 100644
--- a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java
@@ -38,8 +38,8 @@
import org.opensearch.action.get.GetResponse;
import org.opensearch.client.node.NodeClient;
import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.rest.BaseRestHandler;
import org.opensearch.rest.BytesRestResponse;
import org.opensearch.rest.RestChannel;
@@ -108,10 +108,10 @@ static class RestGetSourceResponseListener extends RestResponseListener sourceTuple = restRequest.contentOrSourceParam();
+ Tuple sourceTuple = restRequest.contentOrSourceParam();
PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1());
request.clusterManagerNodeTimeout(restRequest.paramAsTime("cluster_manager_timeout", request.clusterManagerNodeTimeout()));
parseDeprecatedMasterTimeoutParameter(request, restRequest, deprecationLogger, getName());
diff --git a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java
index ec8cc2c00e65a..1aa2c11e3f05e 100644
--- a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java
@@ -36,7 +36,7 @@
import org.opensearch.client.node.NodeClient;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.rest.BaseRestHandler;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.action.RestToXContentListener;
@@ -75,7 +75,7 @@ public String getName() {
@Override
public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
- Tuple sourceTuple = restRequest.contentOrSourceParam();
+ Tuple sourceTuple = restRequest.contentOrSourceParam();
SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1());
request.setId(restRequest.param("id"));
request.setVerbose(restRequest.paramAsBoolean("verbose", false));
diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java
index 189b1e93e6444..5be5a54edddf5 100644
--- a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java
@@ -45,9 +45,9 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.XContent;
import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.rest.BaseRestHandler;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.action.RestCancellableNodeClient;
@@ -190,7 +190,7 @@ public static void parseMultiLineRequest(
boolean ccsMinimizeRoundtrips = request.paramAsBoolean("ccs_minimize_roundtrips", true);
String routing = request.param("routing");
- final Tuple sourceTuple = request.contentOrSourceParam();
+ final Tuple sourceTuple = request.contentOrSourceParam();
final XContent xContent = sourceTuple.v1().xContent();
final BytesReference data = sourceTuple.v2();
MultiSearchRequest.readMultiLineFormat(
diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java
index 2f398a6bea9ff..73eead1b91d57 100644
--- a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java
@@ -12,7 +12,7 @@
import org.opensearch.client.node.NodeClient;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.rest.BaseRestHandler;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.action.RestToXContentListener;
@@ -40,7 +40,7 @@ public List routes() {
@Override
protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
- Tuple sourceTuple = restRequest.contentOrSourceParam();
+ Tuple sourceTuple = restRequest.contentOrSourceParam();
PutSearchPipelineRequest request = new PutSearchPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1());
request.clusterManagerNodeTimeout(restRequest.paramAsTime("cluster_manager_timeout", request.clusterManagerNodeTimeout()));
request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
diff --git a/server/src/main/java/org/opensearch/script/ScriptCache.java b/server/src/main/java/org/opensearch/script/ScriptCache.java
index 874888b53b978..439953f816cbb 100644
--- a/server/src/main/java/org/opensearch/script/ScriptCache.java
+++ b/server/src/main/java/org/opensearch/script/ScriptCache.java
@@ -34,8 +34,8 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.cache.Cache;
import org.opensearch.common.cache.CacheBuilder;
import org.opensearch.common.cache.RemovalListener;
diff --git a/server/src/main/java/org/opensearch/script/StoredScriptSource.java b/server/src/main/java/org/opensearch/script/StoredScriptSource.java
index 1d6f2b8e4ccea..9d3af9e4c9caf 100644
--- a/server/src/main/java/org/opensearch/script/StoredScriptSource.java
+++ b/server/src/main/java/org/opensearch/script/StoredScriptSource.java
@@ -44,6 +44,7 @@
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.common.logging.DeprecationLogger;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.ObjectParser.ValueType;
@@ -197,71 +198,72 @@ private StoredScriptSource build(boolean ignoreEmpty) {
/**
* This will parse XContent into a {@link StoredScriptSource}. The following formats can be parsed:
- *
+ *
* The simple script format with no compiler options or user-defined params:
- *
+ *
* Example:
* {@code
* {"script": "return Math.log(doc.popularity) * 100;"}
* }
- *
+ *
* The above format requires the lang to be specified using the deprecated stored script namespace
* (as a url parameter during a put request). See {@link ScriptMetadata} for more information about
* the stored script namespaces.
- *
+ *
* The complex script format using the new stored script namespace
* where lang and source are required but options is optional:
- *
+ *
* {@code
* {
- * "script" : {
- * "lang" : "",
- * "source" : "",
- * "options" : {
- * "option0" : "",
- * "option1" : "",
- * ...
- * }
- * }
+ * "script" : {
+ * "lang" : "",
+ * "source" : "",
+ * "options" : {
+ * "option0" : "",
+ * "option1" : "",
+ * ...
* }
* }
- *
+ * }
+ * }
+ *
* Example:
* {@code
* {
- * "script": {
- * "lang" : "painless",
- * "source" : "return Math.log(doc.popularity) * params.multiplier"
- * }
+ * "script": {
+ * "lang" : "painless",
+ * "source" : "return Math.log(doc.popularity) * params.multiplier"
* }
* }
- *
+ * }
+ *
* The use of "source" may also be substituted with "code" for backcompat with 5.3 to 5.5 format. For example:
- *
+ *
* {@code
* {
- * "script" : {
- * "lang" : "",
- * "code" : "",
- * "options" : {
- * "option0" : "",
- * "option1" : "",
- * ...
- * }
- * }
+ * "script" : {
+ * "lang" : "",
+ * "code" : "",
+ * "options" : {
+ * "option0" : "",
+ * "option1" : "",
+ * ...
* }
* }
- *
+ * }
+ * }
+ *
* Note that the "source" parameter can also handle template parsing including from
* a complex JSON object.
*
- * @param content The content from the request to be parsed as described above.
- * @return The parsed {@link StoredScriptSource}.
+ * @param content The content from the request to be parsed as described above.
+ * @param mediaType The media type of the request
+ * @return The parsed {@link StoredScriptSource}.
*/
- public static StoredScriptSource parse(BytesReference content, XContentType xContentType) {
+ public static StoredScriptSource parse(BytesReference content, MediaType mediaType) {
try (
InputStream stream = content.streamInput();
- XContentParser parser = xContentType.xContent()
+ XContentParser parser = mediaType.xContent()
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)
) {
Token token = parser.nextToken();
diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java
index c9c70ed52c376..b244290e8ae74 100644
--- a/server/src/main/java/org/opensearch/search/SearchService.java
+++ b/server/src/main/java/org/opensearch/search/SearchService.java
@@ -53,8 +53,8 @@
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.CheckedSupplier;
import org.opensearch.common.UUIDs;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.lucene.Lucene;
@@ -87,7 +87,7 @@
import org.opensearch.index.shard.SearchOperationListener;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.indices.IndicesService;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
import org.opensearch.node.ResponseCollectorService;
import org.opensearch.script.FieldScript;
@@ -1270,7 +1270,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
context.minimumScore(source.minScore());
}
if (source.profile()) {
- context.setProfilers(new Profilers(context.searcher()));
+ context.setProfilers(new Profilers(context.searcher(), context.isConcurrentSegmentSearchEnabled()));
}
if (source.timeout() != null) {
context.timeout(source.timeout());
diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java
index 1d315980512b4..392c65ce27aea 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java
@@ -34,9 +34,9 @@
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreMode;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.SearchShardTarget;
import org.opensearch.search.aggregations.support.ValuesSourceConfig;
import org.opensearch.search.internal.SearchContext;
diff --git a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java
index 83a94afbe8e02..f1416fddebfa2 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java
@@ -32,7 +32,7 @@
package org.opensearch.search.aggregations;
import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.settings.Setting;
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java
index e5cc3f9dbaabd..7f5804c8b9561 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java
@@ -148,7 +148,6 @@ protected Aggregator createInternal(
: includeExclude.convertToStringFilter(DocValueFormat.RAW, maxRegexLength);
MapStringTermsAggregator.CollectorSource collectorSource = new SignificantTextCollectorSource(
- queryShardContext.lookup().source(),
queryShardContext.bigArrays(),
fieldType,
sourceFieldNames,
@@ -186,13 +185,14 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg
private ObjectArray dupSequenceSpotters;
SignificantTextCollectorSource(
- SourceLookup sourceLookup,
BigArrays bigArrays,
MappedFieldType fieldType,
String[] sourceFieldNames,
boolean filterDuplicateText
) {
- this.sourceLookup = sourceLookup;
+ // Create a new SourceLookup instance per aggregator instead of use the shared one from SearchLookup. This is fine because it
+ // will only be accessed by this Aggregator instance and not anywhere else.
+ this.sourceLookup = new SourceLookup();
this.bigArrays = bigArrays;
this.fieldType = fieldType;
this.sourceFieldNames = sourceFieldNames;
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index 246078e7a8eda..4f6c2c327509d 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -13,7 +13,7 @@
import org.opensearch.ExceptionsHelper;
import org.opensearch.action.search.SearchShardTask;
import org.opensearch.action.search.SearchTask;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.monitor.jvm.JvmStats;
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java
index fbd2155c0ef89..ed2ae0bde86c3 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java
@@ -14,7 +14,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.settings.Setting;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.MovingAverage;
import org.opensearch.monitor.jvm.JvmStats;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java
index b82764c38f747..67c6eeae02271 100644
--- a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java
@@ -52,6 +52,7 @@
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.support.XContentMapValues;
+import org.opensearch.core.xcontent.MediaType;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.fieldvisitor.CustomFieldsVisitor;
import org.opensearch.index.fieldvisitor.FieldsVisitor;
@@ -159,7 +160,7 @@ public void execute(SearchContext context) {
SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) currentReaderContext.reader();
fieldReader = lf.getSequentialStoredFieldsReader()::document;
} else {
- fieldReader = currentReaderContext.reader()::document;
+ fieldReader = currentReaderContext.reader().storedFields()::document;
}
for (FetchSubPhaseProcessor processor : processors) {
processor.setNextReader(currentReaderContext);
@@ -377,7 +378,7 @@ private HitContext prepareNestedHitContext(
String rootId;
Map rootSourceAsMap = null;
- XContentType rootSourceContentType = null;
+ MediaType rootSourceContentType = null;
int nestedDocId = nestedTopDocId - subReaderContext.docBase;
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
index 7a358b7e4b252..2238554a12149 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
@@ -72,7 +72,7 @@ public static List loadFieldValues(
) throws IOException {
if (forceSource == false && fieldType.isStored()) {
CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(singleton(fieldType.name()), false);
- hitContext.reader().document(hitContext.docId(), fieldVisitor);
+ hitContext.reader().storedFields().document(hitContext.docId(), fieldVisitor);
List textsToHighlight = fieldVisitor.fields().get(fieldType.name());
return textsToHighlight != null ? textsToHighlight : Collections.emptyList();
}
diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
index e3ca932eb4699..5384b47cc69ec 100644
--- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java
@@ -343,6 +343,10 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto
}
}
}
+
+ // Note: this is called if collection ran successfully, including the above special cases of
+ // CollectionTerminatedException and TimeExceededException, but no other exception.
+ leafCollector.finish();
}
private Weight wrapWeight(Weight weight) {
diff --git a/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java b/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java
index 007ee49b3e697..47bb8754a5803 100644
--- a/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java
+++ b/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java
@@ -153,7 +153,7 @@ private FieldLookup loadFieldData(String name) {
List values = new ArrayList<>(2);
SingleFieldsVisitor visitor = new SingleFieldsVisitor(data.fieldType(), values);
try {
- reader.document(docId, visitor);
+ reader.storedFields().document(docId, visitor);
} catch (IOException e) {
throw new OpenSearchParseException("failed to load field [{}]", e, name);
}
diff --git a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java
index 1d4a424550c12..00fdca4e143ee 100644
--- a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java
+++ b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java
@@ -75,8 +75,8 @@ public Map source() {
return source;
}
- public XContentType sourceContentType() {
- return XContentType.fromMediaType(sourceContentType);
+ public MediaType sourceContentType() {
+ return sourceContentType;
}
public int docId() {
@@ -140,7 +140,7 @@ public void setSegmentAndDocument(LeafReaderContext context, int docId) {
SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) context.reader();
fieldReader = lf.getSequentialStoredFieldsReader()::document;
} else {
- fieldReader = context.reader()::document;
+ fieldReader = context.reader().storedFields()::document;
}
} catch (IOException e) {
throw new UncheckedIOException(e);
@@ -155,7 +155,7 @@ public void setSource(BytesReference source) {
this.sourceAsBytes = source;
}
- public void setSourceContentType(XContentType sourceContentType) {
+ public void setSourceContentType(MediaType sourceContentType) {
this.sourceContentType = sourceContentType;
}
diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
index 512e400c54784..b4f6549c83390 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java
@@ -8,6 +8,7 @@
package org.opensearch.search.pipeline;
+import org.opensearch.Version;
import org.opensearch.cluster.AbstractDiffable;
import org.opensearch.cluster.Diff;
import org.opensearch.common.Strings;
@@ -56,7 +57,7 @@ private static class Builder {
private String id;
private BytesReference config;
- private XContentType xContentType;
+ private MediaType mediaType;
void setId(String id) {
this.id = id;
@@ -67,11 +68,11 @@ void setConfig(BytesReference config, MediaType mediaType) {
throw new IllegalArgumentException("PipelineConfiguration does not support media type [" + mediaType.getClass() + "]");
}
this.config = config;
- this.xContentType = XContentType.fromMediaType(mediaType);
+ this.mediaType = mediaType;
}
PipelineConfiguration build() {
- return new PipelineConfiguration(id, config, xContentType);
+ return new PipelineConfiguration(id, config, mediaType);
}
}
@@ -80,16 +81,12 @@ PipelineConfiguration build() {
// and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options)
// also the get pipeline api just directly returns this to the caller
private final BytesReference config;
- private final XContentType xContentType;
+ private final MediaType mediaType;
- public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) {
+ public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) {
this.id = Objects.requireNonNull(id);
this.config = Objects.requireNonNull(config);
- this.xContentType = Objects.requireNonNull(xContentType);
- }
-
- public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) {
- this(id, config, XContentType.fromMediaType(mediaType));
+ this.mediaType = Objects.requireNonNull(mediaType);
}
public String getId() {
@@ -97,12 +94,12 @@ public String getId() {
}
public Map getConfigAsMap() {
- return XContentHelper.convertToMap(config, true, xContentType).v2();
+ return XContentHelper.convertToMap(config, true, mediaType).v2();
}
// pkg-private for tests
- XContentType getXContentType() {
- return xContentType;
+ MediaType getMediaType() {
+ return mediaType;
}
// pkg-private for tests
@@ -120,7 +117,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
public static PipelineConfiguration readFrom(StreamInput in) throws IOException {
- return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class));
+ return new PipelineConfiguration(
+ in.readString(),
+ in.readBytesReference(),
+ in.getVersion().onOrAfter(Version.V_3_0_0) ? in.readMediaType() : in.readEnum(XContentType.class)
+ );
}
public static Diff readDiffFrom(StreamInput in) throws IOException {
@@ -136,7 +137,11 @@ public String toString() {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeBytesReference(config);
- out.writeEnum(xContentType);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ mediaType.writeTo(out);
+ } else {
+ out.writeEnum((XContentType) mediaType);
+ }
}
@Override
diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
index 7384b494d71fc..1066d836e5183 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
@@ -259,7 +259,7 @@ static ClusterState innerPut(PutSearchPipelineRequest request, ClusterState curr
} else {
pipelines = new HashMap<>();
}
- pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType()));
+ pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getMediaType()));
ClusterState.Builder newState = ClusterState.builder(currentState);
newState.metadata(
Metadata.builder(currentState.getMetadata())
@@ -273,7 +273,7 @@ void validatePipeline(Map searchPipelineInfos
if (searchPipelineInfos.isEmpty()) {
throw new IllegalStateException("Search pipeline info is empty");
}
- Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
+ Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getMediaType()).v2();
Pipeline pipeline = PipelineWithMetrics.create(
request.getId(),
pipelineConfig,
diff --git a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java
index a29d4f9a0ee20..67ab062c0e3ca 100644
--- a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java
+++ b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java
@@ -50,8 +50,10 @@ public abstract class AbstractProfileBreakdown> {
/**
* The accumulated timings for this query node
*/
- private final Timer[] timings;
- private final T[] timingTypes;
+ protected final Timer[] timings;
+ protected final T[] timingTypes;
+ public static final String TIMING_TYPE_COUNT_SUFFIX = "_count";
+ public static final String TIMING_TYPE_START_TIME_SUFFIX = "_start_time";
/** Sole constructor. */
public AbstractProfileBreakdown(Class clazz) {
@@ -74,17 +76,10 @@ public void setTimer(T timing, Timer timer) {
* Build a timing count breakdown for current instance
*/
public Map toBreakdownMap() {
- return buildBreakdownMap(this);
- }
-
- /**
- * Build a timing count breakdown for arbitrary instance
- */
- protected final Map buildBreakdownMap(AbstractProfileBreakdown breakdown) {
- Map map = new HashMap<>(breakdown.timings.length * 2);
- for (T timingType : breakdown.timingTypes) {
- map.put(timingType.toString(), breakdown.timings[timingType.ordinal()].getApproximateTiming());
- map.put(timingType.toString() + "_count", breakdown.timings[timingType.ordinal()].getCount());
+ Map map = new HashMap<>(this.timings.length * 3);
+ for (T timingType : this.timingTypes) {
+ map.put(timingType.toString(), this.timings[timingType.ordinal()].getApproximateTiming());
+ map.put(timingType + TIMING_TYPE_COUNT_SUFFIX, this.timings[timingType.ordinal()].getCount());
}
return Collections.unmodifiableMap(map);
}
diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java
index 89c3d7504de66..d96db1d2dd8da 100644
--- a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java
+++ b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java
@@ -32,6 +32,7 @@
package org.opensearch.search.profile;
+import org.opensearch.Version;
import org.opensearch.core.ParseField;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,10 @@
import java.io.IOException;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.LinkedHashMap;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
@@ -69,7 +72,13 @@ public final class ProfileResult implements Writeable, ToXContentObject {
static final ParseField BREAKDOWN = new ParseField("breakdown");
static final ParseField DEBUG = new ParseField("debug");
static final ParseField NODE_TIME = new ParseField("time");
+ static final ParseField MAX_SLICE_NODE_TIME = new ParseField("max_slice_time");
+ static final ParseField MIN_SLICE_NODE_TIME = new ParseField("min_slice_time");
+ static final ParseField AVG_SLICE_NODE_TIME = new ParseField("avg_slice_time");
static final ParseField NODE_TIME_RAW = new ParseField("time_in_nanos");
+ static final ParseField MAX_SLICE_NODE_TIME_RAW = new ParseField("max_slice_time_in_nanos");
+ static final ParseField MIN_SLICE_NODE_TIME_RAW = new ParseField("min_slice_time_in_nanos");
+ static final ParseField AVG_SLICE_NODE_TIME_RAW = new ParseField("avg_slice_time_in_nanos");
static final ParseField CHILDREN = new ParseField("children");
private final String type;
@@ -77,6 +86,9 @@ public final class ProfileResult implements Writeable, ToXContentObject {
private final Map breakdown;
private final Map debug;
private final long nodeTime;
+ private Long maxSliceNodeTime;
+ private Long minSliceNodeTime;
+ private Long avgSliceNodeTime;
private final List children;
public ProfileResult(
@@ -86,6 +98,20 @@ public ProfileResult(
Map debug,
long nodeTime,
List children
+ ) {
+ this(type, description, breakdown, debug, nodeTime, children, null, null, null);
+ }
+
+ public ProfileResult(
+ String type,
+ String description,
+ Map breakdown,
+ Map debug,
+ long nodeTime,
+ List children,
+ Long maxSliceNodeTime,
+ Long minSliceNodeTime,
+ Long avgSliceNodeTime
) {
this.type = type;
this.description = description;
@@ -93,6 +119,9 @@ public ProfileResult(
this.debug = debug == null ? Map.of() : debug;
this.children = children == null ? List.of() : children;
this.nodeTime = nodeTime;
+ this.maxSliceNodeTime = maxSliceNodeTime;
+ this.minSliceNodeTime = minSliceNodeTime;
+ this.avgSliceNodeTime = avgSliceNodeTime;
}
/**
@@ -105,6 +134,15 @@ public ProfileResult(StreamInput in) throws IOException {
breakdown = in.readMap(StreamInput::readString, StreamInput::readLong);
debug = in.readMap(StreamInput::readString, StreamInput::readGenericValue);
children = in.readList(ProfileResult::new);
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ this.maxSliceNodeTime = in.readOptionalLong();
+ this.minSliceNodeTime = in.readOptionalLong();
+ this.avgSliceNodeTime = in.readOptionalLong();
+ } else {
+ this.maxSliceNodeTime = null;
+ this.minSliceNodeTime = null;
+ this.avgSliceNodeTime = null;
+ }
}
@Override
@@ -115,6 +153,11 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeMap(breakdown, StreamOutput::writeString, StreamOutput::writeLong);
out.writeMap(debug, StreamOutput::writeString, StreamOutput::writeGenericValue);
out.writeList(children);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ out.writeOptionalLong(maxSliceNodeTime);
+ out.writeOptionalLong(minSliceNodeTime);
+ out.writeOptionalLong(avgSliceNodeTime);
+ }
}
/**
@@ -154,6 +197,18 @@ public long getTime() {
return nodeTime;
}
+ public Long getMaxSliceTime() {
+ return maxSliceNodeTime;
+ }
+
+ public Long getMinSliceTime() {
+ return minSliceNodeTime;
+ }
+
+ public Long getAvgSliceTime() {
+ return avgSliceNodeTime;
+ }
+
/**
* Returns a list of all profiled children queries
*/
@@ -168,9 +223,27 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(DESCRIPTION.getPreferredName(), description);
if (builder.humanReadable()) {
builder.field(NODE_TIME.getPreferredName(), new TimeValue(getTime(), TimeUnit.NANOSECONDS).toString());
+ if (getMaxSliceTime() != null) {
+ builder.field(MAX_SLICE_NODE_TIME.getPreferredName(), new TimeValue(getMaxSliceTime(), TimeUnit.NANOSECONDS).toString());
+ }
+ if (getMinSliceTime() != null) {
+ builder.field(MIN_SLICE_NODE_TIME.getPreferredName(), new TimeValue(getMinSliceTime(), TimeUnit.NANOSECONDS).toString());
+ }
+ if (getAvgSliceTime() != null) {
+ builder.field(AVG_SLICE_NODE_TIME.getPreferredName(), new TimeValue(getAvgSliceTime(), TimeUnit.NANOSECONDS).toString());
+ }
}
builder.field(NODE_TIME_RAW.getPreferredName(), getTime());
- builder.field(BREAKDOWN.getPreferredName(), breakdown);
+ if (getMaxSliceTime() != null) {
+ builder.field(MAX_SLICE_NODE_TIME_RAW.getPreferredName(), getMaxSliceTime());
+ }
+ if (getMinSliceTime() != null) {
+ builder.field(MIN_SLICE_NODE_TIME_RAW.getPreferredName(), getMinSliceTime());
+ }
+ if (getAvgSliceTime() != null) {
+ builder.field(AVG_SLICE_NODE_TIME_RAW.getPreferredName(), getAvgSliceTime());
+ }
+ createBreakdownView(builder);
if (false == debug.isEmpty()) {
builder.field(DEBUG.getPreferredName(), debug);
}
@@ -186,6 +259,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder.endObject();
}
+ private void createBreakdownView(XContentBuilder builder) throws IOException {
+ Map modifiedBreakdown = new LinkedHashMap<>(breakdown);
+ removeStartTimeFields(modifiedBreakdown);
+ builder.field(BREAKDOWN.getPreferredName(), modifiedBreakdown);
+ }
+
+ static void removeStartTimeFields(Map modifiedBreakdown) {
+ Iterator> iterator = modifiedBreakdown.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry entry = iterator.next();
+ if (entry.getKey().endsWith(AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX)) {
+ iterator.remove();
+ }
+ }
+ }
+
private static final InstantiatingObjectParser PARSER;
static {
InstantiatingObjectParser.Builder parser = InstantiatingObjectParser.builder(
@@ -199,6 +288,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
parser.declareObject(optionalConstructorArg(), (p, c) -> p.map(), DEBUG);
parser.declareLong(constructorArg(), NODE_TIME_RAW);
parser.declareObjectArray(optionalConstructorArg(), (p, c) -> fromXContent(p), CHILDREN);
+ parser.declareLong(optionalConstructorArg(), MAX_SLICE_NODE_TIME_RAW);
+ parser.declareLong(optionalConstructorArg(), MIN_SLICE_NODE_TIME_RAW);
+ parser.declareLong(optionalConstructorArg(), AVG_SLICE_NODE_TIME_RAW);
PARSER = parser.build();
}
diff --git a/server/src/main/java/org/opensearch/search/profile/Profilers.java b/server/src/main/java/org/opensearch/search/profile/Profilers.java
index 2bc2f3a5a3920..8e87c7ff4acd4 100644
--- a/server/src/main/java/org/opensearch/search/profile/Profilers.java
+++ b/server/src/main/java/org/opensearch/search/profile/Profilers.java
@@ -34,6 +34,7 @@
import org.opensearch.search.internal.ContextIndexSearcher;
import org.opensearch.search.profile.aggregation.AggregationProfiler;
+import org.opensearch.search.profile.aggregation.ConcurrentAggregationProfiler;
import org.opensearch.search.profile.query.QueryProfiler;
import java.util.ArrayList;
@@ -50,18 +51,20 @@ public final class Profilers {
private final ContextIndexSearcher searcher;
private final List queryProfilers;
private final AggregationProfiler aggProfiler;
+ private final boolean isConcurrentSegmentSearchEnabled;
/** Sole constructor. This {@link Profilers} instance will initially wrap one {@link QueryProfiler}. */
- public Profilers(ContextIndexSearcher searcher) {
+ public Profilers(ContextIndexSearcher searcher, boolean isConcurrentSegmentSearchEnabled) {
this.searcher = searcher;
+ this.isConcurrentSegmentSearchEnabled = isConcurrentSegmentSearchEnabled;
this.queryProfilers = new ArrayList<>();
- this.aggProfiler = new AggregationProfiler();
+ this.aggProfiler = isConcurrentSegmentSearchEnabled ? new ConcurrentAggregationProfiler() : new AggregationProfiler();
addQueryProfiler();
}
/** Switch to a new profile. */
public QueryProfiler addQueryProfiler() {
- QueryProfiler profiler = new QueryProfiler(searcher.getExecutor() != null);
+ QueryProfiler profiler = new QueryProfiler(isConcurrentSegmentSearchEnabled);
searcher.setProfiler(profiler);
queryProfilers.add(profiler);
return profiler;
diff --git a/server/src/main/java/org/opensearch/search/profile/Timer.java b/server/src/main/java/org/opensearch/search/profile/Timer.java
index 231324b4a5598..172762cabeb6a 100644
--- a/server/src/main/java/org/opensearch/search/profile/Timer.java
+++ b/server/src/main/java/org/opensearch/search/profile/Timer.java
@@ -51,7 +51,7 @@
public class Timer {
private boolean doTiming;
- private long timing, count, lastCount, start;
+ private long timing, count, lastCount, start, earliestTimerStartTime;
/** pkg-private for testing */
long nanoTime() {
@@ -71,6 +71,9 @@ public final void start() {
doTiming = (count - lastCount) >= Math.min(lastCount >>> 8, 1024);
if (doTiming) {
start = nanoTime();
+ if (count == 0) {
+ earliestTimerStartTime = start;
+ }
}
count++;
}
@@ -92,6 +95,14 @@ public final long getCount() {
return count;
}
+ /** Return the timer start time in nanoseconds.*/
+ public final long getEarliestTimerStartTime() {
+ if (start != 0) {
+ throw new IllegalStateException("#start call misses a matching #stop call");
+ }
+ return earliestTimerStartTime;
+ }
+
/** Return an approximation of the total time spent between consecutive calls of #start and #stop. */
public final long getApproximateTiming() {
if (start != 0) {
diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java
index 24eccba575e77..d0c67915e6d8d 100644
--- a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java
+++ b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java
@@ -34,6 +34,7 @@
import org.opensearch.search.profile.AbstractProfileBreakdown;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -62,4 +63,18 @@ public void addDebugInfo(String key, Object value) {
protected Map toDebugMap() {
return unmodifiableMap(extra);
}
+
+ /**
+ * Build a timing count startTime breakdown for aggregation timing types
+ */
+ @Override
+ public Map toBreakdownMap() {
+ Map map = new HashMap<>(timings.length * 3);
+ for (AggregationTimingType timingType : timingTypes) {
+ map.put(timingType.toString(), timings[timingType.ordinal()].getApproximateTiming());
+ map.put(timingType + TIMING_TYPE_COUNT_SUFFIX, timings[timingType.ordinal()].getCount());
+ map.put(timingType + TIMING_TYPE_START_TIME_SUFFIX, timings[timingType.ordinal()].getEarliestTimerStartTime());
+ }
+ return Collections.unmodifiableMap(map);
+ }
}
diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java
index 1d2cf424ee5a7..39620c25dc5a3 100644
--- a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java
+++ b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java
@@ -36,8 +36,6 @@
import org.opensearch.search.profile.AbstractProfiler;
import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
import java.util.Map;
/**
@@ -47,29 +45,25 @@
*/
public class AggregationProfiler extends AbstractProfiler {
- private final Map, AggregationProfileBreakdown> profileBreakdownLookup = new HashMap<>();
+ private final Map profileBreakdownLookup = new HashMap<>();
public AggregationProfiler() {
super(new InternalAggregationProfileTree());
}
+ /**
+ * This method does not need to be thread safe for concurrent search use case as well.
+ * The {@link AggregationProfileBreakdown} for each Aggregation operator is created in sync path when
+ * {@link org.opensearch.search.aggregations.BucketCollector#preCollection()} is called
+ * on the Aggregation collector instances during construction.
+ */
@Override
public AggregationProfileBreakdown getQueryBreakdown(Aggregator agg) {
- List path = getAggregatorPath(agg);
- AggregationProfileBreakdown aggregationProfileBreakdown = profileBreakdownLookup.get(path);
+ AggregationProfileBreakdown aggregationProfileBreakdown = profileBreakdownLookup.get(agg);
if (aggregationProfileBreakdown == null) {
aggregationProfileBreakdown = super.getQueryBreakdown(agg);
- profileBreakdownLookup.put(path, aggregationProfileBreakdown);
+ profileBreakdownLookup.put(agg, aggregationProfileBreakdown);
}
return aggregationProfileBreakdown;
}
-
- public static List getAggregatorPath(Aggregator agg) {
- LinkedList path = new LinkedList<>();
- while (agg != null) {
- path.addFirst(agg.name());
- agg = agg.parent();
- }
- return path;
- }
}
diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java b/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java
new file mode 100644
index 0000000000000..7c9bf55a97de5
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java
@@ -0,0 +1,199 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.search.profile.aggregation;
+
+import org.opensearch.search.profile.ProfileResult;
+import org.opensearch.search.profile.AbstractProfileBreakdown;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Main class to profile aggregations with concurrent execution
+ *
+ * @opensearch.internal
+ */
+public class ConcurrentAggregationProfiler extends AggregationProfiler {
+
+ private static final String MAX_PREFIX = "max_";
+ private static final String MIN_PREFIX = "min_";
+ private static final String AVG_PREFIX = "avg_";
+ private static final String START_TIME_KEY = AggregationTimingType.INITIALIZE + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX;
+ private static final String[] breakdownCountStatsTypes = { "build_leaf_collector_count", "collect_count" };
+
+ @Override
+ public List getTree() {
+ List tree = profileTree.getTree();
+ List reducedTree = new LinkedList<>();
+ Map> sliceLevelAggregationMap = getSliceLevelAggregationMap(tree);
+ for (List profileResultsAcrossSlices : sliceLevelAggregationMap.values()) {
+ reducedTree.addAll(reduceProfileResultsTree(profileResultsAcrossSlices));
+ }
+ return reducedTree;
+ }
+
+ private List reduceProfileResultsTree(List profileResultsAcrossSlices) {
+ String type = profileResultsAcrossSlices.get(0).getQueryName();
+ String description = profileResultsAcrossSlices.get(0).getLuceneDescription();
+ long maxSliceNodeEndTime = Long.MIN_VALUE;
+ long minSliceNodeStartTime = Long.MAX_VALUE;
+ long maxSliceNodeTime = Long.MIN_VALUE;
+ long minSliceNodeTime = Long.MAX_VALUE;
+ long avgSliceNodeTime = 0L;
+ Map breakdown = new HashMap<>();
+ Map timeStatsMap = new HashMap<>();
+ Map minSliceStartTimeMap = new HashMap<>();
+ Map maxSliceEndTimeMap = new HashMap<>();
+ Map countStatsMap = new HashMap<>();
+ Map debug = new HashMap<>();
+ List children = new LinkedList<>();
+
+ for (ProfileResult profileResult : profileResultsAcrossSlices) {
+ long profileNodeTime = profileResult.getTime();
+ long sliceStartTime = profileResult.getTimeBreakdown().get(START_TIME_KEY);
+
+ // Profiled total time
+ maxSliceNodeEndTime = Math.max(maxSliceNodeEndTime, sliceStartTime + profileNodeTime);
+ minSliceNodeStartTime = Math.min(minSliceNodeStartTime, sliceStartTime);
+
+ // Profiled total time stats
+ maxSliceNodeTime = Math.max(maxSliceNodeTime, profileNodeTime);
+ minSliceNodeTime = Math.min(minSliceNodeTime, profileNodeTime);
+ avgSliceNodeTime += profileNodeTime;
+
+ // Profiled breakdown time stats
+ for (AggregationTimingType timingType : AggregationTimingType.values()) {
+ buildBreakdownStatsMap(timeStatsMap, profileResult, timingType.toString());
+ }
+
+ // Profiled breakdown total time
+ for (AggregationTimingType timingType : AggregationTimingType.values()) {
+ String breakdownTimingType = timingType.toString();
+ Long startTime = profileResult.getTimeBreakdown()
+ .get(breakdownTimingType + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX);
+ Long endTime = startTime + profileResult.getTimeBreakdown().get(breakdownTimingType);
+ minSliceStartTimeMap.put(
+ breakdownTimingType,
+ Math.min(minSliceStartTimeMap.getOrDefault(breakdownTimingType, Long.MAX_VALUE), startTime)
+ );
+ maxSliceEndTimeMap.put(
+ breakdownTimingType,
+ Math.max(maxSliceEndTimeMap.getOrDefault(breakdownTimingType, Long.MIN_VALUE), endTime)
+ );
+ }
+
+ // Profiled breakdown count stats
+ for (String breakdownCountType : breakdownCountStatsTypes) {
+ buildBreakdownStatsMap(countStatsMap, profileResult, breakdownCountType);
+ }
+
+ // Profiled breakdown count
+ for (AggregationTimingType timingType : AggregationTimingType.values()) {
+ String breakdownType = timingType.toString();
+ String breakdownTypeCount = breakdownType + AbstractProfileBreakdown.TIMING_TYPE_COUNT_SUFFIX;
+ breakdown.put(
+ breakdownTypeCount,
+ breakdown.getOrDefault(breakdownTypeCount, 0L) + profileResult.getTimeBreakdown().get(breakdownTypeCount)
+ );
+ }
+
+ debug = profileResult.getDebugInfo();
+ children.addAll(profileResult.getProfiledChildren());
+ }
+ // nodeTime
+ long nodeTime = maxSliceNodeEndTime - minSliceNodeStartTime;
+ avgSliceNodeTime /= profileResultsAcrossSlices.size();
+
+ // Profiled breakdown time stats
+ for (AggregationTimingType breakdownTimingType : AggregationTimingType.values()) {
+ buildBreakdownMap(profileResultsAcrossSlices.size(), breakdown, timeStatsMap, breakdownTimingType.toString());
+ }
+
+ // Profiled breakdown total time
+ for (AggregationTimingType breakdownTimingType : AggregationTimingType.values()) {
+ String breakdownType = breakdownTimingType.toString();
+ breakdown.put(breakdownType, maxSliceEndTimeMap.get(breakdownType) - minSliceStartTimeMap.get(breakdownType));
+ }
+
+ // Profiled breakdown count stats
+ for (String breakdownCountType : breakdownCountStatsTypes) {
+ buildBreakdownMap(profileResultsAcrossSlices.size(), breakdown, countStatsMap, breakdownCountType);
+ }
+
+ // children
+ List reducedChildrenTree = new LinkedList<>();
+ if (!children.isEmpty()) {
+ Map> sliceLevelAggregationMap = getSliceLevelAggregationMap(children);
+ for (List profileResults : sliceLevelAggregationMap.values()) {
+ reducedChildrenTree.addAll(reduceProfileResultsTree(profileResults));
+ }
+ }
+
+ ProfileResult reducedResult = new ProfileResult(
+ type,
+ description,
+ breakdown,
+ debug,
+ nodeTime,
+ reducedChildrenTree,
+ maxSliceNodeTime,
+ minSliceNodeTime,
+ avgSliceNodeTime
+ );
+ return List.of(reducedResult);
+ }
+
+ static void buildBreakdownMap(int treeSize, Map breakdown, Map statsMap, String breakdownType) {
+ String maxBreakdownType = MAX_PREFIX + breakdownType;
+ String minBreakdownType = MIN_PREFIX + breakdownType;
+ String avgBreakdownType = AVG_PREFIX + breakdownType;
+ breakdown.put(maxBreakdownType, statsMap.get(maxBreakdownType));
+ breakdown.put(minBreakdownType, statsMap.get(minBreakdownType));
+ breakdown.put(avgBreakdownType, statsMap.get(avgBreakdownType) / treeSize);
+ }
+
+ static void buildBreakdownStatsMap(Map statsMap, ProfileResult result, String breakdownType) {
+ String maxBreakdownType = MAX_PREFIX + breakdownType;
+ String minBreakdownType = MIN_PREFIX + breakdownType;
+ String avgBreakdownType = AVG_PREFIX + breakdownType;
+ statsMap.put(
+ maxBreakdownType,
+ Math.max(statsMap.getOrDefault(maxBreakdownType, Long.MIN_VALUE), result.getTimeBreakdown().get(breakdownType))
+ );
+ statsMap.put(
+ minBreakdownType,
+ Math.min(statsMap.getOrDefault(minBreakdownType, Long.MAX_VALUE), result.getTimeBreakdown().get(breakdownType))
+ );
+ statsMap.put(avgBreakdownType, statsMap.getOrDefault(avgBreakdownType, 0L) + result.getTimeBreakdown().get(breakdownType));
+ }
+
+ /**
+ * @return a slice level aggregation map where the key is the description of the aggregation and
+ * the value is a list of ProfileResult across all slices.
+ */
+ static Map> getSliceLevelAggregationMap(List tree) {
+ Map> sliceLevelAggregationMap = new HashMap<>();
+ for (ProfileResult result : tree) {
+ String description = result.getLuceneDescription();
+ final List sliceLevelAggregationList = sliceLevelAggregationMap.computeIfAbsent(
+ description,
+ k -> new LinkedList<>()
+ );
+ sliceLevelAggregationList.add(result);
+ }
+ return sliceLevelAggregationMap;
+ }
+}
diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java b/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java
index 36cfc53f41ccd..34716b87c7c9c 100644
--- a/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java
+++ b/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java
@@ -62,6 +62,10 @@ protected String getTypeFromElement(Aggregator element) {
return element.getClass().getSimpleName();
}
+ /**
+ * @return is used to group aggregations with same name across slices.
+ * So the name returned here should be same across slices for an aggregation operator.
+ */
@Override
protected String getDescriptionFromElement(Aggregator element) {
return element.name();
diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
index 74ef78bc93c5f..6f0c78e8b307d 100644
--- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
+++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java
@@ -44,10 +44,10 @@ public AbstractProfileBreakdown context(Object context) {
@Override
public Map toBreakdownMap() {
- final Map map = new HashMap<>(buildBreakdownMap(this));
+ final Map map = new HashMap<>(super.toBreakdownMap());
for (final AbstractProfileBreakdown context : contexts.values()) {
- for (final Map.Entry entry : buildBreakdownMap(context).entrySet()) {
+ for (final Map.Entry entry : context.toBreakdownMap().entrySet()) {
map.merge(entry.getKey(), entry.getValue(), Long::sum);
}
}
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java
index 2ca642292a7c0..b54f988218ac8 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java
@@ -36,7 +36,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
index c7498635d2553..fabb67d7fb841 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java
@@ -35,6 +35,7 @@
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.CollectionTerminatedException;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.suggest.document.CompletionQuery;
import org.apache.lucene.search.suggest.document.TopSuggestDocs;
@@ -103,17 +104,24 @@ protected Suggest.Suggestion extends Suggest.Suggestion.Entry extends Sugges
}
private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSuggestDocsCollector collector) throws IOException {
- query = (CompletionQuery) query.rewrite(searcher.getIndexReader());
+ query = (CompletionQuery) query.rewrite(searcher);
Weight weight = query.createWeight(searcher, collector.scoreMode(), 1f);
for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
BulkScorer scorer = weight.bulkScorer(context);
if (scorer != null) {
+ LeafCollector leafCollector = null;
try {
- scorer.score(collector.getLeafCollector(context), context.reader().getLiveDocs());
+ leafCollector = collector.getLeafCollector(context);
+ scorer.score(leafCollector, context.reader().getLiveDocs());
} catch (CollectionTerminatedException e) {
// collection was terminated prematurely
// continue with the following leaf
}
+ // Note: this is called if collection ran successfully, including the above special cases of
+ // CollectionTerminatedException and TimeExceededException, but no other exception.
+ if (leafCollector != null) {
+ leafCollector.finish();
+ }
}
}
}
diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java
index 6f3c3f471b47c..1276bc824c05b 100644
--- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java
+++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggestionBuilder.java
@@ -38,11 +38,11 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.unit.Fuzziness;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ObjectParser;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -111,7 +111,7 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder {
// Copy the current structure. We will parse, once the mapping is provided
- XContentBuilder builder = XContentFactory.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE);
builder.copyCurrentStructure(p);
v.contextBytes = BytesReference.bytes(builder);
p.skipChildren();
@@ -216,7 +216,7 @@ public CompletionSuggestionBuilder regex(String regex, RegexOptions regexOptions
public CompletionSuggestionBuilder contexts(Map> queryContexts) {
Objects.requireNonNull(queryContexts, "contexts must not be null");
try {
- XContentBuilder contentBuilder = XContentFactory.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE);
+ XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE);
contentBuilder.startObject();
for (Map.Entry> contextEntry : queryContexts.entrySet()) {
contentBuilder.startArray(contextEntry.getKey());
diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java
index 4b1c43bf7a317..6992af62b6626 100644
--- a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java
+++ b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java
@@ -44,7 +44,7 @@
import org.opensearch.common.lucene.Lucene;
import org.opensearch.core.common.text.Text;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
-import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.query.AbstractQueryBuilder;
import org.opensearch.index.query.ParsedQuery;
@@ -160,7 +160,8 @@ public Suggestion extends Entry extends Option>> innerExecute(
QueryShardContext shardContext = suggestion.getShardContext();
final String querySource = scriptFactory.newInstance(vars).execute();
try (
- XContentParser parser = XContentFactory.xContent(querySource)
+ XContentParser parser = MediaTypeRegistry.xContent(querySource)
+ .xContent()
.createParser(shardContext.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, querySource)
) {
QueryBuilder innerQueryBuilder = AbstractQueryBuilder.parseInnerQueryBuilder(parser);
diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java
index 54d55f67ccdcd..119e632928cc7 100644
--- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java
@@ -41,6 +41,7 @@
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
import org.opensearch.action.support.IndicesOptions;
import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterInfo;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.ClusterStateApplier;
import org.opensearch.cluster.ClusterStateTaskConfig;
@@ -62,12 +63,14 @@
import org.opensearch.cluster.metadata.MetadataIndexUpgradeService;
import org.opensearch.cluster.metadata.RepositoriesMetadata;
import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.routing.IndexShardRoutingTable;
import org.opensearch.cluster.routing.RecoverySource;
import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
import org.opensearch.cluster.routing.RecoverySource.RemoteStoreRecoverySource;
import org.opensearch.cluster.routing.RoutingChangesObserver;
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardsIterator;
import org.opensearch.cluster.routing.UnassignedInfo;
import org.opensearch.cluster.routing.allocation.AllocationService;
import org.opensearch.cluster.service.ClusterManagerTaskKeys;
@@ -86,6 +89,9 @@
import org.opensearch.index.IndexSettings;
import org.opensearch.index.shard.IndexShard;
import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.snapshots.IndexShardSnapshotStatus;
+import org.opensearch.index.store.remote.filecache.FileCacheStats;
+import org.opensearch.indices.IndicesService;
import org.opensearch.indices.ShardLimitValidator;
import org.opensearch.repositories.IndexId;
import org.opensearch.repositories.RepositoriesService;
@@ -103,6 +109,7 @@
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import static java.util.Collections.unmodifiableSet;
@@ -118,6 +125,8 @@
import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
import static org.opensearch.common.util.set.Sets.newHashSet;
import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
+import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING;
+import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING;
import static org.opensearch.snapshots.SnapshotUtils.filterIndices;
/**
@@ -176,6 +185,10 @@ public class RestoreService implements ClusterStateApplier {
private final ClusterSettings clusterSettings;
+ private final IndicesService indicesService;
+
+ private final Supplier clusterInfoSupplier;
+
private final ClusterManagerTaskThrottler.ThrottlingKey restoreSnapshotTaskKey;
private static final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor = new CleanRestoreStateTaskExecutor();
@@ -186,8 +199,9 @@ public RestoreService(
AllocationService allocationService,
MetadataCreateIndexService createIndexService,
MetadataIndexUpgradeService metadataIndexUpgradeService,
- ClusterSettings clusterSettings,
- ShardLimitValidator shardLimitValidator
+ ShardLimitValidator shardLimitValidator,
+ IndicesService indicesService,
+ Supplier clusterInfoSupplier
) {
this.clusterService = clusterService;
this.repositoriesService = repositoriesService;
@@ -199,6 +213,8 @@ public RestoreService(
}
this.clusterSettings = clusterService.getClusterSettings();
this.shardLimitValidator = shardLimitValidator;
+ this.indicesService = indicesService;
+ this.clusterInfoSupplier = clusterInfoSupplier;
// Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
restoreSnapshotTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.RESTORE_SNAPSHOT_KEY, true);
@@ -234,21 +250,34 @@ public ClusterState execute(ClusterState currentState) {
continue;
}
if (currentIndexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false)) {
- if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) {
- throw new IllegalStateException(
- "cannot restore index ["
- + index
- + "] because an open index "
- + "with same name already exists in the cluster. Close the existing index"
- );
+ IndexMetadata updatedIndexMetadata = currentIndexMetadata;
+ Map activeInitializingShards = new HashMap<>();
+ if (request.restoreAllShards()) {
+ if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) {
+ throw new IllegalStateException(
+ "cannot restore index ["
+ + index
+ + "] because an open index "
+ + "with same name already exists in the cluster. Close the existing index"
+ );
+ }
+ updatedIndexMetadata = IndexMetadata.builder(currentIndexMetadata)
+ .state(IndexMetadata.State.OPEN)
+ .version(1 + currentIndexMetadata.getVersion())
+ .mappingVersion(1 + currentIndexMetadata.getMappingVersion())
+ .settingsVersion(1 + currentIndexMetadata.getSettingsVersion())
+ .aliasesVersion(1 + currentIndexMetadata.getAliasesVersion())
+ .build();
+ } else {
+ activeInitializingShards = currentState.routingTable()
+ .index(index)
+ .shards()
+ .values()
+ .stream()
+ .map(IndexShardRoutingTable::primaryShard)
+ .filter(shardRouting -> shardRouting.unassigned() == false)
+ .collect(Collectors.toMap(ShardRouting::shardId, Function.identity()));
}
- IndexMetadata updatedIndexMetadata = IndexMetadata.builder(currentIndexMetadata)
- .state(IndexMetadata.State.OPEN)
- .version(1 + currentIndexMetadata.getVersion())
- .mappingVersion(1 + currentIndexMetadata.getMappingVersion())
- .settingsVersion(1 + currentIndexMetadata.getSettingsVersion())
- .aliasesVersion(1 + currentIndexMetadata.getAliasesVersion())
- .build();
IndexId indexId = new IndexId(index, updatedIndexMetadata.getIndexUUID());
@@ -257,7 +286,7 @@ public ClusterState execute(ClusterState currentState) {
updatedIndexMetadata.getCreationVersion(),
indexId
);
- rtBuilder.addAsRemoteStoreRestore(updatedIndexMetadata, recoverySource);
+ rtBuilder.addAsRemoteStoreRestore(updatedIndexMetadata, recoverySource, activeInitializingShards);
blocks.updateBlocks(updatedIndexMetadata);
mdBuilder.put(updatedIndexMetadata, true);
indicesToBeRestored.add(index);
@@ -401,7 +430,6 @@ public ClusterManagerTaskThrottler.ThrottlingKey getClusterManagerThrottlingKey(
@Override
public ClusterState execute(ClusterState currentState) {
- RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY);
// Check if the snapshot to restore is currently being deleted
SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(
SnapshotDeletionsInProgress.TYPE,
@@ -422,7 +450,9 @@ public ClusterState execute(ClusterState currentState) {
ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
final Map shards;
+ final boolean isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(request.storageType().toString());
Set aliases = new HashSet<>();
+ long totalRestorableRemoteIndexesSize = 0;
if (indices.isEmpty() == false) {
// We have some indices to restore
@@ -433,17 +463,14 @@ public ClusterState execute(ClusterState currentState) {
String index = indexEntry.getValue();
boolean partial = checkPartial(index);
+ IndexId snapshotIndexId = repositoryData.resolveIndexId(index);
IndexMetadata snapshotIndexMetadata = updateIndexSettings(
metadata.index(index),
request.indexSettings(),
request.ignoreIndexSettings()
);
- if (IndexModule.Type.REMOTE_SNAPSHOT.match(request.storageType().toString())) {
- snapshotIndexMetadata = addSnapshotToIndexSettings(
- snapshotIndexMetadata,
- snapshot,
- repositoryData.resolveIndexId(index)
- );
+ if (isRemoteSnapshot) {
+ snapshotIndexMetadata = addSnapshotToIndexSettings(snapshotIndexMetadata, snapshot, snapshotIndexId);
}
final boolean isSearchableSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(
snapshotIndexMetadata.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey())
@@ -469,7 +496,7 @@ public ClusterState execute(ClusterState currentState) {
restoreUUID,
snapshot,
snapshotInfo.version(),
- repositoryData.resolveIndexId(index),
+ snapshotIndexId,
isSearchableSnapshot,
isRemoteStoreShallowCopy,
request.getSourceRemoteStoreRepository()
@@ -588,6 +615,14 @@ public ClusterState execute(ClusterState currentState) {
}
for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) {
+ if (isRemoteSnapshot) {
+ IndexShardSnapshotStatus.Copy shardStatus = repository.getShardSnapshotStatus(
+ snapshotInfo.snapshotId(),
+ snapshotIndexId,
+ new ShardId(metadata.index(index).getIndex(), shard)
+ ).asCopy();
+ totalRestorableRemoteIndexesSize += shardStatus.getTotalSize();
+ }
if (!ignoreShards.contains(shard)) {
shardsBuilder.put(
new ShardId(renamedIndex, shard),
@@ -624,6 +659,9 @@ public ClusterState execute(ClusterState currentState) {
}
checkAliasNameConflicts(indices, aliases);
+ if (isRemoteSnapshot) {
+ validateSearchableSnapshotRestorable(totalRestorableRemoteIndexesSize);
+ }
Map updatedDataStreams = new HashMap<>(currentState.metadata().dataStreams());
updatedDataStreams.putAll(
@@ -823,6 +861,45 @@ private IndexMetadata updateIndexSettings(
return builder.settings(settingsBuilder).build();
}
+ private void validateSearchableSnapshotRestorable(long totalRestorableRemoteIndexesSize) {
+ ClusterInfo clusterInfo = clusterInfoSupplier.get();
+ double remoteDataToFileCacheRatio = DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.get(clusterService.getSettings());
+ Map nodeFileCacheStats = clusterInfo.getNodeFileCacheStats();
+ if (nodeFileCacheStats.isEmpty() || remoteDataToFileCacheRatio <= 0.01f) {
+ return;
+ }
+
+ long totalNodeFileCacheSize = clusterInfo.getNodeFileCacheStats()
+ .values()
+ .stream()
+ .map(fileCacheStats -> fileCacheStats.getTotal().getBytes())
+ .mapToLong(Long::longValue)
+ .sum();
+
+ Predicate isRemoteSnapshotShard = shardRouting -> shardRouting.primary()
+ && indicesService.indexService(shardRouting.index()).getIndexSettings().isRemoteSnapshot();
+
+ ShardsIterator shardsIterator = clusterService.state()
+ .routingTable()
+ .allShardsSatisfyingPredicate(isRemoteSnapshotShard);
+
+ long totalRestoredRemoteIndexesSize = shardsIterator.getShardRoutings()
+ .stream()
+ .map(clusterInfo::getShardSize)
+ .mapToLong(Long::longValue)
+ .sum();
+
+ if (totalRestoredRemoteIndexesSize + totalRestorableRemoteIndexesSize > remoteDataToFileCacheRatio
+ * totalNodeFileCacheSize) {
+ throw new SnapshotRestoreException(
+ snapshot,
+ "Size of the indexes to be restored exceeds the file cache bounds. Increase the file cache capacity on the cluster nodes using "
+ + NODE_SEARCH_CACHE_SIZE_SETTING.getKey()
+ + " setting."
+ );
+ }
+ }
+
@Override
public void onFailure(String source, Exception e) {
logger.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e);
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java
index 0fcd3761c98ba..d8f9b8e1b984f 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java
@@ -47,7 +47,7 @@
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.Nullable;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.concurrent.GatedCloseable;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
index cd9c2c4ce325a..2772f061b4c4a 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
@@ -82,7 +82,7 @@
import org.opensearch.common.Strings;
import org.opensearch.common.UUIDs;
import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.regex.Regex;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
diff --git a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java
index 28a09fd80b408..041a5d32b027c 100644
--- a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java
+++ b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java
@@ -36,8 +36,8 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import java.io.IOException;
@@ -76,7 +76,7 @@ public void writeTo(StreamOutput out) throws IOException {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
try (InputStream stream = status.streamInput()) {
- return builder.rawValue(stream, XContentHelper.xContentType(status));
+ return builder.rawValue(stream, MediaTypeRegistry.xContentType(status));
}
}
diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java
index 5b512af56e195..343d4571593a7 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java
@@ -11,7 +11,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.search.SearchShardTask;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.threadpool.Scheduler;
import org.opensearch.threadpool.ThreadPool;
diff --git a/server/src/main/java/org/opensearch/tasks/TaskManager.java b/server/src/main/java/org/opensearch/tasks/TaskManager.java
index bcd3004188bb6..ec59a13dd1999 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskManager.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskManager.java
@@ -50,7 +50,7 @@
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
diff --git a/server/src/main/java/org/opensearch/tasks/TaskResult.java b/server/src/main/java/org/opensearch/tasks/TaskResult.java
index b6006636da2e9..34efbe3ec572b 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskResult.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskResult.java
@@ -42,10 +42,10 @@
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.InstantiatingObjectParser;
import org.opensearch.common.xcontent.ObjectParserHelper;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
@@ -238,7 +238,7 @@ public int hashCode() {
}
private static BytesReference toXContent(Exception error) throws IOException {
- try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
builder.startObject();
OpenSearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, error);
builder.endObject();
diff --git a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java
index 1feb115cb585a..01be4eaaaf732 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java
@@ -54,9 +54,9 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.util.io.Streams;
import org.opensearch.threadpool.ThreadPool;
@@ -169,7 +169,7 @@ private int getTaskResultMappingVersion(IndexMetadata metadata) {
private void doStoreResult(TaskResult taskResult, ActionListener listener) {
IndexRequestBuilder index = client.prepareIndex(TASK_INDEX).setId(taskResult.getTask().getTaskId().toString());
- try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
taskResult.toXContent(builder, ToXContent.EMPTY_PARAMS);
index.setSource(builder);
} catch (IOException e) {
diff --git a/server/src/main/java/org/opensearch/transport/InboundAggregator.java b/server/src/main/java/org/opensearch/transport/InboundAggregator.java
index 60d45c8b3ad2b..060ccec3563e2 100644
--- a/server/src/main/java/org/opensearch/transport/InboundAggregator.java
+++ b/server/src/main/java/org/opensearch/transport/InboundAggregator.java
@@ -32,8 +32,8 @@
package org.opensearch.transport;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.bytes.CompositeBytesReference;
diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java
index 4a4f928860819..9103692e358e4 100644
--- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java
+++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java
@@ -33,7 +33,7 @@
package org.opensearch.transport;
import org.opensearch.Version;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.bytes.CompositeBytesReference;
import org.opensearch.common.bytes.ReleasableBytesReference;
import org.opensearch.common.util.PageCacheRecycler;
diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java
index 75676307f4026..8733cb5fe7e8e 100644
--- a/server/src/main/java/org/opensearch/transport/TcpTransport.java
+++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java
@@ -40,11 +40,11 @@
import org.opensearch.action.support.ThreadedActionListener;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.common.Booleans;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.component.AbstractLifecycleComponent;
-import org.opensearch.common.component.Lifecycle;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.Lifecycle;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.metrics.MeanMetric;
@@ -57,14 +57,14 @@
import org.opensearch.common.transport.BoundTransportAddress;
import org.opensearch.common.transport.PortsRange;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
import org.opensearch.common.util.concurrent.CountDown;
import org.opensearch.core.common.Strings;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.monitor.jvm.JvmInfo;
import org.opensearch.node.Node;
import org.opensearch.core.rest.RestStatus;
diff --git a/server/src/main/java/org/opensearch/transport/Transport.java b/server/src/main/java/org/opensearch/transport/Transport.java
index fbf6b3ef350d8..3bf855f847685 100644
--- a/server/src/main/java/org/opensearch/transport/Transport.java
+++ b/server/src/main/java/org/opensearch/transport/Transport.java
@@ -36,7 +36,7 @@
import org.opensearch.action.ActionListener;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.common.collect.MapBuilder;
-import org.opensearch.common.component.LifecycleComponent;
+import org.opensearch.common.lifecycle.LifecycleComponent;
import org.opensearch.common.transport.BoundTransportAddress;
import org.opensearch.common.transport.TransportAddress;
import org.opensearch.common.unit.TimeValue;
diff --git a/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java b/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java
index e48267d943739..5b542e4080641 100644
--- a/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java
+++ b/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java
@@ -37,7 +37,7 @@
import org.opensearch.action.ActionListener;
import org.opensearch.common.AsyncBiFunction;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.component.Lifecycle;
+import org.opensearch.common.lifecycle.Lifecycle;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.common.unit.TimeValue;
diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java
index c2163ae1d490a..3b59c99c03d3b 100644
--- a/server/src/main/java/org/opensearch/transport/TransportService.java
+++ b/server/src/main/java/org/opensearch/transport/TransportService.java
@@ -43,7 +43,7 @@
import org.opensearch.cluster.ClusterName;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.common.Nullable;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Streamables;
@@ -169,6 +169,7 @@ public void close() {}
* over the {@link StreamOutput} and {@link StreamInput} wire
*/
Streamables.registerStreamables();
+ /** Registers OpenSearch server specific exceptions (exceptions outside of core library) */
OpenSearchServerException.registerExceptions();
}
diff --git a/server/src/main/java/org/opensearch/transport/TransportSettings.java b/server/src/main/java/org/opensearch/transport/TransportSettings.java
index b4d72ab40409b..05887f6f6d131 100644
--- a/server/src/main/java/org/opensearch/transport/TransportSettings.java
+++ b/server/src/main/java/org/opensearch/transport/TransportSettings.java
@@ -35,7 +35,7 @@
import org.opensearch.common.network.NetworkService;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import java.util.Arrays;
diff --git a/server/src/main/java/org/opensearch/transport/TransportStats.java b/server/src/main/java/org/opensearch/transport/TransportStats.java
index 29544754a5729..e3c4773f4a472 100644
--- a/server/src/main/java/org/opensearch/transport/TransportStats.java
+++ b/server/src/main/java/org/opensearch/transport/TransportStats.java
@@ -35,7 +35,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
index 2656810af0379..629fe9ebd4b99 100644
--- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java
@@ -64,8 +64,8 @@
import org.opensearch.core.common.ParsingException;
import org.opensearch.common.Strings;
import org.opensearch.common.UUIDs;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -73,7 +73,7 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.common.transport.TransportAddress;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.CancellableThreadsTests;
import org.opensearch.common.util.set.Sets;
import org.opensearch.common.xcontent.XContentType;
@@ -852,7 +852,7 @@ public void testIds() {
ids.put(130, org.opensearch.action.NoShardAvailableActionException.class);
ids.put(131, org.opensearch.action.UnavailableShardsException.class);
ids.put(132, org.opensearch.index.engine.FlushFailedEngineException.class);
- ids.put(133, org.opensearch.common.breaker.CircuitBreakingException.class);
+ ids.put(133, org.opensearch.core.common.breaker.CircuitBreakingException.class);
ids.put(134, org.opensearch.transport.NodeNotConnectedException.class);
ids.put(135, org.opensearch.index.mapper.StrictDynamicMappingException.class);
ids.put(136, org.opensearch.action.support.replication.TransportReplicationAction.RetryOnReplicaException.class);
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index cbf7032b50ca5..32f27f3a60e42 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -42,8 +42,8 @@
import org.opensearch.cluster.coordination.PendingClusterStateStats;
import org.opensearch.cluster.coordination.PublishClusterStateStats;
import org.opensearch.http.HttpStats;
-import org.opensearch.indices.breaker.AllCircuitBreakerStats;
-import org.opensearch.indices.breaker.CircuitBreakerStats;
+import org.opensearch.core.indices.breaker.AllCircuitBreakerStats;
+import org.opensearch.core.indices.breaker.CircuitBreakerStats;
import org.opensearch.ingest.IngestStats;
import org.opensearch.monitor.fs.FsInfo;
import org.opensearch.monitor.jvm.JvmStats;
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java
index 96f2365412e7b..485aae433aa16 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java
@@ -672,8 +672,8 @@ private void assertTasksRequestFinishedSuccessfully(NodesResponse nodesResponse,
}
private void assertMemoryUsageWithinLimits(long actual, long expected) {
- // 5% buffer up to 200 KB to account for classloading overhead.
- long maxOverhead = Math.min(200000, expected * 5 / 100);
+ // 5% buffer up to 500 KB to account for classloading overhead.
+ long maxOverhead = Math.min(500000, expected * 5 / 100);
assertThat(actual, lessThanOrEqualTo(expected + maxOverhead));
}
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index abbaf3a3f7b96..e0676cc9ddbdd 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -56,7 +56,7 @@
import org.opensearch.common.transport.BoundTransportAddress;
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.common.lease.Releasable;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.tasks.TaskCancellationService;
import org.opensearch.tasks.TaskManager;
import org.opensearch.tasks.TaskResourceTrackingService;
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index 6885278b08413..c463f7a8f4b8c 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -56,9 +56,9 @@
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.common.settings.Settings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.tasks.Task;
@@ -761,7 +761,7 @@ public void testTasksToXContentGrouping() throws Exception {
}
private Map serialize(ListTasksResponse response, boolean byParents) throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject();
if (byParents) {
DiscoveryNodes nodes = testNodes[0].clusterService.state().nodes();
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java
index 81d7074977253..2edfa23286658 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java
@@ -38,6 +38,7 @@ private RestoreRemoteStoreRequest randomState(RestoreRemoteStoreRequest instance
}
instance.waitForCompletion(randomBoolean());
+ instance.restoreAllShards(randomBoolean());
if (randomBoolean()) {
instance.masterNodeTimeout(randomTimeValue());
@@ -76,6 +77,7 @@ public void testSource() throws IOException {
RestoreRemoteStoreRequest processed = new RestoreRemoteStoreRequest();
processed.masterNodeTimeout(original.masterNodeTimeout());
processed.waitForCompletion(original.waitForCompletion());
+ processed.restoreAllShards(original.restoreAllShards());
processed.source(map);
assertEquals(original, processed);
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
index 6ba7961ccecf3..c9b57d0c41b45 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
@@ -46,10 +46,10 @@
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.network.NetworkModule;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.action.admin.cluster.RestClusterRerouteAction;
@@ -212,7 +212,7 @@ private ClusterRerouteRequest roundTripThroughRestRequest(ClusterRerouteRequest
private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException {
Map params = new HashMap<>();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
boolean hasBody = false;
if (randomBoolean()) {
builder.prettyPrint();
@@ -244,7 +244,7 @@ private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOExcep
FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
requestBuilder.withParams(params);
if (hasBody) {
- requestBuilder.withContent(BytesReference.bytes(builder), XContentType.fromMediaType(builder.contentType()));
+ requestBuilder.withContent(BytesReference.bytes(builder), builder.contentType());
}
return requestBuilder.build();
}
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
index 5c20b3b262730..82b2cfb2e3e51 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java
@@ -112,6 +112,10 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) {
instance.snapshotUuid(randomBoolean() ? null : randomAlphaOfLength(10));
}
+ instance.storageType(
+ randomBoolean() ? RestoreSnapshotRequest.StorageType.LOCAL : RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT
+ );
+
if (randomBoolean()) {
instance.setSourceRemoteStoreRepository(randomAlphaOfLengthBetween(5, 10));
}
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
index 2eb6fa504baf2..cfdd776e60832 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
@@ -56,13 +56,13 @@ public void testSerialization() throws IOException {
new StoredScriptSource("foo", "bar", Collections.emptyMap())
);
- assertEquals(XContentType.JSON, storedScriptRequest.xContentType());
+ assertEquals(XContentType.JSON, storedScriptRequest.mediaType());
try (BytesStreamOutput output = new BytesStreamOutput()) {
storedScriptRequest.writeTo(output);
try (StreamInput in = output.bytes().streamInput()) {
PutStoredScriptRequest serialized = new PutStoredScriptRequest(in);
- assertEquals(XContentType.JSON, serialized.xContentType());
+ assertEquals(XContentType.JSON, serialized.mediaType());
assertEquals(storedScriptRequest.id(), serialized.id());
assertEquals(storedScriptRequest.context(), serialized.context());
}
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java
index 265f02304ce00..2c4462b964d65 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java
@@ -37,9 +37,9 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -195,7 +195,7 @@ public void testParseAdd() throws IOException {
if (filter == null || filter.isEmpty()) {
assertNull(action.filter());
} else {
- assertEquals(Strings.toString(XContentFactory.contentBuilder(XContentType.JSON).map(filter)), action.filter());
+ assertEquals(Strings.toString(MediaTypeRegistry.contentBuilder(XContentType.JSON).map(filter)), action.filter());
}
assertEquals(Objects.toString(searchRouting, null), action.searchRouting());
assertEquals(Objects.toString(indexRouting, null), action.indexRouting());
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java
index 030ee6649bba5..f370528e6f770 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java
@@ -17,7 +17,7 @@
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java
index e4114a622602c..c8a24fd0482cf 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java
@@ -38,9 +38,9 @@
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.json.JsonXContent;
@@ -101,7 +101,7 @@ public void testMappingKeyedByType() throws IOException {
CreateIndexRequest request1 = new CreateIndexRequest("foo");
CreateIndexRequest request2 = new CreateIndexRequest("bar");
{
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
builder.startObject()
.startObject("properties")
.startObject("field1")
@@ -117,7 +117,7 @@ public void testMappingKeyedByType() throws IOException {
.endObject()
.endObject();
request1.mapping(builder);
- builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
builder.startObject()
.startObject(MapperService.SINGLE_MAPPING_NAME)
.startObject("properties")
@@ -140,7 +140,7 @@ public void testMappingKeyedByType() throws IOException {
}
public void testSettingsType() throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
builder.startObject().startArray("settings").endArray().endObject();
CreateIndexRequest parsedCreateIndexRequest = new CreateIndexRequest();
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java
index 5e4e2346a6e5b..c83263e4e22c6 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java
@@ -35,7 +35,7 @@
import org.opensearch.OpenSearchException;
import org.opensearch.core.action.support.DefaultShardOperationFailedException;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.test.AbstractWireSerializingTestCase;
import java.util.ArrayList;
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java
index db61cac534f89..b4512595a1dfe 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java
@@ -32,8 +32,8 @@
package org.opensearch.action.admin.indices.rollover;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.EqualsHashCodeTestUtils;
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java
index 4b23a68f8b383..f65d39583336b 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java
@@ -40,9 +40,10 @@
import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentHelper;
@@ -179,7 +180,7 @@ public void testSerialize() throws Exception {
public void testUnknownFields() throws IOException {
final RolloverRequest request = new RolloverRequest();
XContentType xContentType = randomFrom(XContentType.values());
- final XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+ final XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType);
builder.startObject();
{
builder.startObject("conditions");
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java
index 233e6166d1caf..2406952af6a3b 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java
@@ -33,7 +33,7 @@
package org.opensearch.action.admin.indices.rollover;
import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.test.AbstractSerializingTestCase;
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 65d7c0fc62629..76ddb9facd192 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -59,8 +59,8 @@
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.UUIDs;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.set.Sets;
import org.opensearch.index.cache.query.QueryCacheStats;
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java
index 3d21af584ea04..05922a6c7419b 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java
@@ -39,7 +39,7 @@
import org.opensearch.common.Strings;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.RandomCreateIndexGenerator;
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java
index ef49820192e9b..848df5f8e4979 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java
@@ -52,7 +52,7 @@
import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.index.shard.DocsStats;
import org.opensearch.index.store.StoreStats;
import org.opensearch.snapshots.EmptySnapshotsInfoService;
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
index dbca883506b0d..f069ff3bf7ff2 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
@@ -35,6 +35,7 @@
import org.opensearch.action.admin.indices.alias.Alias;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.common.settings.Settings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
@@ -76,7 +77,7 @@ public void testMappingKeyedByType() throws IOException {
PutIndexTemplateRequest request1 = new PutIndexTemplateRequest("foo");
PutIndexTemplateRequest request2 = new PutIndexTemplateRequest("bar");
{
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
builder.startObject()
.startObject("properties")
.startObject("field1")
@@ -92,7 +93,7 @@ public void testMappingKeyedByType() throws IOException {
.endObject()
.endObject();
request1.mapping(builder);
- builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
builder.startObject()
.startObject("properties")
.startObject("field1")
diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java
index 40d0df61cbb9f..0111c8d6e3132 100644
--- a/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java
@@ -38,8 +38,8 @@
import org.opensearch.action.ActionListener;
import org.opensearch.action.index.IndexRequest;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.common.xcontent.XContentType;
diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java
index c674be8dbba38..2f2c9b2be9e50 100644
--- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java
@@ -39,14 +39,14 @@
import org.opensearch.action.support.WriteRequest.RefreshPolicy;
import org.opensearch.action.update.UpdateRequest;
import org.opensearch.client.Requests;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.ParsingException;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentHelper;
-import org.opensearch.common.xcontent.XContentType;
import org.opensearch.script.Script;
import org.opensearch.test.OpenSearchTestCase;
@@ -340,7 +340,7 @@ public void testSmileIsSupported() throws IOException {
XContentType xContentType = XContentType.SMILE;
BytesReference data;
try (BytesStreamOutput out = new BytesStreamOutput()) {
- try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) {
builder.startObject();
builder.startObject("index");
builder.field("_index", "index");
@@ -349,7 +349,7 @@ public void testSmileIsSupported() throws IOException {
builder.endObject();
}
out.write(xContentType.xContent().streamSeparator());
- try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) {
builder.startObject();
builder.field("field", "value");
builder.endObject();
@@ -375,7 +375,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException
XContentType xContentType = XContentType.SMILE;
BytesReference data;
try (BytesStreamOutput out = new BytesStreamOutput()) {
- try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) {
builder.startObject();
builder.startObject("update");
builder.field("_index", "index");
@@ -386,7 +386,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException
builder.endObject();
}
out.write(xContentType.xContent().streamSeparator());
- try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) {
builder.startObject();
builder.startObject("doc").endObject();
Map values = new HashMap<>();
diff --git a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java
index 866e13c7faf0e..14f2a5c94bcd0 100644
--- a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java
@@ -37,9 +37,9 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.document.DocumentField;
import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.get.GetResult;
@@ -114,7 +114,7 @@ public void testToXContent() throws IOException {
);
ExplainResponse response = new ExplainResponse(index, id, exist, explanation, getResult);
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
String generatedResponse = BytesReference.bytes(builder).utf8ToString().replaceAll("\\s+", "");
diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java
index 382425ce4e57c..d76ed86ed490e 100644
--- a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java
@@ -34,9 +34,9 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.test.AbstractSerializingTestCase;
@@ -125,7 +125,7 @@ protected Predicate getRandomFieldsExcludeFilter() {
public void testToXContent() throws IOException {
FieldCapabilitiesResponse response = createSimpleResponse();
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
String generatedResponse = BytesReference.bytes(builder).utf8ToString();
diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java
index 5498d685dbb6c..cc1e963bc110d 100644
--- a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java
@@ -126,7 +126,7 @@ public void testXContentSerialization() throws IOException {
MultiGetRequest expected = createTestInstance();
XContentType xContentType = randomFrom(XContentType.values());
BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false);
- try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) {
+ try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) {
MultiGetRequest actual = new MultiGetRequest();
actual.add(null, null, null, null, parser, true);
assertThat(parser.nextToken(), nullValue());
diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java
index 71ada43bef116..c2a5f2def31ab 100644
--- a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java
@@ -33,7 +33,6 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.get.GetResult;
@@ -53,7 +52,7 @@ public void testFromXContent() throws IOException {
XContentType xContentType = randomFrom(XContentType.values());
BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false);
MultiGetResponse parsed;
- try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) {
+ try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) {
parsed = MultiGetResponse.fromXContent(parser);
assertNull(parser.nextToken());
}
diff --git a/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java b/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java
index bda9a49bf797b..c4d3b17de1a5a 100644
--- a/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java
@@ -39,7 +39,7 @@
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.VersionType;
import org.opensearch.index.seqno.SequenceNumbers;
diff --git a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java
index a86257fd741c0..336ec67546dc5 100644
--- a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java
@@ -49,14 +49,14 @@ public class PutPipelineRequestTests extends OpenSearchTestCase {
public void testSerializationWithXContent() throws IOException {
PutPipelineRequest request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON);
- assertEquals(XContentType.JSON, request.getXContentType());
+ assertEquals(XContentType.JSON, request.getMediaType());
BytesStreamOutput output = new BytesStreamOutput();
request.writeTo(output);
StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes);
PutPipelineRequest serialized = new PutPipelineRequest(in);
- assertEquals(XContentType.JSON, serialized.getXContentType());
+ assertEquals(XContentType.JSON, serialized.getMediaType());
assertEquals("{}", serialized.getSource().utf8ToString());
}
diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
index 61dd2ac8c14ae..2e50c6225577d 100644
--- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
+++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
@@ -64,7 +64,7 @@
import org.opensearch.index.translog.Translog;
import org.opensearch.indices.IndicesService;
import org.opensearch.indices.SystemIndices;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.tasks.Task;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.transport.MockTransportService;
diff --git a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java
index db371eb9467ba..afb342967bd69 100644
--- a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java
@@ -38,8 +38,8 @@
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.tests.store.MockDirectoryWrapper;
import org.opensearch.action.OriginalIndices;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.common.util.concurrent.AtomicArray;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
diff --git a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java
index 24a018d05a16a..bf18c22f9b7ee 100644
--- a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java
@@ -37,8 +37,8 @@
import org.apache.lucene.tests.store.MockDirectoryWrapper;
import org.opensearch.action.OriginalIndices;
import org.opensearch.common.UUIDs;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
import org.opensearch.core.index.shard.ShardId;
diff --git a/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java b/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java
index 27d04cba204fb..794554c1ec013 100644
--- a/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java
+++ b/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java
@@ -36,8 +36,8 @@
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.opensearch.action.OriginalIndices;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java
index 93f9f69e68de0..dc946c7963a17 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java
@@ -46,9 +46,9 @@
import org.apache.lucene.util.BytesRef;
import org.opensearch.action.OriginalIndices;
import org.opensearch.common.UUIDs;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java
index 1b67d805fe814..d0a1307d33235 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -41,8 +41,8 @@
import org.opensearch.action.OriginalIndices;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.routing.GroupShardsIterator;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
diff --git a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java
index 0e25091d845b5..fafd34602c310 100644
--- a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java
+++ b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java
@@ -38,10 +38,10 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent.MapParams;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.test.OpenSearchTestCase;
@@ -422,7 +422,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException {
final boolean allowNoIndices = randomBoolean();
BytesReference xContentBytes;
- try (XContentBuilder builder = XContentFactory.contentBuilder(type)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) {
builder.startObject();
builder.field("expand_wildcards", "all");
builder.field("ignore_unavailable", ignoreUnavailable);
@@ -441,7 +441,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException {
assertTrue(fromXContentOptions.expandWildcardsHidden());
assertTrue(fromXContentOptions.expandWildcardsOpen());
- try (XContentBuilder builder = XContentFactory.contentBuilder(type)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) {
builder.startObject();
builder.field("expand_wildcards", "none");
builder.field("ignore_unavailable", ignoreUnavailable);
@@ -461,7 +461,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException {
}
private BytesReference toXContentBytes(IndicesOptions indicesOptions, XContentType type) throws IOException {
- try (XContentBuilder builder = XContentFactory.contentBuilder(type)) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) {
builder.startObject();
indicesOptions.toXContent(builder, new MapParams(Collections.emptyMap()));
builder.endObject();
diff --git a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java
index 330d43e791555..9c02bacbadda2 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java
@@ -58,8 +58,8 @@
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.indices.breaker.CircuitBreakerService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.tasks.Task;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java
index 78081a8f83ce3..21d14fb31059d 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java
@@ -49,8 +49,8 @@
import org.opensearch.cluster.routing.IndexShardRoutingTable;
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.cluster.routing.ShardRoutingState;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.transport.TransportAddress;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java
index 5e32193415bd8..46af25c49f262 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java
@@ -35,8 +35,8 @@
import org.opensearch.OpenSearchException;
import org.opensearch.action.support.replication.ReplicationResponse.ShardInfo;
import org.opensearch.common.Strings;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.collect.Tuple;
import org.opensearch.core.xcontent.ToXContent;
diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
index 089bcf77afbae..5af358cae18a8 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
@@ -89,7 +89,7 @@
import org.opensearch.index.shard.ShardNotInPrimaryModeException;
import org.opensearch.indices.IndexClosedException;
import org.opensearch.indices.IndicesService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.indices.cluster.ClusterStateChanges;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java
index 511c1555f1159..a284269dc4151 100644
--- a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java
@@ -45,8 +45,8 @@
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.common.CheckedRunnable;
import org.opensearch.common.UUIDs;
-import org.opensearch.common.component.Lifecycle;
-import org.opensearch.common.component.LifecycleListener;
+import org.opensearch.common.lifecycle.Lifecycle;
+import org.opensearch.common.lifecycle.LifecycleListener;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.transport.BoundTransportAddress;
import org.opensearch.common.transport.TransportAddress;
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java
index ea615c7cfc09a..f0459240c2bbe 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java
@@ -43,7 +43,7 @@
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.set.Sets;
import org.opensearch.common.xcontent.XContentType;
diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java
index 6b869ffed7d23..53f1d71947f7c 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java
@@ -49,10 +49,13 @@
import org.junit.Before;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.function.Predicate;
+import static org.mockito.Mockito.mock;
import static org.opensearch.cluster.routing.ShardRoutingState.UNASSIGNED;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@@ -228,6 +231,49 @@ public void testShardsMatchingPredicateCount() {
assertThat(clusterState.routingTable().shardsMatchingPredicateCount(predicate), is(2));
}
+ public void testAllShardsMatchingPredicate() {
+ MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
+ Metadata metadata = Metadata.builder()
+ .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+ .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+ .build();
+ ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+ .metadata(metadata)
+ .routingTable(RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build())
+ .build();
+ clusterState = ClusterState.builder(clusterState)
+ .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
+ .build();
+ clusterState = allocation.reroute(clusterState, "reroute");
+
+ Predicate predicate = s -> s.state() == ShardRoutingState.UNASSIGNED && s.unassignedInfo().isDelayed();
+ assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(predicate).size(), is(0));
+
+ // starting primaries
+ clusterState = startInitializingShardsAndReroute(allocation, clusterState);
+ // starting replicas
+ clusterState = startInitializingShardsAndReroute(allocation, clusterState);
+ // remove node2 and reroute
+ clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build();
+ // make sure both replicas are marked as delayed (i.e. not reallocated)
+ clusterState = allocation.disassociateDeadNodes(clusterState, true, "reroute");
+ assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(predicate).size(), is(2));
+
+ // Verifies true against all shards on the node (active/inactive)
+ assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(shard -> true).size(), is(4));
+ // Verifies false against all shards on the node (active/inactive)
+ assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(shard -> false).size(), is(0));
+ // Verifies against all primary shards on the node
+ assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(ShardRouting::primary).size(), is(2));
+ // Verifies a predicate which tests for inactive replicas
+ assertThat(
+ clusterState.routingTable()
+ .allShardsSatisfyingPredicate(shardRouting -> !shardRouting.primary() && !shardRouting.active())
+ .size(),
+ is(2)
+ );
+ }
+
public void testActivePrimaryShardsGrouped() {
assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], true).size(), is(0));
assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], false).size(), is(0));
@@ -502,13 +548,40 @@ public void testAddAsRemoteStoreRestore() {
Version.CURRENT,
new IndexId(TEST_INDEX_1, "1")
);
- final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore(indexMetadata, remoteStoreRecoverySource)
- .build();
+ final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore(
+ indexMetadata,
+ remoteStoreRecoverySource,
+ new HashMap<>()
+ ).build();
assertTrue(routingTable.hasIndex(TEST_INDEX_1));
assertEquals(this.numberOfShards, routingTable.allShards(TEST_INDEX_1).size());
assertEquals(this.numberOfShards, routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size());
}
+ public void testAddAsRemoteStoreRestoreWithActiveShards() {
+ final IndexMetadata indexMetadata = createIndexMetadata(TEST_INDEX_1).state(IndexMetadata.State.OPEN).build();
+ final RemoteStoreRecoverySource remoteStoreRecoverySource = new RemoteStoreRecoverySource(
+ "restore_uuid",
+ Version.CURRENT,
+ new IndexId(TEST_INDEX_1, "1")
+ );
+ Map activeInitializingShards = new HashMap<>();
+ for (int i = 0; i < randomIntBetween(1, this.numberOfShards); i++) {
+ activeInitializingShards.put(new ShardId(indexMetadata.getIndex(), i), mock(ShardRouting.class));
+ }
+ final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore(
+ indexMetadata,
+ remoteStoreRecoverySource,
+ activeInitializingShards
+ ).build();
+ assertTrue(routingTable.hasIndex(TEST_INDEX_1));
+ assertEquals(this.numberOfShards, routingTable.allShards(TEST_INDEX_1).size());
+ assertEquals(
+ this.numberOfShards - activeInitializingShards.size(),
+ routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size()
+ );
+ }
+
/** reverse engineer the in sync aid based on the given indexRoutingTable **/
public static IndexMetadata updateActiveAllocations(IndexRoutingTable indexRoutingTable, IndexMetadata indexMetadata) {
IndexMetadata.Builder imdBuilder = IndexMetadata.builder(indexMetadata);
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java
index d23b079e35ef9..22f7b9ecd5206 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java
@@ -34,7 +34,7 @@
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.test.OpenSearchTestCase;
import java.util.Locale;
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 2180a14f5bf30..bde8a45359814 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -69,6 +69,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.core.index.Index;
import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.store.remote.filecache.FileCache;
import org.opensearch.index.store.remote.filecache.FileCacheStats;
import org.opensearch.repositories.IndexId;
import org.opensearch.snapshots.EmptySnapshotsInfoService;
@@ -405,6 +406,7 @@ public void testFileCacheRemoteShardsDecisions() {
DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings);
Metadata metadata = Metadata.builder()
.put(IndexMetadata.builder("test").settings(remoteIndexSettings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0))
+ .persistentSettings(Settings.builder().put(FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.getKey(), 5).build())
.build();
RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
index 3c27748daa87d..d4804b18bd160 100644
--- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
@@ -86,6 +86,8 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import com.carrotsearch.randomizedtesting.annotations.Timeout;
+
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.Matchers.anyOf;
@@ -863,6 +865,7 @@ public ClusterManagerTaskThrottler.ThrottlingKey getClusterManagerThrottlingKey(
AtomicInteger throttledTask3 = new AtomicInteger();
AtomicInteger succeededTask1 = new AtomicInteger();
AtomicInteger succeededTask2 = new AtomicInteger();
+ AtomicInteger succeededTask3 = new AtomicInteger();
AtomicInteger timedOutTask3 = new AtomicInteger();
final ClusterStateTaskListener listener = new ClusterStateTaskListener() {
@@ -880,6 +883,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
succeededTask1.incrementAndGet();
} else if (source.equals(task2)) {
succeededTask2.incrementAndGet();
+ } else if (source.equals(task3)) {
+ succeededTask3.incrementAndGet();
}
latch.countDown();
}
@@ -955,7 +960,7 @@ public void run() {
assertEquals(numberOfTask1, throttledTask1.get() + succeededTask1.get());
assertEquals(numberOfTask2, succeededTask2.get());
assertEquals(0, throttledTask2.get());
- assertEquals(numberOfTask3, throttledTask3.get() + timedOutTask3.get());
+ assertEquals(numberOfTask3, throttledTask3.get() + timedOutTask3.get() + succeededTask3.get());
masterService.close();
}
@@ -1378,6 +1383,76 @@ public void testDeprecatedMasterServiceUpdateTaskThreadName() {
assertThrows(AssertionError.class, () -> MasterService.assertClusterManagerUpdateThread());
}
+ @Timeout(millis = 5_000)
+ public void testTaskTimeout() throws InterruptedException {
+ try (ClusterManagerService clusterManagerService = createClusterManagerService(true)) {
+ final AtomicInteger failureCount = new AtomicInteger();
+ final AtomicInteger successCount = new AtomicInteger();
+ final CountDownLatch taskStartLatch = new CountDownLatch(1);
+ final CountDownLatch blockingTaskLatch = new CountDownLatch(1);
+ final CountDownLatch timeoutLatch = new CountDownLatch(1);
+ final ClusterStateTaskListener blockingListener = new ClusterStateTaskListener() {
+ @Override
+ public void onFailure(String source, Exception e) {
+ fail("Unexpected failure");
+ }
+
+ @Override
+ public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+ successCount.incrementAndGet();
+ taskStartLatch.countDown();
+ try {
+ blockingTaskLatch.await();
+ } catch (InterruptedException e) {
+ fail("Interrupted");
+ }
+ }
+ };
+ final ClusterStateTaskListener timeoutListener = new ClusterStateTaskListener() {
+ @Override
+ public void onFailure(String source, Exception e) {
+ assertEquals("timeout", source);
+ failureCount.incrementAndGet();
+ timeoutLatch.countDown();
+ }
+
+ @Override
+ public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+ fail("Unexpected success");
+ }
+ };
+
+ final ClusterStateTaskExecutor executor = (currentState, tasks) -> ClusterStateTaskExecutor.ClusterTasksResult.builder()
+ .successes(tasks)
+ .build(currentState);
+
+ // start a task and wait for it to start and block on the clusterStateProcessed callback
+ clusterManagerService.submitStateUpdateTask(
+ "success",
+ new Object(),
+ ClusterStateTaskConfig.build(randomFrom(Priority.values())),
+ executor,
+ blockingListener
+ );
+ taskStartLatch.await();
+
+ // start a second task that is guaranteed to timeout as the first task is still running
+ clusterManagerService.submitStateUpdateTask(
+ "timeout",
+ new Object(),
+ ClusterStateTaskConfig.build(randomFrom(Priority.values()), TimeValue.timeValueMillis(1L)),
+ executor,
+ timeoutListener
+ );
+
+ // wait for the timeout to happen, then unblock and assert one success and one failure
+ timeoutLatch.await();
+ blockingTaskLatch.countDown();
+ assertEquals(1, failureCount.get());
+ assertEquals(1, successCount.get());
+ }
+ }
+
/**
* Returns the cluster state that the cluster-manager service uses (and that is provided by the discovery layer)
*/
diff --git a/server/src/test/java/org/opensearch/common/UUIDTests.java b/server/src/test/java/org/opensearch/common/UUIDTests.java
index a71b68bfe859e..02595d19781ab 100644
--- a/server/src/test/java/org/opensearch/common/UUIDTests.java
+++ b/server/src/test/java/org/opensearch/common/UUIDTests.java
@@ -45,7 +45,7 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.util.TestUtil;
import org.opensearch.common.lucene.Lucene;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.test.OpenSearchTestCase;
import org.hamcrest.Matchers;
diff --git a/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java b/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java
index 920684f06f83c..ede5658f0decd 100644
--- a/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java
+++ b/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java
@@ -36,7 +36,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.MockBigArrays;
import org.opensearch.common.util.MockPageCacheRecycler;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.io.IOException;
diff --git a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java
index 417a79da68ae1..4968f7740b7c9 100644
--- a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java
+++ b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java
@@ -37,9 +37,9 @@
import org.opensearch.core.common.io.stream.NamedWriteable;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.common.io.stream.Writeable.Reader;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.test.OpenSearchTestCase;
@@ -86,7 +86,7 @@ public static void afterClass() throws Exception {
public void testFromXContent() throws IOException {
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
SB testShape = createTestShapeBuilder();
- XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
contentBuilder.prettyPrint();
}
diff --git a/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java b/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java
index 20f202149b17d..92ed8fac5aae0 100644
--- a/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java
+++ b/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java
@@ -35,7 +35,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.MockBigArrays;
import org.opensearch.common.util.MockPageCacheRecycler;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.io.IOException;
diff --git a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java
index ef0bedac72541..f7be2c4876e6f 100644
--- a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java
+++ b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java
@@ -35,6 +35,7 @@
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.index.StandardDirectoryReader;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.document.Document;
@@ -565,12 +566,13 @@ public void testWrapAllDocsLive() throws Exception {
}
try (DirectoryReader unwrapped = DirectoryReader.open(writer)) {
DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped);
+ StoredFields storedFields = reader.storedFields();
assertThat(reader.numDocs(), equalTo(liveDocs.size()));
IndexSearcher searcher = new IndexSearcher(reader);
Set actualDocs = new HashSet<>();
TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
- actualDocs.add(reader.document(scoreDoc.doc).get("id"));
+ actualDocs.add(storedFields.document(scoreDoc.doc).get("id"));
}
assertThat(actualDocs, equalTo(liveDocs));
}
@@ -609,13 +611,14 @@ public void testWrapLiveDocsNotExposeAbortedDocuments() throws Exception {
}
try (DirectoryReader unwrapped = DirectoryReader.open(writer)) {
DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped);
+ StoredFields storedFields = reader.storedFields();
assertThat(reader.maxDoc(), equalTo(numDocs + abortedDocs));
assertThat(reader.numDocs(), equalTo(liveDocs.size()));
IndexSearcher searcher = new IndexSearcher(reader);
List actualDocs = new ArrayList<>();
TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE);
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
- actualDocs.add(reader.document(scoreDoc.doc).get("id"));
+ actualDocs.add(storedFields.document(scoreDoc.doc).get("id"));
}
assertThat(actualDocs, equalTo(liveDocs));
}
diff --git a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java
index 60f39208d008a..9423d3c17c98a 100644
--- a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java
+++ b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java
@@ -43,6 +43,7 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.Query;
@@ -149,9 +150,10 @@ public void setUp() throws Exception {
// now go over each doc, build the relevant references and filter
reader = DirectoryReader.open(iw);
+ StoredFields storedFields = reader.storedFields();
List filterTerms = new ArrayList<>();
for (int docId = 0; docId < reader.maxDoc(); docId++) {
- Document doc = reader.document(docId);
+ Document doc = storedFields.document(docId);
addFreqs(doc, referenceAll);
if (!deletedIds.contains(doc.getField("id").stringValue())) {
addFreqs(doc, referenceNotDeleted);
diff --git a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
index 48a8e5799993d..f795df2f48b22 100644
--- a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
+++ b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
@@ -32,7 +32,7 @@
package org.opensearch.common.network;
-import org.opensearch.common.component.AbstractLifecycleComponent;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
@@ -45,7 +45,7 @@
import org.opensearch.http.HttpServerTransport;
import org.opensearch.http.HttpStats;
import org.opensearch.http.NullDispatcher;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.plugins.NetworkPlugin;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.TestThreadPool;
diff --git a/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java b/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java
index 2c7251818e2bc..94fb6b010309d 100644
--- a/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java
+++ b/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java
@@ -33,7 +33,7 @@
package org.opensearch.common.settings;
import org.opensearch.common.settings.Setting.Property;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.PageCacheRecycler;
import org.opensearch.indices.IndexingMemoryController;
import org.opensearch.indices.IndicesQueryCache;
diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java
index 3d5a5090cdc82..27f9c027e2641 100644
--- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java
+++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java
@@ -51,8 +51,8 @@
import org.opensearch.common.settings.Setting.MinTimeValueParser;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Setting.RegexValidator;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.IndexSettings;
import org.opensearch.monitor.jvm.JvmInfo;
diff --git a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java
index 0c6352d118be2..1fd8658a09e67 100644
--- a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java
+++ b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java
@@ -38,8 +38,8 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.common.settings.SecureString;
import org.opensearch.core.xcontent.ToXContent;
diff --git a/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java b/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java
index 804a53d687c95..33cfcfcd79d87 100644
--- a/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java
+++ b/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java
@@ -13,8 +13,8 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.BytesStreamInput;
import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/common/util/BigArraysTests.java b/server/src/test/java/org/opensearch/common/util/BigArraysTests.java
index c973a76623452..22aff6c55d39a 100644
--- a/server/src/test/java/org/opensearch/common/util/BigArraysTests.java
+++ b/server/src/test/java/org/opensearch/common/util/BigArraysTests.java
@@ -33,15 +33,15 @@
package org.opensearch.common.util;
import org.apache.lucene.util.BytesRef;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.core.common.util.BigArray;
import org.opensearch.core.common.util.ByteArray;
import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.junit.Before;
diff --git a/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java b/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java
index bc64a4980947c..98837091d9550 100644
--- a/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java
+++ b/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java
@@ -34,7 +34,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.common.lease.Releasables;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.junit.Before;
diff --git a/server/src/test/java/org/opensearch/common/util/BitArrayTests.java b/server/src/test/java/org/opensearch/common/util/BitArrayTests.java
index 16e89b81c695c..e88e2b20c6155 100644
--- a/server/src/test/java/org/opensearch/common/util/BitArrayTests.java
+++ b/server/src/test/java/org/opensearch/common/util/BitArrayTests.java
@@ -32,12 +32,12 @@
package org.opensearch.common.util;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.util.ArrayList;
diff --git a/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java b/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java
index 8b719283ed71d..905fbfe4e244b 100644
--- a/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java
+++ b/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java
@@ -36,7 +36,7 @@
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.tests.util.TestUtil;
import org.opensearch.common.settings.Settings;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.util.HashMap;
diff --git a/server/src/test/java/org/opensearch/common/util/LongHashTests.java b/server/src/test/java/org/opensearch/common/util/LongHashTests.java
index 295497b9f188c..3716c7ad277b3 100644
--- a/server/src/test/java/org/opensearch/common/util/LongHashTests.java
+++ b/server/src/test/java/org/opensearch/common/util/LongHashTests.java
@@ -33,7 +33,7 @@
package org.opensearch.common.util;
import org.opensearch.common.settings.Settings;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.util.HashMap;
diff --git a/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java b/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java
index fd0f1b5bad4a7..e0b0182216c27 100644
--- a/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java
+++ b/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java
@@ -33,7 +33,7 @@
package org.opensearch.common.util;
import org.opensearch.common.settings.Settings;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.util.ArrayList;
diff --git a/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java b/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java
index 73a6fc5d54883..dc4743189f4cf 100644
--- a/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java
+++ b/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java
@@ -33,7 +33,7 @@
package org.opensearch.common.util;
import org.opensearch.common.settings.Settings;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import java.util.HashMap;
diff --git a/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java b/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
index d771fdbef38d6..20e8916565daf 100644
--- a/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
+++ b/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
@@ -33,7 +33,7 @@
import org.apache.logging.log4j.Logger;
import org.opensearch.common.SuppressLoggerChecks;
-import org.opensearch.common.component.Lifecycle;
+import org.opensearch.common.lifecycle.Lifecycle;
import org.opensearch.test.OpenSearchTestCase;
import org.mockito.InOrder;
diff --git a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java
index 274b6477e3fbd..00ab96dab6a5c 100644
--- a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java
+++ b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java
@@ -49,6 +49,7 @@
import org.opensearch.common.unit.DistanceUnit;
import org.opensearch.common.util.CollectionUtils;
import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedObjectNotFoundException;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
@@ -865,7 +866,7 @@ public void testBasics() throws IOException {
generator.writeEndObject();
}
byte[] data = os.toByteArray();
- assertEquals(xcontentType(), XContentFactory.xContentType(data));
+ assertEquals(xcontentType(), MediaTypeRegistry.xContent(data));
}
public void testMissingEndObject() throws IOException {
diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java
index 866def49af991..1b7c535a550b0 100644
--- a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java
+++ b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java
@@ -37,6 +37,7 @@
import org.opensearch.common.Strings;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.test.OpenSearchTestCase;
@@ -64,14 +65,14 @@ public void testGuessCbor() throws IOException {
}
private void testGuessType(XContentType type) throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(type);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(type);
builder.startObject();
builder.field("field1", "value1");
builder.endObject();
final BytesReference bytes;
if (type == XContentType.JSON && randomBoolean()) {
- final int length = randomIntBetween(0, 8 * XContentFactory.GUESS_HEADER_LENGTH);
+ final int length = randomIntBetween(0, 8 * MediaTypeRegistry.GUESS_HEADER_LENGTH);
final String content = Strings.toString(builder);
final StringBuilder sb = new StringBuilder(length + content.length());
final char[] chars = new char[length];
@@ -82,24 +83,24 @@ private void testGuessType(XContentType type) throws IOException {
bytes = BytesReference.bytes(builder);
}
- assertThat(XContentHelper.xContentType(bytes), equalTo(type));
- assertThat(XContentFactory.xContentType(bytes.streamInput()), equalTo(type));
+ assertThat(MediaTypeRegistry.xContentType(bytes), equalTo(type));
+ assertThat(MediaTypeRegistry.xContentType(bytes.streamInput()), equalTo(type));
// CBOR is binary, cannot use String
if (type != XContentType.CBOR && type != XContentType.SMILE) {
- assertThat(XContentFactory.xContentType(Strings.toString(builder)), equalTo(type));
+ assertThat(MediaTypeRegistry.xContentType(Strings.toString(builder)), equalTo(type));
}
}
public void testCBORBasedOnMajorObjectDetection() {
// for this {"f "=> 5} perl encoder for example generates:
byte[] bytes = new byte[] { (byte) 0xA1, (byte) 0x43, (byte) 0x66, (byte) 6f, (byte) 6f, (byte) 0x5 };
- assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR));
+ assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR));
// assertThat(((Number) XContentHelper.convertToMap(bytes, true).v2().get("foo")).intValue(), equalTo(5));
// this if for {"foo" : 5} in python CBOR
bytes = new byte[] { (byte) 0xA1, (byte) 0x63, (byte) 0x66, (byte) 0x6f, (byte) 0x6f, (byte) 0x5 };
- assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR));
+ assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR));
assertThat(((Number) XContentHelper.convertToMap(new BytesArray(bytes), true).v2().get("foo")).intValue(), equalTo(5));
// also make sure major type check doesn't collide with SMILE and JSON, just in case
@@ -111,36 +112,36 @@ public void testCBORBasedOnMajorObjectDetection() {
public void testCBORBasedOnMagicHeaderDetection() {
byte[] bytes = new byte[] { (byte) 0xd9, (byte) 0xd9, (byte) 0xf7 };
- assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR));
+ assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR));
}
public void testEmptyStream() throws Exception {
ByteArrayInputStream is = new ByteArrayInputStream(new byte[0]);
- assertNull(XContentFactory.xContentType(is));
+ assertNull(MediaTypeRegistry.xContentType(is));
is = new ByteArrayInputStream(new byte[] { (byte) 1 });
- assertNull(XContentFactory.xContentType(is));
+ assertNull(MediaTypeRegistry.xContentType(is));
}
public void testInvalidStream() throws Exception {
byte[] bytes = new byte[] { (byte) '"' };
- assertNull(XContentFactory.xContentType(bytes));
+ assertNull(MediaTypeRegistry.mediaTypeFromBytes(bytes, 0, bytes.length));
bytes = new byte[] { (byte) 'x' };
- assertNull(XContentFactory.xContentType(bytes));
+ assertNull(MediaTypeRegistry.mediaTypeFromBytes(bytes, 0, bytes.length));
}
public void testJsonFromBytesOptionallyPrecededByUtf8Bom() throws Exception {
byte[] bytes = new byte[] { (byte) '{', (byte) '}' };
- assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON));
+ assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON));
bytes = new byte[] { (byte) 0x20, (byte) '{', (byte) '}' };
- assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON));
+ assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON));
bytes = new byte[] { (byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) '{', (byte) '}' };
- assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON));
+ assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON));
bytes = new byte[] { (byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) 0x20, (byte) '{', (byte) '}' };
- assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON));
+ assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON));
}
}
diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java
index 53607f432bbd5..8c53d7edebca8 100644
--- a/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java
+++ b/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java
@@ -44,81 +44,80 @@ public class XContentTypeTests extends OpenSearchTestCase {
public void testFromJson() throws Exception {
String mediaType = "application/json";
XContentType expectedXContentType = XContentType.JSON;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
}
public void testFromNdJson() throws Exception {
String mediaType = "application/x-ndjson";
XContentType expectedXContentType = XContentType.JSON;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
}
public void testFromJsonUppercase() throws Exception {
String mediaType = "application/json".toUpperCase(Locale.ROOT);
XContentType expectedXContentType = XContentType.JSON;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
}
public void testFromYaml() throws Exception {
String mediaType = "application/yaml";
XContentType expectedXContentType = XContentType.YAML;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType));
}
public void testFromSmile() throws Exception {
String mediaType = "application/smile";
XContentType expectedXContentType = XContentType.SMILE;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
}
public void testFromCbor() throws Exception {
String mediaType = "application/cbor";
XContentType expectedXContentType = XContentType.CBOR;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
}
public void testFromWildcard() throws Exception {
String mediaType = "application/*";
XContentType expectedXContentType = XContentType.JSON;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
}
public void testFromWildcardUppercase() throws Exception {
String mediaType = "APPLICATION/*";
XContentType expectedXContentType = XContentType.JSON;
- assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType));
- assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType));
+ assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType));
}
public void testFromRubbish() throws Exception {
- assertThat(XContentType.fromMediaType((String) null), nullValue());
- assertThat(XContentType.fromMediaType((MediaType) null), nullValue());
- assertThat(XContentType.fromMediaType(""), nullValue());
- assertThat(XContentType.fromMediaType("text/plain"), nullValue());
- assertThat(XContentType.fromMediaType("gobbly;goop"), nullValue());
+ assertThat(MediaType.fromMediaType(null), nullValue());
+ assertThat(MediaType.fromMediaType(""), nullValue());
+ assertThat(MediaType.fromMediaType("text/plain"), nullValue());
+ assertThat(MediaType.fromMediaType("gobbly;goop"), nullValue());
}
public void testVersionedMediaType() throws Exception {
- assertThat(XContentType.fromMediaType("application/vnd.opensearch+json;compatible-with=7"), equalTo(XContentType.JSON));
- assertThat(XContentType.fromMediaType("application/vnd.opensearch+yaml;compatible-with=7"), equalTo(XContentType.YAML));
- assertThat(XContentType.fromMediaType("application/vnd.opensearch+cbor;compatible-with=7"), equalTo(XContentType.CBOR));
- assertThat(XContentType.fromMediaType("application/vnd.opensearch+smile;compatible-with=7"), equalTo(XContentType.SMILE));
+ assertThat(MediaType.fromMediaType("application/vnd.opensearch+json;compatible-with=7"), equalTo(XContentType.JSON));
+ assertThat(MediaType.fromMediaType("application/vnd.opensearch+yaml;compatible-with=7"), equalTo(XContentType.YAML));
+ assertThat(MediaType.fromMediaType("application/vnd.opensearch+cbor;compatible-with=7"), equalTo(XContentType.CBOR));
+ assertThat(MediaType.fromMediaType("application/vnd.opensearch+smile;compatible-with=7"), equalTo(XContentType.SMILE));
- assertThat(XContentType.fromMediaType("application/vnd.opensearch+json ;compatible-with=7"), equalTo(XContentType.JSON));
+ assertThat(MediaType.fromMediaType("application/vnd.opensearch+json ;compatible-with=7"), equalTo(XContentType.JSON));
String mthv = "application/vnd.opensearch+json ;compatible-with=7;charset=utf-8";
- assertThat(XContentType.fromMediaType(mthv), equalTo(XContentType.JSON));
- assertThat(XContentType.fromMediaType(mthv.toUpperCase(Locale.ROOT)), equalTo(XContentType.JSON));
+ assertThat(MediaType.fromMediaType(mthv), equalTo(XContentType.JSON));
+ assertThat(MediaType.fromMediaType(mthv.toUpperCase(Locale.ROOT)), equalTo(XContentType.JSON));
}
}
diff --git a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java
index 6c97297ce43e7..dc98000d93899 100644
--- a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java
+++ b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java
@@ -39,13 +39,13 @@
import org.opensearch.common.geo.GeoPoint;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentOpenSearchExtension;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.XContentGenerator;
-import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentGenerator;
+import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.test.OpenSearchTestCase;
import java.io.ByteArrayOutputStream;
@@ -68,7 +68,7 @@
public class XContentBuilderTests extends OpenSearchTestCase {
public void testPrettyWithLfAtEnd() throws Exception {
ByteArrayOutputStream os = new ByteArrayOutputStream();
- XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os);
+ XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os);
generator.usePrettyPrint();
generator.usePrintLineFeedAtEnd();
@@ -87,7 +87,7 @@ public void testPrettyWithLfAtEnd() throws Exception {
public void testReuseJsonGenerator() throws Exception {
ByteArrayOutputStream os = new ByteArrayOutputStream();
- XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os);
+ XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os);
generator.writeStartObject();
generator.writeStringField("test", "value");
generator.writeEndObject();
@@ -107,14 +107,14 @@ public void testReuseJsonGenerator() throws Exception {
public void testRaw() throws IOException {
{
- XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
xContentBuilder.startObject();
xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput());
xContentBuilder.endObject();
assertThat(Strings.toString(xContentBuilder), equalTo("{\"foo\":{\"test\":\"value\"}}"));
}
{
- XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
xContentBuilder.startObject();
xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput());
xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}").streamInput());
@@ -122,7 +122,7 @@ public void testRaw() throws IOException {
assertThat(Strings.toString(xContentBuilder), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}"));
}
{
- XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
xContentBuilder.startObject();
xContentBuilder.field("test", "value");
xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput());
@@ -130,7 +130,7 @@ public void testRaw() throws IOException {
assertThat(Strings.toString(xContentBuilder), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}"));
}
{
- XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
xContentBuilder.startObject();
xContentBuilder.field("test", "value");
xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput());
@@ -142,7 +142,7 @@ public void testRaw() throws IOException {
);
}
{
- XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
xContentBuilder.startObject();
xContentBuilder.field("test", "value");
xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput());
@@ -157,17 +157,17 @@ public void testRaw() throws IOException {
}
public void testSimpleGenerator() throws Exception {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject().field("test", "value").endObject();
assertThat(Strings.toString(builder), equalTo("{\"test\":\"value\"}"));
- builder = XContentFactory.contentBuilder(XContentType.JSON);
+ builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject().field("test", "value").endObject();
assertThat(Strings.toString(builder), equalTo("{\"test\":\"value\"}"));
}
public void testOverloadedList() throws Exception {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject().field("test", Arrays.asList("1", "2")).endObject();
assertThat(Strings.toString(builder), equalTo("{\"test\":[\"1\",\"2\"]}"));
}
@@ -175,7 +175,7 @@ public void testOverloadedList() throws Exception {
public void testWritingBinaryToStream() throws Exception {
BytesStreamOutput bos = new BytesStreamOutput();
- XContentGenerator gen = XContentFactory.xContent(XContentType.JSON).createGenerator(bos);
+ XContentGenerator gen = XContentType.JSON.xContent().createGenerator(bos);
gen.writeStartObject();
gen.writeStringField("name", "something");
gen.flush();
@@ -189,7 +189,7 @@ public void testWritingBinaryToStream() throws Exception {
}
public void testByteConversion() throws Exception {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject().field("test_name", (Byte) (byte) 120).endObject();
assertThat(BytesReference.bytes(builder).utf8ToString(), equalTo("{\"test_name\":120}"));
}
@@ -199,21 +199,21 @@ public void testDateTypesConversion() throws Exception {
String expectedDate = XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(date.getTime());
Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT);
String expectedCalendar = XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(calendar.getTimeInMillis());
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject().timeField("date", date).endObject();
assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}"));
- builder = XContentFactory.contentBuilder(XContentType.JSON);
+ builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject().field("calendar", calendar).endObject();
assertThat(Strings.toString(builder), equalTo("{\"calendar\":\"" + expectedCalendar + "\"}"));
- builder = XContentFactory.contentBuilder(XContentType.JSON);
+ builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
Map map = new HashMap<>();
map.put("date", date);
builder.map(map);
assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}"));
- builder = XContentFactory.contentBuilder(XContentType.JSON);
+ builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
map = new HashMap<>();
map.put("calendar", calendar);
builder.map(map);
@@ -221,7 +221,7 @@ public void testDateTypesConversion() throws Exception {
}
public void testCopyCurrentStructure() throws Exception {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject().field("test", "test field").startObject("filter").startObject("terms");
// up to 20k random terms
@@ -248,7 +248,7 @@ public void testCopyCurrentStructure() throws Exception {
}
} else if (token == XContentParser.Token.START_OBJECT) {
if ("filter".equals(currentFieldName)) {
- filterBuilder = XContentFactory.contentBuilder(parser.contentType());
+ filterBuilder = MediaTypeRegistry.contentBuilder(parser.contentType());
filterBuilder.copyCurrentStructure(parser);
}
}
@@ -288,10 +288,10 @@ public void testHandlingOfPath_absolute() throws IOException {
}
private void checkPathSerialization(Path path) throws IOException {
- XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
pathBuilder.startObject().field("file", path).endObject();
- XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
stringBuilder.startObject().field("file", path.toString()).endObject();
assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder)));
@@ -301,10 +301,10 @@ public void testHandlingOfPath_StringName() throws IOException {
Path path = PathUtils.get("path");
String name = new String("file");
- XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
pathBuilder.startObject().field(name, path).endObject();
- XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
stringBuilder.startObject().field(name, path.toString()).endObject();
assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder)));
@@ -313,36 +313,36 @@ public void testHandlingOfPath_StringName() throws IOException {
public void testHandlingOfCollectionOfPaths() throws IOException {
Path path = PathUtils.get("path");
- XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
pathBuilder.startObject().field("file", Arrays.asList(path)).endObject();
- XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
stringBuilder.startObject().field("file", Arrays.asList(path.toString())).endObject();
assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder)));
}
public void testIndentIsPlatformIndependent() throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint();
builder.startObject().field("test", "foo").startObject("foo").field("foobar", "boom").endObject().endObject();
String string = Strings.toString(builder);
assertEquals("{\n" + " \"test\" : \"foo\",\n" + " \"foo\" : {\n" + " \"foobar\" : \"boom\"\n" + " }\n" + "}", string);
- builder = XContentFactory.contentBuilder(XContentType.YAML).prettyPrint();
+ builder = MediaTypeRegistry.contentBuilder(XContentType.YAML).prettyPrint();
builder.startObject().field("test", "foo").startObject("foo").field("foobar", "boom").endObject().endObject();
string = Strings.toString(builder);
assertEquals("---\n" + "test: \"foo\"\n" + "foo:\n" + " foobar: \"boom\"\n", string);
}
public void testRenderGeoPoint() throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint();
builder.startObject().field("foo").value(new GeoPoint(1, 2)).endObject();
String string = Strings.toString(builder);
assertEquals("{\n" + " \"foo\" : {\n" + " \"lat\" : 1.0,\n" + " \"lon\" : 2.0\n" + " }\n" + "}", string.trim());
}
public void testWriteMapWithNullKeys() throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
try {
builder.map(Collections.singletonMap(null, "test"));
fail("write map should have failed");
@@ -352,7 +352,7 @@ public void testWriteMapWithNullKeys() throws IOException {
}
public void testWriteMapValueWithNullKeys() throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
try {
builder.map(Collections.singletonMap(null, "test"));
fail("write map should have failed");
@@ -362,7 +362,7 @@ public void testWriteMapValueWithNullKeys() throws IOException {
}
public void testWriteFieldMapWithNullKeys() throws IOException {
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
try {
builder.startObject();
builder.field("map", Collections.singletonMap(null, "test"));
@@ -374,7 +374,7 @@ public void testWriteFieldMapWithNullKeys() throws IOException {
public void testMissingEndObject() throws IOException {
IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
- try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) {
builder.startObject();
builder.field("foo", true);
}
@@ -385,7 +385,7 @@ public void testMissingEndObject() throws IOException {
public void testMissingEndArray() throws IOException {
IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
- try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) {
+ try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) {
builder.startObject();
builder.startArray("foo");
builder.value(0);
diff --git a/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java b/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java
index e304798a0ff0c..89add6ea78722 100644
--- a/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java
+++ b/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java
@@ -33,7 +33,6 @@
package org.opensearch.common.xcontent.cbor;
import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentGenerator;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -48,10 +47,10 @@
public class JsonVsCborTests extends OpenSearchTestCase {
public void testCompareParsingTokens() throws IOException {
BytesStreamOutput xsonOs = new BytesStreamOutput();
- XContentGenerator xsonGen = XContentFactory.xContent(XContentType.CBOR).createGenerator(xsonOs);
+ XContentGenerator xsonGen = XContentType.CBOR.xContent().createGenerator(xsonOs);
BytesStreamOutput jsonOs = new BytesStreamOutput();
- XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs);
+ XContentGenerator jsonGen = XContentType.JSON.xContent().createGenerator(jsonOs);
xsonGen.writeStartObject();
jsonGen.writeStartObject();
diff --git a/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java b/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java
index 4f26bdc7e9c26..a96031f2a1dad 100644
--- a/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java
+++ b/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java
@@ -33,7 +33,6 @@
package org.opensearch.common.xcontent.smile;
import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentGenerator;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -48,10 +47,10 @@
public class JsonVsSmileTests extends OpenSearchTestCase {
public void testCompareParsingTokens() throws IOException {
BytesStreamOutput xsonOs = new BytesStreamOutput();
- XContentGenerator xsonGen = XContentFactory.xContent(XContentType.SMILE).createGenerator(xsonOs);
+ XContentGenerator xsonGen = XContentType.SMILE.xContent().createGenerator(xsonOs);
BytesStreamOutput jsonOs = new BytesStreamOutput();
- XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs);
+ XContentGenerator jsonGen = XContentType.JSON.xContent().createGenerator(jsonOs);
xsonGen.writeStartObject();
jsonGen.writeStartObject();
diff --git a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
index c318dd89f573c..0a8df4406b5cd 100644
--- a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
+++ b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
@@ -39,7 +39,6 @@
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.support.AbstractFilteringTestCase;
@@ -89,7 +88,7 @@ static void assertXContentBuilderAsString(final XContentBuilder expected, final
}
static void assertXContentBuilderAsBytes(final XContentBuilder expected, final XContentBuilder actual) {
- XContent xContent = XContentFactory.xContent(actual.contentType());
+ XContent xContent = actual.contentType().xContent();
try (
XContentParser jsonParser = xContent.createParser(
NamedXContentRegistry.EMPTY,
diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
index 0f831ff28af79..880eaeca99bb0 100644
--- a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
+++ b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
@@ -125,7 +125,7 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
assertThat(fragment, nullValue());
prefixQuery = new PrefixQuery(new Term("content", "ba"), PrefixQuery.SCORING_BOOLEAN_REWRITE);
- Query rewriteQuery = prefixQuery.rewrite(reader);
+ Query rewriteQuery = prefixQuery.rewrite(searcher);
fragment = highlighter.getBestFragment(highlighter.getFieldQuery(rewriteQuery), reader, topDocs.scoreDocs[0].doc, "content", 30);
assertThat(fragment, notNullValue());
diff --git a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java
index 61aac8247647a..acf94483c8116 100644
--- a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java
+++ b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java
@@ -41,7 +41,7 @@
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.CancellableThreads;
import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.transport.MockTransportService;
import org.opensearch.threadpool.TestThreadPool;
diff --git a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java
index 95f84a70986e1..07491cc0d6435 100644
--- a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java
+++ b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java
@@ -47,7 +47,7 @@
import org.opensearch.common.util.concurrent.OpenSearchExecutors;
import org.opensearch.common.util.concurrent.FutureUtils;
import org.opensearch.common.util.io.IOUtils;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.TestThreadPool;
import org.opensearch.threadpool.ThreadPool;
diff --git a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java
index 382698a300591..a55d89ed37cd7 100644
--- a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java
+++ b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java
@@ -46,8 +46,8 @@
import org.opensearch.common.CheckedRunnable;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.gateway.PersistedClusterStateService;
import org.opensearch.core.index.Index;
diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java
index 41b841aec7be8..9f1050351c7b5 100644
--- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java
+++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java
@@ -66,7 +66,7 @@
import org.opensearch.extensions.settings.RegisterCustomSettingsRequest;
import org.opensearch.extensions.ExtensionsSettings.Extension;
import org.opensearch.identity.IdentityService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.plugins.ExtensionAwarePlugin;
import org.opensearch.rest.RestController;
import org.opensearch.test.FeatureFlagSetter;
diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java
index e46e05cd2dbec..bc216bd3bbbb8 100644
--- a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java
+++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java
@@ -25,7 +25,7 @@
import org.opensearch.extensions.DiscoveryExtensionNode;
import org.opensearch.extensions.AcknowledgedResponse;
import org.opensearch.extensions.rest.RestSendToExtensionActionTests;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.client.NoOpNodeClient;
import org.opensearch.test.transport.MockTransportService;
diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java
index 030df865c4e4a..1faa3bd9aec55 100644
--- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java
+++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java
@@ -34,7 +34,7 @@
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.extensions.ExtensionsManager;
import org.opensearch.extensions.ExtensionsSettings;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.rest.RestRequest;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java
index 8695e409de0bc..b59513fc0045d 100644
--- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java
+++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java
@@ -46,7 +46,7 @@
import org.opensearch.extensions.action.ExtensionAction;
import org.opensearch.extensions.action.ExtensionTransportAction;
import org.opensearch.identity.IdentityService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.rest.NamedRoute;
import org.opensearch.rest.RestHandler.Route;
import org.opensearch.rest.RestRequest.Method;
diff --git a/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java b/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java
index bf9234a7f129d..22d07d7f18920 100644
--- a/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java
+++ b/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java
@@ -16,8 +16,8 @@
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
index 830a8a9ad8ab7..714a2389e64e1 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
@@ -58,7 +58,7 @@
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.env.TestEnvironment;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.node.Node;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.TestThreadPool;
diff --git a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java
index 211c13e00e6db..14ed610942e15 100644
--- a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java
@@ -62,7 +62,7 @@
import org.opensearch.env.NodeMetadata;
import org.opensearch.gateway.PersistedClusterStateService.Writer;
import org.opensearch.core.index.Index;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.MockLogAppender;
import org.opensearch.test.junit.annotations.TestLogging;
diff --git a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java
index b36b3110b2123..fbf0e8cd42c72 100644
--- a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java
+++ b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java
@@ -46,7 +46,7 @@
import org.opensearch.common.util.MockPageCacheRecycler;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.rest.RestChannel;
import org.opensearch.rest.RestRequest;
import org.opensearch.rest.RestResponse;
diff --git a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java
index 1ffe0538edaad..fb545e8db582e 100644
--- a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java
+++ b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java
@@ -47,7 +47,7 @@
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.common.lease.Releasable;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.rest.BytesRestResponse;
import org.opensearch.rest.RestChannel;
import org.opensearch.rest.RestRequest;
diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java
index 95d0ae706b286..2b89558e394fc 100644
--- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java
@@ -56,7 +56,7 @@
import org.opensearch.common.SetOnce;
import org.opensearch.common.SetOnce.AlreadySetException;
import org.opensearch.common.UUIDs;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
@@ -99,8 +99,8 @@
import org.opensearch.indices.IndicesModule;
import org.opensearch.indices.IndicesQueryCache;
import org.opensearch.indices.analysis.AnalysisModule;
-import org.opensearch.indices.breaker.CircuitBreakerService;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.opensearch.indices.mapper.MapperRegistry;
diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java
index e42e9b4970081..f91905fea9561 100644
--- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java
+++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java
@@ -41,7 +41,7 @@
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsException;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.index.translog.Translog;
diff --git a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java
index b1dba0c22c075..387997892ee30 100644
--- a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java
+++ b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java
@@ -33,8 +33,8 @@
import org.apache.lucene.index.NoMergePolicy;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.index.shard.ShardId;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
index 612bebf74bc70..d58d166415446 100644
--- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
@@ -111,7 +111,7 @@
import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver;
import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.BigArrays;
import org.opensearch.common.util.concurrent.AbstractRunnable;
@@ -151,7 +151,7 @@
import org.opensearch.index.translog.TranslogDeletionPolicyFactory;
import org.opensearch.index.translog.TranslogException;
import org.opensearch.index.translog.listener.TranslogEventListener;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.IndexSettingsModule;
import org.opensearch.test.VersionUtils;
import org.opensearch.threadpool.ThreadPool;
@@ -2541,6 +2541,7 @@ class OpAndVersion {
final Term uidTerm = newUid(doc);
engine.index(indexForDoc(doc));
final BiFunction searcherFactory = engine::acquireSearcher;
+
for (int i = 0; i < thread.length; i++) {
thread[i] = new Thread(() -> {
startGun.countDown();
@@ -2549,10 +2550,12 @@ class OpAndVersion {
} catch (InterruptedException e) {
throw new AssertionError(e);
}
+
for (int op = 0; op < opsPerThread; op++) {
try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) {
+
FieldsVisitor visitor = new FieldsVisitor(true);
- get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
+ get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor);
List values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString()));
String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null;
String added = "v_" + idGenerator.incrementAndGet();
@@ -2608,7 +2611,7 @@ class OpAndVersion {
try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) {
FieldsVisitor visitor = new FieldsVisitor(true);
- get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor);
+ get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor);
List values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString()));
assertThat(currentValues, equalTo(new HashSet<>(values)));
}
diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java
index 98f1a416731e4..d635b38e811c4 100644
--- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java
@@ -46,43 +46,4 @@ public void testCreateNRTreaderManager() throws IOException {
}
}
}
-
- public void testUpdateSegmentsWhileRefreshing() throws IOException, InterruptedException {
- try (final Store store = createStore()) {
- store.createEmpty(Version.LATEST);
- final DirectoryReader reader = DirectoryReader.open(store.directory());
- NRTReplicationReaderManager readerManager = new NRTReplicationReaderManager(
- OpenSearchDirectoryReader.wrap(reader, shardId),
- (files) -> {},
- (files) -> {}
- );
-
- final SegmentInfos infos_2 = readerManager.getSegmentInfos().clone();
- infos_2.changed();
-
- Thread refreshThread = new Thread(() -> {
- try {
- readerManager.maybeRefresh();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- Thread updateThread = new Thread(() -> {
- try {
- readerManager.updateSegments(infos_2);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- refreshThread.start();
- updateThread.start();
- refreshThread.join();
- updateThread.join();
- try (final OpenSearchDirectoryReader acquire = readerManager.acquire()) {
- final StandardDirectoryReader standardReader = NRTReplicationReaderManager.unwrapStandardReader(acquire);
- assertEquals(infos_2.version, standardReader.getSegmentInfos().version);
- }
- assertEquals(infos_2, readerManager.getSegmentInfos());
- }
- }
}
diff --git a/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java
index b9fe69c282471..5fe6c2d08cdc7 100644
--- a/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java
@@ -44,7 +44,7 @@
import org.opensearch.cluster.routing.TestShardRouting;
import org.opensearch.common.concurrent.GatedCloseable;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.index.IndexSettings;
diff --git a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java
index a8279ed908779..fed521e2d5ed9 100644
--- a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java
@@ -49,6 +49,7 @@
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.ShuffleForcedMergePolicy;
import org.apache.lucene.index.StandardDirectoryReader;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -89,8 +90,9 @@ public void testPruneAll() throws IOException {
writer.forceMerge(1);
writer.commit();
try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ StoredFields storedFields = reader.storedFields();
for (int i = 0; i < reader.maxDoc(); i++) {
- Document document = reader.document(i);
+ Document document = storedFields.document(i);
assertEquals(1, document.getFields().size());
assertEquals("source", document.getFields().get(0).name());
}
@@ -157,11 +159,12 @@ public void testPruneSome() throws IOException {
writer.forceMerge(1);
writer.commit();
try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ StoredFields storedFields = reader.storedFields();
assertEquals(1, reader.leaves().size());
NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source");
assertNotNull(extra_source);
for (int i = 0; i < reader.maxDoc(); i++) {
- Document document = reader.document(i);
+ Document document = storedFields.document(i);
Set collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet());
assertTrue(collect.contains("source"));
assertTrue(collect.contains("even"));
@@ -197,11 +200,12 @@ public void testPruneNone() throws IOException {
writer.forceMerge(1);
writer.commit();
try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ StoredFields storedFields = reader.storedFields();
assertEquals(1, reader.leaves().size());
NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source");
assertNotNull(extra_source);
for (int i = 0; i < reader.maxDoc(); i++) {
- Document document = reader.document(i);
+ Document document = storedFields.document(i);
Set collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet());
assertTrue(collect.contains("source"));
assertTrue(collect.contains("extra_source"));
diff --git a/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java b/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java
index 48ab9d60727b7..babf0782cdec0 100644
--- a/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java
+++ b/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java
@@ -49,7 +49,7 @@
import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
import org.opensearch.index.mapper.TextFieldMapper;
import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.search.aggregations.support.CoreValuesSourceType;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.FieldMaskingReader;
diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
index d360a2a767e8a..510dd3456924b 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
@@ -40,6 +40,7 @@
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexOrDocValuesQuery;
@@ -259,7 +260,10 @@ public void testRangeQuery() throws IOException {
LongPoint.newRangeQuery("field", instant1, instant2),
SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)
);
- assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new MultiReader()));
+ assertEquals(
+ expected,
+ ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new IndexSearcher(new MultiReader()))
+ );
instant1 = nowInMillis;
instant2 = instant1 + 100;
diff --git a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java
index 365d3738f4022..0e04e57a290b6 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java
@@ -367,9 +367,7 @@ private void doTestDefaultFloatingPointMappings(DocumentMapper mapper, XContentB
.field("quux", "3.2") // float detected through numeric detection
.endObject()
);
- ParsedDocument parsedDocument = mapper.parse(
- new SourceToParse("index", "id", source, XContentType.fromMediaType(builder.contentType()))
- );
+ ParsedDocument parsedDocument = mapper.parse(new SourceToParse("index", "id", source, builder.contentType()));
Mapping update = parsedDocument.dynamicMappingsUpdate();
assertNotNull(update);
assertThat(((FieldMapper) update.root().getMapper("foo")).fieldType().typeName(), equalTo("float"));
diff --git a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java
index 518e5e880f4f8..9e467dca17409 100644
--- a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java
@@ -38,7 +38,7 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.json.JsonXContent;
@@ -75,7 +75,7 @@ public void testNoFormat() throws Exception {
)
);
- assertThat(XContentFactory.xContentType(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON));
+ assertThat(MediaTypeRegistry.xContent(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON));
documentMapper = parser.parse("type", new CompressedXContent(mapping));
doc = documentMapper.parse(
@@ -87,7 +87,7 @@ public void testNoFormat() throws Exception {
)
);
- assertThat(XContentHelper.xContentType(doc.source()), equalTo(XContentType.SMILE));
+ assertThat(MediaTypeRegistry.xContentType(doc.source()), equalTo(XContentType.SMILE));
}
public void testIncludes() throws Exception {
diff --git a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java
index 92efd19732ef0..54afde41daa5b 100644
--- a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java
@@ -37,8 +37,8 @@
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
@@ -181,7 +181,7 @@ public void testIllegalArguments() {
// https://github.com/elastic/elasticsearch/issues/7240
public void testEmptyBooleanQuery() throws Exception {
- XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
contentBuilder.startObject().startObject("bool").endObject().endObject();
try (XContentParser xParser = createParser(contentBuilder)) {
Query parsedQuery = parseQuery(xParser).toQuery(createShardContext());
diff --git a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java
index cb75889a0aa24..5e6226561582e 100644
--- a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java
@@ -34,9 +34,9 @@
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -133,7 +133,7 @@ public void testSerializationOrder() throws Exception {
public void testFromAndToXContent() throws Exception {
for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
InnerHitBuilder innerHit = randomInnerHits();
- XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()));
innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS);
// fields is printed out as an object but parsed into a List where order matters, we disable shuffling
XContentBuilder shuffled = shuffleXContent(builder, "fields");
diff --git a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java
index e6ec132a10474..2d950f0994976 100644
--- a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -268,7 +268,7 @@ private static Fields generateFields(String[] fieldNames, String text) throws IO
for (String fieldName : fieldNames) {
index.addField(fieldName, text, new WhitespaceAnalyzer());
}
- return index.createSearcher().getIndexReader().getTermVectors(0);
+ return index.createSearcher().getIndexReader().termVectors().get(0);
}
@Override
diff --git a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
index bc514208c803d..be87f7ca15907 100644
--- a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -284,8 +284,9 @@ public void testTermExpansionExceptionOnSpanFailure() throws Exception {
BooleanQuery.setMaxClauseCount(1);
try {
QueryBuilder queryBuilder = new SpanMultiTermQueryBuilder(QueryBuilders.prefixQuery("body", "bar"));
- Query query = queryBuilder.toQuery(createShardContext(new IndexSearcher(reader)));
- RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(reader));
+ IndexSearcher searcher = new IndexSearcher(reader);
+ Query query = queryBuilder.toQuery(createShardContext(searcher));
+ RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(searcher));
assertThat(exc.getMessage(), containsString("maxClauseCount"));
} finally {
BooleanQuery.setMaxClauseCount(origBoolMaxClauseCount);
diff --git a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
index 39160b1ac49bc..45172c946ab36 100644
--- a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
+++ b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java
@@ -34,6 +34,7 @@
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -112,7 +113,7 @@ public void testRewrite() throws IOException {
ScoreMode.Avg,
"nested"
);
- Query rewritten = q.rewrite(new MultiReader());
+ Query rewritten = q.rewrite(new IndexSearcher(new MultiReader()));
assertEquals(expected, rewritten);
}
}
diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
index d56cbda2fe6e4..03a6fc3df824d 100644
--- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
@@ -1795,12 +1795,8 @@ public void testSegmentReplicationCheckpointTracking() {
assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds)));
assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable));
assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
- // get insync ids, filter out the primary.
- final Set inSyncAllocationIds = tracker.getReplicationGroup()
- .getInSyncAllocationIds()
- .stream()
- .filter(id -> tracker.shardAllocationId.equals(id) == false)
- .collect(Collectors.toSet());
+
+ initializingIds.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED));
final ReplicationCheckpoint initialCheckpoint = new ReplicationCheckpoint(
tracker.shardId(),
@@ -1831,8 +1827,10 @@ public void testSegmentReplicationCheckpointTracking() {
tracker.setLatestReplicationCheckpoint(secondCheckpoint);
tracker.setLatestReplicationCheckpoint(thirdCheckpoint);
+ final Set expectedIds = ids(initializingIds);
+
Set groupStats = tracker.getSegmentReplicationStats();
- assertEquals(inSyncAllocationIds.size(), groupStats.size());
+ assertEquals(expectedIds.size(), groupStats.size());
for (SegmentReplicationShardStats shardStat : groupStats) {
assertEquals(3, shardStat.getCheckpointsBehindCount());
assertEquals(100L, shardStat.getBytesBehindCount());
@@ -1840,7 +1838,7 @@ public void testSegmentReplicationCheckpointTracking() {
// simulate replicas moved up to date.
final Map checkpoints = tracker.checkpoints;
- for (String id : inSyncAllocationIds) {
+ for (String id : expectedIds) {
final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id);
assertEquals(3, checkpointState.checkpointTimers.size());
tracker.updateVisibleCheckpointForShard(id, initialCheckpoint);
@@ -1848,13 +1846,13 @@ public void testSegmentReplicationCheckpointTracking() {
}
groupStats = tracker.getSegmentReplicationStats();
- assertEquals(inSyncAllocationIds.size(), groupStats.size());
+ assertEquals(expectedIds.size(), groupStats.size());
for (SegmentReplicationShardStats shardStat : groupStats) {
assertEquals(2, shardStat.getCheckpointsBehindCount());
assertEquals(99L, shardStat.getBytesBehindCount());
}
- for (String id : inSyncAllocationIds) {
+ for (String id : expectedIds) {
final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id);
assertEquals(2, checkpointState.checkpointTimers.size());
tracker.updateVisibleCheckpointForShard(id, thirdCheckpoint);
@@ -1862,7 +1860,7 @@ public void testSegmentReplicationCheckpointTracking() {
}
groupStats = tracker.getSegmentReplicationStats();
- assertEquals(inSyncAllocationIds.size(), groupStats.size());
+ assertEquals(expectedIds.size(), groupStats.size());
for (SegmentReplicationShardStats shardStat : groupStats) {
assertEquals(0, shardStat.getCheckpointsBehindCount());
assertEquals(0L, shardStat.getBytesBehindCount());
@@ -1883,19 +1881,24 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() {
AllocationId primaryId = activeAllocationIds.iterator().next();
IndexShardRoutingTable routingTable = routingTable(initializingIds, primaryId);
final ReplicationTracker tracker = newTracker(primaryId, settings);
-
tracker.updateFromClusterManager(initialClusterStateVersion, ids(activeAllocationIds), routingTable);
tracker.activatePrimaryMode(NO_OPS_PERFORMED);
- assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds)));
- assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable));
- assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
- // get insync ids, filter out the primary.
- final Set inSyncAllocationIds = tracker.getReplicationGroup()
- .getInSyncAllocationIds()
- .stream()
- .filter(id -> tracker.shardAllocationId.equals(id) == false)
- .collect(Collectors.toSet());
+ initializingIds.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED));
+
+ assertEquals(tracker.getReplicationGroup().getRoutingTable(), routingTable);
+ assertEquals(
+ "All active & initializing ids are now marked in-sync",
+ Sets.union(ids(activeAllocationIds), ids(initializingIds)),
+ tracker.getReplicationGroup().getInSyncAllocationIds()
+ );
+
+ assertEquals(
+ "Active ids are in-sync but still unavailable",
+ tracker.getReplicationGroup().getUnavailableInSyncShards(),
+ Sets.difference(ids(activeAllocationIds), Set.of(primaryId.getId()))
+ );
+ assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync));
final ReplicationCheckpoint initialCheckpoint = new ReplicationCheckpoint(
tracker.shardId(),
@@ -1907,15 +1910,20 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() {
);
tracker.setLatestReplicationCheckpoint(initialCheckpoint);
+ // we expect that the only returned ids from getSegmentReplicationStats will be the initializing ids we marked with
+ // markAsTrackingAndInSyncQuietly.
+ // This is because the ids marked active initially are still unavailable (don't have an associated routing entry).
+ final Set expectedIds = ids(initializingIds);
Set groupStats = tracker.getSegmentReplicationStats();
- assertEquals(inSyncAllocationIds.size(), groupStats.size());
+ final Set actualIds = groupStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet());
+ assertEquals(expectedIds, actualIds);
for (SegmentReplicationShardStats shardStat : groupStats) {
assertEquals(1, shardStat.getCheckpointsBehindCount());
}
// simulate replicas moved up to date.
final Map checkpoints = tracker.checkpoints;
- for (String id : inSyncAllocationIds) {
+ for (String id : expectedIds) {
final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id);
assertEquals(1, checkpointState.checkpointTimers.size());
tracker.updateVisibleCheckpointForShard(id, initialCheckpoint);
diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index 915a15da6cb1d..1340ff1868a11 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -141,7 +141,7 @@
import org.opensearch.index.translog.TranslogStats;
import org.opensearch.index.translog.listener.TranslogEventListener;
import org.opensearch.indices.IndicesQueryCache;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.opensearch.indices.recovery.RecoveryState;
import org.opensearch.indices.recovery.RecoveryTarget;
diff --git a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java
index 282f4c09b982d..b51ceb2250e8f 100644
--- a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java
@@ -47,7 +47,7 @@
import org.opensearch.common.lucene.uid.Versions;
import org.opensearch.common.network.NetworkModule;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.common.xcontent.XContentFactory;
diff --git a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java
index f812312004b21..f3cffa4721ee5 100644
--- a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java
@@ -36,6 +36,7 @@
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
@@ -70,7 +71,7 @@
import org.opensearch.index.store.Store;
import org.opensearch.index.translog.Translog;
import org.opensearch.index.translog.TranslogConfig;
-import org.opensearch.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
import org.opensearch.test.DummyShardLock;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.IndexSettingsModule;
@@ -369,9 +370,8 @@ public void testLotsOfThreads() throws Exception {
try (Engine.GetResult getResult = engine.get(get, engine::acquireSearcher)) {
assertTrue("document not found", getResult.exists());
assertEquals(iteration, getResult.version());
- org.apache.lucene.document.Document document = getResult.docIdAndVersion().reader.document(
- getResult.docIdAndVersion().docId
- );
+ StoredFields storedFields = getResult.docIdAndVersion().reader.storedFields();
+ org.apache.lucene.document.Document document = storedFields.document(getResult.docIdAndVersion().docId);
assertThat(document.getValues("test"), arrayContaining(testFieldValue));
}
} catch (Exception t) {
diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
index d988e34ef18dc..9107606326150 100644
--- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
@@ -120,6 +120,12 @@ public void testReplicationCheckpointNotNullForSegRep() throws IOException {
closeShards(indexShard);
}
+ public void testNRTReplicasDoNotAcceptRefreshListeners() throws IOException {
+ final IndexShard indexShard = newStartedShard(false, settings, new NRTReplicationEngineFactory());
+ indexShard.addRefreshListener(mock(Translog.Location.class), Assert::assertFalse);
+ closeShards(indexShard);
+ }
+
public void testSegmentInfosAndReplicationCheckpointTuple() throws Exception {
try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) {
shards.startAll();
@@ -184,7 +190,6 @@ public void testIsSegmentReplicationAllowed_WrongEngineType() throws IOException
* reader close operation on replica shard deletes the segment files copied in current round of segment replication.
* It does this by blocking the finalizeReplication on replica shard and performing close operation on acquired
* searcher that triggers the reader close operation.
- * @throws Exception
*/
public void testSegmentReplication_With_ReaderClosedConcurrently() throws Exception {
String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}";
@@ -234,7 +239,6 @@ public void testSegmentReplication_With_ReaderClosedConcurrently() throws Except
/**
* Similar to test above, this test shows the issue where an engine close operation during active segment replication
* can result in Lucene CorruptIndexException.
- * @throws Exception
*/
public void testSegmentReplication_With_EngineClosedConcurrently() throws Exception {
String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}";
@@ -283,7 +287,6 @@ public void testSegmentReplication_With_EngineClosedConcurrently() throws Except
/**
* Verifies that commits on replica engine resulting from engine or reader close does not cleanup the temporary
* replication files from ongoing round of segment replication
- * @throws Exception
*/
public void testTemporaryFilesNotCleanup() throws Exception {
String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}";
@@ -884,7 +887,6 @@ public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitRefresh() throws
testNRTReplicaWithRemoteStorePromotedAsPrimary(true, false);
}
- @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8817")
public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitCommit() throws Exception {
testNRTReplicaWithRemoteStorePromotedAsPrimary(true, true);
}
diff --git a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java
index 04dcea210640c..940d9a4ead5f9 100644
--- a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java
@@ -326,7 +326,7 @@ void assertSplit(Directory dir, IndexMetadata metadata, int targetShardId, boole
}
assertEquals(shard_id.docID(), doc);
long shardID = shard_id.nextValue();
- BytesRef id = reader.document(doc).getBinaryValue("_id");
+ BytesRef id = reader.storedFields().document(doc).getBinaryValue("_id");
String actualId = Uid.decodeId(id.bytes, id.offset, id.length);
assertNotEquals(ctx.reader() + " docID: " + doc + " actualID: " + actualId, shardID, targetShardId);
}
diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java
index 0f957517d48e1..566a53c2508c8 100644
--- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java
+++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java
@@ -35,10 +35,10 @@
import org.apache.lucene.util.Version;
import org.opensearch.OpenSearchParseException;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.json.JsonXContent;
@@ -73,7 +73,7 @@ public void testToFromXContent() throws IOException {
);
ByteSizeValue size = new ByteSizeValue(Math.abs(randomLong()));
BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size);
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint();
BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS);
byte[] xcontent = BytesReference.toBytes(BytesReference.bytes(shuffleXContent(builder)));
@@ -126,7 +126,7 @@ public void testInvalidFieldsInFromXContent() throws IOException {
fail("shouldn't be here");
}
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject();
builder.field(FileInfo.NAME, name);
builder.field(FileInfo.PHYSICAL_NAME, physicalName);
diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java
index ccb89ec3d1547..eb2006124662a 100644
--- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java
+++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java
@@ -10,9 +10,9 @@
import org.opensearch.common.Strings;
import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
@@ -177,7 +177,7 @@ public void testFromXContentInvalid() throws IOException {
fail("shouldn't be here");
}
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+ XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON);
builder.startObject();
builder.field(RemoteStoreShardShallowCopySnapshot.VERSION, version);
builder.field(RemoteStoreShardShallowCopySnapshot.NAME, snapshot);
diff --git a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java
index ab0046c9554f5..350fc384458ab 100644
--- a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java
+++ b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java
@@ -18,7 +18,7 @@
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.Version;
import org.junit.Before;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
import org.opensearch.index.store.StoreFileMetadata;
import org.opensearch.index.store.remote.utils.BlobFetchRequest;
diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java
index eec9666cc06ba..7a912ecf98292 100644
--- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java
+++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java
@@ -20,8 +20,8 @@
import org.junit.Before;
import org.opensearch.Version;
import org.opensearch.cluster.metadata.IndexMetadata;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.index.IndexSettings;
diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java
index 02b6a48b6f48e..19bfc38056af1 100644
--- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java
+++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java
@@ -14,9 +14,9 @@
import org.opensearch.common.breaker.TestCircuitBreaker;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.index.store.remote.directory.RemoteSnapshotDirectoryFactory;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.index.store.remote.utils.cache.CacheUsage;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java
index 962b3a58c0658..ddf61de292eed 100644
--- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java
@@ -25,8 +25,8 @@
import org.junit.After;
import org.junit.Before;
import org.opensearch.common.blobstore.BlobContainer;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
import org.opensearch.index.store.remote.filecache.FileCache;
import org.opensearch.index.store.remote.filecache.FileCacheFactory;
diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java
index 788391f0e42c0..cd10b7bde3a73 100644
--- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java
@@ -63,8 +63,8 @@
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
import org.opensearch.common.util.concurrent.ReleasableLock;
diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java
index d26379eaefa5c..349f78c3c2d7b 100644
--- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java
@@ -30,8 +30,8 @@
import org.opensearch.common.bytes.ReleasableBytesReference;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
import org.opensearch.common.util.io.IOUtils;
diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
index b7091f3f4f8a6..3d622d6bdf8b8 100644
--- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
@@ -44,6 +44,7 @@
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anySet;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
@@ -205,11 +206,12 @@ public void testReadMetadataNoFile() throws IOException {
null
);
doAnswer(invocation -> {
- LatchedActionListener> latchedActionListener = invocation.getArgument(2);
+ LatchedActionListener> latchedActionListener = invocation.getArgument(3);
List bmList = new LinkedList<>();
latchedActionListener.onResponse(bmList);
return null;
- }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class));
+ }).when(transferService)
+ .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class));
assertNull(translogTransferManager.readMetadata());
}
@@ -225,12 +227,13 @@ public void testReadMetadataSingleFile() throws IOException {
TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2);
String mdFilename = tm.getFileName();
doAnswer(invocation -> {
- LatchedActionListener> latchedActionListener = invocation.getArgument(2);
+ LatchedActionListener> latchedActionListener = invocation.getArgument(3);
List bmList = new LinkedList<>();
bmList.add(new PlainBlobMetadata(mdFilename, 1));
latchedActionListener.onResponse(bmList);
return null;
- }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class));
+ }).when(transferService)
+ .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class));
TranslogTransferMetadata metadata = createTransferSnapshot().getTranslogTransferMetadata();
when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenReturn(
@@ -252,12 +255,13 @@ public void testReadMetadataReadException() throws IOException {
String mdFilename = tm.getFileName();
doAnswer(invocation -> {
- LatchedActionListener> latchedActionListener = invocation.getArgument(2);
+ LatchedActionListener> latchedActionListener = invocation.getArgument(3);
List bmList = new LinkedList<>();
bmList.add(new PlainBlobMetadata(mdFilename, 1));
latchedActionListener.onResponse(bmList);
return null;
- }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class));
+ }).when(transferService)
+ .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class));
when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenThrow(new IOException("Something went wrong"));
@@ -283,10 +287,11 @@ public void testReadMetadataListException() throws IOException {
);
doAnswer(invocation -> {
- LatchedActionListener> latchedActionListener = invocation.getArgument(2);
+ LatchedActionListener> latchedActionListener = invocation.getArgument(3);
latchedActionListener.onFailure(new IOException("Issue while listing"));
return null;
- }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class));
+ }).when(transferService)
+ .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class));
when(transferService.downloadBlob(any(BlobPath.class), any(String.class))).thenThrow(new IOException("Something went wrong"));
@@ -416,7 +421,7 @@ public void testDeleteStaleTranslogMetadata() {
String tm2 = new TranslogTransferMetadata(1, 2, 1, 2).getFileName();
String tm3 = new TranslogTransferMetadata(2, 3, 1, 2).getFileName();
doAnswer(invocation -> {
- ActionListener> actionListener = invocation.getArgument(3);
+ ActionListener> actionListener = invocation.getArgument(4);
List bmList = new LinkedList<>();
bmList.add(new PlainBlobMetadata(tm1, 1));
bmList.add(new PlainBlobMetadata(tm2, 1));
@@ -424,12 +429,19 @@ public void testDeleteStaleTranslogMetadata() {
actionListener.onResponse(bmList);
return null;
}).when(transferService)
- .listAllInSortedOrderAsync(eq(ThreadPool.Names.REMOTE_PURGE), any(BlobPath.class), anyInt(), any(ActionListener.class));
+ .listAllInSortedOrderAsync(
+ eq(ThreadPool.Names.REMOTE_PURGE),
+ any(BlobPath.class),
+ anyString(),
+ anyInt(),
+ any(ActionListener.class)
+ );
List files = List.of(tm2, tm3);
translogTransferManager.deleteStaleTranslogMetadataFilesAsync(() -> {
verify(transferService).listAllInSortedOrderAsync(
eq(ThreadPool.Names.REMOTE_PURGE),
any(BlobPath.class),
+ eq(TranslogTransferMetadata.METADATA_PREFIX),
eq(Integer.MAX_VALUE),
any()
);
diff --git a/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java b/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java
index 62984f43744c8..19ed7122741aa 100644
--- a/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java
@@ -36,8 +36,8 @@
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.common.SetOnce;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.codec.CodecService;
import org.opensearch.index.engine.EngineConfig;
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 3508020cb23d7..2a03ddc6f46e4 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -51,7 +51,7 @@
import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.util.io.IOUtils;
@@ -353,7 +353,7 @@ public BytesReference get() {
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1);
assertEquals(1, topDocs.totalHits.value);
- Document document = reader.document(topDocs.scoreDocs[0].doc);
+ Document document = reader.storedFields().document(topDocs.scoreDocs[0].doc);
out.writeString(document.get("value"));
loadedFromCache = false;
return out.bytes();
diff --git a/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java b/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java
index 93c3b0f48315d..7b1aee370ddb7 100644
--- a/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java
+++ b/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java
@@ -32,7 +32,7 @@
package org.opensearch.indices.breaker;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.settings.Settings;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
index 7bdff59e7c334..7ed072306e2cb 100644
--- a/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java
@@ -33,13 +33,14 @@
package org.opensearch.indices.breaker;
import org.opensearch.common.breaker.ChildMemoryCircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreaker;
-import org.opensearch.common.breaker.CircuitBreakingException;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreakingException;
import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.monitor.jvm.JvmInfo;
import org.opensearch.search.aggregations.MultiBucketConsumerService;
import org.opensearch.test.OpenSearchTestCase;
diff --git a/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java b/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java
index a2705a0730aaf..fad619f40222b 100644
--- a/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java
+++ b/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java
@@ -32,7 +32,7 @@
package org.opensearch.indices.memory.breaker;
-import org.opensearch.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.CircuitBreaker;
import org.opensearch.common.settings.Settings;
import org.opensearch.indices.breaker.BreakerSettings;
import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java
index 3b289114f5ca1..84a53ae22a6bc 100644
--- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java
@@ -403,4 +403,38 @@ public void testCancelForMissingIds() throws IOException {
assertEquals(0, replications.cachedCopyStateSize());
closeShards(replica_2);
}
+
+ public void testPrepareForReplicationAlreadyReplicating() throws IOException {
+ OngoingSegmentReplications replications = new OngoingSegmentReplications(mockIndicesService, recoverySettings);
+ final String replicaAllocationId = replica.routingEntry().allocationId().getId();
+ final CheckpointInfoRequest request = new CheckpointInfoRequest(1L, replicaAllocationId, primaryDiscoveryNode, testCheckpoint);
+
+ final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class));
+
+ final SegmentReplicationSourceHandler handler = replications.getHandlers().get(replicaAllocationId);
+ assertEquals(handler.getCopyState(), copyState);
+ assertEquals(1, copyState.refCount());
+
+ ReplicationCheckpoint secondCheckpoint = new ReplicationCheckpoint(
+ testCheckpoint.getShardId(),
+ testCheckpoint.getPrimaryTerm(),
+ testCheckpoint.getSegmentsGen(),
+ testCheckpoint.getSegmentInfosVersion() + 1,
+ testCheckpoint.getCodec()
+ );
+
+ final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest(
+ 1L,
+ replicaAllocationId,
+ primaryDiscoveryNode,
+ secondCheckpoint
+ );
+
+ final CopyState secondCopyState = replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class));
+ final SegmentReplicationSourceHandler secondHandler = replications.getHandlers().get(replicaAllocationId);
+ assertEquals(secondHandler.getCopyState(), secondCopyState);
+ assertEquals("New copy state is incref'd", 1, secondCopyState.refCount());
+ assertEquals("Old copy state is cleaned up", 0, copyState.refCount());
+
+ }
}
diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
index 4643615d45d7e..94e57f4a0d3e4 100644
--- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
@@ -32,6 +32,7 @@
import org.opensearch.indices.recovery.ForceSyncRequest;
import org.opensearch.indices.recovery.RecoverySettings;
import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
+import org.opensearch.indices.replication.common.CopyState;
import org.opensearch.indices.replication.common.ReplicationCollection;
import org.opensearch.indices.replication.common.ReplicationFailedException;
import org.opensearch.indices.replication.common.ReplicationLuceneIndex;
@@ -49,6 +50,7 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.atLeastOnce;
@@ -70,10 +72,7 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase {
private IndexShard replicaShard;
private IndexShard primaryShard;
private ReplicationCheckpoint checkpoint;
- private SegmentReplicationSource replicationSource;
private SegmentReplicationTargetService sut;
-
- private ReplicationCheckpoint initialCheckpoint;
private ReplicationCheckpoint aheadCheckpoint;
private ReplicationCheckpoint newPrimaryCheckpoint;
@@ -83,11 +82,10 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase {
private DiscoveryNode localNode;
private IndicesService indicesService;
- private ClusterService clusterService;
private SegmentReplicationState state;
- private static long TRANSPORT_TIMEOUT = 30000;// 30sec
+ private static final long TRANSPORT_TIMEOUT = 30000;// 30sec
@Override
public void setUp() throws Exception {
@@ -107,9 +105,6 @@ public void setUp() throws Exception {
0L,
replicaShard.getLatestReplicationCheckpoint().getCodec()
);
- SegmentReplicationSourceFactory replicationSourceFactory = mock(SegmentReplicationSourceFactory.class);
- replicationSource = mock(SegmentReplicationSource.class);
- when(replicationSourceFactory.get(replicaShard)).thenReturn(replicationSource);
testThreadPool = new TestThreadPool("test", Settings.EMPTY);
localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT);
@@ -126,7 +121,7 @@ public void setUp() throws Exception {
transportService.acceptIncomingRequests();
indicesService = mock(IndicesService.class);
- clusterService = mock(ClusterService.class);
+ ClusterService clusterService = mock(ClusterService.class);
ClusterState clusterState = mock(ClusterState.class);
RoutingTable mockRoutingTable = mock(RoutingTable.class);
when(clusterService.state()).thenReturn(clusterState);
@@ -135,7 +130,7 @@ public void setUp() throws Exception {
when(clusterState.nodes()).thenReturn(DiscoveryNodes.builder().add(localNode).build());
sut = prepareForReplication(primaryShard, replicaShard, transportService, indicesService, clusterService);
- initialCheckpoint = replicaShard.getLatestReplicationCheckpoint();
+ ReplicationCheckpoint initialCheckpoint = replicaShard.getLatestReplicationCheckpoint();
aheadCheckpoint = new ReplicationCheckpoint(
initialCheckpoint.getShardId(),
initialCheckpoint.getPrimaryTerm(),
@@ -242,7 +237,46 @@ public void testAlreadyOnNewCheckpoint() {
}
public void testShardAlreadyReplicating() {
- sut.startReplication(replicaShard, mock(SegmentReplicationTargetService.SegmentReplicationListener.class));
+ CountDownLatch blockGetCheckpointMetadata = new CountDownLatch(1);
+ SegmentReplicationSource source = new TestReplicationSource() {
+ @Override
+ public void getCheckpointMetadata(
+ long replicationId,
+ ReplicationCheckpoint checkpoint,
+ ActionListener listener
+ ) {
+ try {
+ blockGetCheckpointMetadata.await();
+ final CopyState copyState = new CopyState(
+ ReplicationCheckpoint.empty(primaryShard.shardId(), primaryShard.getLatestReplicationCheckpoint().getCodec()),
+ primaryShard
+ );
+ listener.onResponse(
+ new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes())
+ );
+ } catch (InterruptedException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void getSegmentFiles(
+ long replicationId,
+ ReplicationCheckpoint checkpoint,
+ List