diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 31b1cb5efe3a1..8cfd636b9fd2e 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -21,4 +21,5 @@ BWC_VERSION: - "2.8.0" - "2.8.1" - "2.9.0" + - "2.9.1" - "2.10.0" diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index fdf42a9a2731e..764a365e7411c 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -54,8 +54,8 @@ jobs: echo " - \"$CURRENT_VERSION\"" >> .ci/bwcVersions sed -i "s/opensearch = $CURRENT_VERSION/opensearch = $NEXT_VERSION/g" buildSrc/version.properties echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE - sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java - sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" server/src/main/java/org/opensearch/Version.java + sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java + sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java - name: Create Pull Request uses: peter-evans/create-pull-request@v3 @@ -82,7 +82,7 @@ jobs: echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION sed -i "s/- \"$CURRENT_VERSION\"/\0\n - \"$NEXT_VERSION\"/g" .ci/bwcVersions echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE - sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java + sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java - name: Create Pull Request uses: peter-evans/create-pull-request@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ab07011aefee..9e4a26caa2b10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) - Remote Segment Store Repository setting moved from `index.remote_store.repository` to `index.remote_store.segment.repository` and `cluster.remote_store.repository` to `cluster.remote_store.segment.repository` respectively for Index and Cluster level settings ([#8719](https://github.com/opensearch-project/OpenSearch/pull/8719)) +- [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792)) +- Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792)) ### Deprecated @@ -76,26 +78,42 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Add server version as REST response header [#6583](https://github.com/opensearch-project/OpenSearch/issues/6583) - Start replication checkpointTimers on primary before segments upload to remote store. ([#8221]()https://github.com/opensearch-project/OpenSearch/pull/8221) +- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195) +- Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726)) - Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724)) - Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725)) +- Bump `org.apache.zookeeper:zookeeper` from 3.8.1 to 3.8.2 ([#8844](https://github.com/opensearch-project/OpenSearch/pull/8844)) +- Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842)) +- Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838)) +- Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840)) +- OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868) +- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995)) +- Bump `com.gradle.enterprise` from 3.13.3 to 3.14.1 ([#8996](https://github.com/opensearch-project/OpenSearch/pull/8996)) +- Bump `org.apache.commons:commons-lang3` from 3.12.0 to 3.13.0 ([#8995](https://github.com/opensearch-project/OpenSearch/pull/8995)) +- Bump `com.google.cloud:google-cloud-core-http` from 2.21.0 to 2.21.1 ([#8999](https://github.com/opensearch-project/OpenSearch/pull/8999)) ### Changed - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303)) - Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620)) - Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) - Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805)) +- [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636)) +- Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807)) +- Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801)) +- [Remove] Deprecated Fractional ByteSizeValue support #9005 ([#9005](https://github.com/opensearch-project/OpenSearch/pull/9005)) ### Deprecated ### Removed ### Fixed +- Fix flaky ResourceAwareTasksTests.testBasicTaskResourceTracking test ([#8993](https://github.com/opensearch-project/OpenSearch/pull/8993)) ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0baf626142238..1dce1f8a75035 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -264,7 +264,10 @@ This repository is split into many top level directories. The most important one ### `distribution` -Builds our tar and zip archives and our rpm and deb packages. +Builds our tar and zip archives and our rpm and deb packages. There are several flavors of the distributions, with the classifier included in the name of the final deliverable (archive or package): + - default (no classifier), the distribution with bundled JDK + - `-no-jdk-` - the distribution without bundled JDK/JRE, assumes the JDK/JRE is going to be pre-installed on the target systems + - `-jre-` - the distribution bundled with JRE (smaller footprint), supported as experimental feature for some platforms ### `libs` diff --git a/README.md b/README.md index 55ff067e4d779..b5fc45509b002 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues) [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls) -[![2.8 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.8.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.8.0") +[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0") [![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0") [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml) [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml) diff --git a/TESTING.md b/TESTING.md index 93b4615da6f0b..a76ee07e1faac 100644 --- a/TESTING.md +++ b/TESTING.md @@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Test groups](#test-groups) - [Load balancing and caches](#load-balancing-and-caches) - [Test compatibility](#test-compatibility) + - [Retries](#retries) - [Miscellaneous](#miscellaneous) - [Running verification tasks](#running-verification-tasks) - [Testing the REST layer](#testing-the-rest-layer) @@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to ./gradlew test -Dtests.compatibility=1.0.0 +## Retries + +The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label. + ## Miscellaneous Run all tests without stopping on errors (inspect log files). diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java index 76851881730a3..b18ea4327cbc2 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/search/aggregations/TermsReduceBenchmark.java @@ -40,14 +40,14 @@ import org.opensearch.action.search.SearchPhaseController; import org.opensearch.action.search.SearchProgressListener; import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchModule; import org.opensearch.search.SearchShardTarget; diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java index d3bfc9348cdb3..4f6b431cc56fd 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java @@ -27,8 +27,8 @@ import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.index.store.remote.filecache.CachedIndexInput; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheFactory; diff --git a/build.gradle b/build.gradle index 6a14ab231894b..4d85638270eca 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ plugins { id 'opensearch.docker-support' id 'opensearch.global-build-info' id "com.diffplug.spotless" version "6.19.0" apply false - id "org.gradle.test-retry" version "1.5.3" apply false + id "org.gradle.test-retry" version "1.5.4" apply false id "test-report-aggregation" id 'jacoco-report-aggregation' } @@ -470,6 +470,81 @@ subprojects { maxFailures = 10 } failOnPassedAfterRetry = false + filter { + includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests") + includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT") + includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT") + includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT") + includeClasses.add("org.opensearch.aliases.IndexAliasesIT") + includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.blocks.SimpleBlocksIT") + includeClasses.add("org.opensearch.client.PitIT") + includeClasses.add("org.opensearch.client.ReindexIT") + includeClasses.add("org.opensearch.cluster.ClusterHealthIT") + includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT") + includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT") + includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT") + includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests") + includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests") + includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT") + includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests") + includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT") + includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT") + includeClasses.add("org.opensearch.http.SearchRestCancellationIT") + includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests") + includeClasses.add("org.opensearch.index.IndexServiceTests") + includeClasses.add("org.opensearch.index.IndexSettingsTests") + includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT") + includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests") + includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests") + includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests") + includeClasses.add("org.opensearch.index.shard.IndexShardIT") + includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests") + includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests") + includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT") + includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests") + includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT") + includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests") + includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT") + includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests") + includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT") + includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT") + includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT") + includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT") + includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT") + includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT") + includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT") + includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT") + includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT") + includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests") + } } } } diff --git a/server/src/main/java/org/opensearch/common/component/package-info.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java similarity index 69% rename from server/src/main/java/org/opensearch/common/component/package-info.java rename to buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java index 34d034b5a3ffb..2acc335d80df0 100644 --- a/server/src/main/java/org/opensearch/common/component/package-info.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java @@ -6,5 +6,10 @@ * compatible open source license. */ -/** Base Lifecycle Component package. */ -package org.opensearch.common.component; +package org.opensearch.gradle; + +public enum JavaPackageType { + NONE, + JRE, + JDK +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java new file mode 100644 index 0000000000000..5f576984627a8 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.gradle.api.Buildable; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.TaskDependency; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +abstract class JavaVariant implements Buildable, Iterable { + + private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList( + Arrays.asList("aarch64", "x64", "s390x", "ppc64le") + ); + private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); + private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList( + Arrays.asList("darwin", "freebsd", "linux", "mac", "windows") + ); + private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); + private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?"); + + private final String name; + private final Configuration configuration; + + private final Property vendor; + private final Property version; + private final Property platform; + private final Property architecture; + private String baseVersion; + private String major; + private String build; + private String hash; + + JavaVariant(String name, Configuration configuration, ObjectFactory objectFactory) { + this.name = name; + this.configuration = configuration; + this.vendor = objectFactory.property(String.class); + this.version = objectFactory.property(String.class); + this.platform = objectFactory.property(String.class); + this.architecture = objectFactory.property(String.class); + } + + public String getName() { + return name; + } + + public String getVendor() { + return vendor.get(); + } + + public void setVendor(final String vendor) { + if (ALLOWED_VENDORS.contains(vendor) == false) { + throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS); + } + this.vendor.set(vendor); + } + + public String getVersion() { + return version.get(); + } + + public void setVersion(String version) { + if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) { + throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]"); + } + parseVersion(version); + this.version.set(version); + } + + public String getPlatform() { + return platform.get(); + } + + public void setPlatform(String platform) { + if (ALLOWED_PLATFORMS.contains(platform) == false) { + throw new IllegalArgumentException( + "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS + ); + } + this.platform.set(platform); + } + + public String getArchitecture() { + return architecture.get(); + } + + public void setArchitecture(final String architecture) { + String jdkArchitecture = translateJdkArchitecture(architecture); + if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) { + throw new IllegalArgumentException( + "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES + ); + } + this.architecture.set(jdkArchitecture); + } + + public String getBaseVersion() { + return baseVersion; + } + + public String getMajor() { + return major; + } + + public String getBuild() { + return build; + } + + public String getHash() { + return hash; + } + + public String getPath() { + return configuration.getSingleFile().toString(); + } + + public String getConfigurationName() { + return configuration.getName(); + } + + @Override + public String toString() { + return getPath(); + } + + @Override + public TaskDependency getBuildDependencies() { + return configuration.getBuildDependencies(); + } + + // internal, make this jdks configuration unmodifiable + void finalizeValues() { + if (version.isPresent() == false) { + throw new IllegalArgumentException("version not specified for jdk [" + name + "]"); + } + if (platform.isPresent() == false) { + throw new IllegalArgumentException("platform not specified for jdk [" + name + "]"); + } + if (vendor.isPresent() == false) { + throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]"); + } + if (architecture.isPresent() == false) { + throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]"); + } + version.finalizeValue(); + platform.finalizeValue(); + vendor.finalizeValue(); + architecture.finalizeValue(); + } + + @Override + public Iterator iterator() { + return configuration.iterator(); + } + + private void parseVersion(String version) { + // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build] + // Note the "patch" version is not yet handled here, as it has not yet been used by java. + Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version); + if (jdkVersionMatcher.matches() == false) { + // Try again with the pre-Java9 version format + jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version); + + if (jdkVersionMatcher.matches() == false) { + throw new IllegalArgumentException("Malformed jdk version [" + version + "]"); + } + } + + baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : ""); + major = jdkVersionMatcher.group(1); + build = jdkVersionMatcher.group(3); + hash = jdkVersionMatcher.group(5); + } + + private String translateJdkArchitecture(String architecture) { + /* + * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. + */ + return "arm64".equals(architecture) ? "aarch64" : architecture; + } + +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java index 06e857744be2d..3218abe726639 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java @@ -32,140 +32,13 @@ package org.opensearch.gradle; -import org.gradle.api.Buildable; import org.gradle.api.artifacts.Configuration; import org.gradle.api.model.ObjectFactory; -import org.gradle.api.provider.Property; -import org.gradle.api.tasks.TaskDependency; import org.gradle.internal.os.OperatingSystem; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class Jdk implements Buildable, Iterable { - - private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList( - Arrays.asList("aarch64", "x64", "s390x", "ppc64le") - ); - private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); - private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList( - Arrays.asList("darwin", "freebsd", "linux", "mac", "windows") - ); - private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); - private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?"); - - private final String name; - private final Configuration configuration; - - private final Property vendor; - private final Property version; - private final Property platform; - private final Property architecture; - private String baseVersion; - private String major; - private String build; - private String hash; - +public class Jdk extends JavaVariant { Jdk(String name, Configuration configuration, ObjectFactory objectFactory) { - this.name = name; - this.configuration = configuration; - this.vendor = objectFactory.property(String.class); - this.version = objectFactory.property(String.class); - this.platform = objectFactory.property(String.class); - this.architecture = objectFactory.property(String.class); - } - - public String getName() { - return name; - } - - public String getVendor() { - return vendor.get(); - } - - public void setVendor(final String vendor) { - if (ALLOWED_VENDORS.contains(vendor) == false) { - throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS); - } - this.vendor.set(vendor); - } - - public String getVersion() { - return version.get(); - } - - public void setVersion(String version) { - if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) { - throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]"); - } - parseVersion(version); - this.version.set(version); - } - - public String getPlatform() { - return platform.get(); - } - - public void setPlatform(String platform) { - if (ALLOWED_PLATFORMS.contains(platform) == false) { - throw new IllegalArgumentException( - "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS - ); - } - this.platform.set(platform); - } - - public String getArchitecture() { - return architecture.get(); - } - - public void setArchitecture(final String architecture) { - String jdkArchitecture = translateJdkArchitecture(architecture); - if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) { - throw new IllegalArgumentException( - "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES - ); - } - this.architecture.set(jdkArchitecture); - } - - public String getBaseVersion() { - return baseVersion; - } - - public String getMajor() { - return major; - } - - public String getBuild() { - return build; - } - - public String getHash() { - return hash; - } - - public String getPath() { - return configuration.getSingleFile().toString(); - } - - public String getConfigurationName() { - return configuration.getName(); - } - - @Override - public String toString() { - return getPath(); - } - - @Override - public TaskDependency getBuildDependencies() { - return configuration.getBuildDependencies(); + super(name, configuration, objectFactory); } public Object getBinJavaPath() { @@ -190,56 +63,4 @@ private String getHomeRoot() { boolean isOSX = "mac".equals(getPlatform()) || "darwin".equals(getPlatform()); return getPath() + (isOSX ? "/Contents/Home" : ""); } - - // internal, make this jdks configuration unmodifiable - void finalizeValues() { - if (version.isPresent() == false) { - throw new IllegalArgumentException("version not specified for jdk [" + name + "]"); - } - if (platform.isPresent() == false) { - throw new IllegalArgumentException("platform not specified for jdk [" + name + "]"); - } - if (vendor.isPresent() == false) { - throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]"); - } - if (architecture.isPresent() == false) { - throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]"); - } - version.finalizeValue(); - platform.finalizeValue(); - vendor.finalizeValue(); - architecture.finalizeValue(); - } - - @Override - public Iterator iterator() { - return configuration.iterator(); - } - - private void parseVersion(String version) { - // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build] - // Note the "patch" version is not yet handled here, as it has not yet been used by java. - Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version); - if (jdkVersionMatcher.matches() == false) { - // Try again with the pre-Java9 version format - jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version); - - if (jdkVersionMatcher.matches() == false) { - throw new IllegalArgumentException("Malformed jdk version [" + version + "]"); - } - } - - baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : ""); - major = jdkVersionMatcher.group(1); - build = jdkVersionMatcher.group(3); - hash = jdkVersionMatcher.group(5); - } - - private String translateJdkArchitecture(String architecture) { - /* - * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. - */ - return "arm64".equals(architecture) ? "aarch64" : architecture; - } - } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jre.java b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java new file mode 100644 index 0000000000000..473bfc4860b80 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.model.ObjectFactory; + +public class Jre extends JavaVariant { + Jre(String name, Configuration configuration, ObjectFactory objectFactory) { + super(name, configuration, objectFactory); + } +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java new file mode 100644 index 0000000000000..5a00f41f07a60 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.opensearch.gradle.transform.SymbolicLinkPreservingUntarTransform; +import org.opensearch.gradle.transform.UnzipTransform; +import org.gradle.api.GradleException; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.dsl.RepositoryHandler; +import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.internal.artifacts.ArtifactAttributes; + +public class JreDownloadPlugin implements Plugin { + public static final String VENDOR_ADOPTIUM = "adoptium"; + + private static final String REPO_NAME_PREFIX = "jre_repo_"; + private static final String EXTENSION_NAME = "jres"; + public static final String JRE_TRIMMED_PREFIX = "jdk-?\\d.*-jre"; + + @Override + public void apply(Project project) { + Attribute jreAttribute = Attribute.of("jre", Boolean.class); + project.getDependencies().getAttributesSchema().attribute(jreAttribute); + project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE); + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE) + .attribute(jreAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jreAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX)); + }); + + ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz"); + project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> { + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName()) + .attribute(jreAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jreAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX)); + }); + + NamedDomainObjectContainer jresContainer = project.container(Jre.class, name -> { + Configuration configuration = project.getConfigurations().create("jre_" + name); + configuration.setCanBeConsumed(false); + configuration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + configuration.getAttributes().attribute(jreAttribute, true); + Jre jre = new Jre(name, configuration, project.getObjects()); + configuration.defaultDependencies(dependencies -> { + jre.finalizeValues(); + setupRepository(project, jre); + dependencies.add(project.getDependencies().create(dependencyNotation(jre))); + }); + return jre; + }); + project.getExtensions().add(EXTENSION_NAME, jresContainer); + } + + private void setupRepository(Project project, Jre jre) { + RepositoryHandler repositories = project.getRepositories(); + + /* + * Define the appropriate repository for the given JRE vendor and version + * + * For Oracle/OpenJDK/AdoptOpenJDK we define a repository per-version. + */ + String repoName = REPO_NAME_PREFIX + jre.getVendor() + "_" + jre.getVersion(); + String repoUrl; + String artifactPattern; + + if (jre.getVendor().equals(VENDOR_ADOPTIUM)) { + repoUrl = "https://github.com/adoptium/temurin" + jre.getMajor() + "-binaries/releases/download/"; + + if (jre.getMajor().equals("8")) { + // JDK-8 updates are always suffixed with 'U' (fe OpenJDK8U). + artifactPattern = "jdk" + + jre.getBaseVersion() + + "-" + + jre.getBuild() + + "/OpenJDK" + + jre.getMajor() + + "U" + + "-jre_[classifier]_[module]_hotspot_" + + jre.getBaseVersion() + + jre.getBuild() + + ".[ext]"; + } else { + // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17). + // To distinguish between those, the GA releases have only major version component (fe 17+32), + // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of + // version separator '.' should be enough. + artifactPattern = "jdk-" + jre.getBaseVersion() + "+" + jre.getBuild() + "/OpenJDK" + jre.getMajor() + // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release + + (jre.getBaseVersion().contains(".") || jre.getBaseVersion().matches("^2\\d+$") ? "U" : "") + + "-jre_[classifier]_[module]_hotspot_" + + jre.getBaseVersion() + + "_" + + jre.getBuild() + + ".[ext]"; + } + } else { + throw new GradleException("Unknown JDK vendor [" + jre.getVendor() + "]"); + } + + // Define the repository if we haven't already + if (repositories.findByName(repoName) == null) { + repositories.ivy(repo -> { + repo.setName(repoName); + repo.setUrl(repoUrl); + repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact); + repo.patternLayout(layout -> layout.artifact(artifactPattern)); + repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeGroup(groupName(jre))); + }); + } + } + + @SuppressWarnings("unchecked") + public static NamedDomainObjectContainer getContainer(Project project) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(EXTENSION_NAME); + } + + private static String dependencyNotation(Jre jre) { + String platformDep = jre.getPlatform().equals("darwin") || jre.getPlatform().equals("mac") ? "mac" : jre.getPlatform(); + String extension = jre.getPlatform().equals("windows") ? "zip" : "tar.gz"; + + return groupName(jre) + ":" + platformDep + ":" + jre.getBaseVersion() + ":" + jre.getArchitecture() + "@" + extension; + } + + private static String groupName(Jre jre) { + return jre.getVendor() + "_" + jre.getMajor() + "_jre"; + } + +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java index 968bd13bd4011..0575c23fee9f6 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java @@ -101,7 +101,7 @@ public boolean shouldExtract() { private final Property version; private final Property type; private final Property platform; - private final Property bundledJdk; + private final Property bundledJdk; private final Property failIfUnavailable; private final Configuration extracted; @@ -120,7 +120,7 @@ public boolean shouldExtract() { this.type = objectFactory.property(Type.class); this.type.convention(Type.ARCHIVE); this.platform = objectFactory.property(Platform.class); - this.bundledJdk = objectFactory.property(Boolean.class); + this.bundledJdk = objectFactory.property(JavaPackageType.class); this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true); this.extracted = extractedConfiguration; } @@ -154,8 +154,8 @@ public void setType(Type type) { this.type.set(type); } - public boolean getBundledJdk() { - return bundledJdk.getOrElse(true); + public JavaPackageType getBundledJdk() { + return bundledJdk.getOrElse(JavaPackageType.JDK); } public boolean isDocker() { @@ -163,7 +163,7 @@ public boolean isDocker() { return type == Type.DOCKER; } - public void setBundledJdk(Boolean bundledJdk) { + public void setBundledJdk(JavaPackageType bundledJdk) { this.bundledJdk.set(bundledJdk); } @@ -266,7 +266,7 @@ void finalizeValues() { } if (bundledJdk.isPresent() == false) { - bundledJdk.set(true); + bundledJdk.set(JavaPackageType.JDK); } version.finalizeValue(); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java index 7c942358e12c3..4d8b62d95dff1 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java @@ -76,6 +76,10 @@ public static String getBundledJdk(final String platform) { return getBundledJdk(platform, null); } + public static String getBundledJre(final String platform, final String arch) { + return getBundledJdk(platform, arch); + } + public static String getBundledJdkVendor() { return bundledJdkVendor; } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java index f4368b1cecc59..6a54612320c6c 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -37,6 +37,7 @@ import org.opensearch.gradle.DistributionDependency; import org.opensearch.gradle.DistributionDownloadPlugin; import org.opensearch.gradle.DistributionResolution; +import org.opensearch.gradle.JavaPackageType; import org.opensearch.gradle.OpenSearchDistribution; import org.opensearch.gradle.Version; import org.opensearch.gradle.VersionProperties; @@ -99,7 +100,7 @@ private void registerInternalDistributionResolutions(NamedDomainObjectContainer< resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> { BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion())); if (unreleasedInfo != null) { - if (!distribution.getBundledJdk()) { + if (distribution.getBundledJdk() == JavaPackageType.NONE) { throw new GradleException( "Configuring a snapshot bwc distribution ('" + distribution.getName() @@ -167,8 +168,10 @@ private static String distributionProjectName(OpenSearchDistribution distributio ? "" : "-" + architecture.toString().toLowerCase(); - if (distribution.getBundledJdk() == false) { + if (distribution.getBundledJdk() == JavaPackageType.NONE) { projectName += "no-jdk-"; + } else if (distribution.getBundledJdk() == JavaPackageType.JRE) { + projectName += "jre-"; } switch (distribution.getType()) { case ARCHIVE: diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index 7e0dc72cb9108..a311d89583acb 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -34,6 +34,7 @@ import org.opensearch.gradle.Architecture; import org.opensearch.gradle.DistributionDownloadPlugin; +import org.opensearch.gradle.JavaPackageType; import org.opensearch.gradle.OpenSearchDistribution; import org.opensearch.gradle.Jdk; import org.opensearch.gradle.JdkDownloadPlugin; @@ -71,13 +72,14 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; public class DistroTestPlugin implements Plugin { - private static final String SYSTEM_JDK_VERSION = "11.0.19+7"; + private static final String SYSTEM_JDK_VERSION = "11.0.20+8"; private static final String SYSTEM_JDK_VENDOR = "adoptium"; - private static final String GRADLE_JDK_VERSION = "17.0.7+7"; + private static final String GRADLE_JDK_VERSION = "17.0.8+7"; private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution @@ -137,7 +139,7 @@ public void apply(Project project) { } if ((distribution.getType() == OpenSearchDistribution.Type.DEB || distribution.getType() == OpenSearchDistribution.Type.RPM) - && distribution.getBundledJdk()) { + && distribution.getBundledJdk() != JavaPackageType.NONE) { for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { if (version.before("6.3.0")) { continue; // before opening xpack @@ -379,8 +381,8 @@ private List configureDistributions(Project project) { OpenSearchDistribution.Type.RPM, OpenSearchDistribution.Type.DOCKER )) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false) { + for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) { + if (bundledJdk == JavaPackageType.NONE) { // We'll never publish an ARM (arm64) build without a bundled JDK. if (architecture == Architecture.ARM64) { continue; @@ -403,8 +405,8 @@ private List configureDistributions(Project project) { OpenSearchDistribution.Platform.LINUX, OpenSearchDistribution.Platform.WINDOWS )) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false && architecture != Architecture.X64) { + for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) { + if (bundledJdk == JavaPackageType.NONE && architecture != Architecture.X64) { // We will never publish distributions for non-x86 (amd64) platforms // without a bundled JDK continue; @@ -432,7 +434,7 @@ private static OpenSearchDistribution createDistro( Architecture architecture, OpenSearchDistribution.Type type, OpenSearchDistribution.Platform platform, - boolean bundledJdk, + JavaPackageType bundledJdk, String version ) { String name = distroId(type, platform, bundledJdk, architecture) + "-" + version; @@ -466,11 +468,12 @@ private static boolean isWindows(Project project) { private static String distroId( OpenSearchDistribution.Type type, OpenSearchDistribution.Platform platform, - boolean bundledJdk, + JavaPackageType bundledJdk, Architecture architecture ) { - return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk") - + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); + return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk != JavaPackageType.NONE + ? (bundledJdk == JavaPackageType.JDK ? "" : "-jre") + : "-no-jdk") + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); } private static String destructiveDistroTestTaskName(OpenSearchDistribution distro) { diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties new file mode 100644 index 0000000000000..e9253488ffbeb --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties @@ -0,0 +1,12 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +implementation-class=org.opensearch.gradle.JreDownloadPlugin diff --git a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java index 1a9647573f948..e439e8a047f34 100644 --- a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java @@ -45,6 +45,7 @@ import java.util.Arrays; import java.util.TreeSet; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.core.StringContains.containsString; public class DistributionDownloadPluginTests extends GradleUnitTestCase { @@ -76,7 +77,14 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase { ); public void testVersionDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", null, Type.ARCHIVE, Platform.LINUX, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + null, + Type.ARCHIVE, + Platform.LINUX, + JavaPackageType.JDK + ); assertEquals(distro.getVersion(), VersionProperties.getOpenSearch()); } @@ -123,18 +131,32 @@ public void testBadVersionFormat() { "badversion", Type.ARCHIVE, Platform.LINUX, - true, + JavaPackageType.JDK, "Invalid version format: 'badversion'" ); } public void testTypeDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", null, Platform.LINUX, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + null, + Platform.LINUX, + JavaPackageType.JDK + ); assertEquals(distro.getType(), Type.ARCHIVE); } public void testPlatformDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, null, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + Type.ARCHIVE, + null, + JavaPackageType.JDK + ); assertEquals(distro.getPlatform(), OpenSearchDistribution.CURRENT_PLATFORM); } @@ -151,8 +173,15 @@ public void testPlatformForIntegTest() { } public void testBundledJdkDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, true); - assertTrue(distro.getBundledJdk()); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + Type.ARCHIVE, + Platform.LINUX, + JavaPackageType.JDK + ); + assertThat(distro.getBundledJdk(), equalTo(JavaPackageType.JDK)); } public void testBundledJdkForIntegTest() { @@ -162,7 +191,7 @@ public void testBundledJdkForIntegTest() { "5.0.0", Type.INTEG_TEST_ZIP, null, - true, + JavaPackageType.JDK, "bundledJdk cannot be set on opensearch distribution [testdistro]" ); } @@ -178,7 +207,7 @@ public void testLocalCurrentVersionIntegTestZip() { public void testLocalCurrentVersionArchives() { for (Platform platform : Platform.values()) { - for (boolean bundledJdk : new boolean[] { true, false }) { + for (JavaPackageType bundledJdk : JavaPackageType.values()) { for (Architecture architecture : Architecture.values()) { // create a new project in each iteration, so that we know we are resolving the only additional project being created Project project = createProject(BWC_MINOR, true); @@ -204,7 +233,7 @@ public void testLocalCurrentVersionArchives() { public void testLocalCurrentVersionPackages() { for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { - for (boolean bundledJdk : new boolean[] { true, false }) { + for (JavaPackageType bundledJdk : JavaPackageType.values()) { Project project = createProject(BWC_MINOR, true); String projectName = projectName(packageType.toString(), bundledJdk); Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build(); @@ -219,7 +248,7 @@ public void testLocalCurrentVersionPackages() { public void testLocalBwcArchives() { for (Platform platform : Platform.values()) { // note: no non bundled jdk for bwc - String configName = projectName(platform.toString(), true); + String configName = projectName(platform.toString(), JavaPackageType.JDK); configName += (platform == Platform.WINDOWS ? "-zip" : "-tar"); checkBwc("minor", configName, BWC_MINOR_VERSION, Type.ARCHIVE, platform, BWC_MINOR, true); @@ -232,7 +261,7 @@ public void testLocalBwcArchives() { public void testLocalBwcPackages() { for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { // note: no non bundled jdk for bwc - String configName = projectName(packageType.toString(), true); + String configName = projectName(packageType.toString(), JavaPackageType.JDK); checkBwc("minor", configName, BWC_MINOR_VERSION, packageType, null, BWC_MINOR, true); checkBwc("staged", configName, BWC_STAGED_VERSION, packageType, null, BWC_STAGED, true); @@ -247,7 +276,7 @@ private void assertDistroError( String version, Type type, Platform platform, - Boolean bundledJdk, + JavaPackageType bundledJdk, String message ) { IllegalArgumentException e = expectThrows( @@ -263,7 +292,7 @@ private OpenSearchDistribution createDistro( String version, Type type, Platform platform, - Boolean bundledJdk + JavaPackageType bundledJdk ) { NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project); return distros.create(name, distro -> { @@ -289,7 +318,7 @@ private OpenSearchDistribution checkDistro( String version, Type type, Platform platform, - Boolean bundledJdk + JavaPackageType bundledJdk ) { OpenSearchDistribution distribution = createDistro(project, name, version, type, platform, bundledJdk); distribution.finalizeValues(); @@ -315,7 +344,7 @@ private void checkBwc( Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build(); archiveProject.getConfigurations().create(config); archiveProject.getArtifacts().add(config, new File("doesnotmatter")); - final OpenSearchDistribution distro = createDistro(project, "distro", version.toString(), type, platform, true); + final OpenSearchDistribution distro = createDistro(project, "distro", version.toString(), type, platform, JavaPackageType.JDK); distro.setArchitecture(Architecture.current()); checkPlugin(project); } @@ -335,7 +364,7 @@ private Project createProject(BwcVersions bwcVersions, boolean isInternal) { return project; } - private static String projectName(String base, boolean bundledJdk) { - return bundledJdk ? base : ("no-jdk-" + base); + private static String projectName(String base, JavaPackageType bundledJdk) { + return (bundledJdk == JavaPackageType.JDK) ? base : ((bundledJdk == JavaPackageType.NONE) ? ("no-jdk-" + base) : "jre-" + base); } } diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 246f6e3444224..e4a9293c59b8f 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,8 +1,8 @@ opensearch = 3.0.0 -lucene = 9.7.0 +lucene = 9.8.0-snapshot-4373c3b bundled_jdk_vendor = adoptium -bundled_jdk = 20.0.1+9 +bundled_jdk = 20.0.2+9 # See please https://github.com/adoptium/temurin-build/issues/3371 bundled_jdk_linux_ppc64le = 20+36 diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java index 6a75837f8a0fd..332e089ad9e76 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java @@ -99,7 +99,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC defaultPipeline, defaultRequireAlias, true, - request.getXContentType() + request.getMediaType() ); // short circuit the call to the transport layer diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java index f3fbadf07d304..d23a5976fada6 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RequestConverters.java @@ -157,7 +157,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { // Bulk API only supports newline delimited JSON or Smile. Before executing // the bulk, we need to check that all requests have the same content-type // and this content-type is supported by the Bulk API. - XContentType bulkContentType = null; + MediaType bulkContentType = null; for (int i = 0; i < bulkRequest.numberOfActions(); i++) { DocWriteRequest action = bulkRequest.requests().get(i); @@ -245,7 +245,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { IndexRequest indexRequest = (IndexRequest) action; BytesReference indexSource = indexRequest.source(); - XContentType indexXContentType = indexRequest.getContentType(); + MediaType mediaType = indexRequest.getContentType(); try ( XContentParser parser = XContentHelper.createParser( @@ -257,7 +257,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, - indexXContentType + mediaType ) ) { try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) { @@ -266,7 +266,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { } } } else if (opType == DocWriteRequest.OpType.UPDATE) { - source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef(); + source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, ToXContent.EMPTY_PARAMS, false).toBytesRef(); } if (source != null) { @@ -392,30 +392,30 @@ static Request update(UpdateRequest updateRequest) throws IOException { // set for the partial document and the upsert document. This client // only accepts update requests that have the same content types set // for both doc and upsert. - XContentType xContentType = null; + MediaType mediaType = null; if (updateRequest.doc() != null) { - xContentType = updateRequest.doc().getContentType(); + mediaType = updateRequest.doc().getContentType(); } if (updateRequest.upsertRequest() != null) { - XContentType upsertContentType = updateRequest.upsertRequest().getContentType(); - if ((xContentType != null) && (xContentType != upsertContentType)) { + MediaType upsertContentType = updateRequest.upsertRequest().getContentType(); + if ((mediaType != null) && (mediaType != upsertContentType)) { throw new IllegalStateException( "Update request cannot have different content types for doc [" - + xContentType + + mediaType + "]" + " and upsert [" + upsertContentType + "] documents" ); } else { - xContentType = upsertContentType; + mediaType = upsertContentType; } } - if (xContentType == null) { - xContentType = Requests.INDEX_CONTENT_TYPE; + if (mediaType == null) { + mediaType = Requests.INDEX_CONTENT_TYPE; } request.addParameters(parameters.asMap()); - request.setEntity(createEntity(updateRequest, xContentType)); + request.setEntity(createEntity(updateRequest, mediaType)); return request; } @@ -816,14 +816,13 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest) return request; } - static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException { - return createEntity(toXContent, xContentType, ToXContent.EMPTY_PARAMS); + static HttpEntity createEntity(ToXContent toXContent, MediaType mediaType) throws IOException { + return createEntity(toXContent, mediaType, ToXContent.EMPTY_PARAMS); } - static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType, ToXContent.Params toXContentParams) - throws IOException { - BytesRef source = XContentHelper.toXContent(toXContent, xContentType, toXContentParams, false).toBytesRef(); - return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType)); + static HttpEntity createEntity(ToXContent toXContent, MediaType mediaType, ToXContent.Params toXContentParams) throws IOException { + BytesRef source = XContentHelper.toXContent(toXContent, mediaType, toXContentParams, false).toBytesRef(); + return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(mediaType)); } static String endpoint(String index, String id) { @@ -868,20 +867,6 @@ static String endpoint(String[] indices, String endpoint, String type) { return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build(); } - /** - * Returns a {@link ContentType} from a given {@link XContentType}. - * - * @param xContentType the {@link XContentType} - * @return the {@link ContentType} - * - * @deprecated use {@link #createContentType(MediaType)} instead - */ - @Deprecated - @SuppressForbidden(reason = "Only allowed place to convert a XContentType to a ContentType") - public static ContentType createContentType(final XContentType xContentType) { - return ContentType.create(xContentType.mediaTypeWithoutParameters(), (Charset) null); - } - /** * Returns a {@link ContentType} from a given {@link XContentType}. * @@ -1265,8 +1250,8 @@ Params withWaitForEvents(Priority waitForEvents) { * * @return the {@link IndexRequest}'s content type */ - static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) { - XContentType requestContentType = indexRequest.getContentType(); + static MediaType enforceSameContentType(IndexRequest indexRequest, @Nullable MediaType mediaType) { + MediaType requestContentType = indexRequest.getContentType(); if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE) { throw new IllegalArgumentException( "Unsupported content-type found for request with content-type [" @@ -1274,19 +1259,19 @@ static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable + "], only JSON and SMILE are supported" ); } - if (xContentType == null) { + if (mediaType == null) { return requestContentType; } - if (requestContentType != xContentType) { + if (requestContentType != mediaType) { throw new IllegalArgumentException( "Mismatching content-type found for request with content-type [" + requestContentType + "], previous requests have content-type [" - + xContentType + + mediaType + "]" ); } - return xContentType; + return mediaType; } /** diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index 9574d4b17819d..1ff5c81c4a0e3 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -88,9 +88,9 @@ import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.rankeval.RankEvalRequest; import org.opensearch.index.rankeval.RankEvalResponse; import org.opensearch.index.reindex.BulkByScrollResponse; @@ -2227,11 +2227,11 @@ protected final Resp parseEntity(final HttpEntity entity, final CheckedFu if (entity.getContentType() == null) { throw new IllegalStateException("OpenSearch didn't return the [Content-Type] header, unable to parse response body"); } - XContentType xContentType = XContentType.fromMediaType(entity.getContentType()); - if (xContentType == null) { + MediaType medaiType = MediaType.fromMediaType(entity.getContentType()); + if (medaiType == null) { throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType()); } - try (XContentParser parser = xContentType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) { + try (XContentParser parser = medaiType.xContent().createParser(registry, DEPRECATION_HANDLER, entity.getContent())) { return entityParser.apply(parser); } } diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java index 3ed41684b090e..cffed98fce3aa 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexRequest.java @@ -44,10 +44,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -77,7 +77,7 @@ public class CreateIndexRequest extends TimedRequest implements Validatable, ToX private Settings settings = EMPTY_SETTINGS; private BytesReference mappings; - private XContentType mappingsXContentType; + private MediaType mappingsMediaType; private final Set aliases = new HashSet<>(); @@ -123,17 +123,6 @@ public CreateIndexRequest settings(Settings settings) { return this; } - /** - * The settings to create the index with (either json or yaml format) - * - * @deprecated use {@link #settings(String source, MediaType mediaType)} instead - */ - @Deprecated - public CreateIndexRequest settings(String source, XContentType xContentType) { - this.settings = Settings.builder().loadFromSource(source, xContentType).build(); - return this; - } - /** * The settings to create the index with (either json or yaml format) */ @@ -162,23 +151,8 @@ public BytesReference mappings() { return mappings; } - public XContentType mappingsXContentType() { - return mappingsXContentType; - } - - /** - * Adds mapping that will be added when the index gets created. - * - * Note that the definition should *not* be nested under a type name. - * - * @param source The mapping source - * @param xContentType The content type of the source - * - * @deprecated use {@link #mapping(String source, MediaType mediaType)} instead - */ - @Deprecated - public CreateIndexRequest mapping(String source, XContentType xContentType) { - return mapping(new BytesArray(source), xContentType); + public MediaType mappingsMediaType() { + return mappingsMediaType; } /** @@ -213,7 +187,7 @@ public CreateIndexRequest mapping(XContentBuilder source) { */ public CreateIndexRequest mapping(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); builder.map(source); return mapping(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { @@ -221,24 +195,6 @@ public CreateIndexRequest mapping(Map source) { } } - /** - * Adds mapping that will be added when the index gets created. - * - * Note that the definition should *not* be nested under a type name. - * - * @param source The mapping source - * @param xContentType the content type of the mapping source - * - * @deprecated use {@link #mapping(BytesReference source, MediaType mediaType)} instead - */ - @Deprecated - public CreateIndexRequest mapping(BytesReference source, XContentType xContentType) { - Objects.requireNonNull(xContentType); - mappings = source; - mappingsXContentType = xContentType; - return this; - } - /** * Adds mapping that will be added when the index gets created. * @@ -250,7 +206,7 @@ public CreateIndexRequest mapping(BytesReference source, XContentType xContentTy public CreateIndexRequest mapping(BytesReference source, MediaType mediaType) { Objects.requireNonNull(mediaType); mappings = source; - mappingsXContentType = XContentType.fromMediaType(mediaType); + mappingsMediaType = mediaType; return this; } @@ -278,16 +234,6 @@ public CreateIndexRequest aliases(XContentBuilder source) { return aliases(BytesReference.bytes(source), source.contentType()); } - /** - * Sets the aliases that will be associated with the index when it gets created - * - * @deprecated use {@link #aliases(String, MediaType)} instead - */ - @Deprecated - public CreateIndexRequest aliases(String source, XContentType contentType) { - return aliases(new BytesArray(source), contentType); - } - /** * Sets the aliases that will be associated with the index when it gets created */ @@ -295,16 +241,6 @@ public CreateIndexRequest aliases(String source, MediaType mediaType) { return aliases(new BytesArray(source), mediaType); } - /** - * Sets the aliases that will be associated with the index when it gets created - * - * @deprecated use {@link #aliases(BytesReference source, MediaType contentType)} instead - */ - @Deprecated - public CreateIndexRequest aliases(BytesReference source, XContentType contentType) { - return aliases(source, (MediaType) contentType); - } - /** * Sets the aliases that will be associated with the index when it gets created */ @@ -345,18 +281,6 @@ public CreateIndexRequest aliases(Collection aliases) { return this; } - /** - * Sets the settings and mappings as a single source. - * - * Note that the mapping definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(String, MediaType)} instead - */ - @Deprecated - public CreateIndexRequest source(String source, XContentType xContentType) { - return source(new BytesArray(source), xContentType); - } - /** * Sets the settings and mappings as a single source. * @@ -375,20 +299,6 @@ public CreateIndexRequest source(XContentBuilder source) { return source(BytesReference.bytes(source), source.contentType()); } - /** - * Sets the settings and mappings as a single source. - * - * Note that the mapping definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(BytesReference, MediaType)} instead - */ - @Deprecated - public CreateIndexRequest source(BytesReference source, XContentType xContentType) { - Objects.requireNonNull(xContentType); - source(XContentHelper.convertToMap(source, false, xContentType).v2()); - return this; - } - /** * Sets the settings and mappings as a single source. * @@ -458,7 +368,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (mappings != null) { try (InputStream stream = mappings.streamInput()) { - builder.rawField(MAPPINGS.getPreferredName(), stream, mappingsXContentType); + builder.rawField(MAPPINGS.getPreferredName(), stream, mappingsMediaType); } } diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java index 2c90d5a734aa0..327836160cceb 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/DataStreamsStatsResponse.java @@ -34,7 +34,7 @@ import org.opensearch.client.core.BroadcastResponse; import org.opensearch.core.ParseField; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java index fed958955af99..09dbbd63b9479 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutIndexTemplateRequest.java @@ -48,6 +48,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -267,7 +268,7 @@ public PutIndexTemplateRequest mapping(Map source) { private PutIndexTemplateRequest internalMapping(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(source); MediaType mediaType = builder.contentType(); Objects.requireNonNull(mediaType); diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java index a2237efaed25f..6d7e95d191ba6 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/PutMappingRequest.java @@ -38,9 +38,8 @@ import org.opensearch.client.TimedRequest; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -98,17 +97,7 @@ public BytesReference source() { } /** - * The {@link XContentType} of the mapping source. - * - * @deprecated use {@link #mediaType()} instead - */ - @Deprecated - public XContentType xContentType() { - return XContentType.fromMediaType(mediaType); - } - - /** - * The {@link XContentType} of the mapping source. + * The {@link MediaType} of the mapping source. */ public MediaType mediaType() { return mediaType; @@ -121,7 +110,7 @@ public MediaType mediaType() { */ public PutMappingRequest source(Map mappingSource) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.getDefaultMediaType()); builder.map(mappingSource); return source(builder); } catch (IOException e) { @@ -129,20 +118,6 @@ public PutMappingRequest source(Map mappingSource) { } } - /** - * The mapping source definition. - * - * Note that the definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(String, MediaType)} instead - */ - @Deprecated - public PutMappingRequest source(String mappingSource, XContentType xContentType) { - this.source = new BytesArray(mappingSource); - this.mediaType = xContentType; - return this; - } - /** * The mapping source definition. * @@ -165,20 +140,6 @@ public PutMappingRequest source(XContentBuilder builder) { return this; } - /** - * The mapping source definition. - * - * Note that the definition should *not* be nested under a type name. - * - * @deprecated use {@link #source(BytesReference, MediaType)} instead - */ - @Deprecated - public PutMappingRequest source(BytesReference source, XContentType xContentType) { - this.source = source; - this.mediaType = xContentType; - return this; - } - /** * The mapping source definition. * diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java index 61799a83e5df5..2ffb5c605c27f 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java @@ -39,7 +39,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import java.io.IOException; import java.util.Collections; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java index 443cef45e646b..327fccfb35951 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverRequest.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.indices.rollover.MaxSizeCondition; import org.opensearch.client.TimedRequest; import org.opensearch.client.indices.CreateIndexRequest; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java index 707f4246009aa..e9b297d21bcba 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractRequestTestCase.java @@ -36,7 +36,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -60,7 +59,7 @@ public final void testFromXContent() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference bytes = toShuffledXContent(clientTestInstance, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, bytes.streamInput()); final S serverInstance = doParseToServerInstance(parser); assertInstances(serverInstance, clientTestInstance); diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java index 5f7df0cd5860c..b5ee31c87b8d4 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java @@ -36,7 +36,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -59,7 +58,7 @@ public final void testFromXContent() throws IOException { final S serverTestInstance = createServerTestInstance(xContentType); final BytesReference bytes = toShuffledXContent(serverTestInstance, xContentType, getParams(), randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java index 49acbe8cd2bc1..dd793fdbb7ff6 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/BulkProcessorIT.java @@ -44,8 +44,8 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesArray; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java index 2ac0eee407b95..3c7d988e3f01d 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java @@ -61,7 +61,7 @@ import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java index eaf8f4f8efff7..cefe992b58c64 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java @@ -60,8 +60,8 @@ import org.opensearch.client.indices.GetIndexRequest; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java index 8ca5c5fa58742..ea57f1857fbfb 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java @@ -112,8 +112,8 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java index e21619ff15ef6..c672ed6be110d 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java @@ -79,7 +79,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.test.OpenSearchTestCase; import org.junit.Assert; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import java.io.IOException; import java.util.Arrays; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java index 84cd2fb8b20dd..e3c8197dc2c90 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java @@ -161,10 +161,7 @@ private void testSimulatePipeline(boolean isVerbose, boolean isFailure) throws I } builder.endObject(); - SimulatePipelineRequest request = new SimulatePipelineRequest( - BytesReference.bytes(builder), - XContentType.fromMediaType(builder.contentType()) - ); + SimulatePipelineRequest request = new SimulatePipelineRequest(BytesReference.bytes(builder), builder.contentType()); request.setVerbose(isVerbose); SimulatePipelineResponse response = execute( request, diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java index ee6aeeeacf633..15a99b3e91685 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java @@ -76,6 +76,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -809,7 +810,7 @@ public void testUpdate() throws IOException { UpdateRequest parsedUpdateRequest = new UpdateRequest(); - XContentType entityContentType = XContentType.fromMediaType(entity.getContentType()); + MediaType entityContentType = MediaType.fromMediaType(entity.getContentType()); try (XContentParser parser = createParser(entityContentType.xContent(), entity.getContent())) { parsedUpdateRequest.fromXContent(parser); } diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java index e86de6ba718f9..af178ad2a5d47 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java @@ -48,7 +48,7 @@ import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchTestCase; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java index 03d6e6720ffed..376eb3c5e407c 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/GetSourceResponseTests.java @@ -35,9 +35,9 @@ import org.opensearch.client.AbstractResponseTestCase; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -61,7 +61,7 @@ static class SourceOnlyResponse implements ToXContentObject { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { // this implementation copied from RestGetSourceAction.RestGetSourceResponseListener::buildResponse try (InputStream stream = source.streamInput()) { - builder.rawValue(stream, XContentHelper.xContentType(source)); + builder.rawValue(stream, MediaTypeRegistry.xContentType(source)); } return builder; } diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java index a973753aa2032..178d9296bd242 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/CRUDDocumentationIT.java @@ -77,8 +77,8 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java index f85fcae7af365..ccbb64b13b2d2 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java @@ -63,7 +63,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java index 73346bc57646e..de2eb91c6660b 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java @@ -104,8 +104,8 @@ import org.opensearch.cluster.metadata.Template; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java index b83bdab899a41..7babb339e176a 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java @@ -39,7 +39,6 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.index.Index; @@ -194,7 +193,7 @@ public final void testBwcFromXContent() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference bytes = toShuffledXContent(expected, xContentType, getParams(), randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, @@ -215,7 +214,7 @@ public final void testBwcFromXContent() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); final BytesReference bytes = toShuffledXContent(expected, xContentType, getParams(), randomBoolean()); - final XContent xContent = XContentFactory.xContent(xContentType); + final XContent xContent = xContentType.xContent(); final XContentParser parser = xContent.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java index 1ac5ff06d5624..0cfbd8c29fe43 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CreateIndexRequestTests.java @@ -69,8 +69,8 @@ private void assertMappingsEqual(CreateIndexRequest expected, CreateIndexRequest } else { assertNotNull(actual.mappings()); try ( - XContentParser expectedJson = createParser(expected.mappingsXContentType().xContent(), expected.mappings()); - XContentParser actualJson = createParser(actual.mappingsXContentType().xContent(), actual.mappings()) + XContentParser expectedJson = createParser(expected.mappingsMediaType().xContent(), expected.mappings()); + XContentParser actualJson = createParser(actual.mappingsMediaType().xContent(), actual.mappings()) ) { assertEquals(expectedJson.map(), actualJson.map()); } catch (IOException e) { diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java index 96db2fb43c161..ef8655b0226dd 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/DataStreamsStatsResponseTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.indices.datastream.DataStreamsStatsAction; import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.client.AbstractResponseTestCase; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java index 6f924898096a0..8c83791d94d42 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java @@ -39,10 +39,10 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -262,7 +262,7 @@ private static AliasMetadata randomAliasMetadata(String name) { } static XContentBuilder randomMapping(String type, XContentType xContentType) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); builder.startObject().startObject(type); randomMappingFields(builder, true); diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java index 9d51c09ba2e44..c8372d5001fac 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverRequestTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.indices.rollover.MaxAgeCondition; import org.opensearch.action.admin.indices.rollover.MaxDocsCondition; import org.opensearch.action.admin.indices.rollover.MaxSizeCondition; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java index ff2418c5caee2..0fb00f6510147 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/rollover/RolloverResponseTests.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.indices.rollover.MaxAgeCondition; import org.opensearch.action.admin.indices.rollover.MaxDocsCondition; import org.opensearch.action.admin.indices.rollover.MaxSizeCondition; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index e7137127053e2..161b8008525b4 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -28,9 +28,11 @@ * under the License. */ +import org.opensearch.gradle.JavaPackageType + apply plugin: 'opensearch.internal-distribution-archive-setup' -CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean jdk) { +CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, JavaPackageType java) { return copySpec { into("opensearch-${version}") { into('lib') { @@ -39,19 +41,23 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla into('config') { dirMode 0750 fileMode 0660 - with configFiles(distributionType, jdk) + with configFiles(distributionType, java) from { dirMode 0750 jvmOptionsDir.getParent() } } into('bin') { - with binFiles(distributionType, jdk) + with binFiles(distributionType, java) } - if (jdk) { + if (java == JavaPackageType.JDK) { into("darwin".equals(platform) ? 'jdk.app' : 'jdk') { with jdkFiles(project, platform, architecture) } + } else if (java == JavaPackageType.JRE) { + into("darwin".equals(platform) ? 'jre.app' : 'jre') { + with jreFiles(project, platform, architecture) + } } into('') { from { @@ -73,7 +79,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla rename { 'LICENSE.txt' } } - with noticeFile(jdk) + with noticeFile(java) into('modules') { with modulesFiles } @@ -84,77 +90,84 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla distribution_archives { integTestZip { content { - archiveFiles(transportModulesFiles, 'zip', null, 'x64', false) + archiveFiles(transportModulesFiles, 'zip', null, 'x64', JavaPackageType.NONE) } } darwinTar { archiveClassifier = 'darwin-x64' content { - archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', true) + archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.JDK) } } darwinArm64Tar { archiveClassifier = 'darwin-arm64' content { - archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', true) + archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.JDK) } } noJdkDarwinTar { archiveClassifier = 'no-jdk-darwin-x64' content { - archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', false) + archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.NONE) } } noJdkDarwinArm64Tar { archiveClassifier = 'no-jdk-darwin-arm64' content { - archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', false) + archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.NONE) } } freebsdTar { archiveClassifier = 'freebsd-x64' content { - archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false) + archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE) } } noJdkFreebsdTar { archiveClassifier = 'no-jdk-freebsd-x64' content { - archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false) + archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE) } } linuxArm64Tar { archiveClassifier = 'linux-arm64' content { - archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', true) + archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.JDK) } } noJdkLinuxArm64Tar { archiveClassifier = 'no-jdk-linux-arm64' content { - archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', false) + archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.NONE) } } linuxTar { archiveClassifier = 'linux-x64' content { - archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', true) + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JDK) } } noJdkLinuxTar { archiveClassifier = 'no-jdk-linux-x64' content { - archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', false) + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.NONE) + } + } + + jreLinuxTar { + archiveClassifier = 'jre-linux-x64' + content { + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JRE) } } @@ -163,7 +176,7 @@ distribution_archives { linuxS390xTar { archiveClassifier = 'linux-s390x' content { - archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', false) + archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', JavaPackageType.NONE) } } @@ -171,28 +184,28 @@ distribution_archives { linuxPpc64leTar { archiveClassifier = 'linux-ppc64le' content { - archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', true) + archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.JDK) } } noJdkLinuxPpc64leTar { archiveClassifier = 'no-jdk-linux-ppc64le' content { - archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', false) + archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.NONE) } } windowsZip { archiveClassifier = 'windows-x64' content { - archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', true) + archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.JDK) } } noJdkWindowsZip { archiveClassifier = 'no-jdk-windows-x64' content { - archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', false) + archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.NONE) } } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 2378ed833ead7..35ca84ca66dba 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -36,6 +36,7 @@ import org.opensearch.gradle.MavenFilteringHack import org.opensearch.gradle.NoticeTask import org.opensearch.gradle.VersionProperties import org.opensearch.gradle.info.BuildParams +import org.opensearch.gradle.JavaPackageType import java.nio.file.Files import java.nio.file.Path @@ -275,13 +276,14 @@ project(':test:external-modules').subprojects.each { Project testModule -> configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { apply plugin: 'opensearch.jdk-download' + apply plugin: 'opensearch.jre-download' apply plugin: 'opensearch.repositories' // Setup all required JDKs project.jdks { ['darwin', 'linux', 'windows'].each { platform -> (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture -> - "bundled_${platform}_${architecture}" { + "bundled_jdk_${platform}_${architecture}" { it.platform = platform it.version = VersionProperties.getBundledJdk(platform, architecture) it.vendor = VersionProperties.bundledJdkVendor @@ -291,6 +293,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } + // Setup all required JREs + project.jres { + ['darwin', 'linux', 'windows'].each { platform -> + (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture -> + "bundled_jre_${platform}_${architecture}" { + it.platform = platform + it.version = VersionProperties.getBundledJre(platform, architecture) + it.vendor = VersionProperties.bundledJdkVendor + it.architecture = architecture + } + } + } + } + // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run... /***************************************************************************** * Properties to expand when copying packaging files * @@ -377,20 +393,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from buildTransportModulesTaskProvider } - configFiles = { distributionType, jdk -> + configFiles = { distributionType, java -> copySpec { with copySpec { // main config files, processed with distribution specific substitutions from '../src/config' exclude 'log4j2.properties' // this is handled separately below - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } from project(':distribution').buildLog4jConfig from project(':distribution').buildConfig } } - binFiles = { distributionType, jdk -> + binFiles = { distributionType, java -> copySpec { // non-windows files, for all distributions with copySpec { @@ -398,7 +414,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { exclude '*.exe' exclude '*.bat' eachFile { it.setMode(0755) } - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } // windows files, only for zip if (distributionType == 'zip') { @@ -406,7 +422,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from '../src/bin' include '*.bat' filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } with copySpec { from '../src/bin' @@ -424,12 +440,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } - noticeFile = { jdk -> + noticeFile = { java -> copySpec { if (project.name == 'integ-test-zip') { from buildServerNoticeTaskProvider } else { - if (jdk) { + if (java != JavaPackageType.NONE) { from buildNoticeTaskProvider } else { from buildNoJdkNoticeTaskProvider @@ -446,7 +462,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { if ("arm64".equals(architecture)) { architecture = "aarch64" } - from project.jdks."bundled_${platform}_${architecture}" + from project.jdks."bundled_jdk_${platform}_${architecture}" exclude "demo/**" /* * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude @@ -465,6 +481,31 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } } + + jreFiles = { Project project, String platform, String architecture -> + return copySpec { + /* + * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. + */ + if ("arm64".equals(architecture)) { + architecture = "aarch64" + } + from project.jres."bundled_jre_${platform}_${architecture}" + exclude "demo/**" + /* + * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude + * it from the build. + */ + if ("darwin".equals(platform)) { + exclude "Contents/MacOS" + } + eachFile { FileCopyDetails details -> + if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') { + details.mode = 0755 + } + } + } + } } } @@ -583,7 +624,7 @@ subprojects { ], 'opensearch.bundled_jdk': [ - 'def': jdk ? 'true' : 'false' + 'def': jdk != JavaPackageType.NONE ? true : false ], 'license.name': [ diff --git a/distribution/docker/docker-arm64-export/build.gradle b/distribution/docker/docker-arm64-export/build.gradle new file mode 100644 index 0000000000000..3506c4e39c234 --- /dev/null +++ b/distribution/docker/docker-arm64-export/build.gradle @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/distribution/docker/docker-s390x-export/build.gradle b/distribution/docker/docker-s390x-export/build.gradle new file mode 100644 index 0000000000000..3506c4e39c234 --- /dev/null +++ b/distribution/docker/docker-s390x-export/build.gradle @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index a0ae4c5c2472a..7914fcc172ef4 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -63,7 +63,7 @@ import java.util.regex.Pattern */ plugins { - id "com.netflix.nebula.ospackage-base" version "11.3.0" + id "com.netflix.nebula.ospackage-base" version "11.4.0" } void addProcessFilesTask(String type, boolean jdk) { diff --git a/distribution/src/bin/opensearch-env b/distribution/src/bin/opensearch-env index f94824576b122..cd7a0b2b4520b 100644 --- a/distribution/src/bin/opensearch-env +++ b/distribution/src/bin/opensearch-env @@ -46,7 +46,7 @@ fi # now set the classpath OPENSEARCH_CLASSPATH="$OPENSEARCH_HOME/lib/*" -# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JDK +# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JRE -> bundled JDK if [ ! -z "$OPENSEARCH_JAVA_HOME" ]; then JAVA="$OPENSEARCH_JAVA_HOME/bin/java" JAVA_TYPE="OPENSEARCH_JAVA_HOME" @@ -57,13 +57,18 @@ else if [ $OS = "darwin" ]; then # macOS bundled Java JAVA="$OPENSEARCH_HOME/jdk.app/Contents/Home/bin/java" + JAVA_TYPE="bundled jdk" elif [ $OS = "freebsd" ]; then # using FreeBSD default java from ports if JAVA_HOME is not set JAVA="/usr/local/bin/java" + JAVA_TYPE="bundled jdk" + elif [ -d "$OPENSEARCH_HOME/jre" ]; then + JAVA="$OPENSEARCH_HOME/jre/bin/java" + JAVA_TYPE="bundled jre" else JAVA="$OPENSEARCH_HOME/jdk/bin/java" + JAVA_TYPE="bundled jdk" fi - JAVA_TYPE="bundled jdk" fi if [ ! -x "$JAVA" ]; then diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index e15afc0f677c3..952110c6c0289 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -82,3 +82,6 @@ ${error.file} # JDK 20+ Incubating Vector Module for SIMD optimizations; # disabling may reduce performance on vector optimized lucene 20:--add-modules=jdk.incubator.vector + +# HDFS ForkJoinPool.common() support by SecurityManager +-Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory diff --git a/server/src/main/java/org/opensearch/common/component/AbstractLifecycleComponent.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/AbstractLifecycleComponent.java similarity index 98% rename from server/src/main/java/org/opensearch/common/component/AbstractLifecycleComponent.java rename to libs/common/src/main/java/org/opensearch/common/lifecycle/AbstractLifecycleComponent.java index 837f8af44bf13..111556fbe43cf 100644 --- a/server/src/main/java/org/opensearch/common/component/AbstractLifecycleComponent.java +++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/AbstractLifecycleComponent.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.component; +package org.opensearch.common.lifecycle; import java.io.IOException; import java.io.UncheckedIOException; diff --git a/server/src/main/java/org/opensearch/common/component/Lifecycle.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java similarity index 99% rename from server/src/main/java/org/opensearch/common/component/Lifecycle.java rename to libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java index fb12c1fc9ac4b..e76d49cbf49e8 100644 --- a/server/src/main/java/org/opensearch/common/component/Lifecycle.java +++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/Lifecycle.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.component; +package org.opensearch.common.lifecycle; /** * Lifecycle state. Allows the following transitions: diff --git a/server/src/main/java/org/opensearch/common/component/LifecycleComponent.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java similarity index 97% rename from server/src/main/java/org/opensearch/common/component/LifecycleComponent.java rename to libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java index 984d55df1bdfa..f343f9ada01ef 100644 --- a/server/src/main/java/org/opensearch/common/component/LifecycleComponent.java +++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleComponent.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.component; +package org.opensearch.common.lifecycle; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/common/component/LifecycleListener.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleListener.java similarity index 97% rename from server/src/main/java/org/opensearch/common/component/LifecycleListener.java rename to libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleListener.java index 89c344b955bc9..7ac41a5eb0df0 100644 --- a/server/src/main/java/org/opensearch/common/component/LifecycleListener.java +++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/LifecycleListener.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.component; +package org.opensearch.common.lifecycle; /** * Base lifecycle listener. diff --git a/libs/common/src/main/java/org/opensearch/common/lifecycle/package-info.java b/libs/common/src/main/java/org/opensearch/common/lifecycle/package-info.java new file mode 100644 index 0000000000000..1bedde5585e36 --- /dev/null +++ b/libs/common/src/main/java/org/opensearch/common/lifecycle/package-info.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Foundation implementation for a object lifecycle. + * + * See {@link org.opensearch.common.lifecycle.Lifecycle} for example usage + * + * @opensearch.internal + */ +package org.opensearch.common.lifecycle; diff --git a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0.jar.sha1 deleted file mode 100644 index 2b0f77275c0ab..0000000000000 --- a/libs/core/licenses/lucene-core-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad391210ffd806931334be9670a35af00c56f959 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..dc363f2776429 --- /dev/null +++ b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +30c3afcf058532d3d2b8820375043000e7f34a9b \ No newline at end of file diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchException.java b/libs/core/src/main/java/org/opensearch/OpenSearchException.java index f75a225af1b4d..1a1273ad8963c 100644 --- a/libs/core/src/main/java/org/opensearch/OpenSearchException.java +++ b/libs/core/src/main/java/org/opensearch/OpenSearchException.java @@ -118,6 +118,14 @@ public class OpenSearchException extends RuntimeException implements Writeable, UNKNOWN_VERSION_ADDED ) ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.OpenSearchParseException.class, + org.opensearch.OpenSearchParseException::new, + 35, + UNKNOWN_VERSION_ADDED + ) + ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.core.common.ParsingException.class, @@ -134,6 +142,14 @@ public class OpenSearchException extends RuntimeException implements Writeable, UNKNOWN_VERSION_ADDED ) ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.core.common.breaker.CircuitBreakingException.class, + org.opensearch.core.common.breaker.CircuitBreakingException::new, + 133, + UNKNOWN_VERSION_ADDED + ) + ); } /** diff --git a/server/src/main/java/org/opensearch/OpenSearchParseException.java b/libs/core/src/main/java/org/opensearch/OpenSearchParseException.java similarity index 100% rename from server/src/main/java/org/opensearch/OpenSearchParseException.java rename to libs/core/src/main/java/org/opensearch/OpenSearchParseException.java diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index f85da63bdbb1f..3f83282245fd8 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -55,7 +55,7 @@ public class Version implements Comparable, ToXContentFragment { * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99 * indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id - * + *>> * IMPORTANT: Unreleased vs. Released Versions * * All listed versions MUST be released versions, except the last major, the last minor and the last revison. ONLY those are required @@ -90,8 +90,9 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_2_9_1 = new Version(2090199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_10_0 = new Version(2100099, org.apache.lucene.util.Version.LUCENE_9_7_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { diff --git a/libs/core/src/main/java/org/opensearch/core/common/Strings.java b/libs/core/src/main/java/org/opensearch/core/common/Strings.java index d6c484e5e4746..ffb7711f84492 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/Strings.java +++ b/libs/core/src/main/java/org/opensearch/core/common/Strings.java @@ -515,4 +515,27 @@ private static String changeFirstCharacterCase(String str, boolean capitalize) { sb.append(str.substring(1)); return sb.toString(); } + + /** + * Format the double value with a single decimal points, trimming trailing '.0'. + */ + public static String format1Decimals(double value, String suffix) { + String p = String.valueOf(value); + int ix = p.indexOf('.') + 1; + int ex = p.indexOf('E'); + char fraction = p.charAt(ix); + if (fraction == '0') { + if (ex != -1) { + return p.substring(0, ix - 1) + p.substring(ex) + suffix; + } else { + return p.substring(0, ix - 1) + suffix; + } + } else { + if (ex != -1) { + return p.substring(0, ix) + fraction + p.substring(ex) + suffix; + } else { + return p.substring(0, ix) + fraction + suffix; + } + } + } } diff --git a/server/src/main/java/org/opensearch/common/breaker/CircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java similarity index 99% rename from server/src/main/java/org/opensearch/common/breaker/CircuitBreaker.java rename to libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java index 4cbd375e8c1ff..0f75f763d21c1 100644 --- a/server/src/main/java/org/opensearch/common/breaker/CircuitBreaker.java +++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.breaker; +package org.opensearch.core.common.breaker; import java.util.Locale; diff --git a/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java similarity index 98% rename from server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java rename to libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java index 4cab014912970..4869127ee473d 100644 --- a/server/src/main/java/org/opensearch/common/breaker/CircuitBreakingException.java +++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreakingException.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.common.breaker; +package org.opensearch.core.common.breaker; import org.opensearch.OpenSearchException; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/common/breaker/NoopCircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java similarity index 98% rename from server/src/main/java/org/opensearch/common/breaker/NoopCircuitBreaker.java rename to libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java index ddd72280faa4f..86a0a7ccb96fd 100644 --- a/server/src/main/java/org/opensearch/common/breaker/NoopCircuitBreaker.java +++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/NoopCircuitBreaker.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.breaker; +package org.opensearch.core.common.breaker; /** * A CircuitBreaker that doesn't increment or adjust, and all operations are diff --git a/libs/core/src/main/java/org/opensearch/core/common/breaker/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/package-info.java new file mode 100644 index 0000000000000..f9fb83d2207e1 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Foundation classes for the Circuit Breaker + */ +package org.opensearch.core.common.breaker; diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java index 24795204c6dc8..d9040da569345 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java +++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java @@ -53,6 +53,8 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import java.io.ByteArrayInputStream; import java.io.EOFException; @@ -344,6 +346,10 @@ public BigInteger readBigInteger() throws IOException { return new BigInteger(readString()); } + public MediaType readMediaType() throws IOException { + return MediaTypeRegistry.fromMediaType(readString()); + } + @Nullable public Text readOptionalText() throws IOException { int length = readInt(); diff --git a/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java similarity index 99% rename from server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java rename to libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java index b95e39feb8fac..68486dd7c975f 100644 --- a/server/src/main/java/org/opensearch/common/unit/ByteSizeUnit.java +++ b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeUnit.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.unit; +package org.opensearch.core.common.unit; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeValue.java similarity index 88% rename from server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java rename to libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeValue.java index a123c79464727..529501226f5e3 100644 --- a/server/src/main/java/org/opensearch/common/unit/ByteSizeValue.java +++ b/libs/core/src/main/java/org/opensearch/core/common/unit/ByteSizeValue.java @@ -30,16 +30,13 @@ * GitHub history for details. */ -package org.opensearch.common.unit; +package org.opensearch.core.common.unit; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.logging.DeprecationLogger; -import org.opensearch.common.logging.LogConfigurator; -import org.opensearch.common.network.NetworkService; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -54,17 +51,6 @@ */ public class ByteSizeValue implements Writeable, Comparable, ToXContentFragment { - /** - * We have to lazy initialize the deprecation logger as otherwise a static logger here would be constructed before logging is configured - * leading to a runtime failure (see {@link LogConfigurator#checkErrorListener()} ). The premature construction would come from any - * {@link ByteSizeValue} object constructed in, for example, settings in {@link NetworkService}. - * - * @opensearch.internal - */ - static class DeprecationLoggerHolder { - static DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ByteSizeValue.class); - } - public static final ByteSizeValue ZERO = new ByteSizeValue(0, ByteSizeUnit.BYTES); private final long size; @@ -262,14 +248,14 @@ private static ByteSizeValue parse( return new ByteSizeValue(Long.parseLong(s), unit); } catch (final NumberFormatException e) { try { - final double doubleValue = Double.parseDouble(s); - DeprecationLoggerHolder.deprecationLogger.deprecate( - "fractional_byte_values", - "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for setting [{}]", + Double.parseDouble(s); + throw new OpenSearchParseException( + "Failed to parse bytes value [{}]. Fractional bytes values have been " + + "deprecated since Legacy 6.2. Use non-fractional bytes values instead: found for setting [{}]", + e, initialInput, settingName ); - return new ByteSizeValue((long) (doubleValue * unit.toBytes(1))); } catch (final NumberFormatException ignored) { throw new OpenSearchParseException("failed to parse [{}]", e, initialInput); } diff --git a/libs/core/src/main/java/org/opensearch/core/common/unit/package-info.java b/libs/core/src/main/java/org/opensearch/core/common/unit/package-info.java new file mode 100644 index 0000000000000..79b5dcdcba3b6 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/common/unit/package-info.java @@ -0,0 +1,16 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Common units of measurement used by the core library. These units of measurement classes exist + * in the core because they depend on core functionality beyond the common library (e.g., serializable). + * + * @opensearch.api + * @opensearch.experimental + */ +package org.opensearch.core.common.unit; diff --git a/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java similarity index 98% rename from server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java index 83f3f9532948f..ab887acb85a87 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/AllCircuitBreakerStats.java +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/AllCircuitBreakerStats.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.indices.breaker; +package org.opensearch.core.indices.breaker; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerService.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java similarity index 92% rename from server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerService.java rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java index b5cc1a6b1c6c5..ee9c94f432a36 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerService.java +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerService.java @@ -30,12 +30,12 @@ * GitHub history for details. */ -package org.opensearch.indices.breaker; +package org.opensearch.core.indices.breaker; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; +import org.opensearch.core.common.breaker.CircuitBreaker; /** * Interface for Circuit Breaker services, which provide breakers to classes diff --git a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java similarity index 97% rename from server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java index 94e63acd10648..0e53a38908a96 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/CircuitBreakerStats.java +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/CircuitBreakerStats.java @@ -30,12 +30,12 @@ * GitHub history for details. */ -package org.opensearch.indices.breaker; +package org.opensearch.core.indices.breaker; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/indices/breaker/NoneCircuitBreakerService.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java similarity index 91% rename from server/src/main/java/org/opensearch/indices/breaker/NoneCircuitBreakerService.java rename to libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java index bcb47b48a5f14..4095fd32b6d3c 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/NoneCircuitBreakerService.java +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/NoneCircuitBreakerService.java @@ -30,10 +30,10 @@ * GitHub history for details. */ -package org.opensearch.indices.breaker; +package org.opensearch.core.indices.breaker; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; /** * Class that returns a breaker that never breaks diff --git a/libs/core/src/main/java/org/opensearch/core/indices/breaker/package-info.java b/libs/core/src/main/java/org/opensearch/core/indices/breaker/package-info.java new file mode 100644 index 0000000000000..a98f9ab1d9f1e --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/indices/breaker/package-info.java @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Top Level core circuit breaker implementation + * + * @opensearch.internal + * @opensearch.experimental + */ +package org.opensearch.core.indices.breaker; diff --git a/libs/core/src/main/java/org/opensearch/core/indices/package-info.java b/libs/core/src/main/java/org/opensearch/core/indices/package-info.java new file mode 100644 index 0000000000000..c80edf3d2f01a --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/indices/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Top Level Package for implementations used across indexes + */ +package org.opensearch.core.indices; diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java index 82d16315f6527..8e3c115c7ba58 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaType.java @@ -32,12 +32,18 @@ package org.opensearch.core.xcontent; +import org.opensearch.core.common.io.stream.Writeable; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Locale; + /** * Abstracts a Media Type and a format parameter. * Media types are used as values on Content-Type and Accept headers * format is an URL parameter, specifies response media type. */ -public interface MediaType { +public interface MediaType extends Writeable { /** * Returns a type part of a MediaType * i.e. application for application/json @@ -65,9 +71,54 @@ default String typeWithSubtype() { XContent xContent(); + boolean detectedXContent(final byte[] bytes, int offset, int length); + + boolean detectedXContent(final CharSequence content, final int length); + default String mediaType() { return mediaTypeWithoutParameters(); } String mediaTypeWithoutParameters(); + + XContentBuilder contentBuilder() throws IOException; + + XContentBuilder contentBuilder(final OutputStream os) throws IOException; + + /** + * Accepts a format string, which is most of the time is equivalent to {@link MediaType#subtype()} + * and attempts to match the value to an {@link MediaType}. + * The comparisons are done in lower case format. + * This method will return {@code null} if no match is found + */ + static MediaType fromFormat(String mediaType) { + return MediaTypeRegistry.fromFormat(mediaType); + } + + /** + * Attempts to match the given media type with the known {@link MediaType} values. This match is done in a case-insensitive manner. + * The provided media type can optionally has parameters. + * This method is suitable for parsing of the {@code Content-Type} and {@code Accept} HTTP headers. + * This method will return {@code null} if no match is found + */ + static MediaType fromMediaType(String mediaTypeHeaderValue) { + mediaTypeHeaderValue = removeVersionInMediaType(mediaTypeHeaderValue); + return MediaTypeRegistry.fromMediaType(mediaTypeHeaderValue); + } + + /** + * Clients compatible with ES 7.x might start sending media types with versioned media type + * in a form of application/vnd.elasticsearch+json;compatible-with=7. + * This has to be removed in order to be used in 7.x server. + * The same client connecting using that media type will be able to communicate with ES 8 thanks to compatible API. + * @param mediaType - a media type used on Content-Type header, might contain versioned media type. + * + * @return a media type string without + */ + private static String removeVersionInMediaType(String mediaType) { + if (mediaType != null && (mediaType = mediaType.toLowerCase(Locale.ROOT)).contains("vnd.opensearch")) { + return mediaType.replaceAll("vnd.opensearch\\+", "").replaceAll("\\s*;\\s*compatible-with=\\d+", ""); + } + return mediaType; + } } diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java deleted file mode 100644 index cce95e7b4d6b1..0000000000000 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeParser.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.core.xcontent; - -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; - -/** - * Parses supported internet media types - * - * @opensearch.internal - */ -public class MediaTypeParser { - private final Map formatToMediaType; - private final Map typeWithSubtypeToMediaType; - - public MediaTypeParser(T[] acceptedMediaTypes) { - this(acceptedMediaTypes, Map.of()); - } - - public MediaTypeParser(T[] acceptedMediaTypes, Map additionalMediaTypes) { - final int size = acceptedMediaTypes.length + additionalMediaTypes.size(); - Map formatMap = new HashMap<>(size); - Map typeMap = new HashMap<>(size); - for (T mediaType : acceptedMediaTypes) { - typeMap.put(mediaType.typeWithSubtype(), mediaType); - formatMap.put(mediaType.format(), mediaType); - } - for (Map.Entry entry : additionalMediaTypes.entrySet()) { - String typeWithSubtype = entry.getKey(); - T mediaType = entry.getValue(); - - typeMap.put(typeWithSubtype.toLowerCase(Locale.ROOT), mediaType); - formatMap.put(mediaType.format(), mediaType); - } - - this.formatToMediaType = Map.copyOf(formatMap); - this.typeWithSubtypeToMediaType = Map.copyOf(typeMap); - } - - public T fromMediaType(String mediaType) { - ParsedMediaType parsedMediaType = parseMediaType(mediaType); - return parsedMediaType != null ? parsedMediaType.getMediaType() : null; - } - - public T fromFormat(String format) { - if (format == null) { - return null; - } - return formatToMediaType.get(format.toLowerCase(Locale.ROOT)); - } - - /** - * parsing media type that follows https://tools.ietf.org/html/rfc7231#section-3.1.1.1 - * @param headerValue a header value from Accept or Content-Type - * @return a parsed media-type - */ - public ParsedMediaType parseMediaType(String headerValue) { - if (headerValue != null) { - String[] split = headerValue.toLowerCase(Locale.ROOT).split(";"); - - String[] typeSubtype = split[0].trim().split("/"); - if (typeSubtype.length == 2) { - String type = typeSubtype[0]; - String subtype = typeSubtype[1]; - T xContentType = typeWithSubtypeToMediaType.get(type + "/" + subtype); - if (xContentType != null) { - Map parameters = new HashMap<>(); - for (int i = 1; i < split.length; i++) { - // spaces are allowed between parameters, but not between '=' sign - String[] keyValueParam = split[i].trim().split("="); - if (keyValueParam.length != 2 || hasSpaces(keyValueParam[0]) || hasSpaces(keyValueParam[1])) { - return null; - } - parameters.put(keyValueParam[0], keyValueParam[1]); - } - return new ParsedMediaType(xContentType, parameters); - } - } - - } - return null; - } - - private boolean hasSpaces(String s) { - return s.trim().equals(s) == false; - } - - /** - * A media type object that contains all the information provided on a Content-Type or Accept header - */ - public class ParsedMediaType { - private final Map parameters; - private final T mediaType; - - public ParsedMediaType(T mediaType, Map parameters) { - this.parameters = parameters; - this.mediaType = mediaType; - } - - public T getMediaType() { - return mediaType; - } - - public Map getParameters() { - return parameters; - } - } -} diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java new file mode 100644 index 0000000000000..064f154602caf --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/MediaTypeRegistry.java @@ -0,0 +1,406 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.core.xcontent; + +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.spi.MediaTypeProvider; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Parses supported internet media types + * + * @opensearch.internal + */ +public final class MediaTypeRegistry { + private static Map formatToMediaType = Map.of(); + private static Map typeWithSubtypeToMediaType = Map.of(); + + // Default mediaType singleton + private static MediaType DEFAULT_MEDIA_TYPE; + public static final int GUESS_HEADER_LENGTH = 20; + + // JSON is a core type, so we create a static instance for implementations that require JSON format (e.g., tests) + // todo we should explore moving the concrete JSON implementation from the xcontent library to core + public static final MediaType JSON; + + static { + List mediaTypes = new ArrayList<>(); + Map amt = new HashMap<>(); + for (MediaTypeProvider provider : ServiceLoader.load(MediaTypeProvider.class, MediaTypeProvider.class.getClassLoader())) { + mediaTypes.addAll(provider.getMediaTypes()); + amt = Stream.of(amt, provider.getAdditionalMediaTypes()) + .flatMap(map -> map.entrySet().stream()) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + } + register(mediaTypes.toArray(new MediaType[0]), amt); + JSON = fromMediaType("application/json"); + setDefaultMediaType(JSON); + } + + private static void register(MediaType[] acceptedMediaTypes, Map additionalMediaTypes) { + // ensures the map is not overwritten: + Map typeMap = new HashMap<>(typeWithSubtypeToMediaType); + Map formatMap = new HashMap<>(formatToMediaType); + for (MediaType mediaType : acceptedMediaTypes) { + if (formatMap.containsKey(mediaType.format())) { + throw new IllegalArgumentException("unable to register mediaType: [" + mediaType.format() + "]. Type already exists."); + } + typeMap.put(mediaType.typeWithSubtype(), mediaType); + formatMap.put(mediaType.format(), mediaType); + } + for (Map.Entry entry : additionalMediaTypes.entrySet()) { + String typeWithSubtype = entry.getKey().toLowerCase(Locale.ROOT); + if (typeMap.containsKey(typeWithSubtype)) { + throw new IllegalArgumentException( + "unable to register mediaType: [" + + entry.getKey() + + "]. " + + "Type already exists and is mapped to: [." + + entry.getValue().format() + + "]" + ); + } + + MediaType mediaType = entry.getValue(); + typeMap.put(typeWithSubtype, mediaType); + formatMap.putIfAbsent(mediaType.format(), mediaType); // ignore if the additional type mapping already exists + } + + formatToMediaType = Map.copyOf(formatMap); + typeWithSubtypeToMediaType = Map.copyOf(typeMap); + } + + public static MediaType fromMediaType(String mediaType) { + ParsedMediaType parsedMediaType = parseMediaType(mediaType); + return parsedMediaType != null ? parsedMediaType.getMediaType() : null; + } + + public static MediaType fromFormat(String format) { + if (format == null) { + return null; + } + return formatToMediaType.get(format.toLowerCase(Locale.ROOT)); + } + + /** + * Returns a binary content builder for the provided content type. + */ + public static XContentBuilder contentBuilder(MediaType type) throws IOException { + for (var mediaType : formatToMediaType.values()) { + if (type == mediaType) { + return type.contentBuilder(); + } + } + throw new IllegalArgumentException("No matching content type for " + type); + } + + public static XContentBuilder contentBuilder(MediaType type, OutputStream outputStream) throws IOException { + for (var mediaType : formatToMediaType.values()) { + if (type == mediaType) { + return type.contentBuilder(outputStream); + } + } + throw new IllegalArgumentException("No matching content type for " + type); + } + + /** + * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent} + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContent(final byte[] data, int offset, int length) { + MediaType type = mediaTypeFromBytes(data, offset, length); + if (type == null) { + throw new XContentParseException("Failed to derive xcontent"); + } + return type; + } + + /** + * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent} + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContent(byte[] data) { + return xContent(data, 0, data.length); + } + + /** + * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent} + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContent(CharSequence content) { + MediaType type = xContentType(content); + if (type == null) { + throw new XContentParseException("Failed to derive xcontent"); + } + return type; + } + + /** + * Guesses the content type based on the provided char sequence. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContentType(CharSequence content) { + int length = content.length() < GUESS_HEADER_LENGTH ? content.length() : GUESS_HEADER_LENGTH; + if (length == 0) { + return null; + } + for (var mediaType : formatToMediaType.values()) { + if (mediaType.detectedXContent(content, length)) { + return mediaType; + } + } + + // fallback for json + for (int i = 0; i < length; i++) { + char c = content.charAt(i); + if (c == '{') { + return MediaType.fromMediaType("application/json"); + } + if (Character.isWhitespace(c) == false) { + break; + } + } + return null; + } + + /** + * Guesses the content type based on the provided input stream without consuming it. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContentType(InputStream si) throws IOException { + /* + * We need to guess the content type. To do this, we look for the first non-whitespace character and then try to guess the content + * type on the GUESS_HEADER_LENGTH bytes that follow. We do this in a way that does not modify the initial read position in the + * underlying input stream. This is why the input stream must support mark/reset and why we repeatedly mark the read position and + * reset. + */ + if (si.markSupported() == false) { + throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass()); + } + si.mark(Integer.MAX_VALUE); + try { + // scan until we find the first non-whitespace character or the end of the stream + int current; + do { + current = si.read(); + if (current == -1) { + return null; + } + } while (Character.isWhitespace((char) current)); + // now guess the content type off the next GUESS_HEADER_LENGTH bytes including the current byte + final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH]; + firstBytes[0] = (byte) current; + int read = 1; + while (read < GUESS_HEADER_LENGTH) { + final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read); + if (r == -1) { + break; + } + read += r; + } + return mediaTypeFromBytes(firstBytes, 0, read); + } finally { + si.reset(); + } + + } + + /** + * Guesses the content type based on the provided bytes. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType xContentType(BytesReference bytes) { + if (bytes instanceof BytesArray) { + final BytesArray array = (BytesArray) bytes; + return mediaTypeFromBytes(array.array(), array.offset(), array.length()); + } + try { + final InputStream inputStream = bytes.streamInput(); + assert inputStream.markSupported(); + return xContentType(inputStream); + } catch (IOException e) { + assert false : "Should not happen, we're just reading bytes from memory"; + throw new UncheckedIOException(e); + } + } + + /** + * Guesses the content type based on the provided bytes. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static MediaType mediaTypeFromBytes(final byte[] data, int offset, int length) { + int totalLength = data.length; + if (totalLength == 0 || length == 0) { + return null; + } else if ((offset + length) > totalLength) { + return null; + } + for (var mediaType : formatToMediaType.values()) { + if (mediaType.detectedXContent(data, offset, length)) { + return mediaType; + } + } + + // a last chance for JSON + int jsonStart = 0; + // JSON may be preceded by UTF-8 BOM + if (length > 3 && data[offset] == (byte) 0xEF && data[offset + 1] == (byte) 0xBB && data[offset + 2] == (byte) 0xBF) { + jsonStart = 3; + } + + for (int i = jsonStart; i < length; i++) { + byte b = data[offset + i]; + if (b == '{') { + return fromMediaType("application/json"); + } + if (Character.isWhitespace(b) == false) { + break; + } + } + + return null; + } + + /** + * parsing media type that follows https://tools.ietf.org/html/rfc7231#section-3.1.1.1 + * @param headerValue a header value from Accept or Content-Type + * @return a parsed media-type + */ + public static ParsedMediaType parseMediaType(String headerValue) { + if (headerValue != null) { + String[] split = headerValue.toLowerCase(Locale.ROOT).split(";"); + + String[] typeSubtype = split[0].trim().split("/"); + if (typeSubtype.length == 2) { + String type = typeSubtype[0]; + String subtype = typeSubtype[1]; + MediaType mediaType = typeWithSubtypeToMediaType.get(type + "/" + subtype); + if (mediaType != null) { + Map parameters = new HashMap<>(); + for (int i = 1; i < split.length; i++) { + // spaces are allowed between parameters, but not between '=' sign + String[] keyValueParam = split[i].trim().split("="); + if (keyValueParam.length != 2 || hasSpaces(keyValueParam[0]) || hasSpaces(keyValueParam[1])) { + return null; + } + parameters.put(keyValueParam[0], keyValueParam[1]); + } + return new ParsedMediaType(mediaType, parameters); + } + } + + } + return null; + } + + private static boolean hasSpaces(String s) { + return s.trim().equals(s) == false; + } + + /** + * A media type object that contains all the information provided on a Content-Type or Accept header + */ + public static class ParsedMediaType { + private final Map parameters; + private final MediaType mediaType; + + public ParsedMediaType(MediaType mediaType, Map parameters) { + this.parameters = parameters; + this.mediaType = mediaType; + } + + public MediaType getMediaType() { + return mediaType; + } + + public Map getParameters() { + return parameters; + } + } + + private static void setDefaultMediaType(final MediaType mediaType) { + if (DEFAULT_MEDIA_TYPE != null) { + throw new RuntimeException( + "unable to reset the default media type from current default [" + DEFAULT_MEDIA_TYPE + "] to [" + mediaType + "]" + ); + } else { + DEFAULT_MEDIA_TYPE = mediaType; + } + } + + public static MediaType getDefaultMediaType() { + return DEFAULT_MEDIA_TYPE; + } +} diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java new file mode 100644 index 0000000000000..eeaadc1698df6 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/MediaTypeProvider.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.core.xcontent.spi; + +import org.opensearch.core.xcontent.MediaType; + +import java.util.List; +import java.util.Map; + +/** + * Service Provider Interface for plugins, modules, extensions providing + * their own Media Types + * + * @opensearch.experimental + * @opensearch.api + */ +public interface MediaTypeProvider { + /** Extensions that implement their own concrete {@link MediaType}s provide them through this interface method */ + List getMediaTypes(); + + /** Extensions that implement additional {@link MediaType} aliases provide them through this interface method */ + Map getAdditionalMediaTypes(); +} diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java new file mode 100644 index 0000000000000..67ccd981dafa8 --- /dev/null +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/spi/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Service Provider Interface for extensible media types */ +package org.opensearch.core.xcontent.spi; diff --git a/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeUnitTests.java similarity index 91% rename from server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java rename to libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeUnitTests.java index c6bcceec99fbd..07b9131602ac3 100644 --- a/server/src/test/java/org/opensearch/common/unit/ByteSizeUnitTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeUnitTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.unit; +package org.opensearch.core.common.unit; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; @@ -38,12 +38,12 @@ import java.io.IOException; -import static org.opensearch.common.unit.ByteSizeUnit.BYTES; -import static org.opensearch.common.unit.ByteSizeUnit.GB; -import static org.opensearch.common.unit.ByteSizeUnit.KB; -import static org.opensearch.common.unit.ByteSizeUnit.MB; -import static org.opensearch.common.unit.ByteSizeUnit.PB; -import static org.opensearch.common.unit.ByteSizeUnit.TB; +import static org.opensearch.core.common.unit.ByteSizeUnit.BYTES; +import static org.opensearch.core.common.unit.ByteSizeUnit.GB; +import static org.opensearch.core.common.unit.ByteSizeUnit.KB; +import static org.opensearch.core.common.unit.ByteSizeUnit.MB; +import static org.opensearch.core.common.unit.ByteSizeUnit.PB; +import static org.opensearch.core.common.unit.ByteSizeUnit.TB; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeValueTests.java similarity index 97% rename from server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java rename to libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeValueTests.java index 99c1feb78527f..def1694a72ba4 100644 --- a/server/src/test/java/org/opensearch/common/unit/ByteSizeValueTests.java +++ b/libs/core/src/test/java/org/opensearch/core/common/unit/ByteSizeValueTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.common.unit; +package org.opensearch.core.common.unit; import org.opensearch.OpenSearchParseException; import org.opensearch.core.common.io.stream.Writeable.Reader; @@ -336,12 +336,10 @@ public void testParseInvalidNumber() throws IOException { public void testParseFractionalNumber() throws IOException { ByteSizeUnit unit = randomValueOtherThan(ByteSizeUnit.BYTES, () -> randomFrom(ByteSizeUnit.values())); String fractionalValue = "23.5" + unit.getSuffix(); - ByteSizeValue instance = ByteSizeValue.parseBytesSizeValue(fractionalValue, "test"); - assertEquals(fractionalValue, instance.toString()); - assertWarnings( - "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [" - + fractionalValue - + "] found for setting [test]" + // test exception is thrown: fractional byte size values has been deprecated since Legacy 6.2 + OpenSearchParseException e = expectThrows( + OpenSearchParseException.class, + () -> ByteSizeValue.parseBytesSizeValue(fractionalValue, "test") ); } diff --git a/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java new file mode 100644 index 0000000000000..fe239fea8129e --- /dev/null +++ b/libs/secure-sm/src/main/java/org/opensearch/secure_sm/SecuredForkJoinWorkerThreadFactory.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.secure_sm; + +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.Permission; +import java.security.Permissions; +import java.security.PrivilegedAction; +import java.security.ProtectionDomain; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory; +import java.util.concurrent.ForkJoinWorkerThread; + +public class SecuredForkJoinWorkerThreadFactory implements ForkJoinWorkerThreadFactory { + static AccessControlContext contextWithPermissions(Permission... perms) { + Permissions permissions = new Permissions(); + for (Permission perm : perms) + permissions.add(perm); + return new AccessControlContext(new ProtectionDomain[] { new ProtectionDomain(null, permissions) }); + } + + // ACC for access to the factory + private static final AccessControlContext ACC = contextWithPermissions( + new RuntimePermission("getClassLoader"), + new RuntimePermission("setContextClassLoader"), + new RuntimePermission("modifyThreadGroup"), + new RuntimePermission("modifyThread") + ); + + public final ForkJoinWorkerThread newThread(ForkJoinPool pool) { + return AccessController.doPrivileged(new PrivilegedAction<>() { + public ForkJoinWorkerThread run() { + return new ForkJoinWorkerThread(pool) { + + }; + } + }, ACC); + } +} diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java index 94ac264546dbb..9f423bc9abad3 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentFactory.java @@ -32,19 +32,14 @@ package org.opensearch.common.xcontent; -import com.fasterxml.jackson.dataformat.cbor.CBORConstants; -import com.fasterxml.jackson.dataformat.smile.SmileConstants; -import org.opensearch.common.xcontent.cbor.CborXContent; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.smile.SmileXContent; import org.opensearch.common.xcontent.yaml.YamlXContent; -import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParseException; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -52,13 +47,11 @@ */ public class XContentFactory { - static final int GUESS_HEADER_LENGTH = 20; - /** * Returns a content builder using JSON format ({@link org.opensearch.common.xcontent.XContentType#JSON}. */ public static XContentBuilder jsonBuilder() throws IOException { - return contentBuilder(XContentType.JSON); + return MediaTypeRegistry.contentBuilder(XContentType.JSON); } /** @@ -72,7 +65,7 @@ public static XContentBuilder jsonBuilder(OutputStream os) throws IOException { * Returns a content builder using SMILE format ({@link org.opensearch.common.xcontent.XContentType#SMILE}. */ public static XContentBuilder smileBuilder() throws IOException { - return contentBuilder(XContentType.SMILE); + return MediaTypeRegistry.contentBuilder(XContentType.SMILE); } /** @@ -86,7 +79,7 @@ public static XContentBuilder smileBuilder(OutputStream os) throws IOException { * Returns a content builder using YAML format ({@link org.opensearch.common.xcontent.XContentType#YAML}. */ public static XContentBuilder yamlBuilder() throws IOException { - return contentBuilder(XContentType.YAML); + return MediaTypeRegistry.contentBuilder(XContentType.YAML); } /** @@ -100,271 +93,6 @@ public static XContentBuilder yamlBuilder(OutputStream os) throws IOException { * Returns a content builder using CBOR format ({@link org.opensearch.common.xcontent.XContentType#CBOR}. */ public static XContentBuilder cborBuilder() throws IOException { - return contentBuilder(XContentType.CBOR); - } - - /** - * Constructs a new cbor builder that will output the result into the provided output stream. - */ - public static XContentBuilder cborBuilder(OutputStream os) throws IOException { - return new XContentBuilder(CborXContent.cborXContent, os); - } - - /** - * Constructs a xcontent builder that will output the result into the provided output stream. - */ - public static XContentBuilder contentBuilder(XContentType type, OutputStream outputStream) throws IOException { - if (type == XContentType.JSON) { - return jsonBuilder(outputStream); - } else if (type == XContentType.SMILE) { - return smileBuilder(outputStream); - } else if (type == XContentType.YAML) { - return yamlBuilder(outputStream); - } else if (type == XContentType.CBOR) { - return cborBuilder(outputStream); - } - throw new IllegalArgumentException("No matching content type for " + type); - } - - /** - * Returns a binary content builder for the provided media type. - */ - public static XContentBuilder contentBuilder(MediaType type) throws IOException { - if (type instanceof XContentType) { - return contentBuilder(XContentType.fromMediaType(type)); - } - throw new IllegalArgumentException("Content type [" + type.getClass().getName() + "] not supported"); - } - - /** - * Returns a binary content builder for the provided content type. - */ - public static XContentBuilder contentBuilder(XContentType type) throws IOException { - if (type == XContentType.JSON) { - return JsonXContent.contentBuilder(); - } else if (type == XContentType.SMILE) { - return SmileXContent.contentBuilder(); - } else if (type == XContentType.YAML) { - return YamlXContent.contentBuilder(); - } else if (type == XContentType.CBOR) { - return CborXContent.contentBuilder(); - } - throw new IllegalArgumentException("No matching content type for " + type); - } - - /** - * Returns the {@link XContent} for the provided content type. - */ - public static XContent xContent(MediaType type) { - if (type == null) { - throw new IllegalArgumentException("Cannot get xcontent for unknown type"); - } - return type.xContent(); - } - - /** - * Guesses the content type based on the provided char sequence. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(CharSequence content) { - int length = content.length() < GUESS_HEADER_LENGTH ? content.length() : GUESS_HEADER_LENGTH; - if (length == 0) { - return null; - } - char first = content.charAt(0); - if (first == '{') { - return XContentType.JSON; - } - // Should we throw a failure here? Smile idea is to use it in bytes.... - if (length > 2 - && first == SmileConstants.HEADER_BYTE_1 - && content.charAt(1) == SmileConstants.HEADER_BYTE_2 - && content.charAt(2) == SmileConstants.HEADER_BYTE_3) { - return XContentType.SMILE; - } - if (length > 2 && first == '-' && content.charAt(1) == '-' && content.charAt(2) == '-') { - return XContentType.YAML; - } - - // CBOR is not supported - - for (int i = 0; i < length; i++) { - char c = content.charAt(i); - if (c == '{') { - return XContentType.JSON; - } - if (Character.isWhitespace(c) == false) { - break; - } - } - return null; - } - - /** - * Guesses the content (type) based on the provided char sequence and returns the corresponding {@link XContent} - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContent xContent(CharSequence content) { - XContentType type = xContentType(content); - if (type == null) { - throw new XContentParseException("Failed to derive xcontent"); - } - return xContent(type); - } - - /** - * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent} - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContent xContent(byte[] data) { - return xContent(data, 0, data.length); - } - - /** - * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent} - * - * @deprecated guessing the content type should not be needed ideally. We should rather know the content type upfront or read it - * from headers. Till we fixed the REST layer to read the Content-Type header, that should be the only place where guessing is needed. - */ - @Deprecated - public static XContent xContent(byte[] data, int offset, int length) { - XContentType type = xContentType(data, offset, length); - if (type == null) { - throw new XContentParseException("Failed to derive xcontent"); - } - return xContent(type); - } - - /** - * Guesses the content type based on the provided input stream without consuming it. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(InputStream si) throws IOException { - /* - * We need to guess the content type. To do this, we look for the first non-whitespace character and then try to guess the content - * type on the GUESS_HEADER_LENGTH bytes that follow. We do this in a way that does not modify the initial read position in the - * underlying input stream. This is why the input stream must support mark/reset and why we repeatedly mark the read position and - * reset. - */ - if (si.markSupported() == false) { - throw new IllegalArgumentException("Cannot guess the xcontent type without mark/reset support on " + si.getClass()); - } - si.mark(Integer.MAX_VALUE); - try { - // scan until we find the first non-whitespace character or the end of the stream - int current; - do { - current = si.read(); - if (current == -1) { - return null; - } - } while (Character.isWhitespace((char) current)); - // now guess the content type off the next GUESS_HEADER_LENGTH bytes including the current byte - final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH]; - firstBytes[0] = (byte) current; - int read = 1; - while (read < GUESS_HEADER_LENGTH) { - final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read); - if (r == -1) { - break; - } - read += r; - } - return xContentType(firstBytes, 0, read); - } finally { - si.reset(); - } - - } - - /** - * Guesses the content type based on the provided bytes. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(byte[] bytes) { - return xContentType(bytes, 0, bytes.length); - } - - /** - * Guesses the content type based on the provided bytes. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(byte[] bytes, int offset, int length) { - int totalLength = bytes.length; - if (totalLength == 0 || length == 0) { - return null; - } else if ((offset + length) > totalLength) { - return null; - } - byte first = bytes[offset]; - if (first == '{') { - return XContentType.JSON; - } - if (length > 2 - && first == SmileConstants.HEADER_BYTE_1 - && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2 - && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3) { - return XContentType.SMILE; - } - if (length > 2 && first == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-') { - return XContentType.YAML; - } - // CBOR logic similar to CBORFactory#hasCBORFormat - if (first == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1) { - return XContentType.CBOR; - } - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first) && length > 2) { - // Actually, specific "self-describe tag" is a very good indicator - if (first == (byte) 0xD9 && bytes[offset + 1] == (byte) 0xD9 && bytes[offset + 2] == (byte) 0xF7) { - return XContentType.CBOR; - } - } - // for small objects, some encoders just encode as major type object, we can safely - // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort - if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, first)) { - return XContentType.CBOR; - } - - int jsonStart = 0; - // JSON may be preceded by UTF-8 BOM - if (length > 3 && first == (byte) 0xEF && bytes[offset + 1] == (byte) 0xBB && bytes[offset + 2] == (byte) 0xBF) { - jsonStart = 3; - } - - // a last chance for JSON - for (int i = jsonStart; i < length; i++) { - byte b = bytes[offset + i]; - if (b == '{') { - return XContentType.JSON; - } - if (Character.isWhitespace(b) == false) { - break; - } - } - return null; + return MediaTypeRegistry.contentBuilder(XContentType.CBOR); } } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java index b3a8f5e27635d..7026dbd5e78c3 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java @@ -32,16 +32,19 @@ package org.opensearch.common.xcontent; +import com.fasterxml.jackson.dataformat.cbor.CBORConstants; +import com.fasterxml.jackson.dataformat.smile.SmileConstants; import org.opensearch.common.xcontent.cbor.CborXContent; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.smile.SmileXContent; import org.opensearch.common.xcontent.yaml.YamlXContent; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.MediaType; -import org.opensearch.core.xcontent.MediaTypeParser; import org.opensearch.core.xcontent.XContent; +import org.opensearch.core.xcontent.XContentBuilder; -import java.util.Locale; -import java.util.Map; +import java.io.IOException; +import java.io.OutputStream; /** * The content type of {@link XContent}. @@ -71,6 +74,26 @@ public String subtype() { public XContent xContent() { return JsonXContent.jsonXContent; } + + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + return bytes[offset] == '{'; + } + + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return content.charAt(0) == '{'; + } + + @Override + public XContentBuilder contentBuilder() throws IOException { + return JsonXContent.contentBuilder(); + } + + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(JsonXContent.jsonXContent, os); + } }, /** * The jackson based smile binary format. Fast and compact binary format. @@ -90,6 +113,32 @@ public String subtype() { public XContent xContent() { return SmileXContent.smileXContent; } + + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + return length > 2 + && bytes[offset] == SmileConstants.HEADER_BYTE_1 + && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2 + && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3; + } + + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return length > 2 + && content.charAt(0) == SmileConstants.HEADER_BYTE_1 + && content.charAt(1) == SmileConstants.HEADER_BYTE_2 + && content.charAt(2) == SmileConstants.HEADER_BYTE_3; + } + + @Override + public XContentBuilder contentBuilder() throws IOException { + return SmileXContent.contentBuilder(); + } + + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(SmileXContent.smileXContent, os); + } }, /** * A YAML based content type. @@ -109,6 +158,26 @@ public String subtype() { public XContent xContent() { return YamlXContent.yamlXContent; } + + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + return length > 2 && bytes[offset] == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-'; + } + + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return length > 2 && content.charAt(0) == '-' && content.charAt(1) == '-' && content.charAt(2) == '-'; + } + + @Override + public XContentBuilder contentBuilder() throws IOException { + return YamlXContent.contentBuilder(); + } + + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(YamlXContent.yamlXContent, os); + } }, /** * A CBOR based content type. @@ -128,56 +197,42 @@ public String subtype() { public XContent xContent() { return CborXContent.cborXContent; } - }; - /** a parser of media types */ - private static final MediaTypeParser MEDIA_TYPE_PARSER = new MediaTypeParser<>( - XContentType.values(), - Map.of("application/*", JSON, "application/x-ndjson", JSON) - ); - - /** gets the {@link MediaTypeParser} singleton for use outside class */ - @SuppressWarnings("rawtypes") - public static MediaTypeParser getMediaTypeParser() { - return MEDIA_TYPE_PARSER; - } + @Override + public boolean detectedXContent(final byte[] bytes, int offset, int length) { + // CBOR logic similar to CBORFactory#hasCBORFormat + if (bytes[offset] == CBORConstants.BYTE_OBJECT_INDEFINITE && length > 1) { + return true; + } + if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, bytes[offset]) && length > 2) { + // Actually, specific "self-describe tag" is a very good indicator + if (bytes[offset] == (byte) 0xD9 && bytes[offset + 1] == (byte) 0xD9 && bytes[offset + 2] == (byte) 0xF7) { + return true; + } + } + // for small objects, some encoders just encode as major type object, we can safely + // say its CBOR since it doesn't contradict SMILE or JSON, and its a last resort + if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_OBJECT, bytes[offset])) { + return true; + } + return false; + } - /** - * Accepts a format string, which is most of the time is equivalent to {@link XContentType#subtype()} - * and attempts to match the value to an {@link XContentType}. - * The comparisons are done in lower case format. - * This method will return {@code null} if no match is found - */ - public static XContentType fromFormat(String mediaType) { - return MEDIA_TYPE_PARSER.fromFormat(mediaType); - } + @Override + public boolean detectedXContent(final CharSequence content, final int length) { + return false; + } - /** - * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner. - * The provided media type can optionally has parameters. - * This method is suitable for parsing of the {@code Content-Type} and {@code Accept} HTTP headers. - * This method will return {@code null} if no match is found - */ - public static XContentType fromMediaType(String mediaTypeHeaderValue) { - mediaTypeHeaderValue = removeVersionInMediaType(mediaTypeHeaderValue); - return MEDIA_TYPE_PARSER.fromMediaType(mediaTypeHeaderValue); - } + @Override + public XContentBuilder contentBuilder() throws IOException { + return CborXContent.contentBuilder(); + } - /** - * Clients compatible with ES 7.x might start sending media types with versioned media type - * in a form of application/vnd.elasticsearch+json;compatible-with=7. - * This has to be removed in order to be used in 7.x server. - * The same client connecting using that media type will be able to communicate with ES 8 thanks to compatible API. - * @param mediaType - a media type used on Content-Type header, might contain versioned media type. - * - * @return a media type string without - */ - private static String removeVersionInMediaType(String mediaType) { - if (mediaType != null && (mediaType = mediaType.toLowerCase(Locale.ROOT)).contains("vnd.opensearch")) { - return mediaType.replaceAll("vnd.opensearch\\+", "").replaceAll("\\s*;\\s*compatible-with=\\d+", ""); + @Override + public XContentBuilder contentBuilder(final OutputStream os) throws IOException { + return new XContentBuilder(CborXContent.cborXContent, os); } - return mediaType; - } + }; private int index; @@ -199,12 +254,8 @@ public String format() { return subtype(); } - /** Converts from a {@link MediaType} to an explicit {@link XContentType} */ - public static XContentType fromMediaType(MediaType mediaType) { - if (mediaType instanceof XContentType) { - return (XContentType) mediaType; - } else { - return mediaType != null ? MEDIA_TYPE_PARSER.fromMediaType(mediaType.mediaTypeWithoutParameters()) : null; - } + @Override + public void writeTo(StreamOutput output) throws IOException { + output.writeString(this.mediaType()); } } diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java index 9164b5216f9a5..3d4bbba9cc50c 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContentGenerator.java @@ -43,9 +43,9 @@ import com.fasterxml.jackson.core.util.JsonGeneratorDelegate; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentGenerator; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -339,7 +339,7 @@ public void writeRawField(String name, InputStream content) throws IOException { // needed for the XContentFactory.xContentType call content = new BufferedInputStream(content); } - XContentType contentType = XContentFactory.xContentType(content); + MediaType contentType = MediaTypeRegistry.xContentType(content); if (contentType == null) { throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed"); } @@ -354,7 +354,7 @@ public void writeRawField(String name, InputStream content, MediaType mediaType) if (mayWriteRawData(mediaType) == false) { // EMPTY is safe here because we never call namedObject when writing raw data try ( - XContentParser parser = XContentFactory.xContent(mediaType) + XContentParser parser = mediaType.xContent() // It's okay to pass the throwing deprecation handler // because we should not be writing raw fields when // generating JSON diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java new file mode 100644 index 0000000000000..af5ab67507b81 --- /dev/null +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/XContentProvider.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.xcontent.spi; + +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.spi.MediaTypeProvider; + +import java.util.List; +import java.util.Map; + +/** + * Media Type implementations provided by xcontent library + * + * @opensearch.internal + */ +public class XContentProvider implements MediaTypeProvider { + /** Returns the concrete {@link MediaType} provided by the xcontent library */ + @Override + public List getMediaTypes() { + return List.of(XContentType.values()); + } + + /** Returns the additional {@link MediaType} aliases provided by the xcontent library */ + @Override + public Map getAdditionalMediaTypes() { + return Map.of("application/*", XContentType.JSON, "application/x-ndjson", XContentType.JSON); + } +} diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java new file mode 100644 index 0000000000000..c265021f12763 --- /dev/null +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/spi/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** SPI implementation for the xcontent library */ +package org.opensearch.common.xcontent.spi; diff --git a/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider b/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider new file mode 100644 index 0000000000000..ce3fab93087dd --- /dev/null +++ b/libs/x-content/src/main/resources/META-INF/services/org.opensearch.core.xcontent.spi.MediaTypeProvider @@ -0,0 +1,9 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +org.opensearch.common.xcontent.spi.XContentProvider diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java index f5616f2159f77..64d36f0a8b78f 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/MediaTypeParserTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.xcontent; -import org.opensearch.core.xcontent.MediaTypeParser; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.test.OpenSearchTestCase; import java.util.Collections; @@ -44,42 +44,39 @@ public class MediaTypeParserTests extends OpenSearchTestCase { - @SuppressWarnings("unchecked") - MediaTypeParser mediaTypeParser = XContentType.getMediaTypeParser(); - public void testJsonWithParameters() throws Exception { String mediaType = "application/json"; - assertThat(mediaTypeParser.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap())); - assertThat(mediaTypeParser.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8"))); + assertThat(MediaTypeRegistry.parseMediaType(mediaType).getParameters(), equalTo(Collections.emptyMap())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + ";").getParameters(), equalTo(Collections.emptyMap())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8"))); assertThat( - mediaTypeParser.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(), + MediaTypeRegistry.parseMediaType(mediaType + "; custom=123;charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); } public void testWhiteSpaceInTypeSubtype() { String mediaType = " application/json "; - assertThat(mediaTypeParser.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.parseMediaType(mediaType).getMediaType(), equalTo(XContentType.JSON)); assertThat( - mediaTypeParser.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(), + MediaTypeRegistry.parseMediaType(mediaType + "; custom=123; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); assertThat( - mediaTypeParser.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(), + MediaTypeRegistry.parseMediaType(mediaType + "; custom=123;\n charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "custom", "123")) ); mediaType = " application / json "; - assertThat(mediaTypeParser.parseMediaType(mediaType), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType), is(nullValue())); } public void testInvalidParameters() { String mediaType = "application/json"; - assertThat(mediaTypeParser.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; keyvalueNoEqualsSign"), is(nullValue())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; key = value"), is(nullValue())); - assertThat(mediaTypeParser.parseMediaType(mediaType + "; key="), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; key = value"), is(nullValue())); + assertThat(MediaTypeRegistry.parseMediaType(mediaType + "; key="), is(nullValue())); } } diff --git a/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java index c5dc68ff4c800..0384615c2f2f5 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java +++ b/modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java @@ -37,7 +37,7 @@ import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.core.xcontent.ContextParser; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.plugins.SearchPlugin; import org.opensearch.script.ScriptService; import org.opensearch.search.aggregations.Aggregation; diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java index 3bd1137975800..b76fe41c8e67d 100644 --- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java +++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/BytesProcessor.java @@ -32,7 +32,7 @@ package org.opensearch.ingest.common; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import java.util.Map; diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java index bbd9ff4c8b912..ce8c182b60a61 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/BytesProcessorTests.java @@ -33,15 +33,14 @@ package org.opensearch.ingest.common; import org.opensearch.OpenSearchException; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.OpenSearchParseException; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.ingest.IngestDocument; import org.opensearch.ingest.Processor; import org.opensearch.ingest.RandomDocumentPicks; import org.hamcrest.CoreMatchers; -import static org.hamcrest.Matchers.equalTo; - public class BytesProcessorTests extends AbstractStringProcessorTestCase { private String modifiedInput; @@ -101,14 +100,16 @@ public void testMissingUnits() { assertThat(exception.getMessage(), CoreMatchers.containsString("unit is missing or unrecognized")); } - public void testFractional() throws Exception { + public void testFractional() { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "1.1kb"); Processor processor = newProcessor(fieldName, randomBoolean(), fieldName); - processor.execute(ingestDocument); - assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(1126L)); - assertWarnings( - "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for setting " + "[Ingest Field]" + OpenSearchParseException e = expectThrows(OpenSearchParseException.class, () -> processor.execute(ingestDocument)); + assertThat( + e.getMessage(), + CoreMatchers.containsString( + "Fractional bytes values have been deprecated since Legacy 6.2. " + "Use non-fractional bytes values instead:" + ) ); } } diff --git a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java index b189c8ed8905e..87598aa28ce8f 100644 --- a/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/opensearch/ingest/useragent/UserAgentParser.java @@ -35,7 +35,6 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -68,7 +67,7 @@ final class UserAgentParser { private void init(InputStream regexStream) throws IOException { // EMPTY is safe here because we don't use namedObject - XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML) + XContentParser yamlParser = XContentType.YAML.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, regexStream); XContentParser.Token token = yamlParser.nextToken(); diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 deleted file mode 100644 index ecf696b4b3b83..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -297e1cfade4ef71466cc9d4f361d81807c8dc4c8 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..6eaa40708e4ae --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +9f8a34fc3d450343ab05ccb5af318a836a6a5fb3 \ No newline at end of file diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java index 6e56ecf3950bb..da67a0d2dd13a 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java @@ -40,8 +40,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.rest.RestStatus; @@ -104,11 +104,11 @@ public static SearchTemplateResponse fromXContent(XContentParser parser) throws if (contentAsMap.containsKey(TEMPLATE_OUTPUT_FIELD.getPreferredName())) { Object source = contentAsMap.get(TEMPLATE_OUTPUT_FIELD.getPreferredName()); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).value(source); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).value(source); searchTemplateResponse.setSource(BytesReference.bytes(builder)); } else { MediaType contentType = parser.contentType(); - XContentBuilder builder = XContentFactory.contentBuilder(contentType).map(contentAsMap); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType).map(contentAsMap); XContentParser searchResponseParser = contentType.xContent() .createParser(parser.getXContentRegistry(), parser.getDeprecationHandler(), BytesReference.bytes(builder).streamInput()); diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java index bbda8d15d9d41..f6b0cc4eecf9a 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java @@ -42,7 +42,6 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.action.search.RestSearchAction; @@ -132,8 +131,7 @@ static SearchRequest convert( } try ( - XContentParser parser = XContentFactory.xContent(XContentType.JSON) - .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, source) + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, source) ) { SearchSourceBuilder builder = SearchSourceBuilder.searchSource(); builder.parseXContent(parser, false); diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java index 664e87da0a7d8..e3669ef67ff36 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestXContentTests.java @@ -33,9 +33,9 @@ package org.opensearch.script.mustache; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -101,7 +101,7 @@ public void testToXContentWithInlineTemplate() throws IOException { request.setScriptParams(scriptParams); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedRequest = MediaTypeRegistry.contentBuilder(contentType) .startObject() .field("source", "{\"query\": { \"match\" : { \"{{my_field}}\" : \"{{my_value}}\" } } }") .startObject("params") @@ -112,7 +112,7 @@ public void testToXContentWithInlineTemplate() throws IOException { .field("profile", true) .endObject(); - XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); + XContentBuilder actualRequest = MediaTypeRegistry.contentBuilder(contentType); request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType); @@ -131,7 +131,7 @@ public void testToXContentWithStoredTemplate() throws IOException { request.setScriptParams(params); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedRequest = MediaTypeRegistry.contentBuilder(contentType) .startObject() .field("id", "match_template") .startObject("params") @@ -142,7 +142,7 @@ public void testToXContentWithStoredTemplate() throws IOException { .field("profile", false) .endObject(); - XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); + XContentBuilder actualRequest = MediaTypeRegistry.contentBuilder(contentType); request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType); diff --git a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java index 804a72561c10e..fd0a4e9612a8f 100644 --- a/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java +++ b/modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java @@ -36,6 +36,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -164,7 +165,7 @@ public void testSourceToXContent() throws IOException { response.setSource(BytesReference.bytes(source)); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType) .startObject() .startObject("template_output") .startObject("query") @@ -175,7 +176,7 @@ public void testSourceToXContent() throws IOException { .endObject() .endObject(); - XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType); + XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType); response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType); @@ -210,7 +211,7 @@ public void testSearchResponseToXContent() throws IOException { response.setResponse(searchResponse); XContentType contentType = randomFrom(XContentType.values()); - XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType) + XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType) .startObject() .field("took", 0) .field("timed_out", false) @@ -235,7 +236,7 @@ public void testSearchResponseToXContent() throws IOException { .endObject() .endObject(); - XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType); + XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType); response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType); diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 3b3f2a7f9ca38..d7af8621c478a 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -84,8 +84,7 @@ shadowJar { tasks.validateNebulaPom.dependsOn tasks.generatePomFileForShadowPublication tasks.validateShadowPom.dependsOn tasks.generatePomFileForNebulaPublication -tasks.publishNebulaPublicationToMavenLocal.dependsOn tasks.generatePomFileForShadowPublication -tasks.publishShadowPublicationToMavenLocal.dependsOn tasks.generatePomFileForNebulaPublication +tasks.withType(AbstractPublishToMaven)*.dependsOn "generatePomFileForShadowPublication", "generatePomFileForNebulaPublication" tasks.named("dependencyLicenses").configure { mapping from: /asm-.*/, to: 'asm' diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java index faae24083eeca..9a2c8c1f0aa55 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/action/PainlessExecuteAction.java @@ -65,9 +65,9 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -178,11 +178,11 @@ static class ContextSetup implements Writeable, ToXContentObject { private final BytesReference document; private final QueryBuilder query; - private XContentType xContentType; + private MediaType mediaType; static ContextSetup parse(XContentParser parser, Void context) throws IOException { ContextSetup contextSetup = PARSER.parse(parser, null); - contextSetup.setXContentType(XContentType.fromMediaType(parser.contentType())); + contextSetup.setXContentType(parser.contentType()); return contextSetup; } @@ -195,9 +195,9 @@ static ContextSetup parse(XContentParser parser, Void context) throws IOExceptio ContextSetup(StreamInput in) throws IOException { index = in.readOptionalString(); document = in.readOptionalBytesReference(); - String xContentType = in.readOptionalString(); - if (xContentType != null) { - this.xContentType = XContentType.fromMediaType(xContentType); + String mediaType = in.readOptionalString(); + if (mediaType != null) { + this.mediaType = MediaType.fromMediaType(mediaType); } query = in.readOptionalNamedWriteable(QueryBuilder.class); } @@ -214,12 +214,12 @@ public QueryBuilder getQuery() { return query; } - public XContentType getXContentType() { - return xContentType; + public MediaType getXContentType() { + return mediaType; } - public void setXContentType(XContentType xContentType) { - this.xContentType = xContentType; + public void setXContentType(MediaType mediaType) { + this.mediaType = mediaType; } @Override @@ -230,19 +230,19 @@ public boolean equals(Object o) { return Objects.equals(index, that.index) && Objects.equals(document, that.document) && Objects.equals(query, that.query) - && Objects.equals(xContentType, that.xContentType); + && Objects.equals(mediaType, that.mediaType); } @Override public int hashCode() { - return Objects.hash(index, document, query, xContentType); + return Objects.hash(index, document, query, mediaType); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(index); out.writeOptionalBytesReference(document); - out.writeOptionalString(xContentType != null ? xContentType.mediaTypeWithoutParameters() : null); + out.writeOptionalString(mediaType != null ? mediaType.mediaTypeWithoutParameters() : null); out.writeOptionalNamedWriteable(query); } @@ -257,7 +257,7 @@ public String toString() { + ", query=" + query + ", xContentType=" - + xContentType + + mediaType + '}'; } @@ -275,7 +275,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, document, - xContentType + mediaType ) ) { builder.generator().copyCurrentStructure(parser); @@ -594,8 +594,8 @@ private static Response prepareRamIndex( try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(defaultAnalyzer))) { String index = indexService.index().getName(); BytesReference document = request.contextSetup.document; - XContentType xContentType = request.contextSetup.xContentType; - SourceToParse sourceToParse = new SourceToParse(index, "_id", document, xContentType); + MediaType mediaType = request.contextSetup.mediaType; + SourceToParse sourceToParse = new SourceToParse(index, "_id", document, mediaType); ParsedDocument parsedDocument = indexService.mapperService().documentMapper().parse(sourceToParse); indexWriter.addDocuments(parsedDocument.docs()); try (IndexReader indexReader = DirectoryReader.open(indexWriter)) { diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java b/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java index 8084420295280..c8a28158ad8db 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/api/LimitedCharSequence.java @@ -32,8 +32,8 @@ package org.opensearch.painless.api; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.painless.CompilerSettings; import java.util.regex.Pattern; diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java index c3233bc0d924a..26bebfdee2fd0 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/RegexLimitTests.java @@ -34,7 +34,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.Settings; import java.util.Collections; diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java index a7787f4bc3c29..4117eb331197f 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/api/LimitedCharSequenceTests.java @@ -32,7 +32,7 @@ package org.opensearch.painless.api; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.test.OpenSearchTestCase; import java.util.regex.Pattern; diff --git a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java index 4f26a15031dd4..b38bc5d7b45fc 100644 --- a/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java @@ -416,11 +416,12 @@ public void visit(QueryVisitor visitor) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } + IndexReader reader = searcher.getIndexReader(); if (reader instanceof DirectoryReader) { IndexSearcher indexSearcher = new IndexSearcher(reader); indexSearcher.setQueryCache(null); diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java index 86d30f009e709..2702a02a7a951 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQuery.java @@ -32,7 +32,6 @@ package org.opensearch.percolator; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; @@ -89,8 +88,8 @@ final class PercolateQuery extends Query implements Accountable { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = candidateMatchesQuery.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = candidateMatchesQuery.rewrite(searcher); if (rewritten != candidateMatchesQuery) { return new PercolateQuery( name, diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java index d5b61d5c5a517..08d9a4855c473 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java @@ -71,6 +71,8 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -90,8 +92,8 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; import org.opensearch.index.query.Rewriteable; -import org.opensearch.indices.breaker.CircuitBreakerService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -123,7 +125,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder documents; - private final XContentType documentXContentType; + private final MediaType documentXContentType; private final String indexedDocumentIndex; private final String indexedDocumentId; @@ -150,7 +152,7 @@ public PercolateQueryBuilder(String field, BytesReference document, XContentType * @param documents The binary blob containing document to percolate * @param documentXContentType The content type of the binary blob containing the document to percolate */ - public PercolateQueryBuilder(String field, List documents, XContentType documentXContentType) { + public PercolateQueryBuilder(String field, List documents, MediaType documentXContentType) { if (field == null) { throw new IllegalArgumentException("[field] is a required argument"); } @@ -252,7 +254,11 @@ protected PercolateQueryBuilder(String field, Supplier documentS } documents = in.readList(StreamInput::readBytesReference); if (documents.isEmpty() == false) { - documentXContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + documentXContentType = in.readMediaType(); + } else { + documentXContentType = in.readEnum(XContentType.class); + } } else { documentXContentType = null; } @@ -298,7 +304,11 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeBytesReference(document); } if (documents.isEmpty() == false) { - out.writeEnum(documentXContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + documentXContentType.writeTo(out); + } else { + out.writeEnum((XContentType) documentXContentType); + } } } @@ -432,7 +442,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) { PercolateQueryBuilder rewritten = new PercolateQueryBuilder( field, Collections.singletonList(source), - XContentHelper.xContentType(source) + MediaTypeRegistry.xContentType(source) ); if (name != null) { rewritten.setName(name); @@ -560,7 +570,7 @@ public List getDocuments() { } // pkg-private for testing - XContentType getXContentType() { + MediaType getXContentType() { return documentXContentType; } diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java index 3b953fcfe65e1..1040d014483e1 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java @@ -1275,7 +1275,7 @@ private CustomQuery(Term term) { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { return new TermQuery(term); } diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java index ea001de0ee7c6..d97d5a3a7dcd5 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -36,9 +36,9 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -274,7 +274,7 @@ public static DiscountedCumulativeGain createTestItem() { public void testXContentRoundtrip() throws IOException { DiscountedCumulativeGain testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java index 014f52faa9d57..32c7edc845e32 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -166,7 +166,7 @@ public static ExpectedReciprocalRank createTestItem() { public void testXContentRoundtrip() throws IOException { ExpectedReciprocalRank testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java index 3df79acfa6ce1..d34e6ba8bd72e 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/MeanReciprocalRankTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -183,7 +183,7 @@ public void testNoResults() throws Exception { public void testXContentRoundtrip() throws IOException { MeanReciprocalRank testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java index 3317a2d2f00f1..41340cfd1003d 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/PrecisionAtKTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -199,7 +199,7 @@ public static PrecisionAtK createTestItem() { public void testXContentRoundtrip() throws IOException { PrecisionAtK testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java index 7c0590566bba9..b021d739cc6a6 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java @@ -39,14 +39,14 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.coordination.NoClusterManagerBlockService; import org.opensearch.core.common.ParsingException; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -177,7 +177,7 @@ public void testToXContent() throws IOException { Collections.singletonMap("coffee_query", coffeeQueryQuality), Collections.singletonMap("beer_query", new ParsingException(new XContentLocation(0, 0), "someMsg")) ); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); String xContent = BytesReference.bytes(response.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString(); assertEquals( ("{" diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java index 988784b6e57a3..01f5a3a12da01 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedDocumentTests.java @@ -34,9 +34,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -57,7 +57,7 @@ public static RatedDocument createRatedDocument() { public void testXContentParsing() throws IOException { RatedDocument testItem = createRatedDocument(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { RatedDocument parsedItem = RatedDocument.fromXContent(itemParser); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java index ac2401f30e6f0..6e99e31a2b819 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RatedRequestsTests.java @@ -35,10 +35,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -134,7 +134,7 @@ public static RatedRequest createTestItem(boolean forceRequest) { public void testXContentRoundtrip() throws IOException { RatedRequest testItem = createTestItem(randomBoolean()); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java index 85e024f6bb1e9..37f778fbc5059 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RecallAtKTests.java @@ -35,9 +35,9 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -185,7 +185,7 @@ public static RecallAtK createTestItem() { public void testXContentRoundtrip() throws IOException { RecallAtK testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); try (XContentParser itemParser = createParser(shuffled)) { itemParser.nextToken(); diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 6170c1adabbea..a0ad02899ea27 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -53,7 +53,7 @@ import org.opensearch.action.support.TransportAction; import org.opensearch.client.ParentTaskAssigningClient; import org.opensearch.common.Nullable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.index.VersionType; diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java index 6f36fafd852ef..4b1e416ac63ea 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/AbstractBulkByQueryRestHandler.java @@ -37,8 +37,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.search.RestSearchAction; @@ -106,7 +106,7 @@ private XContentParser extractRequestSpecificFields(RestRequest restRequest, Map } try ( XContentParser parser = restRequest.contentOrSourceParamParser(); - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()) + XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType()) ) { Map body = parser.map(); diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java index ba974d248ac9b..cb07e593f8155 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/Reindexer.java @@ -61,10 +61,10 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.VersionFieldMapper; import org.opensearch.index.reindex.remote.RemoteScrollableHitSource; @@ -378,27 +378,24 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) index.id(doc.getId()); // the source xcontent type and destination could be different - final XContentType sourceXContentType = doc.getXContentType(); - final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType(); - if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) { + final MediaType sourceMediaType = doc.getMediaType(); + final MediaType mainRequestMediaType = mainRequest.getDestination().getContentType(); + if (mainRequestMediaType != null && doc.getMediaType() != mainRequestMediaType) { // we need to convert try ( InputStream stream = doc.getSource().streamInput(); - XContentParser parser = sourceXContentType.xContent() + XContentParser parser = sourceMediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream); - XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent()) + XContentBuilder builder = XContentBuilder.builder(mainRequestMediaType.xContent()) ) { parser.nextToken(); builder.copyCurrentStructure(parser); index.source(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { - throw new UncheckedIOException( - "failed to convert hit from " + sourceXContentType + " to " + mainRequestXContentType, - e - ); + throw new UncheckedIOException("failed to convert hit from " + sourceMediaType + " to " + mainRequestMediaType, e); } } else { - index.source(doc.getSource(), doc.getXContentType()); + index.source(doc.getSource(), doc.getMediaType()); } /* diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java index 5297723ab3687..1ea9ec5fb7beb 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportUpdateByQueryAction.java @@ -138,7 +138,7 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) IndexRequest index = new IndexRequest(); index.index(doc.getIndex()); index.id(doc.getId()); - index.source(doc.getSource(), doc.getXContentType()); + index.source(doc.getSource(), doc.getMediaType()); index.setIfSeqNo(doc.getSeqNo()); index.setIfPrimaryTerm(doc.getPrimaryTerm()); index.setPipeline(mainRequest.getPipeline()); diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java index 3d606b6b77c44..aaca04641d76a 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteResponseParsers.java @@ -41,12 +41,12 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentLocation; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.reindex.ScrollableHitSource.BasicHit; import org.opensearch.index.reindex.ScrollableHitSource.Hit; import org.opensearch.index.reindex.ScrollableHitSource.Response; @@ -72,7 +72,7 @@ private RemoteResponseParsers() {} /** * Parser for an individual {@code hit} element. */ - public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> { + public static final ConstructingObjectParser HIT_PARSER = new ConstructingObjectParser<>("hit", true, a -> { int i = 0; String index = (String) a[i++]; String id = (String) a[i++]; @@ -106,7 +106,7 @@ private RemoteResponseParsers() {} class Fields { String routing; } - ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); + ObjectParser fieldsParser = new ObjectParser<>("fields", Fields::new); HIT_PARSER.declareObject((hit, fields) -> { hit.setRouting(fields.routing); }, fieldsParser, new ParseField("fields")); fieldsParser.declareString((fields, routing) -> fields.routing = routing, routingField); fieldsParser.declareLong((fields, ttl) -> {}, ttlField); // ignore ttls since they have been removed @@ -116,7 +116,7 @@ class Fields { /** * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List)]}. */ - public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a); + public static final ConstructingObjectParser HITS_PARSER = new ConstructingObjectParser<>("hits", true, a -> a); static { HITS_PARSER.declareField(constructorArg(), (p, c) -> { if (p.currentToken() == XContentParser.Token.START_OBJECT) { @@ -184,7 +184,7 @@ class Fields { SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures")); } - public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser RESPONSE_PARSER = new ConstructingObjectParser<>( "search_response", true, a -> { @@ -296,13 +296,13 @@ public void setCausedBy(Throwable causedBy) { /** * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out. */ - public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser MAIN_ACTION_PARSER = new ConstructingObjectParser<>( "/", true, a -> (Version) a[0] ); static { - ConstructingObjectParser versionParser = new ConstructingObjectParser<>( + ConstructingObjectParser versionParser = new ConstructingObjectParser<>( "version", true, a -> a[0] == null diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java index 39d3adcd82ad5..3c305c2a8239d 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSource.java @@ -55,10 +55,10 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.reindex.RejectAwareActionListener; import org.opensearch.index.reindex.ScrollableHitSource; import org.opensearch.core.rest.RestStatus; @@ -182,7 +182,7 @@ protected void cleanup(Runnable onCompletion) { private void execute( Request request, - BiFunction parser, + BiFunction parser, RejectAwareActionListener listener ) { // Preserve the thread context so headers survive after the call @@ -198,12 +198,12 @@ public void onSuccess(org.opensearch.client.Response response) { try { HttpEntity responseEntity = response.getEntity(); InputStream content = responseEntity.getContent(); - XContentType xContentType = null; + MediaType mediaType = null; if (responseEntity.getContentType() != null) { final String mimeType = ContentType.parse(responseEntity.getContentType()).getMimeType(); - xContentType = XContentType.fromMediaType(mimeType); + mediaType = MediaType.fromMediaType(mimeType); } - if (xContentType == null) { + if (mediaType == null) { try { logger.debug("Response didn't include Content-Type: " + bodyMessage(response.getEntity())); throw new OpenSearchException( @@ -217,10 +217,10 @@ public void onSuccess(org.opensearch.client.Response response) { } // EMPTY is safe here because we don't call namedObject try ( - XContentParser xContentParser = xContentType.xContent() + XContentParser xContentParser = mediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, content) ) { - parsedResponse = parser.apply(xContentParser, xContentType); + parsedResponse = parser.apply(xContentParser, mediaType); } catch (XContentParseException e) { /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it * is totally wrong and we're probably not talking to Elasticsearch. */ diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java index 2946544a8f901..651ccd47df7bd 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/RestReindexActionTests.java @@ -74,7 +74,7 @@ public void testPipelineQueryParameterIsError() throws IOException { body.endObject(); } body.endObject(); - request.withContent(BytesReference.bytes(body), XContentType.fromMediaType(body.contentType())); + request.withContent(BytesReference.bytes(body), body.contentType()); } request.withParams(singletonMap("pipeline", "doesn't matter")); Exception e = expectThrows( diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java index ebbd2da776ace..e671fec1fedee 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/remote/RemoteScrollableHitSourceTests.java @@ -44,8 +44,8 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.common.io.Streams; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; diff --git a/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java b/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java index 1bf461d67862b..9c61bca316a56 100644 --- a/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java +++ b/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java @@ -38,7 +38,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.plugin.repository.url.URLRepositoryModulePlugin; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.fs.FsRepository; diff --git a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java index fbfbf5e006fee..0fad0cbe21033 100644 --- a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java +++ b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java @@ -37,8 +37,8 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import java.net.MalformedURLException; import java.net.URL; diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java index db76c0b145840..95440e28686e7 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/opensearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -39,8 +39,8 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.http.HttpServerTransport; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; diff --git a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java index 2584b768707cd..f7e1c6106cf5a 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/opensearch/rest/Netty4BadRequestIT.java @@ -38,7 +38,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.http.HttpTransportSettings; import org.opensearch.test.rest.OpenSearchRestTestCase; import org.opensearch.test.rest.yaml.ObjectPath; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java index 124bc02527bd1..998c89590c53c 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java @@ -78,8 +78,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java index 9a5459a5ab572..a30cf00ce4047 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/CopyBytesSocketChannel.java @@ -52,7 +52,7 @@ import io.netty.channel.RecvByteBufAllocator; import io.netty.channel.socket.nio.NioSocketChannel; import org.opensearch.common.SuppressForbidden; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java index d7f2f6eb6acbb..ef60797bca067 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java @@ -45,7 +45,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.netty4.Netty4HttpServerTransport; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.threadpool.ThreadPool; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java index f2f6538d305d9..d32a43e3eb9eb 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java @@ -42,7 +42,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.Booleans; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.monitor.jvm.JvmInfo; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java index 561cac2facbff..637cb10c383bd 100644 --- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/Netty4Transport.java @@ -57,13 +57,13 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.net.NetUtils; import org.opensearch.common.lease.Releasables; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Netty4NioSocketChannel; import org.opensearch.transport.NettyAllocator; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java index ef014aa39367b..5fcfc4ee5e151 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4BadRequestTests.java @@ -44,7 +44,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpTransportSettings; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java index cad2e50327023..9f359002e441a 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java @@ -72,8 +72,8 @@ import io.netty.util.AttributeKey; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.tasks.Task; import org.opensearch.transport.NettyAllocator; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java index adf4d59a0c139..6c8cf69afb148 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerPipeliningTests.java @@ -52,7 +52,7 @@ import org.opensearch.http.HttpResponse; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.NullDispatcher; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java index 5a43057b1b7d1..d23edfda829f9 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpServerTransportTests.java @@ -65,7 +65,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; @@ -75,7 +75,7 @@ import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpTransportSettings; import org.opensearch.http.NullDispatcher; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java index f80d7f41b5f55..5c1c5970a7cfb 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.SharedGroupFactory; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java index d3fa8ea56ffe7..5ee194b7bb513 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/Netty4UtilsTests.java @@ -36,14 +36,14 @@ import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.Unpooled; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.bytes.AbstractBytesReferenceTestCase; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java index 5d7841df4bf33..c10a92ba1900b 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/NettyTransportMultiPortTests.java @@ -32,14 +32,14 @@ package org.opensearch.transport.netty4; import org.opensearch.Version; -import org.opensearch.common.component.Lifecycle; +import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.network.NetworkService; import org.opensearch.common.network.NetworkUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java index 619f473b8bef2..ee2bf12a4246f 100644 --- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/SimpleNetty4TransportTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.util.net.NetUtils; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.transport.MockTransportService; import org.opensearch.test.transport.StubbableTransport; import org.opensearch.transport.AbstractSimpleTransportTestCase; diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 deleted file mode 100644 index 0ed030926ab93..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -94293b169fb8572f440a5a4a523320ecf9778ffe \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..8a3332c950b6d --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +fde64e3b23bc9a0849b9897febfe9f13c5113143 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 deleted file mode 100644 index ddd67276606a5..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2df800a38b64867b8dcd61fc2cd986114e4a80cb \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..33c2afacf2395 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +b01a791705fa01fce48dd02ea79fa8045de8dd5e \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 deleted file mode 100644 index 0cd68af98e724..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a01e8153f34d72e8c8c0180c1dea5b10f677dd3a \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..1e7986dafa11e --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +43d19320b1b9cd18638b1602fa87d5f21ee043bc \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 deleted file mode 100644 index c7b4d2dc6da75..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b7d47d54683b0b1e09b271c32d1b7d3eb1990f49 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..14880d9c2d243 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +9244dc232f175010b480d4d88e13945c17a0b28b \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 deleted file mode 100644 index 8df7245044171..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e68b9816e6cff8ee15f5b350cf2ffa54f9828b7 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..edc4de3fffe28 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +3101a4f79820c1ca3dfb8f49b74c5fb5b32940e1 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 deleted file mode 100644 index 974e4202f5ffb..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d23b1f05b471e05d0d6068b3ece7c8c65672eae7 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..54c310277b09b --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +f12b2a22cd5ebcd84f40a40e78fdd4e268b3b26d \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 deleted file mode 100644 index dce408a7d40ef..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dfb4313f3c68d337310522840d7144c1605d084a \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..358db9ea3f0f5 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +7dbf5cc3dff93cc1ffe45d79b129859590d001dd \ No newline at end of file diff --git a/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java index 9dbf08a3e1a01..23db0a74dc3fc 100644 --- a/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java +++ b/plugins/discovery-azure-classic/src/main/java/org/opensearch/cloud/azure/classic/management/AzureComputeServiceImpl.java @@ -51,7 +51,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.SpecialPermission; import org.opensearch.cloud.azure.classic.AzureServiceRemoteException; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java index 861926a9e67c9..afa35f63ae4dc 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java @@ -46,7 +46,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.Transport; import org.opensearch.transport.TransportService; diff --git a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java index cc65e92b1485a..ea10d03576d94 100644 --- a/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java @@ -45,7 +45,7 @@ import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.discovery.SeedHostsProvider; import org.opensearch.discovery.SeedHostsResolver; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.transport.MockTransportService; import org.opensearch.transport.TransportService; import org.opensearch.transport.nio.MockNioTransport; diff --git a/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java b/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java index 4873cb6dcbf7a..b7ae7f8b404be 100644 --- a/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java +++ b/plugins/discovery-gce/src/main/java/org/opensearch/cloud/gce/GceMetadataService.java @@ -46,7 +46,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.cloud.gce.util.Access; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 62651216c8144..17213b96a25f3 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -89,7 +89,7 @@ dependencies { api "org.apache.james:apache-mime4j-core:${versions.mime4j}" api "org.apache.james:apache-mime4j-dom:${versions.mime4j}" // EPUB books - api 'org.apache.commons:commons-lang3:3.12.0' + api 'org.apache.commons:commons-lang3:3.13.0' // Microsoft Word files with visio diagrams api 'org.apache.commons:commons-math3:3.6.1' // POIs dependency diff --git a/plugins/ingest-attachment/licenses/commons-lang3-3.12.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.12.0.jar.sha1 deleted file mode 100644 index 9273d8c01aaba..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-lang3-3.12.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c6842c86792ff03b9f1d1fe2aab8dc23aa6c6f0e \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 new file mode 100644 index 0000000000000..d0c2f2486ee1f --- /dev/null +++ b/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 @@ -0,0 +1 @@ +b7263237aa89c1f99b327197c41d0669707a462e \ No newline at end of file diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 67e79addfedc5..48285f80150be 100644 --- a/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -46,7 +46,7 @@ import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.blobstore.OpenSearchMockAPIBasedRepositoryIntegTestCase; import org.opensearch.core.rest.RestStatus; diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java index 2677604ecb622..3846dd14559b5 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepository.java @@ -41,7 +41,7 @@ import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java index c518cc2716db6..88e9a63384f7a 100644 --- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java @@ -61,8 +61,8 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import java.net.Authenticator; diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java index 8b68ccebf8c53..ab16edc7b1a2d 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java @@ -53,7 +53,7 @@ import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.rest.RestStatus; diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java index 24a226290985a..bfae0a3c4438c 100644 --- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureRepositorySettingsTests.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 1e8e358833cc9..0fb98c8b9abff 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -67,7 +67,7 @@ dependencies { api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}" api 'com.google.cloud:google-cloud-core:2.5.10' - api 'com.google.cloud:google-cloud-core-http:2.21.0' + api 'com.google.cloud:google-cloud-core-http:2.21.1' api 'com.google.cloud:google-cloud-storage:1.113.1' api 'com.google.code.gson:gson:2.9.0' @@ -77,7 +77,7 @@ dependencies { api 'com.google.http-client:google-http-client:1.43.2' api 'com.google.http-client:google-http-client-appengine:1.43.2' - api 'com.google.http-client:google-http-client-gson:1.43.2' + api 'com.google.http-client:google-http-client-gson:1.43.3' api 'com.google.http-client:google-http-client-jackson2:1.43.2' api 'com.google.oauth-client:google-oauth-client:1.34.1' diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 deleted file mode 100644 index 2ef0a9bf9b33e..0000000000000 --- a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -07da4710ccdbcfee253672c0b9e00e7370626c26 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 new file mode 100644 index 0000000000000..cc5e7a53098ac --- /dev/null +++ b/plugins/repository-gcs/licenses/google-cloud-core-http-2.21.1.jar.sha1 @@ -0,0 +1 @@ +88dd2b413dd06826c611e39e6e3259e069f02f66 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 deleted file mode 100644 index df0374aa27c70..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b1c2e3e89804c113dba7b342aa8e0fc2cf3d9378 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 new file mode 100644 index 0000000000000..43f4fe4a127e1 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 @@ -0,0 +1 @@ +252e267acf720ef6333488740a696a1d5e204639 \ No newline at end of file diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index f0f0fb7681c1a..92a2f99f4a441 100644 --- a/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -55,8 +55,8 @@ import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java index 226aec437fc0f..f5c20003ea7b6 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -55,8 +55,8 @@ import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.io.Streams; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java index a743ac72bdb8b..05d9739c00d50 100644 --- a/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Setting; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index 488376d36cdc4..466344668d966 100644 --- a/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -53,7 +53,7 @@ import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.common.Strings; diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 0b13e18ba3f4d..6626bfccc6662 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -48,7 +48,7 @@ opensearchplugin { } versions << [ - 'hadoop3': '3.3.4' + 'hadoop3': '3.3.6' ] testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs" @@ -440,3 +440,7 @@ thirdPartyAudit { 'org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField', ) } + +tasks.withType(JavaForkOptions) { + systemProperty "java.util.concurrent.ForkJoinPool.common.threadFactory", "org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory" +} diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 deleted file mode 100644 index dd79b8a10cebc..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6339a8f7279310c8b1f7ef314b592d8c71ca72ef \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..d99793bc56522 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-api-3.3.6.jar.sha1 @@ -0,0 +1 @@ +12ac6f103a0ff29fce17a078c7c64d25320b6165 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 deleted file mode 100644 index 32d58d1dc501a..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..ea22d763b7bfa --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.6.jar.sha1 @@ -0,0 +1 @@ +81065531e63fccbe85fb04a3274709593fb00d3c \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 deleted file mode 100644 index 532d25a44531f..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -036ef2f86dc44410d2bb5d54ce40435d2484d9a5 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 new file mode 100644 index 0000000000000..fe60968056eb7 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.6.jar.sha1 @@ -0,0 +1 @@ +ba40aca60f39599d5b1f1d32b35295bfde1f3c8b \ No newline at end of file diff --git a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java index 88c58942e9bbf..10c3bc2d0364b 100644 --- a/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/opensearch/repositories/hdfs/HdfsRepository.java @@ -48,7 +48,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; diff --git a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java index b9b0e9e87dd0c..2758bd020e979 100644 --- a/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java +++ b/plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsClientThreadLeakFilter.java @@ -43,6 +43,9 @@ * to ignore the offending thread until a version of Hadoop is released that addresses the incorrect * interrupt handling. * + * In Hadoop 3.3.6, the org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap uses ForkJoinPool + * to perform statistics calculation, leaving dangling workers. + * * @see https://issues.apache.org/jira/browse/HADOOP-12829 * @see "org.apache.hadoop.fs.FileSystem.Statistics.StatisticsDataReferenceCleaner" * @see "org.apache.hadoop.fs.FileSystem.Statistics" @@ -53,6 +56,6 @@ public final class HdfsClientThreadLeakFilter implements ThreadFilter { @Override public boolean reject(Thread t) { - return t.getName().equals(OFFENDING_THREAD_NAME); + return t.getName().equals(OFFENDING_THREAD_NAME) || t.getName().startsWith("ForkJoinPool.commonPool-"); } } diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index 3070c654a96ee..805af9874b552 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -45,7 +45,7 @@ import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.plugins.Plugin; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java index 81a902a6992d8..1a644934245cb 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java @@ -51,8 +51,8 @@ import org.opensearch.common.blobstore.support.AbstractBlobContainer; import org.opensearch.common.blobstore.support.PlainBlobMetadata; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java index 30040e182cbc9..ed1ebf1f531f8 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java @@ -39,7 +39,7 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.BlobStoreException; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import software.amazon.awssdk.services.s3.model.ObjectCannedACL; import software.amazon.awssdk.services.s3.model.StorageClass; import org.opensearch.repositories.s3.async.AsyncExecutorContainer; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java index d42bfc0be7e4f..f98b775d9ce4b 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java @@ -46,8 +46,8 @@ import org.opensearch.common.settings.SecureSetting; import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.recovery.RecoverySettings; diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java index 5b43ae84c51dc..cb6851652f208 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java @@ -17,7 +17,7 @@ import org.opensearch.common.blobstore.exception.CorruptFileException; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.io.InputStreamContainer; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.util.ByteUtils; import org.opensearch.repositories.s3.io.CheckedContainer; import org.opensearch.repositories.s3.SocketAccess; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java index 10137f0475177..8bb446fff0b61 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java @@ -24,7 +24,7 @@ import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.s3.async.AsyncExecutorContainer; import org.opensearch.repositories.s3.async.AsyncTransferManager; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index 1a1fb123aa5ea..016be07e5d533 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -57,8 +57,8 @@ import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.common.util.io.IOUtils; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java index a2a7ca8d8bdd5..f88b3616d2f0a 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java @@ -40,7 +40,7 @@ import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.blobstore.DeleteResult; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.test.OpenSearchTestCase; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java index 84d56c7ae2854..e5fd9e5caab9c 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.repositories.RepositoryException; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java index 596291a1d94fb..2fc10c65eaa65 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java @@ -14,7 +14,7 @@ import org.opensearch.common.blobstore.exception.CorruptFileException; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.io.InputStreamContainer; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.repositories.blobstore.ZeroInputStream; import org.opensearch.test.OpenSearchTestCase; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java index 1befc110eb6a5..9dd13144bc454 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/http/nio/NioHttpServerTransport.java @@ -41,7 +41,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java index 1509e0b179bfe..0990f97124e53 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransport.java @@ -44,7 +44,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.nio.BytesChannelContext; import org.opensearch.nio.ChannelFactory; import org.opensearch.nio.Config; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java index 67598aec154fa..bd8aec534f608 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/NioTransportPlugin.java @@ -47,7 +47,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.http.HttpServerTransport; import org.opensearch.http.nio.NioHttpServerTransport; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.threadpool.ThreadPool; diff --git a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java index 29ef19a2aec87..ee96e9d85c03b 100644 --- a/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/opensearch/transport/nio/TcpReadWriteHandler.java @@ -32,7 +32,7 @@ package org.opensearch.transport.nio; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java index c606a4818a324..ca62c0dedd452 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/HttpReadWriteHandlerTests.java @@ -49,7 +49,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.http.CorsHandler; import org.opensearch.http.HttpChannel; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java index edaee15507df9..2922f28e3be18 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpClient.java @@ -51,8 +51,8 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.nio.BytesChannelContext; import org.opensearch.nio.ChannelFactory; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java index c69fe23002dfe..4d0db18d433ec 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpServerTransportTests.java @@ -54,7 +54,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; @@ -64,7 +64,7 @@ import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpTransportSettings; import org.opensearch.http.NullDispatcher; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.nio.NioSocketChannel; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; diff --git a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java index 4b06c4e15bce7..d0e779edded7d 100644 --- a/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/opensearch/transport/nio/SimpleNioTransportTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.util.net.NetUtils; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.transport.MockTransportService; import org.opensearch.test.transport.StubbableTransport; import org.opensearch.transport.AbstractSimpleTransportTestCase; diff --git a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 deleted file mode 100644 index 45d8f459573b1..0000000000000 --- a/server/licenses/lucene-analysis-common-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -27ba6caaa4587a982cd451f7217b5a982bcfc44a \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..e7c7dc2bbc046 --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +1446b7641743a1082b566179d1bf2960f5a0724b \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 deleted file mode 100644 index 3981ea4fa226e..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6389463bfbfcf902c8d31d12e9513a6818ac9d5e \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..d0f64519cd6ff --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +127032ea137d2501b24f0e35e5f9a2e1c7864633 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.7.0.jar.sha1 b/server/licenses/lucene-core-9.7.0.jar.sha1 deleted file mode 100644 index 2b0f77275c0ab..0000000000000 --- a/server/licenses/lucene-core-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad391210ffd806931334be9670a35af00c56f959 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..dc363f2776429 --- /dev/null +++ b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +30c3afcf058532d3d2b8820375043000e7f34a9b \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.7.0.jar.sha1 b/server/licenses/lucene-grouping-9.7.0.jar.sha1 deleted file mode 100644 index 90acbf6dcee8d..0000000000000 --- a/server/licenses/lucene-grouping-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e6f0c229f4861be641047c33b05067176e4279c \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..294beba43f62a --- /dev/null +++ b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +e6f742efe0ef3b383468fe38f88ab2dd69ed3d2c \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0.jar.sha1 deleted file mode 100644 index bfcca0bc6cb5b..0000000000000 --- a/server/licenses/lucene-highlighter-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -facb7c7ee0f75ed457a2d98f10d6430e25a53691 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..c2a2ef5b13946 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +3162856444777130dee2c4cabe1bf6d18710ff63 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.7.0.jar.sha1 b/server/licenses/lucene-join-9.7.0.jar.sha1 deleted file mode 100644 index 0dab3a7ddc41a..0000000000000 --- a/server/licenses/lucene-join-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d041bdc0947a14223cf68357407ee18b21027587 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..7c6adaaba9cf1 --- /dev/null +++ b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +5fe8383516eca7300f978ce38042e327b0a57877 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.7.0.jar.sha1 b/server/licenses/lucene-memory-9.7.0.jar.sha1 deleted file mode 100644 index 357a9c4b2ea26..0000000000000 --- a/server/licenses/lucene-memory-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0fade51ee353e15ddbbc45262aafe6f99ed020f1 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..586702c968a77 --- /dev/null +++ b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +b3e77970485be6d2dd59b999bbaa65a2cb993744 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.7.0.jar.sha1 b/server/licenses/lucene-misc-9.7.0.jar.sha1 deleted file mode 100644 index da5e1921626b2..0000000000000 --- a/server/licenses/lucene-misc-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fcf451e2376526c3a027958812866cc5b0ff13f \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..493598eefff5e --- /dev/null +++ b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +86d667ea2f7fb2142d2acacf801dcea47d014a5e \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.7.0.jar.sha1 b/server/licenses/lucene-queries-9.7.0.jar.sha1 deleted file mode 100644 index fa82e95a7e19f..0000000000000 --- a/server/licenses/lucene-queries-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -126989d4622419aa06fcbf3a342e859cab8c8799 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..1bf937f10d795 --- /dev/null +++ b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +930d004de698f374da8ac5530fd80e241edeba45 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0.jar.sha1 deleted file mode 100644 index 438db0aea66e1..0000000000000 --- a/server/licenses/lucene-queryparser-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e77bde908ff698354e4a2149e6dd4658b56d7b0 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..74458bc93f90b --- /dev/null +++ b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +f62882823d5aa9ed4cf0081a8c18f35e21992080 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0.jar.sha1 deleted file mode 100644 index 38b0b1cccbc29..0000000000000 --- a/server/licenses/lucene-sandbox-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9f3e8e1947f2f1c5784132444af51a060ff0b4bf \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..3231d0e067940 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +f1ec1527e283b423b7ff5e12cd8d889e7247199d \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 deleted file mode 100644 index 48679df469fd1..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01b0bc7a407d8c35a70a1adf7966bb3e7caae928 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..dd47faf91f206 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +de787c052879893e47d21fa161c93413665d55d7 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 deleted file mode 100644 index 55d4d217fa6b9..0000000000000 --- a/server/licenses/lucene-spatial3d-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7c6b1b6e0a70c9cd177371e648648c2f896742a2 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..2b378438bfb14 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +7e541ed960a571f5d9a0ecff5c26fd5ca857581e \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.7.0.jar.sha1 b/server/licenses/lucene-suggest-9.7.0.jar.sha1 deleted file mode 100644 index d4d7e6cd6bed9..0000000000000 --- a/server/licenses/lucene-suggest-9.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5c37fd9a5d71dc87fe1cd4c18ff295ec8cfac170 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 new file mode 100644 index 0000000000000..1e3ed6561e3ef --- /dev/null +++ b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 @@ -0,0 +1 @@ +4b222ef09a5f20896d031a8322f2e69304c16384 \ No newline at end of file diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java index 51598d7775623..a75448dadf427 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/ShrinkIndexIT.java @@ -65,7 +65,7 @@ import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.index.Index; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java index 7f175289f3a88..d6a7dcf4b6152 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java @@ -47,8 +47,8 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java index 850034bc631b1..c77178a5165bc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkProcessorIT.java @@ -41,8 +41,8 @@ import org.opensearch.client.Requests; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java index afa5ac908c137..24a66083d85ff 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/search/TransportSearchIT.java @@ -47,7 +47,7 @@ import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java index 61171b0a817b0..4683535a3a095 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/SimpleClusterStateIT.java @@ -50,7 +50,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index cafce1a194caa..2d2714723802f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -59,8 +59,8 @@ import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.PathUtilsForTesting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java index 79b674b23fd48..f1c34aa4f4141 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/settings/ClusterSettingsIT.java @@ -42,7 +42,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.test.OpenSearchIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java index 23335f6e82ef1..9433bebc24c20 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java @@ -46,7 +46,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.util.io.IOUtils; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.snapshots.SnapshotInfo; diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java index 9a465c2f9121c..e48e13b471ba5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java @@ -38,8 +38,8 @@ import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index ba9f335cd24d4..4b95c2fc4bce5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -58,8 +58,8 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; @@ -84,7 +84,7 @@ import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogStats; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.plugins.Plugin; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java index b6124ff09d992..df2c8c62ca392 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -65,8 +65,8 @@ import org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java index d51e4bbff11b5..89031f68aba97 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java @@ -66,8 +66,8 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.env.NodeEnvironment; import org.opensearch.core.index.Index; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java index 1dd0f6a3d664e..6cc1c51ed65d9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedTranslogIT.java @@ -38,8 +38,8 @@ import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.IndexSettings; import org.opensearch.index.MockEngineFactoryPlugin; import org.opensearch.index.translog.TestTranslog; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java index 2ab44f8318617..10bd179ddc5fd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/CircuitBreakerServiceIT.java @@ -46,17 +46,17 @@ import org.opensearch.client.Requests; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.indices.breaker.CircuitBreakerStats; -import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.indices.breaker.CircuitBreakerStats; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.rest.RestStatus; +import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.search.sort.SortOrder; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; @@ -197,7 +197,7 @@ public void testRamAccountingTermsEnum() throws Exception { prepareCreate("ramtest").setSource( "{\"mappings\": {\"type\": {\"properties\": {\"test\": " + "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}", - XContentType.JSON + MediaTypeRegistry.JSON ) ); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java index 341c0a965f94e..5ce9de7d0eae0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java @@ -44,7 +44,7 @@ import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.common.Strings; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java index c31b5e1f3bc5b..850f08b8136c1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java @@ -74,12 +74,12 @@ import org.opensearch.common.Priority; import org.opensearch.common.SetOnce; import org.opensearch.common.Strings; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index d345578c7de6e..3ab1a2a8564c5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -875,69 +875,84 @@ public void testPressureServiceStats() throws Exception { waitForDocs(initialDocCount, indexer); refresh(INDEX_NAME); + // get shard references. + final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME); + final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME); + logger.info("Replica aid {}", replicaShard.routingEntry().allocationId()); + logger.info("former primary aid {}", primaryShard.routingEntry().allocationId()); + + // fetch pressure stats from the Primary's Node. SegmentReplicationPressureService pressureService = internalCluster().getInstance( SegmentReplicationPressureService.class, primaryNode ); - final Map shardStats = pressureService.nodeStats().getShardStats(); - assertEquals(1, shardStats.size()); - final IndexShard primaryShard = getIndexShard(primaryNode, INDEX_NAME); - IndexShard replica = getIndexShard(replicaNode, INDEX_NAME); - SegmentReplicationPerGroupStats groupStats = shardStats.get(primaryShard.shardId()); - Set replicaStats = groupStats.getReplicaStats(); - assertEquals(1, replicaStats.size()); - - // assert replica node returns nothing. + // Fetch pressure stats from the Replica's Node we will assert replica node returns nothing until it is promoted. SegmentReplicationPressureService replicaNode_service = internalCluster().getInstance( SegmentReplicationPressureService.class, replicaNode ); + + final Map shardStats = pressureService.nodeStats().getShardStats(); + assertEquals("We should have stats returned for the replication group", 1, shardStats.size()); + + SegmentReplicationPerGroupStats groupStats = shardStats.get(primaryShard.shardId()); + Set replicaStats = groupStats.getReplicaStats(); + assertAllocationIdsInReplicaShardStats(Set.of(replicaShard.routingEntry().allocationId().getId()), replicaStats); + assertTrue(replicaNode_service.nodeStats().getShardStats().isEmpty()); - // drop the primary, this won't hand off SR state. + // drop the primary, this won't hand off pressure stats between old/new primary. internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode)); ensureYellowAndNoInitializingShards(INDEX_NAME); - replicaNode_service = internalCluster().getInstance(SegmentReplicationPressureService.class, replicaNode); - replica = getIndexShard(replicaNode, INDEX_NAME); - assertTrue("replica should be promoted as a primary", replica.routingEntry().primary()); - assertEquals(1, replicaNode_service.nodeStats().getShardStats().size()); - // we don't have a replica assigned yet, so this should be 0. - assertEquals(0, replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats().size()); + + assertTrue("replica should be promoted as a primary", replicaShard.routingEntry().primary()); + assertEquals( + "We should have stats returned for the replication group", + 1, + replicaNode_service.nodeStats().getShardStats().size() + ); + // after the primary is dropped and replica is promoted we won't have a replica assigned yet, so stats per replica should return + // empty. + replicaStats = replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats(); + assertTrue(replicaStats.isEmpty()); // start another replica. String replicaNode_2 = internalCluster().startDataOnlyNode(); ensureGreen(INDEX_NAME); - String docId = String.valueOf(initialDocCount + 1); - client().prepareIndex(INDEX_NAME).setId(docId).setSource("foo", "bar").get(); - refresh(INDEX_NAME); - waitForSearchableDocs(initialDocCount + 1, replicaNode_2); + final IndexShard secondReplicaShard = getIndexShard(replicaNode_2, INDEX_NAME); + final String second_replica_aid = secondReplicaShard.routingEntry().allocationId().getId(); + waitForSearchableDocs(initialDocCount, replicaNode_2); - replicaNode_service = internalCluster().getInstance(SegmentReplicationPressureService.class, replicaNode); - replica = getIndexShard(replicaNode_2, INDEX_NAME); - assertEquals(1, replicaNode_service.nodeStats().getShardStats().size()); - replicaStats = replicaNode_service.nodeStats().getShardStats().get(primaryShard.shardId()).getReplicaStats(); - assertEquals(1, replicaStats.size()); + assertEquals( + "We should have stats returned for the replication group", + 1, + replicaNode_service.nodeStats().getShardStats().size() + ); + replicaStats = replicaNode_service.nodeStats().getShardStats().get(replicaShard.shardId()).getReplicaStats(); + assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), replicaStats); + final SegmentReplicationShardStats replica_entry = replicaStats.stream().findFirst().get(); + assertEquals(replica_entry.getCheckpointsBehindCount(), 0); // test a checkpoint without any new segments flush(INDEX_NAME); assertBusy(() -> { - final SegmentReplicationPressureService service = internalCluster().getInstance( - SegmentReplicationPressureService.class, - replicaNode - ); - assertEquals(1, service.nodeStats().getShardStats().size()); - final Set shardStatsSet = service.nodeStats() + assertEquals(1, replicaNode_service.nodeStats().getShardStats().size()); + final Set shardStatsSet = replicaNode_service.nodeStats() .getShardStats() - .get(primaryShard.shardId()) + .get(replicaShard.shardId()) .getReplicaStats(); - assertEquals(1, shardStatsSet.size()); + assertAllocationIdsInReplicaShardStats(Set.of(second_replica_aid), shardStatsSet); final SegmentReplicationShardStats stats = shardStatsSet.stream().findFirst().get(); assertEquals(0, stats.getCheckpointsBehindCount()); }); } } + private void assertAllocationIdsInReplicaShardStats(Set expected, Set replicaStats) { + assertEquals(expected, replicaStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet())); + } + /** * Tests a scroll query on the replica * @throws Exception when issue is encountered @@ -1162,6 +1177,10 @@ public void testScrollWithOngoingSegmentReplication() throws Exception { } public void testPitCreatedOnReplica() throws Exception { + assumeFalse( + "Skipping the test as it is flaky with remote store. Tracking issue https://github.com/opensearch-project/OpenSearch/issues/8850", + segmentReplicationWithRemoteEnabled() + ); final String primary = internalCluster().startDataOnlyNode(); createIndex(INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java index 7426ca2f13f84..3024eeb798b48 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java @@ -22,6 +22,8 @@ import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.SegmentReplicationShardStats; +import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -29,8 +31,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -56,7 +60,7 @@ public void testPrimaryRelocation() throws Exception { createIndex(1); final String replica = internalCluster().startNode(); ensureGreen(INDEX_NAME); - final int initialDocCount = scaledRandomIntBetween(100, 1000); + final int initialDocCount = scaledRandomIntBetween(10, 100); final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); final List> pendingIndexResponses = new ArrayList<>(); for (int i = 0; i < initialDocCount; i++) { @@ -133,7 +137,7 @@ public void testPrimaryRelocationWithSegRepFailure() throws Exception { createIndex(1); final String replica = internalCluster().startNode(); ensureGreen(INDEX_NAME); - final int initialDocCount = scaledRandomIntBetween(100, 1000); + final int initialDocCount = scaledRandomIntBetween(10, 100); final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); final List> pendingIndexResponses = new ArrayList<>(); for (int i = 0; i < initialDocCount; i++) { @@ -528,13 +532,27 @@ public void testFlushAfterRelocation() throws Exception { client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); } - // Verify segment replication event never happened on replica shard + final IndexShard replicaShard = getIndexShard(replicaNode, INDEX_NAME); + + // Verify segment replication event never happened on replica shard other than recovery. + assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setPreference("_only_local").setSize(0).get(), 0); + assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setPreference("_only_local").setSize(0).get(), 0); + SegmentReplicationStatsResponse segmentReplicationStatsResponse = client().admin() .indices() .prepareSegmentReplicationStats(INDEX_NAME) .execute() .actionGet(); - assertTrue(segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME).get(0).getReplicaStats().isEmpty()); + final Set replicaStats = segmentReplicationStatsResponse.getReplicationStats() + .get(INDEX_NAME) + .get(0) + .getReplicaStats(); + assertEquals( + Set.of(replicaShard.routingEntry().allocationId().getId()), + replicaStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet()) + ); + // the primary still has not refreshed to update its checkpoint, so our replica is not yet behind. + assertEquals(0, replicaStats.stream().findFirst().get().getCheckpointsBehindCount()); // Relocate primary to new primary. When new primary starts it does perform a flush. logger.info("--> relocate the shard from primary to newPrimary"); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java index 28bd5a6ae252d..e95f10bd7abdb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseIndexIT.java @@ -46,8 +46,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.set.Sets; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java index b5d7bd476059d..2454f6553951e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java @@ -39,8 +39,8 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.FileChunkRequest; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java index 3fe7f3d553a1b..608d7d9d02581 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java @@ -12,7 +12,7 @@ import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index c5d023bdd7a64..2887fbc56106c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -26,6 +26,7 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -37,6 +38,13 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final int REPLICA_COUNT = 1; protected Path absolutePath; protected Path absolutePath2; + private final List documentKeys = List.of( + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5) + ); @Override protected boolean addMockInternalEngine() { @@ -59,7 +67,7 @@ public Settings indexSettings() { IndexResponse indexSingleDoc(String indexName) { return client().prepareIndex(indexName) .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) + .setSource(documentKeys.get(randomIntBetween(0, documentKeys.size() - 1)), randomAlphaOfLength(5)) .get(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index b4456f887cbaa..4d5648c74ba5c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -104,9 +104,17 @@ private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlus Map indexStats = indexData(numberOfIterations, invokeFlush, flushAfterMerge, deletedDocs); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); - assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + } + client().admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(INDEX_NAME); if (deletedDocs == -1) { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 42bd4b5173fa3..693c4113f8f3b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -11,6 +11,7 @@ import org.hamcrest.MatcherAssert; import org.junit.Before; import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; +import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.recovery.RecoveryResponse; import org.opensearch.action.index.IndexResponse; @@ -18,7 +19,6 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.RecoverySource; -import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.RemoteStoreRefreshListener; import org.opensearch.indices.recovery.RecoveryState; @@ -34,15 +34,16 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import static org.hamcrest.Matchers.comparesEqualTo; -import static org.hamcrest.Matchers.oneOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.comparesEqualTo; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0) public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; @@ -68,13 +69,6 @@ public Settings indexSettings() { return remoteStoreIndexSettings(0); } - private IndexResponse indexSingleDoc() { - return client().prepareIndex(INDEX_NAME) - .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) - .get(); - } - private Map indexData(int numberOfIterations, boolean invokeFlush, String index) { long totalOperations = 0; long refreshedOrFlushedOperations = 0; @@ -93,7 +87,7 @@ private Map indexData(int numberOfIterations, boolean invokeFlush, refreshedOrFlushedOperations = totalOperations; int numberOfOperations = randomIntBetween(20, 50); for (int j = 0; j < numberOfOperations; j++) { - IndexResponse response = INDEX_NAME.equals(index) ? indexSingleDoc() : indexSingleDoc(index); + IndexResponse response = indexSingleDoc(index); maxSeqNo = response.getSeqNo(); shardId = response.getShardId().id(); indexingStats.put(MAX_SEQ_NO_TOTAL + "-shard-" + shardId, maxSeqNo); @@ -109,12 +103,14 @@ private Map indexData(int numberOfIterations, boolean invokeFlush, } private void verifyRestoredData(Map indexStats, boolean checkTotal, String indexName) { + // This is required to get updated number from already active shards which were not restored + refresh(indexName); String statsGranularity = checkTotal ? TOTAL_OPERATIONS : REFRESHED_OR_FLUSHED_OPERATIONS; String maxSeqNoGranularity = checkTotal ? MAX_SEQ_NO_TOTAL : MAX_SEQ_NO_REFRESHED_OR_FLUSHED; ensureYellowAndNoInitializingShards(indexName); ensureGreen(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity)); - IndexResponse response = INDEX_NAME.equals(indexName) ? indexSingleDoc() : indexSingleDoc(indexName); + IndexResponse response = indexSingleDoc(indexName); assertEquals(indexStats.get(maxSeqNoGranularity + "-shard-" + response.getShardId().id()) + 1, response.getSeqNo()); refresh(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); @@ -130,6 +126,28 @@ private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, St } } + private void restore(String... indices) { + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices)); + } + client().admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(indices).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); + } + + private void restoreAndVerify(int shardCount, int replicaCount, Map indexStats) { + restore(INDEX_NAME); + ensureGreen(INDEX_NAME); + // This is required to get updated number from already active shards which were not restored + assertEquals(shardCount * (1 + replicaCount), getNumShards(INDEX_NAME).totalNumShards); + assertEquals(replicaCount, getNumShards(INDEX_NAME).numReplicas); + verifyRestoredData(indexStats, true, INDEX_NAME); + } + /** * Helper function to test restoring an index with no replication from remote store. Only primary node is dropped. * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. @@ -144,23 +162,16 @@ private void testRestoreFlow(int numberOfIterations, boolean invokeFlush, int sh internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); ensureRed(INDEX_NAME); - assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); - - ensureGreen(INDEX_NAME); - assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); - verifyRestoredData(indexStats, true, INDEX_NAME); + restoreAndVerify(shardCount, 0, indexStats); } /** * Helper function to test restoring an index having replicas from remote store when all the nodes housing the primary/replica drop. - * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) - throws IOException { + private void testRestoreFlowBothPrimaryReplicasDown(int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { prepareCluster(1, 2, INDEX_NAME, 1, shardCount); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); @@ -170,14 +181,7 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int ensureRed(INDEX_NAME); internalCluster().startDataOnlyNodes(2); - assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); - - ensureGreen(INDEX_NAME); - - assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); - assertEquals(0, getNumShards(INDEX_NAME).numReplicas); - verifyRestoredData(indexStats, true, INDEX_NAME); + restoreAndVerify(shardCount, 1, indexStats); } /** @@ -212,10 +216,16 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices)); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices)); + } client().admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(indices); for (String index : indices) { assertEquals(shardCount, getNumShards(index).totalNumShards); @@ -223,6 +233,37 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo } } + public void testRestoreFlowAllShardsNoRedIndex() throws InterruptedException { + int shardCount = randomIntBetween(1, 5); + prepareCluster(0, 3, INDEX_NAME, 0, shardCount); + indexData(randomIntBetween(2, 5), true, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + + PlainActionFuture future = PlainActionFuture.newFuture(); + client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(true), future); + try { + future.get(); + } catch (ExecutionException e) { + // If the request goes to co-ordinator, e.getCause() can be RemoteTransportException + assertTrue(e.getCause() instanceof IllegalStateException || e.getCause().getCause() instanceof IllegalStateException); + } + } + + public void testRestoreFlowNoRedIndex() { + int shardCount = randomIntBetween(1, 5); + prepareCluster(0, 3, INDEX_NAME, 0, shardCount); + Map indexStats = indexData(randomIntBetween(2, 5), true, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(false), PlainActionFuture.newFuture()); + + ensureGreen(INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); + verifyRestoredData(indexStats, true, INDEX_NAME); + } + /** * Simulates all data restored using Remote Translog Store. * @throws IOException IO Exception. @@ -265,7 +306,7 @@ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, 1, true, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(1, true, randomIntBetween(1, 5)); } /** @@ -274,7 +315,7 @@ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOExce */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, 1, false, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(1, false, randomIntBetween(1, 5)); } /** @@ -284,7 +325,7 @@ public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOExc */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), false, randomIntBetween(1, 5)); } /** @@ -294,7 +335,7 @@ public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOExcepti */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), true, randomIntBetween(1, 5)); } /** @@ -341,10 +382,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices)); - client().admin() - .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(new String[] {}), PlainActionFuture.newFuture()); + restore(indices); ensureGreen(indices); for (String index : indices) { @@ -381,10 +419,16 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + } client().admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indices[0], indices[1]), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(indices[0], indices[1]).restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(indices[0], indices[1]); assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); @@ -427,10 +471,16 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOExc ensureRed(indices); internalCluster().startDataOnlyNodes(3); - assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + boolean restoreAllShards = randomBoolean(); + if (restoreAllShards) { + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + } client().admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*"), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*").restoreAllShards(restoreAllShards), + PlainActionFuture.newFuture() + ); ensureGreen(indices[0], indices[1]); assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); @@ -490,7 +540,7 @@ private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throw assertEquals(0, recoverySource.get().getIndex().recoveredFileCount()); } - IndexResponse response = indexSingleDoc(); + IndexResponse response = indexSingleDoc(INDEX_NAME); assertEquals(indexStats.get(MAX_SEQ_NO_TOTAL) + 1, response.getSeqNo()); refresh(INDEX_NAME); assertBusy( diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java index 4e7f2ae486c93..d5c744a49c62d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/repositories/fs/FsBlobStoreRepositoryIT.java @@ -37,8 +37,8 @@ import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.blobstore.OpenSearchBlobStoreRepositoryIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java index c7392b260319a..aa8ef3f29c989 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchTimeoutIT.java @@ -79,7 +79,6 @@ public void testSimpleTimeout() throws Exception { .get(); assertTrue(searchResponse.isTimedOut()); assertEquals(0, searchResponse.getFailedShards()); - assertTrue(numDocs > searchResponse.getHits().getTotalHits().value); } public void testSimpleDoesNotTimeout() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java index e554a3be20528..fe62f5902cbba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java @@ -35,8 +35,8 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.WrapperQueryBuilder; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; @@ -59,7 +59,7 @@ public void testWrapperQueryIsRewritten() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); BytesReference bytesReference; - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) { builder.startObject(); { builder.startObject("terms"); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java index be69428453952..a54ca880c3dd0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java @@ -35,7 +35,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.ExceptionsHelper; import org.opensearch.action.index.IndexRequestBuilder; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.Settings; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.search.aggregations.Aggregator; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java index f67b913a75871..3651a7354e5de 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/ExplainableScriptIT.java @@ -179,8 +179,18 @@ public void testExplainScript() throws InterruptedException, IOException, Execut for (SearchHit hit : hits.getHits()) { assertThat(hit.getId(), equalTo(Integer.toString(idCounter))); assertThat(hit.getExplanation().toString(), containsString(Double.toString(idCounter))); - assertThat(hit.getExplanation().toString(), containsString("1 = n")); - assertThat(hit.getExplanation().toString(), containsString("1 = N")); + + // Since Apache Lucene 9.8, the scores are not computed because script (see please ExplainableScriptPlugin) + // says "needs_score() == false" + // 19.0 = min of: + // 19.0 = This script returned 19.0 + // 0.0 = _score: + // 0.0 = weight(text:text in 0) [PerFieldSimilarity], result of: + // 0.0 = score(freq=1.0), with freq of: + // 1.0 = freq, occurrences of term within document + // 3.4028235E38 = maxBoost + + assertThat(hit.getExplanation().toString(), containsString("1.0 = freq, occurrences of term within document")); assertThat(hit.getExplanation().getDetails().length, equalTo(2)); idCounter--; } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java index 7f306b3ee550f..bb14ed1ea5578 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java @@ -406,9 +406,7 @@ public void testBulk() throws Exception { .endObject(); client().admin().indices().prepareCreate("countries").setSettings(settings).setMapping(xContentBuilder).get(); - BulkResponse bulk = client().prepareBulk() - .add(bulkAction, 0, bulkAction.length, null, XContentType.fromMediaType(xContentBuilder.contentType())) - .get(); + BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, xContentBuilder.contentType()).get(); for (BulkItemResponse item : bulk.getItems()) { assertFalse("unable to index data", item.isFailed()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java index 0f08c537d74d8..9d0c30c5a488f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/profile/aggregation/AggregationProfilerIT.java @@ -90,7 +90,52 @@ public class AggregationProfilerIT extends OpenSearchIntegTestCase { COLLECT + "_count", POST_COLLECTION + "_count", BUILD_AGGREGATION + "_count", - REDUCE + "_count" + REDUCE + "_count", + INITIALIZE + "_start_time", + BUILD_LEAF_COLLECTOR + "_start_time", + COLLECT + "_start_time", + POST_COLLECTION + "_start_time", + BUILD_AGGREGATION + "_start_time", + REDUCE + "_start_time" + ); + + private static final Set CONCURRENT_SEARCH_BREAKDOWN_KEYS = Set.of( + INITIALIZE, + BUILD_LEAF_COLLECTOR, + COLLECT, + POST_COLLECTION, + BUILD_AGGREGATION, + REDUCE, + INITIALIZE + "_count", + BUILD_LEAF_COLLECTOR + "_count", + COLLECT + "_count", + POST_COLLECTION + "_count", + BUILD_AGGREGATION + "_count", + REDUCE + "_count", + "max_" + INITIALIZE, + "max_" + BUILD_LEAF_COLLECTOR, + "max_" + COLLECT, + "max_" + POST_COLLECTION, + "max_" + BUILD_AGGREGATION, + "max_" + REDUCE, + "min_" + INITIALIZE, + "min_" + BUILD_LEAF_COLLECTOR, + "min_" + COLLECT, + "min_" + POST_COLLECTION, + "min_" + BUILD_AGGREGATION, + "min_" + REDUCE, + "avg_" + INITIALIZE, + "avg_" + BUILD_LEAF_COLLECTOR, + "avg_" + COLLECT, + "avg_" + POST_COLLECTION, + "avg_" + BUILD_AGGREGATION, + "avg_" + REDUCE, + "max_" + BUILD_LEAF_COLLECTOR + "_count", + "max_" + COLLECT + "_count", + "min_" + BUILD_LEAF_COLLECTOR + "_count", + "min_" + COLLECT + "_count", + "avg_" + BUILD_LEAF_COLLECTOR + "_count", + "avg_" + COLLECT + "_count" ); private static final String TOTAL_BUCKETS = "total_buckets"; @@ -169,7 +214,12 @@ public void testSimpleProfile() { assertThat(histoAggResult.getTime(), greaterThan(0L)); Map breakdown = histoAggResult.getTimeBreakdown(); assertThat(breakdown, notNullValue()); - assertThat(breakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (histoAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(breakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(breakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(breakdown.get(INITIALIZE), greaterThan(0L)); assertThat(breakdown.get(COLLECT), greaterThan(0L)); assertThat(breakdown.get(BUILD_AGGREGATION).longValue(), greaterThan(0L)); @@ -212,7 +262,12 @@ public void testMultiLevelProfile() { assertThat(histoAggResult.getTime(), greaterThan(0L)); Map histoBreakdown = histoAggResult.getTimeBreakdown(); assertThat(histoBreakdown, notNullValue()); - assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (histoAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(histoBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(histoBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(histoBreakdown.get(COLLECT), greaterThan(0L)); assertThat(histoBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); @@ -230,7 +285,12 @@ public void testMultiLevelProfile() { assertThat(termsAggResult.getTime(), greaterThan(0L)); Map termsBreakdown = termsAggResult.getTimeBreakdown(); assertThat(termsBreakdown, notNullValue()); - assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (termsAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(termsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(termsBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(termsBreakdown.get(COLLECT), greaterThan(0L)); assertThat(termsBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); @@ -245,7 +305,12 @@ public void testMultiLevelProfile() { assertThat(avgAggResult.getTime(), greaterThan(0L)); Map avgBreakdown = termsAggResult.getTimeBreakdown(); assertThat(avgBreakdown, notNullValue()); - assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (avgAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); @@ -298,7 +363,12 @@ public void testMultiLevelProfileBreadthFirst() { assertThat(histoAggResult.getTime(), greaterThan(0L)); Map histoBreakdown = histoAggResult.getTimeBreakdown(); assertThat(histoBreakdown, notNullValue()); - assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (histoAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(histoBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(histoBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(histoBreakdown.get(COLLECT), greaterThan(0L)); assertThat(histoBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); @@ -316,7 +386,12 @@ public void testMultiLevelProfileBreadthFirst() { assertThat(termsAggResult.getTime(), greaterThan(0L)); Map termsBreakdown = termsAggResult.getTimeBreakdown(); assertThat(termsBreakdown, notNullValue()); - assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (termsAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(termsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(termsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(termsBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(termsBreakdown.get(COLLECT), greaterThan(0L)); assertThat(termsBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); @@ -331,7 +406,12 @@ public void testMultiLevelProfileBreadthFirst() { assertThat(avgAggResult.getTime(), greaterThan(0L)); Map avgBreakdown = avgAggResult.getTimeBreakdown(); assertThat(avgBreakdown, notNullValue()); - assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (avgAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_AGGREGATION), greaterThan(0L)); @@ -369,7 +449,12 @@ public void testDiversifiedAggProfile() { assertThat(diversifyAggResult.getTime(), greaterThan(0L)); Map diversifyBreakdown = diversifyAggResult.getTimeBreakdown(); assertThat(diversifyBreakdown, notNullValue()); - assertThat(diversifyBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (diversifyAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(diversifyBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(diversifyBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(diversifyBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(diversifyBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(diversifyBreakdown.get(COLLECT), greaterThan(0L)); @@ -386,7 +471,12 @@ public void testDiversifiedAggProfile() { assertThat(maxAggResult.getTime(), greaterThan(0L)); Map maxBreakdown = maxAggResult.getTimeBreakdown(); assertThat(maxBreakdown, notNullValue()); - assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (maxAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(diversifyBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(diversifyBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(diversifyBreakdown.get(COLLECT), greaterThan(0L)); @@ -439,7 +529,12 @@ public void testComplexProfile() { assertThat(histoAggResult.getTime(), greaterThan(0L)); Map histoBreakdown = histoAggResult.getTimeBreakdown(); assertThat(histoBreakdown, notNullValue()); - assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (histoAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(histoBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(histoBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(histoBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(histoBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(histoBreakdown.get(COLLECT), greaterThan(0L)); @@ -462,7 +557,12 @@ public void testComplexProfile() { assertThat(tagsAggResult.getTime(), greaterThan(0L)); Map tagsBreakdown = tagsAggResult.getTimeBreakdown(); assertThat(tagsBreakdown, notNullValue()); - assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (tagsAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(tagsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(tagsBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(tagsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(tagsBreakdown.get(COLLECT), greaterThan(0L)); @@ -482,7 +582,12 @@ public void testComplexProfile() { assertThat(avgAggResult.getTime(), greaterThan(0L)); Map avgBreakdown = avgAggResult.getTimeBreakdown(); assertThat(avgBreakdown, notNullValue()); - assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (avgAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); @@ -498,7 +603,12 @@ public void testComplexProfile() { assertThat(maxAggResult.getTime(), greaterThan(0L)); Map maxBreakdown = maxAggResult.getTimeBreakdown(); assertThat(maxBreakdown, notNullValue()); - assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (maxAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(maxBreakdown.get(COLLECT), greaterThan(0L)); @@ -514,7 +624,12 @@ public void testComplexProfile() { assertThat(stringsAggResult.getTime(), greaterThan(0L)); Map stringsBreakdown = stringsAggResult.getTimeBreakdown(); assertThat(stringsBreakdown, notNullValue()); - assertThat(stringsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (stringsAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(stringsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(stringsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(stringsBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(stringsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(stringsBreakdown.get(COLLECT), greaterThan(0L)); @@ -534,7 +649,12 @@ public void testComplexProfile() { assertThat(avgAggResult.getTime(), greaterThan(0L)); avgBreakdown = avgAggResult.getTimeBreakdown(); assertThat(avgBreakdown, notNullValue()); - assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (avgAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); @@ -550,7 +670,12 @@ public void testComplexProfile() { assertThat(maxAggResult.getTime(), greaterThan(0L)); maxBreakdown = maxAggResult.getTimeBreakdown(); assertThat(maxBreakdown, notNullValue()); - assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (maxAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(maxBreakdown.get(COLLECT), greaterThan(0L)); @@ -567,7 +692,12 @@ public void testComplexProfile() { assertThat(tagsAggResult.getTime(), greaterThan(0L)); tagsBreakdown = tagsAggResult.getTimeBreakdown(); assertThat(tagsBreakdown, notNullValue()); - assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (tagsAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(tagsBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(tagsBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(tagsBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(tagsBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(tagsBreakdown.get(COLLECT), greaterThan(0L)); @@ -587,7 +717,12 @@ public void testComplexProfile() { assertThat(avgAggResult.getTime(), greaterThan(0L)); avgBreakdown = avgAggResult.getTimeBreakdown(); assertThat(avgBreakdown, notNullValue()); - assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (avgAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(avgBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(avgBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(avgBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(avgBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(avgBreakdown.get(COLLECT), greaterThan(0L)); @@ -603,7 +738,12 @@ public void testComplexProfile() { assertThat(maxAggResult.getTime(), greaterThan(0L)); maxBreakdown = maxAggResult.getTimeBreakdown(); assertThat(maxBreakdown, notNullValue()); - assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + if (maxAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertThat(maxBreakdown.keySet(), equalTo(CONCURRENT_SEARCH_BREAKDOWN_KEYS)); + } else { + assertThat(maxBreakdown.keySet(), equalTo(BREAKDOWN_KEYS)); + } assertThat(maxBreakdown.get(INITIALIZE), greaterThan(0L)); assertThat(maxBreakdown.get(BUILD_LEAF_COLLECTOR), greaterThan(0L)); assertThat(maxBreakdown.get(COLLECT), greaterThan(0L)); @@ -700,7 +840,12 @@ public void testGlobalAggWithStatsSubAggregatorProfile() { assertThat(globalAggResult.getTime(), greaterThan(0L)); Map breakdown = globalAggResult.getTimeBreakdown(); assertThat(breakdown, notNullValue()); - assertEquals(BREAKDOWN_KEYS, breakdown.keySet()); + if (globalAggResult.getMaxSliceTime() != null) { + // concurrent segment search enabled + assertEquals(CONCURRENT_SEARCH_BREAKDOWN_KEYS, breakdown.keySet()); + } else { + assertEquals(BREAKDOWN_KEYS, breakdown.keySet()); + } assertThat(breakdown.get(INITIALIZE), greaterThan(0L)); assertThat(breakdown.get(COLLECT), greaterThan(0L)); assertThat(breakdown.get(BUILD_AGGREGATION).longValue(), greaterThan(0L)); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java index 483d698f3c9a4..861279c4000d8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.util.io.IOUtils; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java index f6df7cccf96f7..bbca3bdc417c7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.core.util.FileSystemUtils; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryException; import org.opensearch.repositories.RepositoryVerificationException; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index dbd96a7fd109f..8677e61efeb46 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -52,7 +52,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; @@ -276,7 +276,10 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); client.admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), + PlainActionFuture.newFuture() + ); ensureYellowAndNoInitializingShards(restoredIndexName1); ensureGreen(restoredIndexName1); assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1); @@ -434,7 +437,9 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException { // Re-initialize client to make sure we are not using client from stopped node. client = client(clusterManagerNode); assertAcked(client.admin().indices().prepareClose(indexName1)); - client.admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture()); + client.admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1).restoreAllShards(true), PlainActionFuture.newFuture()); ensureYellowAndNoInitializingShards(indexName1); ensureGreen(indexName1); assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); @@ -515,7 +520,10 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException assertAcked(client.admin().indices().prepareClose(restoredIndexName1)); client.admin() .cluster() - .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture()); + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true), + PlainActionFuture.newFuture() + ); ensureYellowAndNoInitializingShards(restoredIndexName1); ensureGreen(restoredIndexName1); // indexing some new docs and validating diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java index 300e1db09b4c5..efd072fce511d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java @@ -30,7 +30,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexNotFoundException; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java index 4bba25039d376..420df6b4c34c8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -66,7 +66,7 @@ import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.util.BytesRefUtils; import org.opensearch.core.index.Index; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java index c22dd90cc930b..e47a2b94fc715 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java @@ -46,7 +46,7 @@ import org.opensearch.client.Client; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index 3a36a6ff103e0..9d276cfe283eb 100644 --- a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -42,6 +42,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.TermQuery; @@ -93,11 +94,12 @@ public BlendedTermQuery(Term[] terms, float[] boosts) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } + IndexReader reader = searcher.getIndexReader(); IndexReaderContext context = reader.getContext(); TermStates[] ctx = new TermStates[terms.length]; int[] docFreqs = new int[ctx.length]; diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index d53164427debf..b8c6d1e78d25a 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -288,14 +288,6 @@ public static void registerExceptions() { UNKNOWN_VERSION_ADDED ) ); - registerExceptionHandle( - new OpenSearchExceptionHandle( - org.opensearch.OpenSearchParseException.class, - org.opensearch.OpenSearchParseException::new, - 35, - UNKNOWN_VERSION_ADDED - ) - ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.search.SearchException.class, @@ -915,14 +907,6 @@ public static void registerExceptions() { UNKNOWN_VERSION_ADDED ) ); - registerExceptionHandle( - new OpenSearchExceptionHandle( - org.opensearch.common.breaker.CircuitBreakingException.class, - org.opensearch.common.breaker.CircuitBreakingException::new, - 133, - UNKNOWN_VERSION_ADDED - ) - ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.transport.NodeNotConnectedException.class, diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 2ce1d4bcd4b02..b775095861150 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -301,7 +301,7 @@ import org.opensearch.index.seqno.RetentionLeaseActions; import org.opensearch.identity.IdentityService; import org.opensearch.indices.SystemIndices; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.persistent.CompletionPersistentTaskAction; import org.opensearch.persistent.RemovePersistentTaskAction; import org.opensearch.persistent.StartPersistentTaskAction; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java index acf40e3a9de3c..1d2dbf66920f8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/info/NodeInfo.java @@ -40,7 +40,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.http.HttpInfo; import org.opensearch.ingest.IngestInfo; import org.opensearch.monitor.jvm.JvmInfo; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index 3b6c85ee6e091..82e03d1fd79ac 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -49,7 +49,7 @@ import org.opensearch.index.stats.ShardIndexingPressureStats; import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.indices.NodeIndicesStats; -import org.opensearch.indices.breaker.AllCircuitBreakerStats; +import org.opensearch.core.indices.breaker.AllCircuitBreakerStats; import org.opensearch.ingest.IngestStats; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.monitor.jvm.JvmStats; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java index 703b9575a88ad..eb1935158c231 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java @@ -35,7 +35,8 @@ public class RestoreRemoteStoreRequest extends ClusterManagerNodeRequest implements ToXContentObject { private String[] indices = Strings.EMPTY_ARRAY; - private Boolean waitForCompletion; + private Boolean waitForCompletion = false; + private Boolean restoreAllShards = false; public RestoreRemoteStoreRequest() {} @@ -43,6 +44,7 @@ public RestoreRemoteStoreRequest(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); waitForCompletion = in.readOptionalBoolean(); + restoreAllShards = in.readOptionalBoolean(); } @Override @@ -50,6 +52,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(indices); out.writeOptionalBoolean(waitForCompletion); + out.writeOptionalBoolean(restoreAllShards); } @Override @@ -118,6 +121,27 @@ public boolean waitForCompletion() { return waitForCompletion; } + /** + * Set the value for restoreAllShards, denoting whether to restore all shards or only unassigned shards + * + * @param restoreAllShards If true, the operation will restore all the shards of the given indices. + * If false, the operation will restore only the unassigned shards of the given indices. + * @return this request + */ + public RestoreRemoteStoreRequest restoreAllShards(boolean restoreAllShards) { + this.restoreAllShards = restoreAllShards; + return this; + } + + /** + * Returns restoreAllShards setting + * + * @return true if the operation will restore all the shards of the given indices + */ + public boolean restoreAllShards() { + return restoreAllShards; + } + /** * Parses restore definition * @@ -167,12 +191,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RestoreRemoteStoreRequest that = (RestoreRemoteStoreRequest) o; - return waitForCompletion == that.waitForCompletion && Arrays.equals(indices, that.indices); + return waitForCompletion == that.waitForCompletion + && restoreAllShards == that.restoreAllShards + && Arrays.equals(indices, that.indices); } @Override public int hashCode() { - int result = Objects.hash(waitForCompletion); + int result = Objects.hash(waitForCompletion, restoreAllShards); result = 31 * result + Arrays.hashCode(indices); return result; } @@ -181,4 +207,5 @@ public int hashCode() { public String toString() { return org.opensearch.common.Strings.toString(XContentType.JSON, this); } + } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 5da3f2eb01260..3bb552b80084d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -387,7 +388,7 @@ public CreateSnapshotRequest settings(String source, MediaType mediaType) { */ public CreateSnapshotRequest settings(Map source) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(source); settings(Strings.toString(builder), builder.contentType()); } catch (IOException e) { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 840564a4bd7a2..7a142e70305ae 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -517,7 +517,7 @@ public String snapshotUuid() { /** * Sets the storage type for this request. */ - RestoreSnapshotRequest storageType(StorageType storageType) { + public RestoreSnapshotRequest storageType(StorageType storageType) { this.storageType = storageType; return this; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 282585a43183a..8b718aeaf70c7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -36,7 +36,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java index 699884ca0eab3..bddb3fb746eb1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -42,7 +42,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentFragment; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index ecfcd6310dd0a..761373a001ffe 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -32,12 +32,14 @@ package org.opensearch.action.admin.cluster.storedscripts; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; @@ -59,14 +61,18 @@ public class PutStoredScriptRequest extends AcknowledgedRequest filter) { return this; } try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(filter); this.filter = Strings.toString(builder); return this; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java index 142cbe6a0ab0b..6ed9648502ee3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -48,6 +48,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.ToXContent; @@ -428,7 +429,7 @@ public AliasActions filter(Map filter) { return this; } try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(filter); this.filter = org.opensearch.common.Strings.toString(builder); return this; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java index a716959614065..27f20f028ea74 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -39,6 +39,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; @@ -200,8 +201,8 @@ public CreateIndexRequestBuilder addAlias(Alias alias) { /** * Sets the settings and mappings as a single source. */ - public CreateIndexRequestBuilder setSource(String source, XContentType xContentType) { - request.source(source, xContentType); + public CreateIndexRequestBuilder setSource(String source, MediaType mediaType) { + request.source(source, mediaType); return this; } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java index 0b2375850f1fc..2b536d24c946f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java @@ -55,7 +55,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexService; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java index 465a44556c081..2588dfd718e71 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -49,6 +49,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.index.Index; @@ -298,7 +299,7 @@ public PutMappingRequest source(XContentBuilder mappingBuilder) { */ public PutMappingRequest source(Map mappingSource) { try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.map(mappingSource); return source(BytesReference.bytes(builder), builder.contentType()); } catch (IOException e) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java index 280dc307447b7..e014d6d703500 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/Condition.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.core.common.io.stream.NamedWriteable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java index f0ab571ea9f75..faa3558420a5c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/MaxSizeCondition.java @@ -34,8 +34,8 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java index 95a4b6573611d..11a7555d15d3f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java @@ -40,7 +40,7 @@ import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java index ed598c14acec3..1603f95f4d512 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequestBuilder.java @@ -36,7 +36,7 @@ import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; /** diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java index 4ddff1563885a..ca2921ceb70c8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -55,7 +55,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.inject.Inject; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.shard.DocsStats; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java index b2e7ed92e608a..aef2dc8f2c7c8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java index eb05c0a69b78b..855e678c77b9b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java @@ -37,7 +37,7 @@ import org.opensearch.action.support.master.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; /** * Transport request builder for resizing an index diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java index 0d31c90a98f56..328768bc9ae0e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java @@ -57,7 +57,7 @@ import org.opensearch.core.index.shard.ShardId; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.store.StoreStats; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java index 5a3a34e9a2ebe..e4abaef4ddfa8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java @@ -37,7 +37,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java index 7fe663a347ee3..28215475416ba 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/get/UpgradeStatusResponse.java @@ -36,7 +36,7 @@ import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java index 4695b44c4986b..a01fc82fffd01 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkProcessor.java @@ -41,8 +41,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.threadpool.Scheduler; diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java index 07d29413667cc..a3294e160659b 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java @@ -50,8 +50,8 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.search.fetch.subphase.FetchSourceContext; import java.io.IOException; @@ -236,30 +236,30 @@ public long estimatedSizeInBytes() { /** * Adds a framed data in binary format */ - public BulkRequest add(byte[] data, int from, int length, XContentType xContentType) throws IOException { - return add(data, from, length, null, xContentType); + public BulkRequest add(byte[] data, int from, int length, MediaType mediaType) throws IOException { + return add(data, from, length, null, mediaType); } /** * Adds a framed data in binary format */ - public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, XContentType xContentType) throws IOException { - return add(new BytesArray(data, from, length), defaultIndex, xContentType); + public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, MediaType mediaType) throws IOException { + return add(new BytesArray(data, from, length), defaultIndex, mediaType); } /** * Adds a framed data in binary format */ - public BulkRequest add(BytesReference data, @Nullable String defaultIndex, XContentType xContentType) throws IOException { - return add(data, defaultIndex, null, null, null, null, true, xContentType); + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, MediaType mediaType) throws IOException { + return add(data, defaultIndex, null, null, null, null, true, mediaType); } /** * Adds a framed data in binary format */ - public BulkRequest add(BytesReference data, @Nullable String defaultIndex, boolean allowExplicitIndex, XContentType xContentType) + public BulkRequest add(BytesReference data, @Nullable String defaultIndex, boolean allowExplicitIndex, MediaType mediaType) throws IOException { - return add(data, defaultIndex, null, null, null, null, allowExplicitIndex, xContentType); + return add(data, defaultIndex, null, null, null, null, allowExplicitIndex, mediaType); } public BulkRequest add( @@ -269,9 +269,9 @@ public BulkRequest add( @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String defaultPipeline, boolean allowExplicitIndex, - XContentType xContentType + MediaType mediaType ) throws IOException { - return add(data, defaultIndex, defaultRouting, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, xContentType); + return add(data, defaultIndex, defaultRouting, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, mediaType); } public BulkRequest add( @@ -282,7 +282,7 @@ public BulkRequest add( @Nullable String defaultPipeline, @Nullable Boolean defaultRequireAlias, boolean allowExplicitIndex, - XContentType xContentType + MediaType mediaType ) throws IOException { String routing = valueOrDefault(defaultRouting, globalRouting); String pipeline = valueOrDefault(defaultPipeline, globalPipeline); @@ -295,7 +295,7 @@ public BulkRequest add( pipeline, requireAlias, allowExplicitIndex, - xContentType, + mediaType, this::internalAdd, this::internalAdd, this::add diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java index d2e0d558fcc1b..08eee82a53cf9 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequestBuilder.java @@ -46,6 +46,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; /** * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes @@ -124,9 +125,8 @@ public BulkRequestBuilder add(byte[] data, int from, int length, XContentType xC /** * Adds a framed data in binary format */ - public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, XContentType xContentType) - throws Exception { - request.add(data, from, length, defaultIndex, xContentType); + public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, MediaType mediaType) throws Exception { + request.add(data, from, length, defaultIndex, mediaType); return this; } diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java index 569aa8f9e4f34..3a86ccd578991 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentParser; @@ -101,10 +102,10 @@ private static BytesReference sliceTrimmingCarriageReturn( BytesReference bytesReference, int from, int nextMarker, - XContentType xContentType + MediaType mediaType ) { final int length; - if (XContentType.JSON == xContentType && bytesReference.get(nextMarker - 1) == (byte) '\r') { + if (XContentType.JSON == mediaType && bytesReference.get(nextMarker - 1) == (byte) '\r') { length = nextMarker - from - 1; } else { length = nextMarker - from; @@ -125,12 +126,12 @@ public void parse( @Nullable String defaultPipeline, @Nullable Boolean defaultRequireAlias, boolean allowExplicitIndex, - XContentType xContentType, + MediaType mediaType, Consumer indexRequestConsumer, Consumer updateRequestConsumer, Consumer deleteRequestConsumer ) throws IOException { - XContent xContent = xContentType.xContent(); + XContent xContent = mediaType.xContent(); int line = 0; int from = 0; byte marker = xContent.streamSeparator(); @@ -312,7 +313,7 @@ public void parse( .setPipeline(pipeline) .setIfSeqNo(ifSeqNo) .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType) .setRequireAlias(requireAlias) ); } else { @@ -325,7 +326,7 @@ public void parse( .setPipeline(pipeline) .setIfSeqNo(ifSeqNo) .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType) .setRequireAlias(requireAlias) ); } @@ -339,7 +340,7 @@ public void parse( .setPipeline(pipeline) .setIfSeqNo(ifSeqNo) .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), mediaType) .setRequireAlias(requireAlias) ); } else if ("update".equals(action)) { @@ -358,7 +359,7 @@ public void parse( .routing(routing); try ( XContentParser sliceParser = createParser( - sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), + sliceTrimmingCarriageReturn(data, from, nextMarker, mediaType), xContent ) ) { diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java index 0b675fd9125fa..140c1320daa91 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java @@ -78,8 +78,8 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.lease.Releasable; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationPressureService; @@ -752,7 +752,7 @@ static BulkItemResponse processUpdateResponse( if (updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) { final BytesReference indexSourceAsBytes = updateIndexRequest.source(); - final Tuple> sourceAndContent = XContentHelper.convertToMap( + final Tuple> sourceAndContent = XContentHelper.convertToMap( indexSourceAsBytes, true, updateIndexRequest.getContentType() diff --git a/server/src/main/java/org/opensearch/action/index/IndexRequest.java b/server/src/main/java/org/opensearch/action/index/IndexRequest.java index 86ba13dedfa06..ac4c8436aab5a 100644 --- a/server/src/main/java/org/opensearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/opensearch/action/index/IndexRequest.java @@ -53,12 +53,12 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.uid.Versions; -import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; @@ -78,10 +78,10 @@ * created using {@link org.opensearch.client.Requests#indexRequest(String)}. * * The index requires the {@link #index()}, {@link #id(String)} and - * {@link #source(byte[], XContentType)} to be set. + * {@link #source(byte[], MediaType)} to be set. * - * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}), - * its string form ({@link #source(String, XContentType)}) or using a {@link XContentBuilder} + * The source (content to index) can be set in its bytes form using ({@link #source(byte[], MediaType)}), + * its string form ({@link #source(String, MediaType)}) or using a {@link XContentBuilder} * ({@link #source(XContentBuilder)}). * * If the {@link #id(String)} is not set, it will be automatically generated. @@ -116,7 +116,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; - private XContentType contentType; + private MediaType contentType; private String pipeline; private String finalPipeline; @@ -159,7 +159,11 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); if (in.readBoolean()) { - contentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + contentType = in.readMediaType(); + } else { + contentType = in.readEnum(XContentType.class); + } } else { contentType = null; } @@ -174,7 +178,7 @@ public IndexRequest() { /** * Constructs a new index request against the specific index. The - * {@link #source(byte[], XContentType)} must be set. + * {@link #source(byte[], MediaType)} must be set. */ public IndexRequest(String index) { super(NO_SHARD_ID); @@ -249,7 +253,7 @@ public IndicesOptions indicesOptions() { * The content type. This will be used when generating a document from user provided objects like Maps and when parsing the * source at index time */ - public XContentType getContentType() { + public MediaType getContentType() { return contentType; } @@ -372,9 +376,9 @@ public IndexRequest source(Map source) throws OpenSearchGenerationExc * * @param source The map to index */ - public IndexRequest source(Map source, XContentType contentType) throws OpenSearchGenerationException { + public IndexRequest source(Map source, MediaType contentType) throws OpenSearchGenerationException { try { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType); builder.map(source); return source(builder); } catch (IOException e) { @@ -386,10 +390,10 @@ public IndexRequest source(Map source, XContentType contentType) thro * Sets the document source to index. * * Note, its preferable to either set it using {@link #source(XContentBuilder)} - * or using the {@link #source(byte[], XContentType)}. + * or using the {@link #source(byte[], MediaType)}. */ - public IndexRequest source(String source, XContentType xContentType) { - return source(new BytesArray(source), xContentType); + public IndexRequest source(String source, MediaType mediaType) { + return source(new BytesArray(source), mediaType); } /** @@ -419,7 +423,7 @@ public IndexRequest source(Object... source) { * valid String representation. *

*/ - public IndexRequest source(XContentType xContentType, Object... source) { + public IndexRequest source(MediaType mediaType, Object... source) { if (source.length % 2 != 0) { throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]"); } @@ -430,7 +434,7 @@ public IndexRequest source(XContentType xContentType, Object... source) { ); } try { - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(mediaType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); @@ -447,27 +451,28 @@ public IndexRequest source(XContentType xContentType, Object... source) { */ public IndexRequest source(BytesReference source, MediaType mediaType) { this.source = Objects.requireNonNull(source); - this.contentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType)); + this.contentType = Objects.requireNonNull(mediaType); return this; } /** * Sets the document to index in bytes form. */ - public IndexRequest source(byte[] source, XContentType xContentType) { - return source(source, 0, source.length, xContentType); + public IndexRequest source(byte[] source, MediaType mediaType) { + return source(source, 0, source.length, mediaType); } /** * Sets the document to index in bytes form (assumed to be safe to be used from different * threads). * - * @param source The source to index - * @param offset The offset in the byte array - * @param length The length of the data + * @param source The source to index + * @param offset The offset in the byte array + * @param length The length of the data + * @param mediaType The data format over the wire */ - public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) { - return source(new BytesArray(source, offset, length), xContentType); + public IndexRequest source(byte[] source, int offset, int length, MediaType mediaType) { + return source(new BytesArray(source, offset, length), mediaType); } /** @@ -665,7 +670,11 @@ private void writeBody(StreamOutput out) throws IOException { out.writeLong(autoGeneratedTimestamp); if (contentType != null) { out.writeBoolean(true); - out.writeEnum(contentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + contentType.writeTo(out); + } else { + out.writeEnum((XContentType) contentType); + } } else { out.writeBoolean(false); } diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java index a3067fe0ba160..7a88f817c70bf 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java @@ -32,12 +32,13 @@ package org.opensearch.action.ingest; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -54,19 +55,7 @@ public class PutPipelineRequest extends AcknowledgedRequest private String id; private BytesReference source; - private XContentType xContentType; - - /** - * Create a new pipeline request with the id and source along with the content type of the source - * - * @deprecated use {@link #PutPipelineRequest(String, BytesReference, MediaType)} instead - */ - @Deprecated - public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) { - this.id = Objects.requireNonNull(id); - this.source = Objects.requireNonNull(source); - this.xContentType = Objects.requireNonNull(xContentType); - } + private MediaType mediaType; /** * Create a new pipeline request with the id and source along with the content type of the source @@ -74,17 +63,18 @@ public PutPipelineRequest(String id, BytesReference source, XContentType xConten public PutPipelineRequest(String id, BytesReference source, MediaType mediaType) { this.id = Objects.requireNonNull(id); this.source = Objects.requireNonNull(source); - if (mediaType instanceof XContentType == false) { - throw new IllegalArgumentException("PutPipelineRequest found unsupported media type [" + mediaType.getClass().getName() + "]"); - } - this.xContentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public PutPipelineRequest(StreamInput in) throws IOException { super(in); id = in.readString(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } PutPipelineRequest() {} @@ -102,8 +92,8 @@ public BytesReference getSource() { return source; } - public XContentType getXContentType() { - return xContentType; + public MediaType getMediaType() { + return mediaType; } @Override @@ -111,13 +101,17 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (source != null) { - builder.rawValue(source.streamInput(), xContentType); + builder.rawValue(source.streamInput(), mediaType); } else { builder.startObject().endObject(); } diff --git a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java index 31f71e5720daa..1ac441a1afe64 100644 --- a/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/SimulatePipelineRequest.java @@ -32,15 +32,17 @@ package org.opensearch.action.ingest; +import org.opensearch.Version; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.ingest.ConfigurationUtils; import org.opensearch.ingest.IngestDocument; @@ -66,14 +68,14 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent private String id; private boolean verbose; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; /** * Creates a new request with the given source and its content type */ - public SimulatePipelineRequest(BytesReference source, XContentType xContentType) { + public SimulatePipelineRequest(BytesReference source, MediaType mediaType) { this.source = Objects.requireNonNull(source); - this.xContentType = Objects.requireNonNull(xContentType); + this.mediaType = Objects.requireNonNull(mediaType); } SimulatePipelineRequest() {} @@ -83,7 +85,11 @@ public SimulatePipelineRequest(BytesReference source, XContentType xContentType) id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } @Override @@ -111,8 +117,8 @@ public BytesReference getSource() { return source; } - public XContentType getXContentType() { - return xContentType; + public MediaType getXContentType() { + return mediaType; } @Override @@ -121,12 +127,16 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBoolean(verbose); out.writeBytesReference(source); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.rawValue(source.streamInput(), xContentType); + builder.rawValue(source.streamInput(), mediaType); return builder; } diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java index 822d1f2320c9b..d32aab0c8a561 100644 --- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java @@ -8,6 +8,7 @@ package org.opensearch.action.search; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.core.common.bytes.BytesReference; @@ -29,7 +30,7 @@ public class PutSearchPipelineRequest extends AcknowledgedRequest implements ToXContentObject { private String id; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; public PutSearchPipelineRequest(String id, BytesReference source, MediaType mediaType) { this.id = Objects.requireNonNull(id); @@ -39,14 +40,18 @@ public PutSearchPipelineRequest(String id, BytesReference source, MediaType medi PutSearchPipelineRequest.class.getSimpleName() + " found unsupported media type [" + mediaType.getClass().getName() + "]" ); } - this.xContentType = XContentType.fromMediaType(Objects.requireNonNull(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public PutSearchPipelineRequest(StreamInput in) throws IOException { super(in); id = in.readString(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } @Override @@ -62,8 +67,8 @@ public BytesReference getSource() { return source; } - public XContentType getXContentType() { - return xContentType; + public MediaType getMediaType() { + return mediaType; } @Override @@ -71,13 +76,17 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (source != null) { - builder.rawValue(source.streamInput(), xContentType); + builder.rawValue(source.streamInput(), mediaType); } else { builder.startObject().endObject(); } diff --git a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java index 45c2dc4f29403..43c95133f12d6 100644 --- a/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/opensearch/action/search/QueryPhaseResultConsumer.java @@ -35,8 +35,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.TopDocs; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.concurrent.AbstractRunnable; diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index 512d3295c4cfc..2f002d21d9b68 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -45,7 +45,7 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.TotalHits.Relation; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.search.DocValueFormat; diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index c4af9ffa20194..1011f17c98dd6 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -57,7 +57,7 @@ import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.inject.Inject; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.Writeable; @@ -70,7 +70,7 @@ import org.opensearch.core.index.Index; import org.opensearch.index.query.Rewriteable; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.SearchPhaseResult; import org.opensearch.search.SearchService; import org.opensearch.search.SearchShardTarget; diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java index 1affc9202c32b..d3a617853b019 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java +++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationOperation.java @@ -47,7 +47,7 @@ import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Nullable; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; diff --git a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java index d33882fc1d53d..57cc4698cefce 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TermVectorsRequest.java @@ -40,6 +40,7 @@ import org.opensearch.action.get.MultiGetRequest; import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.common.Nullable; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.ParseField; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; @@ -48,10 +49,9 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.util.set.Sets; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.mapper.MapperService; @@ -94,7 +94,7 @@ public class TermVectorsRequest extends SingleShardRequest i private BytesReference doc; - private XContentType xContentType; + private MediaType mediaType; private String routing; @@ -186,7 +186,11 @@ public TermVectorsRequest() {} if (in.readBoolean()) { doc = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } routing = in.readOptionalString(); preference = in.readOptionalString(); @@ -235,7 +239,7 @@ public TermVectorsRequest(TermVectorsRequest other) { this.id = other.id(); if (other.doc != null) { this.doc = new BytesArray(other.doc().toBytesRef(), true); - this.xContentType = other.xContentType; + this.mediaType = other.mediaType; } this.flagsEnum = other.getFlags().clone(); this.preference = other.preference(); @@ -285,8 +289,8 @@ public BytesReference doc() { return doc; } - public XContentType xContentType() { - return xContentType; + public MediaType xContentType() { + return mediaType; } /** @@ -302,19 +306,19 @@ public TermVectorsRequest doc(XContentBuilder documentBuilder) { */ @Deprecated public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) { - return this.doc(doc, generateRandomId, XContentHelper.xContentType(doc)); + return this.doc(doc, generateRandomId, MediaTypeRegistry.xContentType(doc)); } /** * Sets an artificial document from which term vectors are requested for. */ - public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, MediaType xContentType) { + public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, MediaType mediaType) { // assign a random id to this artificial document, for routing if (generateRandomId) { this.id(String.valueOf(randomInt.getAndAdd(1))); } this.doc = doc; - this.xContentType = XContentType.fromMediaType(xContentType); + this.mediaType = mediaType; return this; } @@ -534,7 +538,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeBytesReference(doc); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } out.writeOptionalString(routing); out.writeOptionalString(preference); diff --git a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java index d20d5e229a4c7..95433bc42c2d3 100644 --- a/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java @@ -61,7 +61,7 @@ import org.opensearch.core.common.io.stream.NotSerializableExceptionWrapper; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.engine.VersionConflictEngineException; @@ -238,7 +238,7 @@ protected void shardOperation(final UpdateRequest request, final ActionListener< response.getResult() ); if (request.fetchSource() != null && request.fetchSource().fetchSource()) { - Tuple> sourceAndContent = XContentHelper.convertToMap( + Tuple> sourceAndContent = XContentHelper.convertToMap( upsertSourceBytes, true, upsertRequest.getContentType() diff --git a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java index e3aa4f1161ac7..a5568a838f21f 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateHelper.java @@ -40,10 +40,11 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Requests; import org.opensearch.common.Nullable; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; @@ -352,7 +353,7 @@ public static GetResult extractGetResult( long primaryTerm, long version, final Map source, - XContentType sourceContentType, + MediaType sourceContentType, @Nullable final BytesReference sourceAsBytes ) { if (request.fetchSource() == null || request.fetchSource().fetchSource() == false) { diff --git a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java index 8f529ec855d9b..86ebc0d9b69d6 100644 --- a/server/src/main/java/org/opensearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/opensearch/action/update/UpdateRequest.java @@ -48,11 +48,12 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -106,12 +107,12 @@ public class UpdateRequest extends InstanceShardOperationRequest ); PARSER.declareBoolean(UpdateRequest::scriptedUpsert, SCRIPTED_UPSERT_FIELD); PARSER.declareObject((request, builder) -> request.safeUpsertRequest().source(builder), (parser, context) -> { - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType()); builder.copyCurrentStructure(parser); return builder; }, UPSERT_FIELD); PARSER.declareObject((request, builder) -> request.safeDoc().source(builder), (parser, context) -> { - XContentBuilder docBuilder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder docBuilder = MediaTypeRegistry.contentBuilder(parser.contentType()); docBuilder.copyCurrentStructure(parser); return docBuilder; }, DOC_FIELD); @@ -922,13 +923,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("doc_as_upsert", docAsUpsert); } if (doc != null) { - XContentType xContentType = doc.getContentType(); + MediaType mediaType = doc.getContentType(); try ( XContentParser parser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, doc.source(), - xContentType + mediaType ) ) { builder.field("doc"); @@ -945,13 +946,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("script", script); } if (upsertRequest != null) { - XContentType xContentType = upsertRequest.getContentType(); + MediaType mediaType = upsertRequest.getContentType(); try ( XContentParser parser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, upsertRequest.source(), - xContentType + mediaType ) ) { builder.field("upsert"); diff --git a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java index 3793b5094a4cb..7b1a2f0f12b69 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterInfo.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; diff --git a/server/src/main/java/org/opensearch/cluster/DiskUsage.java b/server/src/main/java/org/opensearch/cluster/DiskUsage.java index 961bfce053243..c472522baee51 100644 --- a/server/src/main/java/org/opensearch/cluster/DiskUsage.java +++ b/server/src/main/java/org/opensearch/cluster/DiskUsage.java @@ -32,11 +32,11 @@ package org.opensearch.cluster; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java index 0014d5c61fb2d..b5a9e4e7b30a0 100644 --- a/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java +++ b/server/src/main/java/org/opensearch/cluster/NodeConnectionsService.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterApplier; import org.opensearch.common.Nullable; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index de751d881bc0e..1c38e68c43466 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -62,7 +62,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.SetOnce; import org.opensearch.common.Strings; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java index 44f6f2d7313a3..27ecca0358bd8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasMetadata.java @@ -45,6 +45,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -307,7 +308,7 @@ public Builder filter(String filter) { this.filter = null; return this; } - return filter(XContentHelper.convertToMap(XContentFactory.xContent(filter), filter, true)); + return filter(XContentHelper.convertToMap(MediaTypeRegistry.xContent(filter).xContent(), filter, true)); } public Builder filter(Map filter) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java index 5a019804f5eac..9b9b91a07a5cf 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/AliasValidator.java @@ -36,10 +36,10 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; @@ -88,7 +88,7 @@ public void validateAliasStandalone(Alias alias) { validateAliasStandalone(alias.name(), alias.indexRouting()); if (Strings.hasLength(alias.filter())) { try { - XContentHelper.convertToMap(XContentFactory.xContent(alias.filter()), alias.filter(), false); + XContentHelper.convertToMap(MediaTypeRegistry.xContent(alias.filter()).xContent(), alias.filter(), false); } catch (Exception e) { throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e); } @@ -134,7 +134,8 @@ public void validateAliasFilter( ) { assert queryShardContext != null; try ( - XContentParser parser = XContentFactory.xContent(filter) + XContentParser parser = MediaTypeRegistry.xContent(filter) + .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter) ) { validateAliasFilter(parser, queryShardContext); @@ -158,7 +159,7 @@ public void validateAliasFilter( try ( InputStream inputStream = filter.streamInput(); - XContentParser parser = XContentFactory.xContentType(inputStream) + XContentParser parser = MediaTypeRegistry.xContentType(inputStream) .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, filter.streamInput()) ) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 1ba38daa40566..bcf9cbd0efef8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -53,9 +53,9 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -1296,7 +1296,7 @@ public Builder putMapping(String source) throws IOException { putMapping( new MappingMetadata( MapperService.SINGLE_MAPPING_NAME, - XContentHelper.convertToMap(XContentFactory.xContent(source), source, true) + XContentHelper.convertToMap(MediaTypeRegistry.xContent(source).xContent(), source, true) ) ); return this; diff --git a/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java b/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java index 844b78dccc59b..74a3342addc50 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/DelayedAllocationService.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.routing.allocation.RoutingAllocation; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.inject.Inject; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java index af348c1c98f2d..781ca5bb2255a 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java @@ -450,7 +450,11 @@ public Builder initializeAsRestore(IndexMetadata indexMetadata, SnapshotRecovery /** * Initializes an existing index, to be restored from remote store */ - public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, RemoteStoreRecoverySource recoverySource) { + public Builder initializeAsRemoteStoreRestore( + IndexMetadata indexMetadata, + RemoteStoreRecoverySource recoverySource, + Map activeInitializingShards + ) { final UnassignedInfo unassignedInfo = new UnassignedInfo( UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, "restore_source[remote_store]" @@ -462,7 +466,11 @@ public Builder initializeAsRemoteStoreRestore(IndexMetadata indexMetadata, Remot for (int shardNumber = 0; shardNumber < indexMetadata.getNumberOfShards(); shardNumber++) { ShardId shardId = new ShardId(index, shardNumber); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, true, recoverySource, unassignedInfo)); + if (activeInitializingShards.containsKey(shardId)) { + indexShardRoutingBuilder.addShard(activeInitializingShards.get(shardId)); + } else { + indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, true, recoverySource, unassignedInfo)); + } shards.put(shardNumber, indexShardRoutingBuilder.build()); } return this; diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index 1bee5d8176a0f..d6a67bc714689 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -295,6 +295,16 @@ public ShardsIterator allShardsIncludingRelocationTargets(String[] indices) { return allShardsSatisfyingPredicate(indices, shardRouting -> true, true); } + /** + * All the shards on the node which match the predicate + * @param predicate condition to match + * @return iterator over shards matching the predicate + */ + public ShardsIterator allShardsSatisfyingPredicate(Predicate predicate) { + String[] indices = indicesRouting.keySet().toArray(new String[0]); + return allShardsSatisfyingPredicate(indices, predicate, false); + } + private ShardsIterator allShardsSatisfyingPredicate( String[] indices, Predicate predicate, @@ -562,9 +572,13 @@ public Builder addAsFromOpenToClose(IndexMetadata indexMetadata) { return add(indexRoutingBuilder); } - public Builder addAsRemoteStoreRestore(IndexMetadata indexMetadata, RemoteStoreRecoverySource recoverySource) { + public Builder addAsRemoteStoreRestore( + IndexMetadata indexMetadata, + RemoteStoreRecoverySource recoverySource, + Map activeInitializingShards + ) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()) - .initializeAsRemoteStoreRestore(indexMetadata, recoverySource); + .initializeAsRemoteStoreRestore(indexMetadata, recoverySource, activeInitializingShards); add(indexRoutingBuilder); return this; } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java index d54236ada6780..542aa9ca5becf 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettings.java @@ -34,13 +34,13 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.Version; -import org.opensearch.common.Strings; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.RatioValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.Strings; import java.util.Arrays; import java.util.Iterator; @@ -400,13 +400,13 @@ public boolean isCreateIndexBlockAutoReleaseEnabled() { String describeLowThreshold() { return freeBytesThresholdLow.equals(ByteSizeValue.ZERO) - ? Strings.format1Decimals(100.0 - freeDiskThresholdLow, "%") + ? org.opensearch.core.common.Strings.format1Decimals(100.0 - freeDiskThresholdLow, "%") : freeBytesThresholdLow.toString(); } String describeHighThreshold() { return freeBytesThresholdHigh.equals(ByteSizeValue.ZERO) - ? Strings.format1Decimals(100.0 - freeDiskThresholdHigh, "%") + ? org.opensearch.core.common.Strings.format1Decimals(100.0 - freeDiskThresholdHigh, "%") : freeBytesThresholdHigh.toString(); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java index 946e2d5e5f3dc..ad568fc5ffa57 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/NodeAllocationResult.java @@ -39,7 +39,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 4b69c05807ae4..3325b3475fd73 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -47,11 +47,11 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.DiskThresholdSettings; import org.opensearch.cluster.routing.allocation.RoutingAllocation; -import org.opensearch.common.Strings; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.common.Strings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.store.remote.filecache.FileCacheStats; @@ -68,7 +68,7 @@ import static org.opensearch.cluster.routing.RoutingPool.getShardPool; import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING; import static org.opensearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING; -import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO; +import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING; /** * The {@link DiskThresholdDecider} checks that the node a shard is potentially @@ -199,8 +199,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing final FileCacheStats fileCacheStats = clusterInfo.getNodeFileCacheStats().getOrDefault(node.nodeId(), null); final long nodeCacheSize = fileCacheStats != null ? fileCacheStats.getTotal().getBytes() : 0; final long totalNodeRemoteShardSize = currentNodeRemoteShardSize + shardSize; - - if (totalNodeRemoteShardSize > DATA_TO_FILE_CACHE_SIZE_RATIO * nodeCacheSize) { + final double dataToFileCacheSizeRatio = DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.get(allocation.metadata().settings()); + if (dataToFileCacheSizeRatio > 0.0f && totalNodeRemoteShardSize > dataToFileCacheSizeRatio * nodeCacheSize) { return allocation.decision( Decision.NO, NAME, diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java index 7f1c9f01f7e6f..c03f5ae619edf 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java @@ -51,7 +51,7 @@ import org.opensearch.common.Priority; import org.opensearch.common.StopWatch; import org.opensearch.common.StopWatch.TimingHandle; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java index a605c41bdeff8..e097803d86b48 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java @@ -45,7 +45,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.cluster.routing.RerouteService; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java index 790efaef95292..9f76460d2acdd 100644 --- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java @@ -55,7 +55,7 @@ import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.Nullable; import org.opensearch.common.Priority; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java index 1f8a6aba0c883..86a2fe1397cec 100644 --- a/server/src/main/java/org/opensearch/common/FieldMemoryStats.java +++ b/server/src/main/java/org/opensearch/common/FieldMemoryStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/Strings.java b/server/src/main/java/org/opensearch/common/Strings.java index 8e92c86836723..e9f4d32ed2664 100644 --- a/server/src/main/java/org/opensearch/common/Strings.java +++ b/server/src/main/java/org/opensearch/common/Strings.java @@ -155,29 +155,6 @@ public static String[] split(String toSplit, String delimiter) { return new String[] { beforeDelimiter, afterDelimiter }; } - /** - * Format the double value with a single decimal points, trimming trailing '.0'. - */ - public static String format1Decimals(double value, String suffix) { - String p = String.valueOf(value); - int ix = p.indexOf('.') + 1; - int ex = p.indexOf('E'); - char fraction = p.charAt(ix); - if (fraction == '0') { - if (ex != -1) { - return p.substring(0, ix - 1) + p.substring(ex) + suffix; - } else { - return p.substring(0, ix - 1) + suffix; - } - } else { - if (ex != -1) { - return p.substring(0, ix) + fraction + p.substring(ex) + suffix; - } else { - return p.substring(0, ix) + fraction + suffix; - } - } - } - private Strings() {} public static byte[] toUTF8Bytes(CharSequence charSequence) { diff --git a/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java b/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java index 923f592c6bc79..de4e6ad433c55 100644 --- a/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java +++ b/server/src/main/java/org/opensearch/common/breaker/ChildMemoryCircuitBreaker.java @@ -34,7 +34,9 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.indices.breaker.BreakerSettings; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; diff --git a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java index 62ec933fe5f37..ee2d5a650fc70 100644 --- a/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java +++ b/server/src/main/java/org/opensearch/common/compress/CompressorFactory.java @@ -34,8 +34,8 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.compress.NotXContentException; +import org.opensearch.core.xcontent.MediaTypeRegistry; import java.io.IOException; import java.util.Objects; @@ -67,14 +67,14 @@ public static Compressor compressor(BytesReference bytes) { // bytes should be either detected as compressed or as xcontent, // if we have bytes that can be either detected as compressed or // as a xcontent, we have a problem - assert XContentHelper.xContentType(bytes) == null; + assert MediaTypeRegistry.xContentType(bytes) == null; return DEFLATE_COMPRESSOR; } else if (ZSTD_COMPRESSOR.isCompressed(bytes)) { - assert XContentHelper.xContentType(bytes) == null; + assert MediaTypeRegistry.xContentType(bytes) == null; return ZSTD_COMPRESSOR; } - if (XContentHelper.xContentType(bytes) == null) { + if (MediaTypeRegistry.xContentType(bytes) == null) { throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); } diff --git a/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java b/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java index 80b5dd353703c..e853a6ddc34d2 100644 --- a/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java +++ b/server/src/main/java/org/opensearch/common/io/DiskIoBufferPool.java @@ -32,7 +32,7 @@ package org.opensearch.common.io; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.threadpool.ThreadPool; import java.nio.ByteBuffer; diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java index b4440b85e0037..ef07f6ea8052c 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/MoreLikeThisQuery.java @@ -36,12 +36,12 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.similarities.ClassicSimilarity; @@ -144,12 +144,12 @@ public boolean equals(Object obj) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } - XMoreLikeThis mlt = new XMoreLikeThis(reader, similarity == null ? new ClassicSimilarity() : similarity); + XMoreLikeThis mlt = new XMoreLikeThis(searcher.getIndexReader(), similarity == null ? new ClassicSimilarity() : similarity); mlt.setFieldNames(moreLikeFields); mlt.setAnalyzer(analyzer); diff --git a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java index 98ebb34fc040e..cc0468efb243e 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/MultiPhrasePrefixQuery.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.Query; @@ -159,8 +160,8 @@ public int[] getPositions() { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } @@ -177,7 +178,7 @@ public Query rewrite(IndexReader reader) throws IOException { int position = positions.get(sizeMinus1); Set terms = new HashSet<>(); for (Term term : suffixTerms) { - getPrefixTerms(terms, term, reader); + getPrefixTerms(terms, term, searcher.getIndexReader()); if (terms.size() > maxExpansions) { break; } diff --git a/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java b/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java index d7ffa2df943b7..49148890abd55 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/XMoreLikeThis.java @@ -56,7 +56,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; @@ -808,8 +810,10 @@ public String describeParams() { */ private PriorityQueue retrieveTerms(int docNum) throws IOException { Map termFreqMap = new HashMap<>(); + final TermVectors termVectors = ir.termVectors(); + final StoredFields storedFields = ir.storedFields(); for (String fieldName : fieldNames) { - final Fields vectors = ir.getTermVectors(docNum); + final Fields vectors = termVectors.get(docNum); final Terms vector; if (vectors != null) { vector = vectors.terms(fieldName); @@ -819,7 +823,7 @@ private PriorityQueue retrieveTerms(int docNum) throws IOException { // field does not store term vector info if (vector == null) { - Document d = ir.document(docNum); + Document d = storedFields.document(docNum); IndexableField fields[] = d.getFields(fieldName); for (IndexableField field : fields) { final String stringValue = field.stringValue(); diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java index ebd155b8a5679..db92d4f1cba96 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/FunctionScoreQuery.java @@ -128,7 +128,7 @@ protected int doHashCode() { @Override protected ScoreFunction rewrite(IndexReader reader) throws IOException { - Query newFilter = filter.rewrite(reader); + Query newFilter = filter.rewrite(new IndexSearcher(reader)); if (newFilter == filter) { return this; } @@ -322,16 +322,16 @@ public void visit(QueryVisitor visitor) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = super.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query rewritten = super.rewrite(searcher); if (rewritten != this) { return rewritten; } - Query newQ = subQuery.rewrite(reader); + Query newQ = subQuery.rewrite(searcher); ScoreFunction[] newFunctions = new ScoreFunction[functions.length]; boolean needsRewrite = (newQ != subQuery); for (int i = 0; i < functions.length; i++) { - newFunctions[i] = functions[i].rewrite(reader); + newFunctions[i] = functions[i].rewrite(searcher.getIndexReader()); needsRewrite |= (newFunctions[i] != functions[i]); } if (needsRewrite) { diff --git a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java index 8bf5fc0f89d31..07084a6f690e4 100644 --- a/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java +++ b/server/src/main/java/org/opensearch/common/lucene/search/function/ScriptScoreQuery.java @@ -32,7 +32,6 @@ package org.opensearch.common.lucene.search.function; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.DocIdSetIterator; @@ -105,12 +104,12 @@ public ScriptScoreQuery( } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query newQ = subQuery.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query newQ = subQuery.rewrite(searcher); if (newQ != subQuery) { return new ScriptScoreQuery(newQ, queryName, script, scriptBuilder, minScore, indexName, shardId, indexVersion); } - return super.rewrite(reader); + return super.rewrite(searcher); } @Override diff --git a/server/src/main/java/org/opensearch/common/network/NetworkModule.java b/server/src/main/java/org/opensearch/common/network/NetworkModule.java index d93f8d7c98b32..c052af41d66d9 100644 --- a/server/src/main/java/org/opensearch/common/network/NetworkModule.java +++ b/server/src/main/java/org/opensearch/common/network/NetworkModule.java @@ -53,7 +53,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpServerTransport; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.tasks.RawTaskStatus; import org.opensearch.tasks.Task; diff --git a/server/src/main/java/org/opensearch/common/network/NetworkService.java b/server/src/main/java/org/opensearch/common/network/NetworkService.java index 0fb299ef66e70..1a893283eae21 100644 --- a/server/src/main/java/org/opensearch/common/network/NetworkService.java +++ b/server/src/main/java/org/opensearch/common/network/NetworkService.java @@ -34,7 +34,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 46a43842451d9..9da9e1b14d307 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -44,6 +44,7 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.search.backpressure.settings.NodeDuressSettings; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; import org.opensearch.search.backpressure.settings.SearchShardTaskSettings; @@ -643,6 +644,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Settings related to Searchable Snapshots Node.NODE_SEARCH_CACHE_SIZE_SETTING, + FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING, // Settings related to Remote Refresh Segment Pressure RemoteRefreshSegmentPressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED, diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java index 3bf2988e88e5a..ae390f1cc60ff 100644 --- a/server/src/main/java/org/opensearch/common/settings/Setting.java +++ b/server/src/main/java/org/opensearch/common/settings/Setting.java @@ -40,20 +40,21 @@ import org.opensearch.common.Nullable; import org.opensearch.common.Strings; import org.opensearch.common.collect.Tuple; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.regex.Regex; -import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; @@ -2047,7 +2048,7 @@ static boolean parseBoolean(String b, String key, boolean isFiltered) { } public static Setting byteSizeSetting(String key, ByteSizeValue value, Property... properties) { - return byteSizeSetting(key, (s) -> value.toString(), properties); + return byteSizeSetting(key, (s) -> value.getBytes() + ByteSizeUnit.BYTES.getSuffix(), properties); } public static Setting byteSizeSetting(String key, Setting fallbackSetting, Property... properties) { diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index a7649a1cd22c5..cd80e9727e0df 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -43,12 +43,12 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.logging.LogConfigurator; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.settings.SecureString; @@ -88,7 +88,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; import static org.opensearch.common.unit.TimeValue.parseTimeValue; /** @@ -1081,7 +1081,7 @@ private void processLegacyLists(Map map) { */ public Builder loadFromMap(Map map) { // TODO: do this without a serialization round-trip - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.map(map); return loadFromSource(Strings.toString(builder), builder.contentType()); } catch (IOException e) { @@ -1094,7 +1094,7 @@ public Builder loadFromMap(Map map) { */ public Builder loadFromSource(String source, MediaType xContentType) { try ( - XContentParser parser = XContentFactory.xContent(xContentType) + XContentParser parser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, source) ) { this.put(fromXContent(parser, true, true)); @@ -1127,7 +1127,7 @@ public Builder loadFromStream(String resourceName, InputStream is, boolean accep } // fromXContent doesn't use named xcontent or deprecation. try ( - XContentParser parser = XContentFactory.xContent(xContentType) + XContentParser parser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, is) ) { if (parser.currentToken() == null) { diff --git a/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java b/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java index 8664b14119694..e197e88b3012e 100644 --- a/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java +++ b/server/src/main/java/org/opensearch/common/settings/WriteableSetting.java @@ -23,7 +23,7 @@ import org.opensearch.common.settings.Setting.MinTimeValueParser; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Setting.RegexValidator; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import java.io.IOException; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java b/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java index 18aae6277c379..96f128cb6dc38 100644 --- a/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java +++ b/server/src/main/java/org/opensearch/common/unit/MemorySizeValue.java @@ -33,11 +33,13 @@ package org.opensearch.common.unit; import org.opensearch.OpenSearchParseException; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.monitor.jvm.JvmInfo; import java.util.Objects; -import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; /** * Utility methods to get memory sizes. diff --git a/server/src/main/java/org/opensearch/common/unit/SizeValue.java b/server/src/main/java/org/opensearch/common/unit/SizeValue.java index 3b73955f8d046..766199ebbc8f8 100644 --- a/server/src/main/java/org/opensearch/common/unit/SizeValue.java +++ b/server/src/main/java/org/opensearch/common/unit/SizeValue.java @@ -33,7 +33,7 @@ package org.opensearch.common.unit; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 45c2092601a7b..3b5a057872bbc 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -36,14 +36,14 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.common.Nullable; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; import org.opensearch.common.recycler.Recycler; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.util.BigArray; import org.opensearch.core.common.util.ByteArray; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java b/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java index f8c690471fb61..65be2a082f084 100644 --- a/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java +++ b/server/src/main/java/org/opensearch/common/util/PageCacheRecycler.java @@ -39,7 +39,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java b/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java index b55280d43a473..12fe437b390bf 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnable.java @@ -32,7 +32,7 @@ package org.opensearch.common.util.concurrent; import org.apache.logging.log4j.Logger; -import org.opensearch.common.component.Lifecycle; +import org.opensearch.common.lifecycle.Lifecycle; import java.util.Objects; diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java index cfe8008a4adca..a0903cbdd9f61 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/opensearch/common/xcontent/XContentHelper.java @@ -41,6 +41,7 @@ import org.opensearch.common.compress.CompressorFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContent.Params; @@ -52,7 +53,6 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -86,14 +86,14 @@ public static XContentParser createParser( if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - final XContentType contentType = XContentFactory.xContentType(compressedInput); - return XContentFactory.xContent(contentType).createParser(xContentRegistry, deprecationHandler, compressedInput); + final MediaType contentType = MediaTypeRegistry.xContentType(compressedInput); + return contentType.xContent().createParser(xContentRegistry, deprecationHandler, compressedInput); } catch (Exception e) { if (compressedInput != null) compressedInput.close(); throw e; } } else { - return XContentFactory.xContent(xContentType(bytes)).createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); + return MediaTypeRegistry.xContentType(bytes).xContent().createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); } } @@ -115,7 +115,7 @@ public static XContentParser createParser( if (compressedInput.markSupported() == false) { compressedInput = new BufferedInputStream(compressedInput); } - return XContentFactory.xContent(mediaType).createParser(xContentRegistry, deprecationHandler, compressedInput); + return mediaType.xContent().createParser(xContentRegistry, deprecationHandler, compressedInput); } catch (Exception e) { if (compressedInput != null) compressedInput.close(); throw e; @@ -179,20 +179,14 @@ public static Tuple> convertToMap( final byte[] raw = arr.array(); final int offset = arr.offset(); final int length = arr.length(); - contentType = xContentType != null ? xContentType : XContentFactory.xContentType(raw, offset, length); - return new Tuple<>( - Objects.requireNonNull(contentType), - convertToMap(XContentFactory.xContent(contentType), raw, offset, length, ordered) - ); + contentType = xContentType != null ? xContentType : MediaTypeRegistry.mediaTypeFromBytes(raw, offset, length); + return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(contentType.xContent(), raw, offset, length, ordered)); } else { input = bytes.streamInput(); } try (InputStream stream = input) { - contentType = xContentType != null ? xContentType : XContentFactory.xContentType(stream); - return new Tuple<>( - Objects.requireNonNull(contentType), - convertToMap(XContentFactory.xContent(contentType), stream, ordered) - ); + contentType = xContentType != null ? xContentType : MediaTypeRegistry.xContentType(stream); + return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(contentType.xContent(), stream, ordered)); } } catch (IOException e) { throw new OpenSearchParseException("Failed to parse content to map", e); @@ -266,7 +260,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson) t @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException { - return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes.toBytesRef().bytes)); + return convertToJson(bytes, reformatJson, prettyPrint, MediaTypeRegistry.xContent(bytes.toBytesRef().bytes)); } public static String convertToJson(BytesReference bytes, boolean reformatJson, MediaType xContentType) throws IOException { @@ -311,7 +305,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson, b if (bytes instanceof BytesArray) { final BytesArray array = (BytesArray) bytes; try ( - XContentParser parser = XContentFactory.xContent(mediaType) + XContentParser parser = mediaType.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, @@ -325,7 +319,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson, b } else { try ( InputStream stream = bytes.streamInput(); - XContentParser parser = XContentFactory.xContent(mediaType) + XContentParser parser = mediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream) ) { return toJsonString(prettyPrint, parser); @@ -497,6 +491,7 @@ public static void writeRawField(String field, BytesReference source, XContentTy * {@link XContentType}. Wraps the output into a new anonymous object according to the value returned * by the {@link ToXContent#isFragment()} method returns. */ + @Deprecated public static BytesReference toXContent(ToXContent toXContent, XContentType xContentType, boolean humanReadable) throws IOException { return toXContent(toXContent, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); } @@ -544,29 +539,6 @@ public static BytesReference toXContent(ToXContent toXContent, MediaType mediaTy } } - /** - * Guesses the content type based on the provided bytes. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(BytesReference bytes) { - if (bytes instanceof BytesArray) { - final BytesArray array = (BytesArray) bytes; - return XContentFactory.xContentType(array.array(), array.offset(), array.length()); - } - try { - final InputStream inputStream = bytes.streamInput(); - assert inputStream.markSupported(); - return XContentFactory.xContentType(inputStream); - } catch (IOException e) { - assert false : "Should not happen, we're just reading bytes from memory"; - throw new UncheckedIOException(e); - } - } - /** * Returns the contents of an object as an unparsed BytesReference * diff --git a/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java b/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java index 924db8bdea1dd..ab295b12302e1 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java +++ b/server/src/main/java/org/opensearch/common/xcontent/XContentOpenSearchExtension.java @@ -35,7 +35,7 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentBuilderExtension; diff --git a/server/src/main/java/org/opensearch/discovery/Discovery.java b/server/src/main/java/org/opensearch/discovery/Discovery.java index 25b3cb6c2b90d..9d6807b6522c9 100644 --- a/server/src/main/java/org/opensearch/discovery/Discovery.java +++ b/server/src/main/java/org/opensearch/discovery/Discovery.java @@ -33,7 +33,7 @@ package org.opensearch.discovery; import org.opensearch.cluster.coordination.ClusterStatePublisher; -import org.opensearch.common.component.LifecycleComponent; +import org.opensearch.common.lifecycle.LifecycleComponent; /** * A pluggable module allowing to implement discovery of other nodes, publishing of the cluster diff --git a/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java b/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java index 14805648c6771..cef7853011b82 100644 --- a/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java +++ b/server/src/main/java/org/opensearch/discovery/SeedHostsResolver.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.SetOnce; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/main/java/org/opensearch/env/NodeEnvironment.java b/server/src/main/java/org/opensearch/env/NodeEnvironment.java index f7d1f6e4343cc..59c5b43a7a904 100644 --- a/server/src/main/java/org/opensearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/opensearch/env/NodeEnvironment.java @@ -57,7 +57,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.util.FileSystemUtils; diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java index 0992345735176..e6df6e964a31b 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java +++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java @@ -9,13 +9,15 @@ package org.opensearch.extensions.rest; import org.opensearch.OpenSearchParseException; +import org.opensearch.Version; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestRequest.Method; import org.opensearch.transport.TransportRequest; @@ -42,7 +44,7 @@ public class ExtensionRestRequest extends TransportRequest { private String path; private Map params; private Map> headers; - private XContentType xContentType = null; + private MediaType mediaType = null; private BytesReference content; // The owner of this request object // Will be replaced with PrincipalIdentifierToken class from feature/identity @@ -56,15 +58,15 @@ public class ExtensionRestRequest extends TransportRequest { /** * This object can be instantiated given method, uri, params, content and identifier * - * @param method of type {@link Method} - * @param uri the REST uri string (excluding the query) - * @param path the REST path - * @param params the REST params - * @param headers the REST headers - * @param xContentType the content type, or null for plain text or no content - * @param content the REST request content + * @param method of type {@link Method} + * @param uri the REST uri string (excluding the query) + * @param path the REST path + * @param params the REST params + * @param headers the REST headers + * @param mediaType the content type, or null for plain text or no content + * @param content the REST request content * @param principalIdentifier the owner of this request - * @param httpVersion the REST HTTP protocol version + * @param httpVersion the REST HTTP protocol version */ public ExtensionRestRequest( Method method, @@ -72,7 +74,7 @@ public ExtensionRestRequest( String path, Map params, Map> headers, - XContentType xContentType, + MediaType mediaType, BytesReference content, String principalIdentifier, HttpRequest.HttpVersion httpVersion @@ -82,7 +84,7 @@ public ExtensionRestRequest( this.path = path; this.params = params; this.headers = headers; - this.xContentType = xContentType; + this.mediaType = mediaType; this.content = content; this.principalIdentifierToken = principalIdentifier; this.httpVersion = httpVersion; @@ -102,7 +104,11 @@ public ExtensionRestRequest(StreamInput in) throws IOException { params = in.readMap(StreamInput::readString, StreamInput::readString); headers = in.readMap(StreamInput::readString, StreamInput::readStringList); if (in.readBoolean()) { - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } content = in.readBytesReference(); principalIdentifierToken = in.readString(); @@ -117,9 +123,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(path); out.writeMap(params, StreamOutput::writeString, StreamOutput::writeString); out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeStringCollection); - out.writeBoolean(xContentType != null); - if (xContentType != null) { - out.writeEnum(xContentType); + out.writeBoolean(mediaType != null); + if (mediaType != null) { + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } out.writeBytesReference(content); out.writeString(principalIdentifierToken); @@ -237,8 +247,8 @@ public Map> headers() { * * @return the content type of the {@link #content()}, or null if the context is plain text or if there is no content. */ - public XContentType getXContentType() { - return xContentType; + public MediaType getXContentType() { + return mediaType; } /** @@ -311,7 +321,7 @@ public String toString() { + ", headers=" + headers.toString() + ", xContentType=" - + xContentType + + mediaType + ", contentLength=" + content.length() + ", requester=" @@ -331,7 +341,7 @@ public boolean equals(Object obj) { && Objects.equals(path, that.path) && Objects.equals(params, that.params) && Objects.equals(headers, that.headers) - && Objects.equals(xContentType, that.xContentType) + && Objects.equals(mediaType, that.mediaType) && Objects.equals(content, that.content) && Objects.equals(principalIdentifierToken, that.principalIdentifierToken) && Objects.equals(httpVersion, that.httpVersion); @@ -339,6 +349,6 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(method, uri, path, params, headers, xContentType, content, principalIdentifierToken, httpVersion); + return Objects.hash(method, uri, path, params, headers, mediaType, content, principalIdentifierToken, httpVersion); } } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index 2a3ad63379556..4b622b841a040 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -80,7 +80,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client Tuple> unreadExtensionTuple = XContentHelper.convertToMap( request.content(), false, - request.getXContentType().xContent().mediaType() + request.getMediaType().xContent().mediaType() ); Map extensionMap = unreadExtensionTuple.v2(); diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index e20a3c6b2814c..a96e87b0200cc 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -14,7 +14,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.rest.BaseRestHandler; @@ -181,7 +181,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC String uri = httpRequest.uri(); Map params = request.params(); Map> headers = request.getHeaders(); - XContentType contentType = request.getXContentType(); + MediaType contentType = request.getMediaType(); BytesReference content = request.content(); HttpRequest.HttpVersion httpVersion = httpRequest.protocolVersion(); diff --git a/server/src/main/java/org/opensearch/gateway/GatewayService.java b/server/src/main/java/org/opensearch/gateway/GatewayService.java index cf105380e98ad..b8fd3d147523b 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayService.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayService.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java index 9943f04c964e1..2a419031b2c5a 100644 --- a/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java @@ -49,12 +49,12 @@ import org.opensearch.common.lucene.store.IndexOutputOutputStream; import org.opensearch.common.lucene.store.InputStreamIndexInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.common.util.io.IOUtils; import java.io.FileNotFoundException; import java.io.IOException; @@ -288,7 +288,7 @@ private long write(final T state, boolean cleanup, final Path... locations) thro } protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException { - return XContentFactory.contentBuilder(type, stream); + return MediaTypeRegistry.contentBuilder(type, stream); } /** @@ -321,7 +321,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) { try ( - XContentParser parser = XContentFactory.xContent(FORMAT) + XContentParser parser = FORMAT.xContent() .createParser( namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, diff --git a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java index 8940b0ed25ed4..c01f51eb7188a 100644 --- a/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java @@ -45,6 +45,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -66,7 +67,6 @@ import org.opensearch.common.CheckedConsumer; import org.opensearch.common.Nullable; import org.opensearch.common.SetOnce; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.RecyclingBytesStreamOutput; import org.opensearch.common.io.Streams; import org.opensearch.common.logging.Loggers; @@ -75,14 +75,15 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; -import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.util.ByteArray; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -447,7 +448,7 @@ private OnDiskState loadOnDiskState(Path dataPath, DirectoryReader reader) throw final SetOnce builderReference = new SetOnce<>(); consumeFromType(searcher, GLOBAL_TYPE_NAME, bytes -> { final Metadata metadata = Metadata.Builder.fromXContent( - XContentFactory.xContent(XContentType.SMILE) + XContentType.SMILE.xContent() .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length) ); logger.trace("found global metadata with last-accepted term [{}]", metadata.coordinationMetadata().term()); @@ -467,7 +468,7 @@ private OnDiskState loadOnDiskState(Path dataPath, DirectoryReader reader) throw final Set indexUUIDs = new HashSet<>(); consumeFromType(searcher, INDEX_TYPE_NAME, bytes -> { final IndexMetadata indexMetadata = IndexMetadata.fromXContent( - XContentFactory.xContent(XContentType.SMILE) + XContentType.SMILE.xContent() .createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, bytes.bytes, bytes.offset, bytes.length) ); logger.trace("found index metadata for {}", indexMetadata.getIndex()); @@ -507,12 +508,11 @@ private static void consumeFromType(IndexSearcher indexSearcher, String type, Ch final Bits liveDocs = leafReaderContext.reader().getLiveDocs(); final IntPredicate isLiveDoc = liveDocs == null ? i -> true : liveDocs::get; final DocIdSetIterator docIdSetIterator = scorer.iterator(); + final StoredFields storedFields = leafReaderContext.reader().storedFields(); while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { if (isLiveDoc.test(docIdSetIterator.docID())) { logger.trace("processing doc {}", docIdSetIterator.docID()); - bytesRefConsumer.accept( - leafReaderContext.reader().document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME) - ); + bytesRefConsumer.accept(storedFields.document(docIdSetIterator.docID()).getBinaryValue(DATA_FIELD_NAME)); } } } @@ -921,7 +921,7 @@ private Document makeDocument(String typeName, ToXContent metadata, DocumentBuff try (RecyclingBytesStreamOutput streamOutput = documentBuffer.streamOutput()) { try ( - XContentBuilder xContentBuilder = XContentFactory.contentBuilder( + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder( XContentType.SMILE, Streams.flushOnCloseStream(streamOutput) ) diff --git a/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java b/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java index 1b4e3d89bdc1e..44521ded150be 100644 --- a/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/ReplicaShardAllocator.java @@ -48,7 +48,7 @@ import org.opensearch.cluster.routing.allocation.decider.Decision; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata; diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 46f296f52ae01..0ba49be01d193 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -37,7 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.network.CloseableChannel; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.network.NetworkService; @@ -47,7 +47,7 @@ import org.opensearch.common.transport.NetworkExceptionHelper; import org.opensearch.common.transport.PortsRange; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/http/HttpInfo.java b/server/src/main/java/org/opensearch/http/HttpInfo.java index a5e981e98e3de..24c29b8dc7444 100644 --- a/server/src/main/java/org/opensearch/http/HttpInfo.java +++ b/server/src/main/java/org/opensearch/http/HttpInfo.java @@ -38,7 +38,7 @@ import org.opensearch.common.network.InetAddresses; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/http/HttpServerTransport.java b/server/src/main/java/org/opensearch/http/HttpServerTransport.java index 425fa23047764..6549f0786fcda 100644 --- a/server/src/main/java/org/opensearch/http/HttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/HttpServerTransport.java @@ -32,7 +32,7 @@ package org.opensearch.http; -import org.opensearch.common.component.LifecycleComponent; +import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.node.ReportingService; diff --git a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java index 4522a59d67c05..acff87442d0a8 100644 --- a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java +++ b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java @@ -37,8 +37,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.transport.PortsRange; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import java.util.Collections; diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index e4e3a79c8e60c..8a0d563d51107 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -76,7 +76,7 @@ import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.IndicesQueryCache; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.RecoveryState; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 1ecc98b7f69f2..aed09710deec8 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -91,7 +91,7 @@ import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogFactory; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 0749ad0876534..d0fdbd9ac4e03 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -41,8 +41,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/index/IndexingPressure.java b/server/src/main/java/org/opensearch/index/IndexingPressure.java index 33be340feb335..9f57514a0751a 100644 --- a/server/src/main/java/org/opensearch/index/IndexingPressure.java +++ b/server/src/main/java/org/opensearch/index/IndexingPressure.java @@ -37,7 +37,7 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.index.stats.IndexingPressureStats; diff --git a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java index 87605969feb6a..86c5d4542e5fa 100644 --- a/server/src/main/java/org/opensearch/index/IndexingSlowLog.java +++ b/server/src/main/java/org/opensearch/index/IndexingSlowLog.java @@ -244,7 +244,7 @@ private static Map prepareMap( return map; } try { - String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); + String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getMediaType()); String trim = Strings.cleanTruncate(source, maxSourceCharsToLog).trim(); StringBuilder sb = new StringBuilder(trim); StringBuilders.escapeJson(sb, 0); @@ -279,7 +279,7 @@ private static String message(Index index, ParsedDocument doc, long tookInNanos, return sb.toString(); } try { - String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); + String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getMediaType()); sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog).trim()).append("]"); } catch (IOException e) { sb.append(", source[_failed_to_convert_[").append(e.getMessage()).append("]]"); diff --git a/server/src/main/java/org/opensearch/index/MergePolicyConfig.java b/server/src/main/java/org/opensearch/index/MergePolicyConfig.java index d0416aaf54a40..fe2af21dfe039 100644 --- a/server/src/main/java/org/opensearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/opensearch/index/MergePolicyConfig.java @@ -38,8 +38,8 @@ import org.apache.lucene.index.TieredMergePolicy; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; /** * A shard in opensearch is a Lucene index, and a Lucene index is broken diff --git a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java index b0e6e5076d03c..3315a059ee783 100644 --- a/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java +++ b/server/src/main/java/org/opensearch/index/SegmentReplicationShardStats.java @@ -12,7 +12,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java index a4f2628b5e5a3..88077aafb1495 100644 --- a/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java +++ b/server/src/main/java/org/opensearch/index/cache/query/QueryCacheStats.java @@ -36,7 +36,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java index ab8b4706e4ebe..24f68899c2ac7 100644 --- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java +++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java index 04dd669d60883..234c67cc637f1 100644 --- a/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java +++ b/server/src/main/java/org/opensearch/index/document/SortedUnsignedLongDocValuesRangeQuery.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -87,12 +86,12 @@ public String toString(String field) { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { if (Long.compareUnsigned(lowerValue, Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG) == 0 && Long.compareUnsigned(upperValue, Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG) == 0) { return new FieldExistsQuery(field); } - return super.rewrite(reader); + return super.rewrite(searcher); } abstract SortedNumericDocValues getValues(LeafReader reader, String field) throws IOException; diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index 326e6aef45b08..74c9c25dc4c1f 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -69,7 +69,7 @@ import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver; import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion; import org.opensearch.common.metrics.CounterMetric; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.lease.Releasable; diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 71bd64426161e..7900e63a95c39 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -43,7 +43,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.MemorySizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexSettings; @@ -57,7 +57,7 @@ import org.opensearch.index.translog.TranslogDeletionPolicyFactory; import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.IndexingMemoryController; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.threadpool.ThreadPool; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index 744df13265250..c606c00228db9 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -30,7 +30,7 @@ import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; import org.opensearch.index.translog.TranslogFactory; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.PluginsService; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 77e2f5cbef7f9..77d63dfaade54 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -80,7 +81,7 @@ import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver; import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo; import org.opensearch.common.metrics.CounterMetric; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.KeyedLock; @@ -2889,6 +2890,7 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead final CombinedDocValues dv = new CombinedDocValues(leaf.reader()); final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor(); final DocIdSetIterator iterator = scorer.iterator(); + final StoredFields storedFields = leaf.reader().storedFields(); int docId; while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { final long primaryTerm = dv.docPrimaryTerm(docId); @@ -2896,7 +2898,7 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead localCheckpointTracker.markSeqNoAsProcessed(seqNo); localCheckpointTracker.markSeqNoAsPersisted(seqNo); idFieldVisitor.reset(); - leaf.reader().document(docId, idFieldVisitor); + storedFields.document(docId, idFieldVisitor); if (idFieldVisitor.getId() == null) { assert dv.isTombstone(docId); continue; diff --git a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java index abde2aff6e9e6..23fe59456887e 100644 --- a/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/opensearch/index/engine/LuceneChangesSnapshot.java @@ -289,7 +289,7 @@ private Translog.Operation readDocAsOp(int docIndex) throws IOException { ? SourceFieldMapper.RECOVERY_SOURCE_NAME : SourceFieldMapper.NAME; final FieldsVisitor fields = new FieldsVisitor(true, sourceField); - leaf.reader().document(segmentDocID, fields); + leaf.reader().storedFields().document(segmentDocID, fields); final Translog.Operation op; final boolean isTombstone = parallelArray.isTombStone[docIndex]; diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index a9f7a2e70884c..e852658d7b3ba 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -16,7 +16,7 @@ import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; @@ -77,9 +77,10 @@ public NRTReplicationEngine(EngineConfig engineConfig) { this.completionStatsCache = new CompletionStatsCache(() -> acquireSearcher("completion_stats")); this.readerManager = readerManager; this.readerManager.addListener(completionStatsCache); - for (ReferenceManager.RefreshListener listener : engineConfig.getExternalRefreshListener()) { - this.readerManager.addListener(listener); - } + // NRT Replicas do not have a concept of Internal vs External reader managers. + // We also do not want to wire up refresh listeners for waitFor & pending refresh location. + // which are the current external listeners set from IndexShard. + // Only wire up the internal listeners. for (ReferenceManager.RefreshListener listener : engineConfig.getInternalRefreshListener()) { this.readerManager.addListener(listener); } @@ -322,22 +323,12 @@ public List segments(boolean verbose) { @Override public void refresh(String source) throws EngineException { - maybeRefresh(source); + // Refresh on this engine should only ever happen in the reader after new segments arrive. } @Override public boolean maybeRefresh(String source) throws EngineException { - ensureOpen(); - try { - return readerManager.maybeRefresh(); - } catch (IOException e) { - try { - failEngine("refresh failed source[" + source + "]", e); - } catch (Exception inner) { - e.addSuppressed(inner); - } - throw new RefreshFailedEngineException(shardId, e); - } + return false; } @Override diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java index 35409437f605a..7b4c93c7235fe 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationReaderManager.java @@ -103,7 +103,7 @@ public void updateSegments(SegmentInfos infos) throws IOException { // is always increased. infos.updateGeneration(currentInfos); currentInfos = infos; - maybeRefreshBlocking(); + maybeRefresh(); } public SegmentInfos getSegmentInfos() { diff --git a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java index ce0f1e85e294d..6736354517ace 100644 --- a/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java +++ b/server/src/main/java/org/opensearch/index/engine/OpenSearchConcurrentMergeScheduler.java @@ -41,7 +41,7 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.metrics.MeanMetric; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/main/java/org/opensearch/index/engine/Segment.java b/server/src/main/java/org/opensearch/index/engine/Segment.java index 035d5bb293303..36b84723f0229 100644 --- a/server/src/main/java/org/opensearch/index/engine/Segment.java +++ b/server/src/main/java/org/opensearch/index/engine/Segment.java @@ -44,7 +44,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import java.io.IOException; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java index dbadd50ebd1a1..1fd7f30237d0b 100644 --- a/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java +++ b/server/src/main/java/org/opensearch/index/engine/SegmentsStats.java @@ -36,7 +36,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java index 5efafb562df74..417cdd5a8f030 100644 --- a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java +++ b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java @@ -221,28 +221,33 @@ public int maxDoc() { @Override public void document(int docID, StoredFieldVisitor visitor) throws IOException { - if (docID != 0) { - throw new IllegalArgumentException("no such doc ID " + docID); - } - if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) { - assert operation.source().toBytesRef().offset == 0; - assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length; - visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes); - } - if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { - visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); - } - if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { - BytesRef bytesRef = Uid.encodeId(operation.id()); - final byte[] id = new byte[bytesRef.length]; - System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length); - visitor.binaryField(FAKE_ID_FIELD, id); - } + storedFields().document(docID, visitor); } @Override public StoredFields storedFields() throws IOException { - throw new UnsupportedOperationException(); + return new StoredFields() { + @Override + public void document(int docID, StoredFieldVisitor visitor) throws IOException { + if (docID != 0) { + throw new IllegalArgumentException("no such doc ID " + docID); + } + if (visitor.needsField(FAKE_SOURCE_FIELD) == StoredFieldVisitor.Status.YES) { + assert operation.source().toBytesRef().offset == 0; + assert operation.source().toBytesRef().length == operation.source().toBytesRef().bytes.length; + visitor.binaryField(FAKE_SOURCE_FIELD, operation.source().toBytesRef().bytes); + } + if (operation.routing() != null && visitor.needsField(FAKE_ROUTING_FIELD) == StoredFieldVisitor.Status.YES) { + visitor.stringField(FAKE_ROUTING_FIELD, operation.routing()); + } + if (visitor.needsField(FAKE_ID_FIELD) == StoredFieldVisitor.Status.YES) { + BytesRef bytesRef = Uid.encodeId(operation.id()); + final byte[] id = new byte[bytesRef.length]; + System.arraycopy(bytesRef.bytes, bytesRef.offset, id, 0, bytesRef.length); + visitor.binaryField(FAKE_ID_FIELD, id); + } + } + }; } @Override diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java index 896039313ea1e..fc7db7d316214 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java @@ -37,7 +37,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java index 36e6a242ecdec..05326b7027cac 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java @@ -50,7 +50,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.util.BigArrays; import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java index 0b370893cd90d..6914f5df31da8 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldDataService.java @@ -41,7 +41,7 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.search.lookup.SearchLookup; diff --git a/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java b/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java index cd858278afaa3..517361e75d3ea 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java +++ b/server/src/main/java/org/opensearch/index/fielddata/RamAccountingTermsEnum.java @@ -34,7 +34,7 @@ import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.index.fielddata.plain.AbstractIndexOrdinalsFieldData; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index e136e649d088a..90ee43d320743 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -39,13 +39,13 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.packed.PackedInts; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.fielddata.IndexOrdinalsFieldData; import org.opensearch.index.fielddata.LeafOrdinalsFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import java.io.IOException; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java index 2c6aabf04d4ee..d7b8179398920 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractGeoShapeIndexFieldData.java @@ -15,7 +15,7 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.fielddata.LeafGeoShapeFieldData; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java index 0b3a5e09064ab..15eae77885686 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java @@ -48,7 +48,7 @@ import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.ordinals.GlobalOrdinalsBuilder; import org.opensearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java index 6e9a13074a445..9cd824f570b05 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/AbstractLatLonPointIndexFieldData.java @@ -44,7 +44,7 @@ import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.fielddata.IndexGeoPointFieldData; import org.opensearch.index.fielddata.LeafGeoPointFieldData; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java index 3fbc605e35e1d..420864fbd7cb5 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/BinaryIndexFieldData.java @@ -40,7 +40,7 @@ import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java index e1a08344e68c7..5386d0eee2108 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/BytesBinaryIndexFieldData.java @@ -40,7 +40,7 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.opensearch.index.fielddata.IndexFieldDataCache; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java index 2b08498f2b58b..25d199e9a79af 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/ConstantIndexFieldData.java @@ -49,7 +49,7 @@ import org.opensearch.index.fielddata.IndexOrdinalsFieldData; import org.opensearch.index.fielddata.LeafOrdinalsFieldData; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java index 8f6a1e46ef417..6f880b678a26a 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/PagedBytesIndexFieldData.java @@ -44,7 +44,7 @@ import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; import org.opensearch.common.Nullable; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.util.BigArrays; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -55,7 +55,7 @@ import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.opensearch.index.fielddata.ordinals.Ordinals; import org.opensearch.index.fielddata.ordinals.OrdinalsBuilder; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java index 812010b44b654..e7995f9188275 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -53,7 +53,7 @@ import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; import org.opensearch.index.mapper.DocValueFetcher; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; diff --git a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java index 801186bbc4379..e52eaeb7dfc89 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/plain/SortedSetOrdinalsIndexFieldData.java @@ -46,7 +46,7 @@ import org.opensearch.index.fielddata.LeafOrdinalsFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/get/ShardGetService.java b/server/src/main/java/org/opensearch/index/get/ShardGetService.java index a815074119fb1..8f81e704ef2d4 100644 --- a/server/src/main/java/org/opensearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/opensearch/index/get/ShardGetService.java @@ -51,10 +51,10 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.metrics.MeanMetric; import org.opensearch.common.util.set.Sets; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.engine.Engine; @@ -276,7 +276,7 @@ private GetResult innerGetLoadFromStoredFields( ); if (fieldVisitor != null) { try { - docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor); + docIdAndVersion.reader.storedFields().document(docIdAndVersion.docId, fieldVisitor); } catch (IOException e) { throw new OpenSearchException("Failed to get id [" + id + "]", e); } @@ -299,7 +299,7 @@ private GetResult innerGetLoadFromStoredFields( shardId.getIndexName(), id, source, - XContentHelper.xContentType(source), + MediaTypeRegistry.xContentType(source), fieldVisitor.routing() ); ParsedDocument doc = indexShard.mapperService().documentMapper().parse(sourceToParse); @@ -378,7 +378,7 @@ private GetResult innerGetLoadFromStoredFields( sourceAsMap = typeMapTuple.v2(); sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes()); try { - source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap)); + source = BytesReference.bytes(MediaTypeRegistry.contentBuilder(sourceContentType).map(sourceAsMap)); } catch (IOException e) { throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e); } @@ -406,7 +406,7 @@ private GetResult innerGetLoadFromStoredFields( sourceAsMap = typeMapTuple.v2(); sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes()); try { - source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap)); + source = BytesReference.bytes(MediaTypeRegistry.contentBuilder(sourceContentType).map(sourceAsMap)); } catch (IOException e) { throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e); } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index c5ee5be491095..f276d6ee2e579 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -43,8 +43,8 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.DynamicTemplate.XContentFieldType; @@ -77,14 +77,14 @@ final class DocumentParser { ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadataFieldsMappers) throws MapperParsingException { final Mapping mapping = docMapper.mapping(); final ParseContext.InternalParseContext context; - final XContentType xContentType = source.getXContentType(); + final MediaType mediaType = source.getMediaType(); try ( XContentParser parser = XContentHelper.createParser( docMapperParser.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, source.source(), - xContentType + mediaType ) ) { context = new ParseContext.InternalParseContext(indexSettings, docMapperParser, docMapper, source, parser); @@ -183,7 +183,7 @@ private static ParsedDocument parsedDocument(SourceToParse source, ParseContext. source.routing(), context.docs(), context.sourceToParse().source(), - context.sourceToParse().getXContentType(), + context.sourceToParse().getMediaType(), update ); } diff --git a/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java index 4b1395ac02bf1..9ac226dcc0b70 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IdFieldMapper.java @@ -54,7 +54,7 @@ import org.opensearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.opensearch.index.query.QueryShardContext; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.CoreValuesSourceType; diff --git a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java index 46f0352976408..1084c08cdf32c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/opensearch/index/mapper/ParsedDocument.java @@ -35,6 +35,7 @@ import org.apache.lucene.document.Field; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.mapper.ParseContext.Document; import org.opensearch.index.mapper.MapperService.MergeReason; @@ -57,7 +58,7 @@ public class ParsedDocument { private final List documents; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; private Mapping dynamicMappingsUpdate; @@ -68,7 +69,7 @@ public ParsedDocument( String routing, List documents, BytesReference source, - XContentType xContentType, + MediaType mediaType, Mapping dynamicMappingsUpdate ) { this.version = version; @@ -78,7 +79,7 @@ public ParsedDocument( this.documents = documents; this.source = source; this.dynamicMappingsUpdate = dynamicMappingsUpdate; - this.xContentType = xContentType; + this.mediaType = mediaType; } public String id() { @@ -122,13 +123,13 @@ public BytesReference source() { return this.source; } - public XContentType getXContentType() { - return this.xContentType; + public MediaType getMediaType() { + return this.mediaType; } public void setSource(BytesReference source, XContentType xContentType) { this.source = source; - this.xContentType = xContentType; + this.mediaType = xContentType; } /** diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java index 95e71f168c2ca..deda9883bc07b 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java @@ -39,15 +39,15 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Nullable; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.CollectionUtils; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; @@ -202,7 +202,7 @@ public boolean isComplete() { @Override public void preParse(ParseContext context) throws IOException { BytesReference originalSource = context.sourceToParse().source(); - XContentType contentType = context.sourceToParse().getXContentType(); + MediaType contentType = context.sourceToParse().getMediaType(); final BytesReference adaptedSource = applyFilters(originalSource, contentType); if (adaptedSource != null) { @@ -219,16 +219,16 @@ public void preParse(ParseContext context) throws IOException { } @Nullable - public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable XContentType contentType) throws IOException { + public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable MediaType contentType) throws IOException { if (enabled && originalSource != null) { // Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data if (filter != null) { // we don't update the context source if we filter, we want to keep it as is... - Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); + Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); Map filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); - XContentType actualContentType = mapTuple.v1(); - XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); + MediaType actualContentType = mapTuple.v1(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(actualContentType, bStream).map(filteredSource); builder.close(); return bStream.bytes(); } else { diff --git a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java index 085e258c5501e..2b24a380c8bb6 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/opensearch/index/mapper/SourceToParse.java @@ -37,7 +37,7 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; /** * Stores the document source @@ -54,20 +54,20 @@ public class SourceToParse { private final @Nullable String routing; - private final XContentType xContentType; + private final MediaType mediaType; - public SourceToParse(String index, String id, BytesReference source, XContentType xContentType, @Nullable String routing) { + public SourceToParse(String index, String id, BytesReference source, MediaType mediaType, @Nullable String routing) { this.index = Objects.requireNonNull(index); this.id = Objects.requireNonNull(id); // we always convert back to byte array, since we store it and Field only supports bytes.. // so, we might as well do it here, and improve the performance of working with direct byte arrays this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef()); - this.xContentType = Objects.requireNonNull(xContentType); + this.mediaType = Objects.requireNonNull(mediaType); this.routing = routing; } - public SourceToParse(String index, String id, BytesReference source, XContentType xContentType) { - this(index, id, source, xContentType, null); + public SourceToParse(String index, String id, BytesReference source, MediaType mediaType) { + this(index, id, source, mediaType, null); } public BytesReference source() { @@ -86,8 +86,8 @@ public String id() { return this.routing; } - public XContentType getXContentType() { - return this.xContentType; + public MediaType getMediaType() { + return this.mediaType; } /** diff --git a/server/src/main/java/org/opensearch/index/merge/MergeStats.java b/server/src/main/java/org/opensearch/index/merge/MergeStats.java index fee0ed904b68a..5bdbd0ab30613 100644 --- a/server/src/main/java/org/opensearch/index/merge/MergeStats.java +++ b/server/src/main/java/org/opensearch/index/merge/MergeStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java index 2a7dbf314103d..001d77facb829 100644 --- a/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java +++ b/server/src/main/java/org/opensearch/index/query/DateRangeIncludingNowQuery.java @@ -32,8 +32,8 @@ package org.opensearch.index.query; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; @@ -60,7 +60,7 @@ public Query getQuery() { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { return in; } diff --git a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java index 36b74c95c3200..7287634ecfacb 100644 --- a/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java @@ -58,6 +58,7 @@ import org.opensearch.common.lucene.search.MoreLikeThisQuery; import org.opensearch.common.lucene.search.XMoreLikeThis; import org.opensearch.common.lucene.uid.Versions; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -170,7 +171,7 @@ public static final class Item implements ToXContentObject, Writeable { private String index; private String id; private BytesReference doc; - private XContentType xContentType; + private MediaType mediaType; private String[] fields; private Map perFieldAnalyzer; private String routing; @@ -187,7 +188,7 @@ public Item() {} this.id = copy.id; this.routing = copy.routing; this.doc = copy.doc; - this.xContentType = copy.xContentType; + this.mediaType = copy.mediaType; this.fields = copy.fields; this.perFieldAnalyzer = copy.perFieldAnalyzer; this.version = copy.version; @@ -220,7 +221,7 @@ public Item(@Nullable String index, XContentBuilder doc) { } this.index = index; this.doc = BytesReference.bytes(doc); - this.xContentType = XContentType.fromMediaType(doc.contentType()); + this.mediaType = doc.contentType(); } /** @@ -234,7 +235,11 @@ public Item(@Nullable String index, XContentBuilder doc) { } if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); - xContentType = in.readEnum(XContentType.class); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType = in.readMediaType(); + } else { + mediaType = in.readEnum(XContentType.class); + } } else { id = in.readString(); } @@ -255,7 +260,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeGenericValue(doc); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } else { out.writeString(id); } @@ -331,8 +340,8 @@ public Item versionType(VersionType versionType) { return this; } - XContentType xContentType() { - return xContentType; + MediaType mediaType() { + return mediaType; } /** @@ -351,7 +360,7 @@ TermVectorsRequest toTermVectorsRequest() { .termStatistics(false); // for artificial docs to make sure that the id has changed in the item too if (doc != null) { - termVectorsRequest.doc(doc, true, xContentType); + termVectorsRequest.doc(doc, true, mediaType); this.id = termVectorsRequest.id(); } return termVectorsRequest; @@ -373,7 +382,7 @@ public static Item parse(XContentParser parser, Item item) throws IOException { item.id = parser.text(); } else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) { item.doc = BytesReference.bytes(jsonBuilder().copyCurrentStructure(parser)); - item.xContentType = XContentType.JSON; + item.mediaType = XContentType.JSON; } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) { if (token == XContentParser.Token.START_ARRAY) { List fields = new ArrayList<>(); @@ -419,7 +428,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (this.doc != null) { try (InputStream stream = this.doc.streamInput()) { - builder.rawField(DOC.getPreferredName(), stream, xContentType); + builder.rawField(DOC.getPreferredName(), stream, mediaType); } } if (this.fields != null) { diff --git a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java index aab9adecb9a62..b0fc62ad8ec71 100644 --- a/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/WrapperQueryBuilder.java @@ -34,15 +34,15 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; @@ -175,7 +175,8 @@ protected boolean doEquals(WrapperQueryBuilder other) { @Override protected QueryBuilder doRewrite(QueryRewriteContext context) throws IOException { try ( - XContentParser qSourceParser = XContentFactory.xContent(source) + XContentParser qSourceParser = MediaTypeRegistry.xContent(source) + .xContent() .createParser(context.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, source) ) { diff --git a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java index 40b15eace2bad..a4c4aa9493288 100644 --- a/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/functionscore/DecayFunctionBuilder.java @@ -49,10 +49,10 @@ import org.opensearch.common.unit.DistanceUnit; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.fielddata.FieldData; import org.opensearch.index.fielddata.IndexGeoPointFieldData; @@ -230,7 +230,8 @@ protected ScoreFunction doToFunction(QueryShardContext context) throws IOExcepti // EMPTY is safe because parseVariable doesn't use namedObject try ( InputStream stream = functionBytes.streamInput(); - XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(functionBytes)) + XContentParser parser = MediaTypeRegistry.xContentType(functionBytes) + .xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream) ) { scoreFunction = parseVariable(fieldName, parser, context, multiValueMode); diff --git a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java index 998427c27c1f6..2f03eb66bd71e 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ClientScrollableHitSource.java @@ -49,8 +49,8 @@ import org.opensearch.common.document.DocumentField; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.search.SearchHit; import org.opensearch.threadpool.ThreadPool; @@ -209,8 +209,8 @@ public BytesReference getSource() { } @Override - public XContentType getXContentType() { - return XContentHelper.xContentType(source); + public MediaType getMediaType() { + return MediaTypeRegistry.xContentType(source); } @Override diff --git a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java index f6b4793f3b87a..8a9a346a5344a 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java +++ b/server/src/main/java/org/opensearch/index/reindex/ReindexRequest.java @@ -44,10 +44,10 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.VersionType; import org.opensearch.index.query.QueryBuilder; @@ -356,7 +356,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws request.getSearchRequest().indices(indices); } request.setRemoteInfo(buildRemoteInfo(source)); - XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(parser.contentType()); builder.map(source); try ( InputStream stream = BytesReference.bytes(builder).streamInput(); diff --git a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java index a4047507fcde1..47e882fc04127 100644 --- a/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java +++ b/server/src/main/java/org/opensearch/index/reindex/ScrollableHitSource.java @@ -46,6 +46,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; @@ -296,7 +297,7 @@ public interface Hit { * The content type of the hit source. Returns null if the source didn't come back from the search. */ @Nullable - XContentType getXContentType(); + MediaType getMediaType(); /** * The routing on the hit if there is any or null if there isn't. @@ -316,7 +317,7 @@ public static class BasicHit implements Hit { private final long version; private BytesReference source; - private XContentType xContentType; + private MediaType mediaType; private String routing; private long seqNo; private long primaryTerm; @@ -358,13 +359,13 @@ public BytesReference getSource() { } @Override - public XContentType getXContentType() { - return xContentType; + public MediaType getMediaType() { + return mediaType; } - public BasicHit setSource(BytesReference source, XContentType xContentType) { + public BasicHit setSource(BytesReference source, MediaType mediaType) { this.source = source; - this.xContentType = xContentType; + this.mediaType = mediaType; return this; } diff --git a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java index 770301629d964..cd28d8b6ef04e 100644 --- a/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java +++ b/server/src/main/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQuery.java @@ -32,7 +32,6 @@ package org.opensearch.index.search; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -82,8 +81,8 @@ public ScoreMode getScoreMode() { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query innerRewrite = query.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query innerRewrite = query.rewrite(searcher); if (innerRewrite != query) { // Right now ToParentBlockJoinQuery always rewrites to a ToParentBlockJoinQuery // so the else block will never be used. It is useful in the case that @@ -97,7 +96,7 @@ public Query rewrite(IndexReader reader) throws IOException { return innerRewrite; } } - return super.rewrite(reader); + return super.rewrite(searcher); } @Override diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java index 4d6cffa58510f..19fe9ee97cd2f 100644 --- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java @@ -247,7 +247,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L private final Consumer onReplicationGroupUpdated; - private volatile ReplicationCheckpoint lastPublishedReplicationCheckpoint; + private volatile ReplicationCheckpoint latestReplicationCheckpoint; /** * Get all retention leases tracked on this shard. @@ -1054,6 +1054,7 @@ public ReplicationTracker( this.fileBasedRecoveryThreshold = IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.get(indexSettings.getSettings()); this.safeCommitInfoSupplier = safeCommitInfoSupplier; this.onReplicationGroupUpdated = onReplicationGroupUpdated; + this.latestReplicationCheckpoint = indexSettings.isSegRepEnabled() ? ReplicationCheckpoint.empty(shardId) : null; assert Version.V_EMPTY.equals(indexSettings.getIndexVersionCreated()) == false; assert invariant(); } @@ -1212,26 +1213,42 @@ public synchronized void updateVisibleCheckpointForShard(final String allocation */ public synchronized void setLatestReplicationCheckpoint(ReplicationCheckpoint checkpoint) { assert indexSettings.isSegRepEnabled(); - assert handoffInProgress == false; - if (checkpoint.equals(lastPublishedReplicationCheckpoint) == false) { - this.lastPublishedReplicationCheckpoint = checkpoint; - for (Map.Entry entry : checkpoints.entrySet()) { - if (entry.getKey().equals(this.shardAllocationId) == false) { - final CheckpointState cps = entry.getValue(); - if (cps.inSync) { - cps.checkpointTimers.computeIfAbsent(checkpoint, ignored -> { - final ReplicationTimer replicationTimer = new ReplicationTimer(); - replicationTimer.start(); - return replicationTimer; - }); - logger.trace( - () -> new ParameterizedMessage( - "updated last published checkpoint to {} - timers [{}]", - checkpoint, - cps.checkpointTimers.keySet() - ) - ); - } + if (checkpoint.equals(latestReplicationCheckpoint) == false) { + this.latestReplicationCheckpoint = checkpoint; + } + if (primaryMode) { + startReplicationLagTimers(); + } + } + + public ReplicationCheckpoint getLatestReplicationCheckpoint() { + return this.latestReplicationCheckpoint; + } + + private void startReplicationLagTimers() { + for (Map.Entry entry : checkpoints.entrySet()) { + final String allocationId = entry.getKey(); + if (allocationId.equals(this.shardAllocationId) == false) { + final CheckpointState cps = entry.getValue(); + // if the shard is in checkpoints but is unavailable or out of sync we will not track its replication state. + // it is possible for a shard to be in-sync but not yet removed from the checkpoints collection after a failover event. + if (cps.inSync + && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false + && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint)) { + cps.checkpointTimers.computeIfAbsent(latestReplicationCheckpoint, ignored -> { + final ReplicationTimer replicationTimer = new ReplicationTimer(); + replicationTimer.start(); + return replicationTimer; + }); + logger.trace( + () -> new ParameterizedMessage( + "updated last published checkpoint for {} at visible cp {} to {} - timers [{}]", + allocationId, + cps.visibleReplicationCheckpoint, + latestReplicationCheckpoint, + cps.checkpointTimers.keySet() + ) + ); } } } @@ -1244,12 +1261,17 @@ public synchronized void setLatestReplicationCheckpoint(ReplicationCheckpoint ch */ public synchronized Set getSegmentReplicationStats() { assert indexSettings.isSegRepEnabled(); - final ReplicationCheckpoint lastPublishedCheckpoint = this.lastPublishedReplicationCheckpoint; - if (primaryMode && lastPublishedCheckpoint != null) { + if (primaryMode) { return this.checkpoints.entrySet() .stream() - .filter(entry -> entry.getKey().equals(this.shardAllocationId) == false && entry.getValue().inSync) - .map(entry -> buildShardStats(lastPublishedCheckpoint.getLength(), entry.getKey(), entry.getValue())) + // filter out this shard's allocation id, any shards that are out of sync or unavailable (shard marked in-sync but has not + // been assigned to a node). + .filter( + entry -> entry.getKey().equals(this.shardAllocationId) == false + && entry.getValue().inSync + && replicationGroup.getUnavailableInSyncShards().contains(entry.getKey()) == false + ) + .map(entry -> buildShardStats(latestReplicationCheckpoint.getLength(), entry.getKey(), entry.getValue())) .collect(Collectors.toUnmodifiableSet()); } return Collections.emptySet(); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index fbb2df7f9640a..bb5088866edb6 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -43,7 +43,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.Term; -import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ReferenceManager; @@ -90,7 +89,7 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.metrics.MeanMetric; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.AbstractRunnable; @@ -99,11 +98,11 @@ import org.opensearch.common.util.concurrent.RunOnce; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.gateway.WriteStateException; import org.opensearch.core.index.Index; import org.opensearch.index.IndexModule; @@ -175,7 +174,7 @@ import org.opensearch.index.warmer.WarmerStats; import org.opensearch.indices.IndexingMemoryController; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.indices.recovery.PeerRecoveryTargetService; import org.opensearch.indices.recovery.RecoveryFailedException; @@ -948,7 +947,7 @@ public Engine.IndexResult applyIndexOperationOnReplica( if (indexSettings.isSegRepEnabled()) { Engine.Index index = new Engine.Index( new Term(IdFieldMapper.NAME, Uid.encodeId(id)), - new ParsedDocument(null, null, id, null, null, sourceToParse.source(), sourceToParse.getXContentType(), null), + new ParsedDocument(null, null, id, null, null, sourceToParse.source(), sourceToParse.getMediaType(), null), seqNo, opPrimaryTerm, version, @@ -2172,7 +2171,7 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o shardId.getIndexName(), index.id(), index.source(), - XContentHelper.xContentType(index.source()), + MediaTypeRegistry.xContentType(index.source()), index.routing() ) ); @@ -4424,7 +4423,8 @@ public void addRefreshListener(Translog.Location location, Consumer lis readAllowed = isReadAllowed(); } } - if (readAllowed) { + // NRT Replicas will not accept refresh listeners. + if (readAllowed && isSegmentReplicationAllowed() == false) { refreshListeners.addOrNotify(location, listener); } else { // we're not yet ready fo ready for reads, just ignore refresh cycles @@ -4809,6 +4809,14 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo return true; } else { logger.warn("Checksum mismatch between local and remote segment file: {}, will override local file", file); + // If there is a checksum mismatch and we are not serving reads it is safe to go ahead and delete the file now. + // Outside of engine resets this method will be invoked during recovery so this is safe. + if (isReadAllowed() == false) { + localDirectory.deleteFile(file); + } else { + // segment conflict with remote store while the shard is serving reads. + failShard("Local copy of segment " + file + " has a different checksum than the version in remote store", null); + } } } catch (NoSuchFileException | FileNotFoundException e) { logger.debug("File {} does not exist in local FS, downloading from remote store", file); diff --git a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java index 4f0affb3035ec..5bc3f5f163352 100644 --- a/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java +++ b/server/src/main/java/org/opensearch/index/shard/PrimaryReplicaSyncer.java @@ -44,8 +44,8 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 3ea8278038ac5..2385b906a7ae5 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -457,11 +457,10 @@ private void updateLocalSizeMapAndTracker(Collection segmentFiles) { private void updateFinalStatusInSegmentTracker(boolean uploadStatus, long bytesBeforeUpload, long startTimeInNS) { if (uploadStatus) { long bytesUploaded = segmentTracker.getUploadBytesSucceeded() - bytesBeforeUpload; - long timeTakenInMS = (System.nanoTime() - startTimeInNS) / 1_000_000L; - + long timeTakenInMS = TimeValue.nsecToMSec(System.nanoTime() - startTimeInNS); segmentTracker.incrementTotalUploadsSucceeded(); segmentTracker.addUploadBytes(bytesUploaded); - segmentTracker.addUploadBytesPerSec((bytesUploaded * 1_000L) / timeTakenInMS); + segmentTracker.addUploadBytesPerSec((bytesUploaded * 1_000L) / Math.max(1, timeTakenInMS)); segmentTracker.addUploadTimeMs(timeTakenInMS); } else { segmentTracker.incrementTotalUploadsFailed(); diff --git a/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java index 6d559a6cc0673..219ead931797a 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardSplittingQuery.java @@ -286,7 +286,7 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { boolean matches(int doc) throws IOException { routing = id = null; leftToVisit = 2; - leafReader.document(doc, this); + leafReader.storedFields().document(doc, this); assert id != null : "docID must not be null - we might have hit a nested document"; int targetShardId = OperationRouting.generateShardId(indexMetadata, id, routing); return targetShardId != shardId; diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index d4e779c83644f..5897fa7d513d7 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -51,7 +51,7 @@ import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.opensearch.common.UUIDs; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.index.Index; diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 2d49b153c39f4..0b27afa5e9517 100644 --- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -38,7 +38,7 @@ import org.opensearch.core.ParseField; import org.opensearch.common.Strings; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java b/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java index 9e8c8d29c2058..d69ae02fa8e34 100644 --- a/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java +++ b/server/src/main/java/org/opensearch/index/stats/IndexingPressurePerShardStats.java @@ -11,7 +11,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java index 8f4f0b661ed33..dcdea8adbbcde 100644 --- a/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java +++ b/server/src/main/java/org/opensearch/index/stats/IndexingPressureStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/store/StoreStats.java b/server/src/main/java/org/opensearch/index/store/StoreStats.java index ba36e6b527031..aa73a2c629515 100644 --- a/server/src/main/java/org/opensearch/index/store/StoreStats.java +++ b/server/src/main/java/org/opensearch/index/store/StoreStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java index 3d23b4d22538c..45d60c8ad84db 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java @@ -9,8 +9,9 @@ package org.opensearch.index.store.remote.filecache; import org.apache.lucene.store.IndexInput; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.common.settings.Setting; import org.opensearch.index.store.remote.utils.cache.CacheUsage; import org.opensearch.index.store.remote.utils.cache.RefCountedCache; import org.opensearch.index.store.remote.utils.cache.SegmentedCache; @@ -49,8 +50,20 @@ public class FileCache implements RefCountedCache { private final CircuitBreaker circuitBreaker; - // TODO: Convert the constant into an integer setting - public static final int DATA_TO_FILE_CACHE_SIZE_RATIO = 5; + /** + * Defines a limit of how much total remote data can be referenced as a ratio of the size of the disk reserved for + * the file cache. For example, if 100GB disk space is configured for use as a file cache and the + * remote_data_ratio of 5 is defined, then a total of 500GB of remote data can be loaded as searchable snapshots. + * This is designed to be a safeguard to prevent oversubscribing a cluster. + * Specify a value of zero for no limit, which is the default for compatibility reasons. + */ + public static final Setting DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING = Setting.doubleSetting( + "cluster.filecache.remote_data_ratio", + 0.0, + 0.0, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); public FileCache(SegmentedCache cache, CircuitBreaker circuitBreaker) { this.theCache = cache; diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java index f23e057196096..9e8b2ee7d1938 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheFactory.java @@ -8,7 +8,7 @@ package org.opensearch.index.store.remote.filecache; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.cache.RemovalReason; import org.opensearch.index.store.remote.utils.cache.SegmentedCache; diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java index 46a81adc1ab45..ebb6202c86ec9 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java @@ -11,7 +11,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java index 3314b2e8d2c8c..4d515fadb5a13 100644 --- a/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/opensearch/index/termvectors/TermVectorsService.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.MultiTerms; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.memory.MemoryIndex; import org.opensearch.OpenSearchException; @@ -50,9 +51,9 @@ import org.opensearch.common.document.DocumentField; import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.engine.Engine; import org.opensearch.index.get.GetResult; import org.opensearch.index.mapper.DocumentMapperForType; @@ -127,7 +128,8 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ /* or from an existing document */ else if (docIdAndVersion != null) { // fields with stored term vectors - termVectorsByField = docIdAndVersion.reader.getTermVectors(docIdAndVersion.docId); + TermVectors termVectors = docIdAndVersion.reader.termVectors(); + termVectorsByField = termVectors.get(docIdAndVersion.docId); Set selectedFields = request.selectedFields(); // generate tvs for fields where analyzer is overridden if (selectedFields == null && request.perFieldAnalyzer() != null) { @@ -322,7 +324,8 @@ private static Fields generateTermVectors( } } /* and read vectors from it */ - return index.createSearcher().getIndexReader().getTermVectors(0); + TermVectors termVectors = index.createSearcher().getIndexReader().termVectors(); + return termVectors.get(0); } private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { @@ -390,13 +393,13 @@ private static ParsedDocument parseDocument( IndexShard indexShard, String index, BytesReference doc, - XContentType xContentType, + MediaType mediaType, String routing ) { MapperService mapperService = indexShard.mapperService(); DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(); ParsedDocument parsedDocument = docMapper.getDocumentMapper() - .parse(new SourceToParse(index, "_id_for_tv_api", doc, xContentType, routing)); + .parse(new SourceToParse(index, "_id_for_tv_api", doc, mediaType, routing)); if (docMapper.getMapping() != null) { parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping()); } diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java index 2862accfedc43..e02381c3f97f3 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java @@ -32,8 +32,8 @@ package org.opensearch.index.translog; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.index.IndexSettings; import org.opensearch.core.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java index c740e29963c4d..935f29a428877 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogStats.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index dd85f8f1f77ea..f3108444688f8 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -44,7 +44,7 @@ import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index 974e8af42b939..95424d86fba34 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -213,17 +213,18 @@ public void listFoldersAsync(String threadpoolName, Iterable path, Actio }); } - public void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener) { - blobStore.blobContainer((BlobPath) path).listBlobsByPrefixInSortedOrder("", limit, LEXICOGRAPHIC, listener); + public void listAllInSortedOrder(Iterable path, String filenamePrefix, int limit, ActionListener> listener) { + blobStore.blobContainer((BlobPath) path).listBlobsByPrefixInSortedOrder(filenamePrefix, limit, LEXICOGRAPHIC, listener); } public void listAllInSortedOrderAsync( String threadpoolName, Iterable path, + String filenamePrefix, int limit, ActionListener> listener ) { - threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, limit, listener); }); + threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, filenamePrefix, limit, listener); }); } } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java index a240fd38cda11..885a2e32e915d 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java @@ -125,8 +125,14 @@ void uploadBlobs( */ InputStream downloadBlob(Iterable path, String fileName) throws IOException; - void listAllInSortedOrder(Iterable path, int limit, ActionListener> listener); - - void listAllInSortedOrderAsync(String threadpoolName, Iterable path, int limit, ActionListener> listener); + void listAllInSortedOrder(Iterable path, String filenamePrefix, int limit, ActionListener> listener); + + void listAllInSortedOrderAsync( + String threadpoolName, + Iterable path, + String filenamePrefix, + int limit, + ActionListener> listener + ); } diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java index e2bb5f74df234..850a0c1a6574c 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java @@ -210,7 +210,12 @@ public TranslogTransferMetadata readMetadata() throws IOException { ); try { - transferService.listAllInSortedOrder(remoteMetadataTransferPath, 1, latchedActionListener); + transferService.listAllInSortedOrder( + remoteMetadataTransferPath, + TranslogTransferMetadata.METADATA_PREFIX, + 1, + latchedActionListener + ); latch.await(); } catch (InterruptedException e) { throw new IOException("Exception while reading/downloading metadafile", e); @@ -367,6 +372,7 @@ public void deleteStaleTranslogMetadataFilesAsync(Runnable onCompletion) { transferService.listAllInSortedOrderAsync( ThreadPool.Names.REMOTE_PURGE, remoteMetadataTransferPath, + TranslogTransferMetadata.METADATA_PREFIX, Integer.MAX_VALUE, new ActionListener<>() { @Override diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java index 75d6549b23f1e..a8b3404d3f2ce 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferMetadata.java @@ -36,6 +36,8 @@ public class TranslogTransferMetadata { public static final String METADATA_SEPARATOR = "__"; + public static final String METADATA_PREFIX = "metadata"; + static final int BUFFER_SIZE = 4096; static final int CURRENT_VERSION = 1; @@ -83,6 +85,7 @@ public String getFileName() { return String.join( METADATA_SEPARATOR, Arrays.asList( + METADATA_PREFIX, RemoteStoreUtils.invertLong(primaryTerm), RemoteStoreUtils.invertLong(generation), RemoteStoreUtils.invertLong(createdAt), diff --git a/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java b/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java index 9a87f1c26fa29..fc580d8517706 100644 --- a/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java +++ b/server/src/main/java/org/opensearch/indices/IndexingMemoryController.java @@ -39,8 +39,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.index.engine.Engine; diff --git a/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java b/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java index 6ebed17437074..8f7df5157d673 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesQueryCache.java @@ -48,7 +48,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.cache.query.QueryCacheStats; import org.opensearch.core.index.shard.ShardId; diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java index c06880db42587..461747a21f3a7 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java +++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java @@ -49,7 +49,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b574ffd1006c0..36b937e7df76a 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -60,9 +60,9 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.common.CheckedSupplier; import org.opensearch.common.Nullable; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -72,7 +72,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.AbstractRefCounted; @@ -82,12 +82,12 @@ import org.opensearch.common.util.iterable.Iterables; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; import org.opensearch.core.util.FileSystemUtils; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.env.NodeEnvironment; @@ -137,7 +137,7 @@ import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory; import org.opensearch.index.translog.TranslogFactory; import org.opensearch.index.translog.TranslogStats; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; @@ -1727,7 +1727,7 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set filterParser = bytes -> { try ( InputStream inputStream = bytes.streamInput(); - XContentParser parser = XContentFactory.xContentType(inputStream) + XContentParser parser = MediaTypeRegistry.xContentType(inputStream) .xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, inputStream) ) { diff --git a/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java b/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java index a421253227dbf..32dfb2c1c7529 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java +++ b/server/src/main/java/org/opensearch/indices/breaker/BreakerSettings.java @@ -32,10 +32,10 @@ package org.opensearch.indices.breaker; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; /** * Settings for a {@link CircuitBreaker} diff --git a/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java b/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java index 40bb4894c7397..34cadd2c828d0 100644 --- a/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/server/src/main/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -37,17 +37,20 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.common.Booleans; import org.opensearch.common.breaker.ChildMemoryCircuitBreaker; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ReleasableLock; +import org.opensearch.core.indices.breaker.AllCircuitBreakerStats; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerStats; import org.opensearch.monitor.jvm.GcNames; import org.opensearch.monitor.jvm.JvmInfo; diff --git a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java index b3fc070d62e58..8f2b6cfde7a34 100644 --- a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java @@ -52,7 +52,7 @@ import org.opensearch.cluster.routing.RecoverySource.Type; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java b/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java index dc18097419904..68364d2c2622a 100644 --- a/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java +++ b/server/src/main/java/org/opensearch/indices/fielddata/cache/IndicesFieldDataCache.java @@ -51,7 +51,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.lease.Releasable; import org.opensearch.core.index.Index; import org.opensearch.index.fielddata.LeafFieldData; diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java index 66b5b3f8b7535..1ff1844d9e6ac 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoverySourceService.java @@ -46,7 +46,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.FutureUtils; diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java index a289c8f8a04b7..0ba57a9ee7f65 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java +++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java @@ -49,7 +49,7 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.AbstractRunnable; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java index 8d60b44c2c6bd..12e628b9b7e7d 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverFilesRecoveryException.java @@ -36,7 +36,7 @@ import org.opensearch.OpenSearchWrapperException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java index 60076f1668af8..a4679a2dbae78 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java @@ -40,8 +40,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; /** diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 0b343fb0b0871..f8ba63def6239 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -49,7 +49,7 @@ import org.opensearch.common.StopWatch; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.logging.Loggers; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.concurrent.FutureUtils; diff --git a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java index 04bee5586cbab..4f1fb42a421f6 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RetryableTransportClient.java @@ -14,7 +14,7 @@ import org.opensearch.action.ActionListenerResponseHandler; import org.opensearch.action.support.RetryableAction; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java index e0e356f1531e1..4712ae6c18759 100644 --- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java +++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java @@ -139,13 +139,25 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener { - if (segrepHandler != null) { - logger.warn("Override handler for allocation id {}", request.getTargetAllocationId()); - cancelHandlers(handler -> handler.getAllocationId().equals(request.getTargetAllocationId()), "cancel due to retry"); - } - return createTargetHandler(request.getTargetNode(), copyState, request.getTargetAllocationId(), fileChunkWriter); - }); + final SegmentReplicationSourceHandler newHandler = createTargetHandler( + request.getTargetNode(), + copyState, + request.getTargetAllocationId(), + fileChunkWriter + ); + final SegmentReplicationSourceHandler existingHandler = allocationIdToHandlers.putIfAbsent( + request.getTargetAllocationId(), + newHandler + ); + // If we are already replicating to this allocation Id, cancel the old and replace with a new execution. + // This will clear the old handler & referenced copy state holding an incref'd indexCommit. + if (existingHandler != null) { + logger.warn("Override handler for allocation id {}", request.getTargetAllocationId()); + cancelHandlers(handler -> handler.getAllocationId().equals(request.getTargetAllocationId()), "cancel due to retry"); + assert allocationIdToHandlers.containsKey(request.getTargetAllocationId()) == false; + allocationIdToHandlers.put(request.getTargetAllocationId(), newHandler); + } + assert allocationIdToHandlers.containsKey(request.getTargetAllocationId()); return copyState; } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java index 9d8b30209ab80..8bb2a61e32e2d 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java @@ -19,7 +19,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexService; import org.opensearch.index.shard.IndexEventListener; diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java index da01023ace47c..7549f3450e7f2 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java @@ -33,6 +33,10 @@ public class ReplicationCheckpoint implements Writeable, Comparable(); } - pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType())); + pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getMediaType())); ClusterState.Builder newState = ClusterState.builder(currentState); newState.metadata( Metadata.builder(currentState.getMetadata()).putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines)).build() @@ -487,7 +487,7 @@ void validatePipeline(Map ingestInfos, PutPipelineReq throw new IllegalStateException("Ingest info is empty"); } - Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); + Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getMediaType()).v2(); Pipeline pipeline = Pipeline.create(request.getId(), pipelineConfig, processorFactories, scriptService); List exceptions = new ArrayList<>(); for (Processor processor : pipeline.flattenAllProcessors()) { diff --git a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java index 08a3708ae60ae..04892e4653065 100644 --- a/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/opensearch/ingest/PipelineConfiguration.java @@ -32,6 +32,7 @@ package org.opensearch.ingest; +import org.opensearch.Version; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.core.ParseField; @@ -77,7 +78,7 @@ private static class Builder { private String id; private BytesReference config; - private XContentType xContentType; + private MediaType mediaType; void setId(String id) { this.id = id; @@ -88,11 +89,11 @@ void setConfig(BytesReference config, MediaType mediaType) { throw new IllegalArgumentException("PipelineConfiguration does not support media type [" + mediaType.getClass() + "]"); } this.config = config; - this.xContentType = XContentType.fromMediaType(mediaType); + this.mediaType = mediaType; } PipelineConfiguration build() { - return new PipelineConfiguration(id, config, xContentType); + return new PipelineConfiguration(id, config, mediaType); } } @@ -101,16 +102,12 @@ PipelineConfiguration build() { // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options) // also the get pipeline api just directly returns this to the caller private final BytesReference config; - private final XContentType xContentType; + private final MediaType mediaType; - public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { + public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { this.id = Objects.requireNonNull(id); this.config = Objects.requireNonNull(config); - this.xContentType = Objects.requireNonNull(xContentType); - } - - public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { - this(id, config, XContentType.fromMediaType(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public String getId() { @@ -118,12 +115,12 @@ public String getId() { } public Map getConfigAsMap() { - return XContentHelper.convertToMap(config, true, xContentType).v2(); + return XContentHelper.convertToMap(config, true, mediaType).v2(); } // pkg-private for tests - XContentType getXContentType() { - return xContentType; + MediaType getMediaType() { + return mediaType; } // pkg-private for tests @@ -141,7 +138,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class)); + return new PipelineConfiguration( + in.readString(), + in.readBytesReference(), + in.getVersion().onOrAfter(Version.V_3_0_0) ? in.readMediaType() : in.readEnum(XContentType.class) + ); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -157,7 +158,11 @@ public String toString() { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override diff --git a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java index 2af0237e800c0..5c904d8a7770d 100644 --- a/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java +++ b/server/src/main/java/org/opensearch/lucene/queries/MinDocQuery.java @@ -86,7 +86,8 @@ public boolean equals(Object obj) { } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(IndexSearcher searcher) throws IOException { + final IndexReader reader = searcher.getIndexReader(); if (Objects.equals(reader.getContext().id(), readerId) == false) { return new MinDocQuery(minDoc, reader.getContext().id()); } diff --git a/server/src/main/java/org/opensearch/monitor/MonitorService.java b/server/src/main/java/org/opensearch/monitor/MonitorService.java index bed638484f7c0..ad02b18366b98 100644 --- a/server/src/main/java/org/opensearch/monitor/MonitorService.java +++ b/server/src/main/java/org/opensearch/monitor/MonitorService.java @@ -32,7 +32,7 @@ package org.opensearch.monitor; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.store.remote.filecache.FileCache; diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index 85a44aa905e08..d8090ada56b93 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -38,7 +38,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java index cfa29436f24a5..114702ff0d351 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsInfo.java @@ -37,7 +37,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java index 50d1d981f3c98..e20d84cd9763e 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsProbe.java @@ -39,7 +39,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.PathUtils; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeEnvironment.NodePath; import org.opensearch.index.store.remote.filecache.FileCache; diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java index 7bdc74d9ae574..78379db6e2cff 100644 --- a/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java +++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmGcMonitorService.java @@ -34,11 +34,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.monitor.jvm.JvmStats.GarbageCollector; import org.opensearch.threadpool.Scheduler.Cancellable; diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java index 1acdf00ca6af0..9eafd80203a87 100644 --- a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java +++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java @@ -39,7 +39,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.ReportingService; @@ -171,7 +171,7 @@ public class JvmInfo implements ReportingService.Info { } final boolean bundledJdk = Booleans.parseBoolean(System.getProperty("opensearch.bundled_jdk", Boolean.FALSE.toString())); - final Boolean usingBundledJdk = bundledJdk ? usingBundledJdk() : null; + final Boolean usingBundledJdkOrJre = bundledJdk ? usingBundledJdkOrJre() : null; INSTANCE = new JvmInfo( JvmPid.getPid(), @@ -180,7 +180,7 @@ public class JvmInfo implements ReportingService.Info { runtimeMXBean.getVmVersion(), runtimeMXBean.getVmVendor(), bundledJdk, - usingBundledJdk, + usingBundledJdkOrJre, runtimeMXBean.getStartTime(), configuredInitialHeapSize, configuredMaxHeapSize, @@ -201,7 +201,7 @@ public class JvmInfo implements ReportingService.Info { } @SuppressForbidden(reason = "PathUtils#get") - private static boolean usingBundledJdk() { + private static boolean usingBundledJdkOrJre() { /* * We are using the bundled JDK if java.home is the jdk sub-directory of our working directory. This is because we always set * the working directory of Elasticsearch to home, and the bundled JDK is in the jdk sub-directory there. @@ -211,7 +211,8 @@ private static boolean usingBundledJdk() { if (Constants.MAC_OS_X) { return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk.app/Contents/Home").toAbsolutePath()); } else { - return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath()); + return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jre").toAbsolutePath()) + || PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath()); } } diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java index 5451821a565cd..b429de614f250 100644 --- a/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java +++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/monitor/os/OsStats.java b/server/src/main/java/org/opensearch/monitor/os/OsStats.java index c684bf10c4a03..697b86c6ba55c 100644 --- a/server/src/main/java/org/opensearch/monitor/os/OsStats.java +++ b/server/src/main/java/org/opensearch/monitor/os/OsStats.java @@ -37,7 +37,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java b/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java index de49c07a5600f..9ab38b41a4648 100644 --- a/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java +++ b/server/src/main/java/org/opensearch/monitor/process/ProcessStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index d768165451a5a..d8b51b0184a66 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -38,8 +38,8 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.common.SetOnce; import org.opensearch.common.settings.SettingsException; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.index.IndexModule; @@ -112,9 +112,9 @@ import org.opensearch.cluster.routing.allocation.DiskThresholdMonitor; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.StopWatch; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.component.LifecycleComponent; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.common.inject.Injector; import org.opensearch.common.inject.Key; import org.opensearch.common.inject.Module; @@ -164,9 +164,9 @@ import org.opensearch.indices.SystemIndices; import org.opensearch.indices.analysis.AnalysisModule; import org.opensearch.indices.breaker.BreakerSettings; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.indices.recovery.PeerRecoverySourceService; import org.opensearch.indices.recovery.PeerRecoveryTargetService; @@ -428,7 +428,7 @@ protected Node( Constants.JVM_VERSION ); if (jvmInfo.getBundledJdk()) { - logger.info("JVM home [{}], using bundled JDK [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk()); + logger.info("JVM home [{}], using bundled JDK/JRE [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk()); } else { logger.info("JVM home [{}]", System.getProperty("java.home")); deprecationLogger.deprecate( @@ -941,8 +941,9 @@ protected Node( clusterModule.getAllocationService(), metadataCreateIndexService, metadataIndexUpgradeService, - clusterService.getClusterSettings(), - shardLimitValidator + shardLimitValidator, + indicesService, + clusterInfoService::getClusterInfo ); final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor( diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java index 6f4fe1e083ad7..95b3825cd992d 100644 --- a/server/src/main/java/org/opensearch/node/NodeService.java +++ b/server/src/main/java/org/opensearch/node/NodeService.java @@ -49,7 +49,7 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.ingest.IngestService; import org.opensearch.monitor.MonitorService; import org.opensearch.plugins.PluginsService; diff --git a/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java b/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java index f52aab5b238b3..ab21d33700d54 100644 --- a/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/CircuitBreakerPlugin.java @@ -32,10 +32,10 @@ package org.opensearch.plugins; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.indices.breaker.BreakerSettings; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; /** * An extension point for {@link Plugin} implementations to add custom circuit breakers diff --git a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java index 8ea8333f4851a..fac29b39ec4c5 100644 --- a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java @@ -45,7 +45,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.http.HttpServerTransport; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; import org.opensearch.transport.TransportInterceptor; diff --git a/server/src/main/java/org/opensearch/plugins/Plugin.java b/server/src/main/java/org/opensearch/plugins/Plugin.java index 02a46f44ef23b..e8c479a48d82b 100644 --- a/server/src/main/java/org/opensearch/plugins/Plugin.java +++ b/server/src/main/java/org/opensearch/plugins/Plugin.java @@ -41,7 +41,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.component.LifecycleComponent; +import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.common.inject.Module; import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; diff --git a/server/src/main/java/org/opensearch/plugins/PluginsService.java b/server/src/main/java/org/opensearch/plugins/PluginsService.java index 3cd226c357367..67ce664c629ec 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginsService.java +++ b/server/src/main/java/org/opensearch/plugins/PluginsService.java @@ -44,7 +44,7 @@ import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; import org.opensearch.bootstrap.JarHell; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.component.LifecycleComponent; +import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.common.inject.Module; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java index 462d105dc0c68..52e7f374507d4 100644 --- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java @@ -40,8 +40,8 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.component.LifecycleListener; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.index.mapper.MapperService; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index e7f7a1d9c0554..e66f8ddee5678 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -58,7 +58,7 @@ import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 8a712b0a0c098..683177b3c3639 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -42,7 +42,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; -import org.opensearch.common.component.LifecycleComponent; +import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.index.mapper.MapperService; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index ae8080da73fa8..70db2e0c0a9bd 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -78,7 +78,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.compress.Compressor; import org.opensearch.common.compress.CompressorFactory; import org.opensearch.common.compress.CompressorType; @@ -89,8 +89,8 @@ import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java index cb3d779ece4a9..a138bfc8a7044 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -43,20 +43,20 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.CheckedFunction; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.Compressor; import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.lucene.store.IndexOutputOutputStream; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.gateway.CorruptStateException; import org.opensearch.snapshots.SnapshotInfo; @@ -185,7 +185,7 @@ public void close() throws IOException { // in order to write the footer we need to prevent closing the actual index input. } }; - XContentBuilder builder = XContentFactory.contentBuilder( + XContentBuilder builder = MediaTypeRegistry.contentBuilder( XContentType.SMILE, compressor.threadLocalOutputStream(indexOutputOutputStream) ) diff --git a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java index 0b9989ff64d9c..6971d8ec00ecf 100644 --- a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java @@ -41,7 +41,7 @@ import org.opensearch.common.blobstore.fs.FsBlobStore; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; diff --git a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java index 2fc6ef86df603..11a116e8c858d 100644 --- a/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/opensearch/rest/AbstractRestChannel.java @@ -35,9 +35,9 @@ import org.opensearch.common.io.Streams; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import java.io.OutputStream; @@ -86,43 +86,40 @@ protected AbstractRestChannel(RestRequest request, boolean detailedErrorsEnabled @Override public XContentBuilder newBuilder() throws IOException { - return newBuilder(request.getXContentType(), true); + return newBuilder(request.getMediaType(), true); } @Override public XContentBuilder newErrorBuilder() throws IOException { // Disable filtering when building error responses - return newBuilder(request.getXContentType(), false); + return newBuilder(request.getMediaType(), false); } /** * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following - * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format - * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map + * logic. If the request has a format parameter that will be used to attempt to map to an {@link MediaType}. If there is no format + * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link MediaType}. If this first attempt to map * fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls * back to JSON. */ @Override - public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException { + public XContentBuilder newBuilder(@Nullable MediaType requestContentType, boolean useFiltering) throws IOException { return newBuilder(requestContentType, null, useFiltering); } /** * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type can be sent as a parameter, - * through {@code responseContentType} or it can fallback to {@link #newBuilder(XContentType, boolean)} logic if the sent type value + * through {@code responseContentType} or it can fallback to {@link #newBuilder(MediaType, boolean)} logic if the sent type value * is {@code null}. */ @Override - public XContentBuilder newBuilder( - @Nullable XContentType requestContentType, - @Nullable XContentType responseContentType, - boolean useFiltering - ) throws IOException { + public XContentBuilder newBuilder(@Nullable MediaType requestContentType, @Nullable MediaType responseContentType, boolean useFiltering) + throws IOException { if (responseContentType == null) { // TODO should format vs acceptHeader always be the same, do we allow overriding? - responseContentType = XContentType.fromFormat(format); + responseContentType = MediaType.fromFormat(format); if (responseContentType == null) { - responseContentType = XContentType.fromMediaType(acceptHeader); + responseContentType = MediaType.fromMediaType(acceptHeader); } } // try to determine the response content type from the media type or the format query string parameter, with the format parameter @@ -134,7 +131,7 @@ public XContentBuilder newBuilder( responseContentType = requestContentType; } else { // default to JSON output when all else fails - responseContentType = XContentType.JSON; + responseContentType = MediaTypeRegistry.getDefaultMediaType(); } } @@ -147,12 +144,7 @@ public XContentBuilder newBuilder( } OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput()); - XContentBuilder builder = new XContentBuilder( - XContentFactory.xContent(responseContentType), - unclosableOutputStream, - includes, - excludes - ); + XContentBuilder builder = new XContentBuilder(responseContentType.xContent(), unclosableOutputStream, includes, excludes); if (pretty) { builder.prettyPrint().lfAtEnd(); } diff --git a/server/src/main/java/org/opensearch/rest/RestChannel.java b/server/src/main/java/org/opensearch/rest/RestChannel.java index 6e9e1559ce104..b8ce3e92e0098 100644 --- a/server/src/main/java/org/opensearch/rest/RestChannel.java +++ b/server/src/main/java/org/opensearch/rest/RestChannel.java @@ -34,8 +34,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -50,9 +50,9 @@ public interface RestChannel { XContentBuilder newErrorBuilder() throws IOException; - XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException; + XContentBuilder newBuilder(@Nullable MediaType mediaType, boolean useFiltering) throws IOException; - XContentBuilder newBuilder(@Nullable XContentType xContentType, @Nullable XContentType responseContentType, boolean useFiltering) + XContentBuilder newBuilder(@Nullable MediaType mediaType, @Nullable MediaType responseContentType, boolean useFiltering) throws IOException; BytesStreamOutput bytesOutput(); diff --git a/server/src/main/java/org/opensearch/rest/RestController.java b/server/src/main/java/org/opensearch/rest/RestController.java index 87d1542102f91..1a8bddc094d26 100644 --- a/server/src/main/java/org/opensearch/rest/RestController.java +++ b/server/src/main/java/org/opensearch/rest/RestController.java @@ -38,7 +38,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.client.node.NodeClient; import org.opensearch.common.Nullable; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -47,6 +47,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentType; @@ -56,7 +57,7 @@ import org.opensearch.identity.Subject; import org.opensearch.identity.tokens.AuthToken; import org.opensearch.identity.tokens.RestTokenExtractor; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.usage.UsageService; import java.io.ByteArrayOutputStream; @@ -282,17 +283,17 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th private void dispatchRequest(RestRequest request, RestChannel channel, RestHandler handler) throws Exception { final int contentLength = request.content().length(); if (contentLength > 0) { - final XContentType xContentType = request.getXContentType(); - if (xContentType == null) { + final MediaType mediaType = request.getMediaType(); + if (mediaType == null) { sendContentTypeErrorMessage(request.getAllHeaderValues("Content-Type"), channel); return; } - if (handler.supportsContentStream() && xContentType != XContentType.JSON && xContentType != XContentType.SMILE) { + if (handler.supportsContentStream() && mediaType != XContentType.JSON && mediaType != XContentType.SMILE) { channel.sendResponse( BytesRestResponse.createSimpleErrorResponse( channel, RestStatus.NOT_ACCEPTABLE, - "Content-Type [" + xContentType + "] does not support stream parsing. Use JSON or SMILE instead" + "Content-Type [" + mediaType + "] does not support stream parsing. Use JSON or SMILE instead" ) ); return; @@ -592,14 +593,13 @@ public XContentBuilder newErrorBuilder() throws IOException { } @Override - public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { - return delegate.newBuilder(xContentType, useFiltering); + public XContentBuilder newBuilder(@Nullable MediaType mediaType, boolean useFiltering) throws IOException { + return delegate.newBuilder(mediaType, useFiltering); } @Override - public XContentBuilder newBuilder(XContentType xContentType, XContentType responseContentType, boolean useFiltering) - throws IOException { - return delegate.newBuilder(xContentType, responseContentType, useFiltering); + public XContentBuilder newBuilder(MediaType mediaType, MediaType responseContentType, boolean useFiltering) throws IOException { + return delegate.newBuilder(mediaType, responseContentType, useFiltering); } @Override diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index 653720391d6a3..fb497a17cde72 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -41,9 +41,10 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; @@ -64,7 +65,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; import static org.opensearch.common.unit.TimeValue.parseTimeValue; /** @@ -84,7 +85,7 @@ public class RestRequest implements ToXContent.Params { private final Map> headers; private final String rawPath; private final Set consumedParams = new HashSet<>(); - private final SetOnce xContentType = new SetOnce<>(); + private final SetOnce mediaType = new SetOnce<>(); private final HttpChannel httpChannel; private HttpRequest httpRequest; @@ -117,14 +118,14 @@ private RestRequest( HttpChannel httpChannel, long requestId ) { - final XContentType xContentType; + final MediaType xContentType; try { xContentType = parseContentType(headers.get("Content-Type")); } catch (final IllegalArgumentException e) { throw new ContentTypeHeaderException(e); } if (xContentType != null) { - this.xContentType.set(xContentType); + this.mediaType.set(xContentType); } this.xContentRegistry = xContentRegistry; this.httpRequest = httpRequest; @@ -295,7 +296,7 @@ protected BytesReference content(final boolean contentConsumed) { public final BytesReference requiredContent() { if (hasContent() == false) { throw new OpenSearchParseException("request body is required"); - } else if (xContentType.get() == null) { + } else if (mediaType.get() == null) { throw new IllegalStateException("unknown content type"); } return content(); @@ -340,8 +341,8 @@ public final long getRequestId() { * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request */ @Nullable - public final XContentType getXContentType() { - return xContentType.get(); + public final MediaType getMediaType() { + return mediaType.get(); } public HttpChannel getHttpChannel() { @@ -486,7 +487,7 @@ public NamedXContentRegistry getXContentRegistry() { */ public final XContentParser contentParser() throws IOException { BytesReference content = requiredContent(); // will throw exception if body or content type missing - return xContentType.get().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput()); + return mediaType.get().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput()); } /** @@ -514,7 +515,7 @@ public final boolean hasContentOrSourceParam() { * if you need to handle the absence request content gracefully. */ public final XContentParser contentOrSourceParamParser() throws IOException { - Tuple tuple = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); return tuple.v1().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, tuple.v2().streamInput()); } @@ -525,12 +526,12 @@ public final XContentParser contentOrSourceParamParser() throws IOException { */ public final void withContentOrSourceParamParserOrNull(CheckedConsumer withParser) throws IOException { if (hasContentOrSourceParam()) { - Tuple tuple = contentOrSourceParam(); + Tuple tuple = contentOrSourceParam(); BytesReference content = tuple.v2(); - XContentType xContentType = tuple.v1(); + MediaType mediaType = tuple.v1(); try ( InputStream stream = content.streamInput(); - XContentParser parser = xContentType.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream) + XContentParser parser = mediaType.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, stream) ) { withParser.accept(parser); } @@ -543,11 +544,11 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer contentOrSourceParam() { + public final Tuple contentOrSourceParam() { if (hasContentOrSourceParam() == false) { throw new OpenSearchParseException("request body or source parameter is required"); } else if (hasContent()) { - return new Tuple<>(xContentType.get(), requiredContent()); + return new Tuple<>(mediaType.get(), requiredContent()); } String source = param("source"); String typeParam = param("source_content_type"); @@ -555,18 +556,18 @@ public final Tuple contentOrSourceParam() { throw new IllegalStateException("source and source_content_type parameters are required"); } BytesArray bytes = new BytesArray(source); - final XContentType xContentType = parseContentType(Collections.singletonList(typeParam)); - if (xContentType == null) { + final MediaType mediaType = parseContentType(Collections.singletonList(typeParam)); + if (mediaType == null) { throw new IllegalStateException("Unknown value for source_content_type [" + typeParam + "]"); } - return new Tuple<>(xContentType, bytes); + return new Tuple<>(mediaType, bytes); } /** * Parses the given content type string for the media type. This method currently ignores parameters. */ // TODO stop ignoring parameters such as charset... - public static XContentType parseContentType(List header) { + public static MediaType parseContentType(List header) { if (header == null || header.isEmpty()) { return null; } else if (header.size() > 1) { @@ -580,7 +581,7 @@ public static XContentType parseContentType(List header) { if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches() && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) { - return XContentType.fromMediaType(elements[0]); + return MediaType.fromMediaType(elements[0]); } else { throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]"); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java index 28381af769a51..47ab7ae585039 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestPutStoredScriptAction.java @@ -35,7 +35,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.logging.DeprecationLogger; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -80,10 +80,10 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client String id = request.param("id"); String context = request.param("context"); BytesReference content = request.requiredContent(); - XContentType xContentType = request.getXContentType(); - StoredScriptSource source = StoredScriptSource.parse(content, xContentType); + MediaType mediaType = request.getMediaType(); + StoredScriptSource source = StoredScriptSource.parse(content, mediaType); - PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, context, content, request.getXContentType(), source); + PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, context, content, request.getMediaType(), source); putRequest.clusterManagerNodeTimeout(request.paramAsTime("cluster_manager_timeout", putRequest.clusterManagerNodeTimeout())); parseDeprecatedMasterTimeoutParameter(putRequest, request, deprecationLogger, getName()); putRequest.timeout(request.paramAsTime("timeout", putRequest.timeout())); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java index fca6745167bb4..414c82b4a470f 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java @@ -44,6 +44,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC request.paramAsTime("cluster_manager_timeout", restoreRemoteStoreRequest.masterNodeTimeout()) ); restoreRemoteStoreRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false)); + restoreRemoteStoreRequest.restoreAllShards(request.paramAsBoolean("restore_all_shards", false)); request.applyContentParser(p -> restoreRemoteStoreRequest.source(p.mapOrdered())); return channel -> client.admin().cluster().restoreRemoteStore(restoreRemoteStoreRequest, new RestToXContentListener<>(channel)); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java index e7335eee89c5a..d987e8fe63506 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -76,7 +76,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2(); sourceAsMap = prepareMappings(sourceAsMap); createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 86b0111e4b064..dde9f470161b9 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -88,7 +88,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2(); sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); putRequest.source(sourceAsMap); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java index fd293dceb4cc1..8fdf000139d89 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java @@ -83,7 +83,7 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getMediaType()).v2(); if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java index 702ffff1c9330..176e37abc0f65 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Table; import org.opensearch.common.logging.DeprecationLogger; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 52890274d4198..2dfed426f35d6 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -14,7 +14,7 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentOpenSearchExtension; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java index a04bac0c30bc9..04bbdeeadc4c4 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java index 6346e5d23cd34..b54c8955283a2 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java @@ -49,7 +49,7 @@ import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.network.NetworkAddress; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.http.HttpInfo; import org.opensearch.index.cache.query.QueryCacheStats; import org.opensearch.index.cache.request.RequestCacheStats; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 556928d4685a0..ec44b06318695 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -38,12 +38,12 @@ import org.opensearch.common.io.UTF8StreamWriter; import org.opensearch.core.common.io.stream.BytesStream; import org.opensearch.common.regex.Regex; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.SizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; @@ -69,18 +69,18 @@ public class RestTable { public static RestResponse buildResponse(Table table, RestChannel channel) throws Exception { RestRequest request = channel.request(); - XContentType xContentType = getXContentType(request); + MediaType xContentType = getXContentType(request); if (xContentType != null) { return buildXContentBuilder(table, channel); } return buildTextPlainResponse(table, channel); } - private static XContentType getXContentType(RestRequest request) { + private static MediaType getXContentType(RestRequest request) { if (request.hasParam("format")) { - return XContentType.fromFormat(request.param("format")); + return MediaType.fromFormat(request.param("format")); } - return XContentType.fromMediaType(request.header("Accept")); + return MediaType.fromMediaType(request.header("Accept")); } public static RestResponse buildXContentBuilder(Table table, RestChannel channel) throws Exception { diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java index 3fc02db0a8365..b046146707885 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java @@ -105,7 +105,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC defaultPipeline, defaultRequireAlias, allowExplicitIndex, - request.getXContentType() + request.getMediaType() ); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java index 7247067755721..ce5e12d6fd163 100644 --- a/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/opensearch/rest/action/document/RestGetSourceAction.java @@ -38,8 +38,8 @@ import org.opensearch.action.get.GetResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; @@ -108,10 +108,10 @@ static class RestGetSourceResponseListener extends RestResponseListener sourceTuple = restRequest.contentOrSourceParam(); + Tuple sourceTuple = restRequest.contentOrSourceParam(); PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1()); request.clusterManagerNodeTimeout(restRequest.paramAsTime("cluster_manager_timeout", request.clusterManagerNodeTimeout())); parseDeprecatedMasterTimeoutParameter(request, restRequest, deprecationLogger, getName()); diff --git a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java index ec8cc2c00e65a..1aa2c11e3f05e 100644 --- a/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java +++ b/server/src/main/java/org/opensearch/rest/action/ingest/RestSimulatePipelineAction.java @@ -36,7 +36,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -75,7 +75,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - Tuple sourceTuple = restRequest.contentOrSourceParam(); + Tuple sourceTuple = restRequest.contentOrSourceParam(); SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1()); request.setId(restRequest.param("id")); request.setVerbose(restRequest.paramAsBoolean("verbose", false)); diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java index 189b1e93e6444..5be5a54edddf5 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestMultiSearchAction.java @@ -45,9 +45,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestCancellableNodeClient; @@ -190,7 +190,7 @@ public static void parseMultiLineRequest( boolean ccsMinimizeRoundtrips = request.paramAsBoolean("ccs_minimize_roundtrips", true); String routing = request.param("routing"); - final Tuple sourceTuple = request.contentOrSourceParam(); + final Tuple sourceTuple = request.contentOrSourceParam(); final XContent xContent = sourceTuple.v1().xContent(); final BytesReference data = sourceTuple.v2(); MultiSearchRequest.readMultiLineFormat( diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java index 2f398a6bea9ff..73eead1b91d57 100644 --- a/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java +++ b/server/src/main/java/org/opensearch/rest/action/search/RestPutSearchPipelineAction.java @@ -12,7 +12,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -40,7 +40,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - Tuple sourceTuple = restRequest.contentOrSourceParam(); + Tuple sourceTuple = restRequest.contentOrSourceParam(); PutSearchPipelineRequest request = new PutSearchPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1()); request.clusterManagerNodeTimeout(restRequest.paramAsTime("cluster_manager_timeout", request.clusterManagerNodeTimeout())); request.timeout(restRequest.paramAsTime("timeout", request.timeout())); diff --git a/server/src/main/java/org/opensearch/script/ScriptCache.java b/server/src/main/java/org/opensearch/script/ScriptCache.java index 874888b53b978..439953f816cbb 100644 --- a/server/src/main/java/org/opensearch/script/ScriptCache.java +++ b/server/src/main/java/org/opensearch/script/ScriptCache.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.cache.Cache; import org.opensearch.common.cache.CacheBuilder; import org.opensearch.common.cache.RemovalListener; diff --git a/server/src/main/java/org/opensearch/script/StoredScriptSource.java b/server/src/main/java/org/opensearch/script/StoredScriptSource.java index 1d6f2b8e4ccea..9d3af9e4c9caf 100644 --- a/server/src/main/java/org/opensearch/script/StoredScriptSource.java +++ b/server/src/main/java/org/opensearch/script/StoredScriptSource.java @@ -44,6 +44,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ObjectParser.ValueType; @@ -197,71 +198,72 @@ private StoredScriptSource build(boolean ignoreEmpty) { /** * This will parse XContent into a {@link StoredScriptSource}. The following formats can be parsed: - * + *

* The simple script format with no compiler options or user-defined params: - * + *

* Example: * {@code * {"script": "return Math.log(doc.popularity) * 100;"} * } - * + *

* The above format requires the lang to be specified using the deprecated stored script namespace * (as a url parameter during a put request). See {@link ScriptMetadata} for more information about * the stored script namespaces. - * + *

* The complex script format using the new stored script namespace * where lang and source are required but options is optional: - * + *

* {@code * { - * "script" : { - * "lang" : "", - * "source" : "", - * "options" : { - * "option0" : "", - * "option1" : "", - * ... - * } - * } + * "script" : { + * "lang" : "", + * "source" : "", + * "options" : { + * "option0" : "", + * "option1" : "", + * ... * } * } - * + * } + * } + *

* Example: * {@code * { - * "script": { - * "lang" : "painless", - * "source" : "return Math.log(doc.popularity) * params.multiplier" - * } + * "script": { + * "lang" : "painless", + * "source" : "return Math.log(doc.popularity) * params.multiplier" * } * } - * + * } + *

* The use of "source" may also be substituted with "code" for backcompat with 5.3 to 5.5 format. For example: - * + *

* {@code * { - * "script" : { - * "lang" : "", - * "code" : "", - * "options" : { - * "option0" : "", - * "option1" : "", - * ... - * } - * } + * "script" : { + * "lang" : "", + * "code" : "", + * "options" : { + * "option0" : "", + * "option1" : "", + * ... * } * } - * + * } + * } + *

* Note that the "source" parameter can also handle template parsing including from * a complex JSON object. * - * @param content The content from the request to be parsed as described above. - * @return The parsed {@link StoredScriptSource}. + * @param content The content from the request to be parsed as described above. + * @param mediaType The media type of the request + * @return The parsed {@link StoredScriptSource}. */ - public static StoredScriptSource parse(BytesReference content, XContentType xContentType) { + public static StoredScriptSource parse(BytesReference content, MediaType mediaType) { try ( InputStream stream = content.streamInput(); - XContentParser parser = xContentType.xContent() + XContentParser parser = mediaType.xContent() .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream) ) { Token token = parser.nextToken(); diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index c9c70ed52c376..b244290e8ae74 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -53,8 +53,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedSupplier; import org.opensearch.common.UUIDs; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.Lucene; @@ -87,7 +87,7 @@ import org.opensearch.index.shard.SearchOperationListener; import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import org.opensearch.node.ResponseCollectorService; import org.opensearch.script.FieldScript; @@ -1270,7 +1270,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc context.minimumScore(source.minScore()); } if (source.profile()) { - context.setProfilers(new Profilers(context.searcher())); + context.setProfilers(new Profilers(context.searcher(), context.isConcurrentSegmentSearchEnabled())); } if (source.timeout() != null) { context.timeout(source.timeout()); diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java index 1d315980512b4..392c65ce27aea 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java @@ -34,9 +34,9 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreMode; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; diff --git a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java index 83a94afbe8e02..f1416fddebfa2 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java +++ b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java @@ -32,7 +32,7 @@ package org.opensearch.search.aggregations; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java index e5cc3f9dbaabd..7f5804c8b9561 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java @@ -148,7 +148,6 @@ protected Aggregator createInternal( : includeExclude.convertToStringFilter(DocValueFormat.RAW, maxRegexLength); MapStringTermsAggregator.CollectorSource collectorSource = new SignificantTextCollectorSource( - queryShardContext.lookup().source(), queryShardContext.bigArrays(), fieldType, sourceFieldNames, @@ -186,13 +185,14 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg private ObjectArray dupSequenceSpotters; SignificantTextCollectorSource( - SourceLookup sourceLookup, BigArrays bigArrays, MappedFieldType fieldType, String[] sourceFieldNames, boolean filterDuplicateText ) { - this.sourceLookup = sourceLookup; + // Create a new SourceLookup instance per aggregator instead of use the shared one from SearchLookup. This is fine because it + // will only be accessed by this Aggregator instance and not anywhere else. + this.sourceLookup = new SourceLookup(); this.bigArrays = bigArrays; this.fieldType = fieldType; this.sourceFieldNames = sourceFieldNames; diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index 246078e7a8eda..4f6c2c327509d 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -13,7 +13,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.search.SearchShardTask; import org.opensearch.action.search.SearchTask; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.monitor.jvm.JvmStats; diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java index fbd2155c0ef89..ed2ae0bde86c3 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/HeapUsageTracker.java @@ -14,7 +14,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.MovingAverage; import org.opensearch.monitor.jvm.JvmStats; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java index b82764c38f747..67c6eeae02271 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java @@ -52,6 +52,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.IndexSettings; import org.opensearch.index.fieldvisitor.CustomFieldsVisitor; import org.opensearch.index.fieldvisitor.FieldsVisitor; @@ -159,7 +160,7 @@ public void execute(SearchContext context) { SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) currentReaderContext.reader(); fieldReader = lf.getSequentialStoredFieldsReader()::document; } else { - fieldReader = currentReaderContext.reader()::document; + fieldReader = currentReaderContext.reader().storedFields()::document; } for (FetchSubPhaseProcessor processor : processors) { processor.setNextReader(currentReaderContext); @@ -377,7 +378,7 @@ private HitContext prepareNestedHitContext( String rootId; Map rootSourceAsMap = null; - XContentType rootSourceContentType = null; + MediaType rootSourceContentType = null; int nestedDocId = nestedTopDocId - subReaderContext.docBase; diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java index 7a358b7e4b252..2238554a12149 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java @@ -72,7 +72,7 @@ public static List loadFieldValues( ) throws IOException { if (forceSource == false && fieldType.isStored()) { CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(singleton(fieldType.name()), false); - hitContext.reader().document(hitContext.docId(), fieldVisitor); + hitContext.reader().storedFields().document(hitContext.docId(), fieldVisitor); List textsToHighlight = fieldVisitor.fields().get(fieldType.name()); return textsToHighlight != null ? textsToHighlight : Collections.emptyList(); } diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index e3ca932eb4699..5384b47cc69ec 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -343,6 +343,10 @@ private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collecto } } } + + // Note: this is called if collection ran successfully, including the above special cases of + // CollectionTerminatedException and TimeExceededException, but no other exception. + leafCollector.finish(); } private Weight wrapWeight(Weight weight) { diff --git a/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java b/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java index 007ee49b3e697..47bb8754a5803 100644 --- a/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/LeafFieldsLookup.java @@ -153,7 +153,7 @@ private FieldLookup loadFieldData(String name) { List values = new ArrayList<>(2); SingleFieldsVisitor visitor = new SingleFieldsVisitor(data.fieldType(), values); try { - reader.document(docId, visitor); + reader.storedFields().document(docId, visitor); } catch (IOException e) { throw new OpenSearchParseException("failed to load field [{}]", e, name); } diff --git a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java index 1d4a424550c12..00fdca4e143ee 100644 --- a/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/opensearch/search/lookup/SourceLookup.java @@ -75,8 +75,8 @@ public Map source() { return source; } - public XContentType sourceContentType() { - return XContentType.fromMediaType(sourceContentType); + public MediaType sourceContentType() { + return sourceContentType; } public int docId() { @@ -140,7 +140,7 @@ public void setSegmentAndDocument(LeafReaderContext context, int docId) { SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) context.reader(); fieldReader = lf.getSequentialStoredFieldsReader()::document; } else { - fieldReader = context.reader()::document; + fieldReader = context.reader().storedFields()::document; } } catch (IOException e) { throw new UncheckedIOException(e); @@ -155,7 +155,7 @@ public void setSource(BytesReference source) { this.sourceAsBytes = source; } - public void setSourceContentType(XContentType sourceContentType) { + public void setSourceContentType(MediaType sourceContentType) { this.sourceContentType = sourceContentType; } diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java index 512e400c54784..b4f6549c83390 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineConfiguration.java @@ -8,6 +8,7 @@ package org.opensearch.search.pipeline; +import org.opensearch.Version; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; import org.opensearch.common.Strings; @@ -56,7 +57,7 @@ private static class Builder { private String id; private BytesReference config; - private XContentType xContentType; + private MediaType mediaType; void setId(String id) { this.id = id; @@ -67,11 +68,11 @@ void setConfig(BytesReference config, MediaType mediaType) { throw new IllegalArgumentException("PipelineConfiguration does not support media type [" + mediaType.getClass() + "]"); } this.config = config; - this.xContentType = XContentType.fromMediaType(mediaType); + this.mediaType = mediaType; } PipelineConfiguration build() { - return new PipelineConfiguration(id, config, xContentType); + return new PipelineConfiguration(id, config, mediaType); } } @@ -80,16 +81,12 @@ PipelineConfiguration build() { // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options) // also the get pipeline api just directly returns this to the caller private final BytesReference config; - private final XContentType xContentType; + private final MediaType mediaType; - public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) { + public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { this.id = Objects.requireNonNull(id); this.config = Objects.requireNonNull(config); - this.xContentType = Objects.requireNonNull(xContentType); - } - - public PipelineConfiguration(String id, BytesReference config, MediaType mediaType) { - this(id, config, XContentType.fromMediaType(mediaType)); + this.mediaType = Objects.requireNonNull(mediaType); } public String getId() { @@ -97,12 +94,12 @@ public String getId() { } public Map getConfigAsMap() { - return XContentHelper.convertToMap(config, true, xContentType).v2(); + return XContentHelper.convertToMap(config, true, mediaType).v2(); } // pkg-private for tests - XContentType getXContentType() { - return xContentType; + MediaType getMediaType() { + return mediaType; } // pkg-private for tests @@ -120,7 +117,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class)); + return new PipelineConfiguration( + in.readString(), + in.readBytesReference(), + in.getVersion().onOrAfter(Version.V_3_0_0) ? in.readMediaType() : in.readEnum(XContentType.class) + ); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -136,7 +137,11 @@ public String toString() { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); - out.writeEnum(xContentType); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + mediaType.writeTo(out); + } else { + out.writeEnum((XContentType) mediaType); + } } @Override diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index 7384b494d71fc..1066d836e5183 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -259,7 +259,7 @@ static ClusterState innerPut(PutSearchPipelineRequest request, ClusterState curr } else { pipelines = new HashMap<>(); } - pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType())); + pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getMediaType())); ClusterState.Builder newState = ClusterState.builder(currentState); newState.metadata( Metadata.builder(currentState.getMetadata()) @@ -273,7 +273,7 @@ void validatePipeline(Map searchPipelineInfos if (searchPipelineInfos.isEmpty()) { throw new IllegalStateException("Search pipeline info is empty"); } - Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2(); + Map pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getMediaType()).v2(); Pipeline pipeline = PipelineWithMetrics.create( request.getId(), pipelineConfig, diff --git a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java index a29d4f9a0ee20..67ab062c0e3ca 100644 --- a/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/AbstractProfileBreakdown.java @@ -50,8 +50,10 @@ public abstract class AbstractProfileBreakdown> { /** * The accumulated timings for this query node */ - private final Timer[] timings; - private final T[] timingTypes; + protected final Timer[] timings; + protected final T[] timingTypes; + public static final String TIMING_TYPE_COUNT_SUFFIX = "_count"; + public static final String TIMING_TYPE_START_TIME_SUFFIX = "_start_time"; /** Sole constructor. */ public AbstractProfileBreakdown(Class clazz) { @@ -74,17 +76,10 @@ public void setTimer(T timing, Timer timer) { * Build a timing count breakdown for current instance */ public Map toBreakdownMap() { - return buildBreakdownMap(this); - } - - /** - * Build a timing count breakdown for arbitrary instance - */ - protected final Map buildBreakdownMap(AbstractProfileBreakdown breakdown) { - Map map = new HashMap<>(breakdown.timings.length * 2); - for (T timingType : breakdown.timingTypes) { - map.put(timingType.toString(), breakdown.timings[timingType.ordinal()].getApproximateTiming()); - map.put(timingType.toString() + "_count", breakdown.timings[timingType.ordinal()].getCount()); + Map map = new HashMap<>(this.timings.length * 3); + for (T timingType : this.timingTypes) { + map.put(timingType.toString(), this.timings[timingType.ordinal()].getApproximateTiming()); + map.put(timingType + TIMING_TYPE_COUNT_SUFFIX, this.timings[timingType.ordinal()].getCount()); } return Collections.unmodifiableMap(map); } diff --git a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java index 89c3d7504de66..d96db1d2dd8da 100644 --- a/server/src/main/java/org/opensearch/search/profile/ProfileResult.java +++ b/server/src/main/java/org/opensearch/search/profile/ProfileResult.java @@ -32,6 +32,7 @@ package org.opensearch.search.profile; +import org.opensearch.Version; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -44,8 +45,10 @@ import java.io.IOException; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.LinkedHashMap; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -69,7 +72,13 @@ public final class ProfileResult implements Writeable, ToXContentObject { static final ParseField BREAKDOWN = new ParseField("breakdown"); static final ParseField DEBUG = new ParseField("debug"); static final ParseField NODE_TIME = new ParseField("time"); + static final ParseField MAX_SLICE_NODE_TIME = new ParseField("max_slice_time"); + static final ParseField MIN_SLICE_NODE_TIME = new ParseField("min_slice_time"); + static final ParseField AVG_SLICE_NODE_TIME = new ParseField("avg_slice_time"); static final ParseField NODE_TIME_RAW = new ParseField("time_in_nanos"); + static final ParseField MAX_SLICE_NODE_TIME_RAW = new ParseField("max_slice_time_in_nanos"); + static final ParseField MIN_SLICE_NODE_TIME_RAW = new ParseField("min_slice_time_in_nanos"); + static final ParseField AVG_SLICE_NODE_TIME_RAW = new ParseField("avg_slice_time_in_nanos"); static final ParseField CHILDREN = new ParseField("children"); private final String type; @@ -77,6 +86,9 @@ public final class ProfileResult implements Writeable, ToXContentObject { private final Map breakdown; private final Map debug; private final long nodeTime; + private Long maxSliceNodeTime; + private Long minSliceNodeTime; + private Long avgSliceNodeTime; private final List children; public ProfileResult( @@ -86,6 +98,20 @@ public ProfileResult( Map debug, long nodeTime, List children + ) { + this(type, description, breakdown, debug, nodeTime, children, null, null, null); + } + + public ProfileResult( + String type, + String description, + Map breakdown, + Map debug, + long nodeTime, + List children, + Long maxSliceNodeTime, + Long minSliceNodeTime, + Long avgSliceNodeTime ) { this.type = type; this.description = description; @@ -93,6 +119,9 @@ public ProfileResult( this.debug = debug == null ? Map.of() : debug; this.children = children == null ? List.of() : children; this.nodeTime = nodeTime; + this.maxSliceNodeTime = maxSliceNodeTime; + this.minSliceNodeTime = minSliceNodeTime; + this.avgSliceNodeTime = avgSliceNodeTime; } /** @@ -105,6 +134,15 @@ public ProfileResult(StreamInput in) throws IOException { breakdown = in.readMap(StreamInput::readString, StreamInput::readLong); debug = in.readMap(StreamInput::readString, StreamInput::readGenericValue); children = in.readList(ProfileResult::new); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.maxSliceNodeTime = in.readOptionalLong(); + this.minSliceNodeTime = in.readOptionalLong(); + this.avgSliceNodeTime = in.readOptionalLong(); + } else { + this.maxSliceNodeTime = null; + this.minSliceNodeTime = null; + this.avgSliceNodeTime = null; + } } @Override @@ -115,6 +153,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(breakdown, StreamOutput::writeString, StreamOutput::writeLong); out.writeMap(debug, StreamOutput::writeString, StreamOutput::writeGenericValue); out.writeList(children); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalLong(maxSliceNodeTime); + out.writeOptionalLong(minSliceNodeTime); + out.writeOptionalLong(avgSliceNodeTime); + } } /** @@ -154,6 +197,18 @@ public long getTime() { return nodeTime; } + public Long getMaxSliceTime() { + return maxSliceNodeTime; + } + + public Long getMinSliceTime() { + return minSliceNodeTime; + } + + public Long getAvgSliceTime() { + return avgSliceNodeTime; + } + /** * Returns a list of all profiled children queries */ @@ -168,9 +223,27 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DESCRIPTION.getPreferredName(), description); if (builder.humanReadable()) { builder.field(NODE_TIME.getPreferredName(), new TimeValue(getTime(), TimeUnit.NANOSECONDS).toString()); + if (getMaxSliceTime() != null) { + builder.field(MAX_SLICE_NODE_TIME.getPreferredName(), new TimeValue(getMaxSliceTime(), TimeUnit.NANOSECONDS).toString()); + } + if (getMinSliceTime() != null) { + builder.field(MIN_SLICE_NODE_TIME.getPreferredName(), new TimeValue(getMinSliceTime(), TimeUnit.NANOSECONDS).toString()); + } + if (getAvgSliceTime() != null) { + builder.field(AVG_SLICE_NODE_TIME.getPreferredName(), new TimeValue(getAvgSliceTime(), TimeUnit.NANOSECONDS).toString()); + } } builder.field(NODE_TIME_RAW.getPreferredName(), getTime()); - builder.field(BREAKDOWN.getPreferredName(), breakdown); + if (getMaxSliceTime() != null) { + builder.field(MAX_SLICE_NODE_TIME_RAW.getPreferredName(), getMaxSliceTime()); + } + if (getMinSliceTime() != null) { + builder.field(MIN_SLICE_NODE_TIME_RAW.getPreferredName(), getMinSliceTime()); + } + if (getAvgSliceTime() != null) { + builder.field(AVG_SLICE_NODE_TIME_RAW.getPreferredName(), getAvgSliceTime()); + } + createBreakdownView(builder); if (false == debug.isEmpty()) { builder.field(DEBUG.getPreferredName(), debug); } @@ -186,6 +259,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } + private void createBreakdownView(XContentBuilder builder) throws IOException { + Map modifiedBreakdown = new LinkedHashMap<>(breakdown); + removeStartTimeFields(modifiedBreakdown); + builder.field(BREAKDOWN.getPreferredName(), modifiedBreakdown); + } + + static void removeStartTimeFields(Map modifiedBreakdown) { + Iterator> iterator = modifiedBreakdown.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (entry.getKey().endsWith(AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX)) { + iterator.remove(); + } + } + } + private static final InstantiatingObjectParser PARSER; static { InstantiatingObjectParser.Builder parser = InstantiatingObjectParser.builder( @@ -199,6 +288,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws parser.declareObject(optionalConstructorArg(), (p, c) -> p.map(), DEBUG); parser.declareLong(constructorArg(), NODE_TIME_RAW); parser.declareObjectArray(optionalConstructorArg(), (p, c) -> fromXContent(p), CHILDREN); + parser.declareLong(optionalConstructorArg(), MAX_SLICE_NODE_TIME_RAW); + parser.declareLong(optionalConstructorArg(), MIN_SLICE_NODE_TIME_RAW); + parser.declareLong(optionalConstructorArg(), AVG_SLICE_NODE_TIME_RAW); PARSER = parser.build(); } diff --git a/server/src/main/java/org/opensearch/search/profile/Profilers.java b/server/src/main/java/org/opensearch/search/profile/Profilers.java index 2bc2f3a5a3920..8e87c7ff4acd4 100644 --- a/server/src/main/java/org/opensearch/search/profile/Profilers.java +++ b/server/src/main/java/org/opensearch/search/profile/Profilers.java @@ -34,6 +34,7 @@ import org.opensearch.search.internal.ContextIndexSearcher; import org.opensearch.search.profile.aggregation.AggregationProfiler; +import org.opensearch.search.profile.aggregation.ConcurrentAggregationProfiler; import org.opensearch.search.profile.query.QueryProfiler; import java.util.ArrayList; @@ -50,18 +51,20 @@ public final class Profilers { private final ContextIndexSearcher searcher; private final List queryProfilers; private final AggregationProfiler aggProfiler; + private final boolean isConcurrentSegmentSearchEnabled; /** Sole constructor. This {@link Profilers} instance will initially wrap one {@link QueryProfiler}. */ - public Profilers(ContextIndexSearcher searcher) { + public Profilers(ContextIndexSearcher searcher, boolean isConcurrentSegmentSearchEnabled) { this.searcher = searcher; + this.isConcurrentSegmentSearchEnabled = isConcurrentSegmentSearchEnabled; this.queryProfilers = new ArrayList<>(); - this.aggProfiler = new AggregationProfiler(); + this.aggProfiler = isConcurrentSegmentSearchEnabled ? new ConcurrentAggregationProfiler() : new AggregationProfiler(); addQueryProfiler(); } /** Switch to a new profile. */ public QueryProfiler addQueryProfiler() { - QueryProfiler profiler = new QueryProfiler(searcher.getExecutor() != null); + QueryProfiler profiler = new QueryProfiler(isConcurrentSegmentSearchEnabled); searcher.setProfiler(profiler); queryProfilers.add(profiler); return profiler; diff --git a/server/src/main/java/org/opensearch/search/profile/Timer.java b/server/src/main/java/org/opensearch/search/profile/Timer.java index 231324b4a5598..172762cabeb6a 100644 --- a/server/src/main/java/org/opensearch/search/profile/Timer.java +++ b/server/src/main/java/org/opensearch/search/profile/Timer.java @@ -51,7 +51,7 @@ public class Timer { private boolean doTiming; - private long timing, count, lastCount, start; + private long timing, count, lastCount, start, earliestTimerStartTime; /** pkg-private for testing */ long nanoTime() { @@ -71,6 +71,9 @@ public final void start() { doTiming = (count - lastCount) >= Math.min(lastCount >>> 8, 1024); if (doTiming) { start = nanoTime(); + if (count == 0) { + earliestTimerStartTime = start; + } } count++; } @@ -92,6 +95,14 @@ public final long getCount() { return count; } + /** Return the timer start time in nanoseconds.*/ + public final long getEarliestTimerStartTime() { + if (start != 0) { + throw new IllegalStateException("#start call misses a matching #stop call"); + } + return earliestTimerStartTime; + } + /** Return an approximation of the total time spent between consecutive calls of #start and #stop. */ public final long getApproximateTiming() { if (start != 0) { diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java index 24eccba575e77..d0c67915e6d8d 100644 --- a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfileBreakdown.java @@ -34,6 +34,7 @@ import org.opensearch.search.profile.AbstractProfileBreakdown; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -62,4 +63,18 @@ public void addDebugInfo(String key, Object value) { protected Map toDebugMap() { return unmodifiableMap(extra); } + + /** + * Build a timing count startTime breakdown for aggregation timing types + */ + @Override + public Map toBreakdownMap() { + Map map = new HashMap<>(timings.length * 3); + for (AggregationTimingType timingType : timingTypes) { + map.put(timingType.toString(), timings[timingType.ordinal()].getApproximateTiming()); + map.put(timingType + TIMING_TYPE_COUNT_SUFFIX, timings[timingType.ordinal()].getCount()); + map.put(timingType + TIMING_TYPE_START_TIME_SUFFIX, timings[timingType.ordinal()].getEarliestTimerStartTime()); + } + return Collections.unmodifiableMap(map); + } } diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java index 1d2cf424ee5a7..39620c25dc5a3 100644 --- a/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java +++ b/server/src/main/java/org/opensearch/search/profile/aggregation/AggregationProfiler.java @@ -36,8 +36,6 @@ import org.opensearch.search.profile.AbstractProfiler; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; import java.util.Map; /** @@ -47,29 +45,25 @@ */ public class AggregationProfiler extends AbstractProfiler { - private final Map, AggregationProfileBreakdown> profileBreakdownLookup = new HashMap<>(); + private final Map profileBreakdownLookup = new HashMap<>(); public AggregationProfiler() { super(new InternalAggregationProfileTree()); } + /** + * This method does not need to be thread safe for concurrent search use case as well. + * The {@link AggregationProfileBreakdown} for each Aggregation operator is created in sync path when + * {@link org.opensearch.search.aggregations.BucketCollector#preCollection()} is called + * on the Aggregation collector instances during construction. + */ @Override public AggregationProfileBreakdown getQueryBreakdown(Aggregator agg) { - List path = getAggregatorPath(agg); - AggregationProfileBreakdown aggregationProfileBreakdown = profileBreakdownLookup.get(path); + AggregationProfileBreakdown aggregationProfileBreakdown = profileBreakdownLookup.get(agg); if (aggregationProfileBreakdown == null) { aggregationProfileBreakdown = super.getQueryBreakdown(agg); - profileBreakdownLookup.put(path, aggregationProfileBreakdown); + profileBreakdownLookup.put(agg, aggregationProfileBreakdown); } return aggregationProfileBreakdown; } - - public static List getAggregatorPath(Aggregator agg) { - LinkedList path = new LinkedList<>(); - while (agg != null) { - path.addFirst(agg.name()); - agg = agg.parent(); - } - return path; - } } diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java b/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java new file mode 100644 index 0000000000000..7c9bf55a97de5 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfiler.java @@ -0,0 +1,199 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.profile.aggregation; + +import org.opensearch.search.profile.ProfileResult; +import org.opensearch.search.profile.AbstractProfileBreakdown; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Main class to profile aggregations with concurrent execution + * + * @opensearch.internal + */ +public class ConcurrentAggregationProfiler extends AggregationProfiler { + + private static final String MAX_PREFIX = "max_"; + private static final String MIN_PREFIX = "min_"; + private static final String AVG_PREFIX = "avg_"; + private static final String START_TIME_KEY = AggregationTimingType.INITIALIZE + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX; + private static final String[] breakdownCountStatsTypes = { "build_leaf_collector_count", "collect_count" }; + + @Override + public List getTree() { + List tree = profileTree.getTree(); + List reducedTree = new LinkedList<>(); + Map> sliceLevelAggregationMap = getSliceLevelAggregationMap(tree); + for (List profileResultsAcrossSlices : sliceLevelAggregationMap.values()) { + reducedTree.addAll(reduceProfileResultsTree(profileResultsAcrossSlices)); + } + return reducedTree; + } + + private List reduceProfileResultsTree(List profileResultsAcrossSlices) { + String type = profileResultsAcrossSlices.get(0).getQueryName(); + String description = profileResultsAcrossSlices.get(0).getLuceneDescription(); + long maxSliceNodeEndTime = Long.MIN_VALUE; + long minSliceNodeStartTime = Long.MAX_VALUE; + long maxSliceNodeTime = Long.MIN_VALUE; + long minSliceNodeTime = Long.MAX_VALUE; + long avgSliceNodeTime = 0L; + Map breakdown = new HashMap<>(); + Map timeStatsMap = new HashMap<>(); + Map minSliceStartTimeMap = new HashMap<>(); + Map maxSliceEndTimeMap = new HashMap<>(); + Map countStatsMap = new HashMap<>(); + Map debug = new HashMap<>(); + List children = new LinkedList<>(); + + for (ProfileResult profileResult : profileResultsAcrossSlices) { + long profileNodeTime = profileResult.getTime(); + long sliceStartTime = profileResult.getTimeBreakdown().get(START_TIME_KEY); + + // Profiled total time + maxSliceNodeEndTime = Math.max(maxSliceNodeEndTime, sliceStartTime + profileNodeTime); + minSliceNodeStartTime = Math.min(minSliceNodeStartTime, sliceStartTime); + + // Profiled total time stats + maxSliceNodeTime = Math.max(maxSliceNodeTime, profileNodeTime); + minSliceNodeTime = Math.min(minSliceNodeTime, profileNodeTime); + avgSliceNodeTime += profileNodeTime; + + // Profiled breakdown time stats + for (AggregationTimingType timingType : AggregationTimingType.values()) { + buildBreakdownStatsMap(timeStatsMap, profileResult, timingType.toString()); + } + + // Profiled breakdown total time + for (AggregationTimingType timingType : AggregationTimingType.values()) { + String breakdownTimingType = timingType.toString(); + Long startTime = profileResult.getTimeBreakdown() + .get(breakdownTimingType + AbstractProfileBreakdown.TIMING_TYPE_START_TIME_SUFFIX); + Long endTime = startTime + profileResult.getTimeBreakdown().get(breakdownTimingType); + minSliceStartTimeMap.put( + breakdownTimingType, + Math.min(minSliceStartTimeMap.getOrDefault(breakdownTimingType, Long.MAX_VALUE), startTime) + ); + maxSliceEndTimeMap.put( + breakdownTimingType, + Math.max(maxSliceEndTimeMap.getOrDefault(breakdownTimingType, Long.MIN_VALUE), endTime) + ); + } + + // Profiled breakdown count stats + for (String breakdownCountType : breakdownCountStatsTypes) { + buildBreakdownStatsMap(countStatsMap, profileResult, breakdownCountType); + } + + // Profiled breakdown count + for (AggregationTimingType timingType : AggregationTimingType.values()) { + String breakdownType = timingType.toString(); + String breakdownTypeCount = breakdownType + AbstractProfileBreakdown.TIMING_TYPE_COUNT_SUFFIX; + breakdown.put( + breakdownTypeCount, + breakdown.getOrDefault(breakdownTypeCount, 0L) + profileResult.getTimeBreakdown().get(breakdownTypeCount) + ); + } + + debug = profileResult.getDebugInfo(); + children.addAll(profileResult.getProfiledChildren()); + } + // nodeTime + long nodeTime = maxSliceNodeEndTime - minSliceNodeStartTime; + avgSliceNodeTime /= profileResultsAcrossSlices.size(); + + // Profiled breakdown time stats + for (AggregationTimingType breakdownTimingType : AggregationTimingType.values()) { + buildBreakdownMap(profileResultsAcrossSlices.size(), breakdown, timeStatsMap, breakdownTimingType.toString()); + } + + // Profiled breakdown total time + for (AggregationTimingType breakdownTimingType : AggregationTimingType.values()) { + String breakdownType = breakdownTimingType.toString(); + breakdown.put(breakdownType, maxSliceEndTimeMap.get(breakdownType) - minSliceStartTimeMap.get(breakdownType)); + } + + // Profiled breakdown count stats + for (String breakdownCountType : breakdownCountStatsTypes) { + buildBreakdownMap(profileResultsAcrossSlices.size(), breakdown, countStatsMap, breakdownCountType); + } + + // children + List reducedChildrenTree = new LinkedList<>(); + if (!children.isEmpty()) { + Map> sliceLevelAggregationMap = getSliceLevelAggregationMap(children); + for (List profileResults : sliceLevelAggregationMap.values()) { + reducedChildrenTree.addAll(reduceProfileResultsTree(profileResults)); + } + } + + ProfileResult reducedResult = new ProfileResult( + type, + description, + breakdown, + debug, + nodeTime, + reducedChildrenTree, + maxSliceNodeTime, + minSliceNodeTime, + avgSliceNodeTime + ); + return List.of(reducedResult); + } + + static void buildBreakdownMap(int treeSize, Map breakdown, Map statsMap, String breakdownType) { + String maxBreakdownType = MAX_PREFIX + breakdownType; + String minBreakdownType = MIN_PREFIX + breakdownType; + String avgBreakdownType = AVG_PREFIX + breakdownType; + breakdown.put(maxBreakdownType, statsMap.get(maxBreakdownType)); + breakdown.put(minBreakdownType, statsMap.get(minBreakdownType)); + breakdown.put(avgBreakdownType, statsMap.get(avgBreakdownType) / treeSize); + } + + static void buildBreakdownStatsMap(Map statsMap, ProfileResult result, String breakdownType) { + String maxBreakdownType = MAX_PREFIX + breakdownType; + String minBreakdownType = MIN_PREFIX + breakdownType; + String avgBreakdownType = AVG_PREFIX + breakdownType; + statsMap.put( + maxBreakdownType, + Math.max(statsMap.getOrDefault(maxBreakdownType, Long.MIN_VALUE), result.getTimeBreakdown().get(breakdownType)) + ); + statsMap.put( + minBreakdownType, + Math.min(statsMap.getOrDefault(minBreakdownType, Long.MAX_VALUE), result.getTimeBreakdown().get(breakdownType)) + ); + statsMap.put(avgBreakdownType, statsMap.getOrDefault(avgBreakdownType, 0L) + result.getTimeBreakdown().get(breakdownType)); + } + + /** + * @return a slice level aggregation map where the key is the description of the aggregation and + * the value is a list of ProfileResult across all slices. + */ + static Map> getSliceLevelAggregationMap(List tree) { + Map> sliceLevelAggregationMap = new HashMap<>(); + for (ProfileResult result : tree) { + String description = result.getLuceneDescription(); + final List sliceLevelAggregationList = sliceLevelAggregationMap.computeIfAbsent( + description, + k -> new LinkedList<>() + ); + sliceLevelAggregationList.add(result); + } + return sliceLevelAggregationMap; + } +} diff --git a/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java b/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java index 36cfc53f41ccd..34716b87c7c9c 100644 --- a/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java +++ b/server/src/main/java/org/opensearch/search/profile/aggregation/InternalAggregationProfileTree.java @@ -62,6 +62,10 @@ protected String getTypeFromElement(Aggregator element) { return element.getClass().getSimpleName(); } + /** + * @return is used to group aggregations with same name across slices. + * So the name returned here should be same across slices for an aggregation operator. + */ @Override protected String getDescriptionFromElement(Aggregator element) { return element.name(); diff --git a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java index 74ef78bc93c5f..6f0c78e8b307d 100644 --- a/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java +++ b/server/src/main/java/org/opensearch/search/profile/query/ConcurrentQueryProfileBreakdown.java @@ -44,10 +44,10 @@ public AbstractProfileBreakdown context(Object context) { @Override public Map toBreakdownMap() { - final Map map = new HashMap<>(buildBreakdownMap(this)); + final Map map = new HashMap<>(super.toBreakdownMap()); for (final AbstractProfileBreakdown context : contexts.values()) { - for (final Map.Entry entry : buildBreakdownMap(context).entrySet()) { + for (final Map.Entry entry : context.toBreakdownMap().entrySet()) { map.merge(entry.getKey(), entry.getValue(), Long::sum); } } diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java index 2ca642292a7c0..b54f988218ac8 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionStats.java @@ -36,7 +36,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java index c7498635d2553..fabb67d7fb841 100644 --- a/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/completion/CompletionSuggester.java @@ -35,6 +35,7 @@ import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Weight; import org.apache.lucene.search.suggest.document.CompletionQuery; import org.apache.lucene.search.suggest.document.TopSuggestDocs; @@ -103,17 +104,24 @@ protected Suggest.Suggestion { // Copy the current structure. We will parse, once the mapping is provided - XContentBuilder builder = XContentFactory.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); builder.copyCurrentStructure(p); v.contextBytes = BytesReference.bytes(builder); p.skipChildren(); @@ -216,7 +216,7 @@ public CompletionSuggestionBuilder regex(String regex, RegexOptions regexOptions public CompletionSuggestionBuilder contexts(Map> queryContexts) { Objects.requireNonNull(queryContexts, "contexts must not be null"); try { - XContentBuilder contentBuilder = XContentFactory.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(CONTEXT_BYTES_XCONTENT_TYPE); contentBuilder.startObject(); for (Map.Entry> contextEntry : queryContexts.entrySet()) { contentBuilder.startArray(contextEntry.getKey()); diff --git a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java index 4b1c43bf7a317..6992af62b6626 100644 --- a/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java +++ b/server/src/main/java/org/opensearch/search/suggest/phrase/PhraseSuggester.java @@ -44,7 +44,7 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.core.common.text.Text; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.ParsedQuery; @@ -160,7 +160,8 @@ public Suggestion> innerExecute( QueryShardContext shardContext = suggestion.getShardContext(); final String querySource = scriptFactory.newInstance(vars).execute(); try ( - XContentParser parser = XContentFactory.xContent(querySource) + XContentParser parser = MediaTypeRegistry.xContent(querySource) + .xContent() .createParser(shardContext.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, querySource) ) { QueryBuilder innerQueryBuilder = AbstractQueryBuilder.parseInnerQueryBuilder(parser); diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 54d55f67ccdcd..119e632928cc7 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -41,6 +41,7 @@ import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterInfo; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateApplier; import org.opensearch.cluster.ClusterStateTaskConfig; @@ -62,12 +63,14 @@ import org.opensearch.cluster.metadata.MetadataIndexUpgradeService; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource; import org.opensearch.cluster.routing.RecoverySource.RemoteStoreRecoverySource; import org.opensearch.cluster.routing.RoutingChangesObserver; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.cluster.service.ClusterManagerTaskKeys; @@ -86,6 +89,9 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.snapshots.IndexShardSnapshotStatus; +import org.opensearch.index.store.remote.filecache.FileCacheStats; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; @@ -103,6 +109,7 @@ import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.Collections.unmodifiableSet; @@ -118,6 +125,8 @@ import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY; import static org.opensearch.common.util.set.Sets.newHashSet; import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION; +import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING; +import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING; import static org.opensearch.snapshots.SnapshotUtils.filterIndices; /** @@ -176,6 +185,10 @@ public class RestoreService implements ClusterStateApplier { private final ClusterSettings clusterSettings; + private final IndicesService indicesService; + + private final Supplier clusterInfoSupplier; + private final ClusterManagerTaskThrottler.ThrottlingKey restoreSnapshotTaskKey; private static final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor = new CleanRestoreStateTaskExecutor(); @@ -186,8 +199,9 @@ public RestoreService( AllocationService allocationService, MetadataCreateIndexService createIndexService, MetadataIndexUpgradeService metadataIndexUpgradeService, - ClusterSettings clusterSettings, - ShardLimitValidator shardLimitValidator + ShardLimitValidator shardLimitValidator, + IndicesService indicesService, + Supplier clusterInfoSupplier ) { this.clusterService = clusterService; this.repositoriesService = repositoriesService; @@ -199,6 +213,8 @@ public RestoreService( } this.clusterSettings = clusterService.getClusterSettings(); this.shardLimitValidator = shardLimitValidator; + this.indicesService = indicesService; + this.clusterInfoSupplier = clusterInfoSupplier; // Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction. restoreSnapshotTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.RESTORE_SNAPSHOT_KEY, true); @@ -234,21 +250,34 @@ public ClusterState execute(ClusterState currentState) { continue; } if (currentIndexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false)) { - if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) { - throw new IllegalStateException( - "cannot restore index [" - + index - + "] because an open index " - + "with same name already exists in the cluster. Close the existing index" - ); + IndexMetadata updatedIndexMetadata = currentIndexMetadata; + Map activeInitializingShards = new HashMap<>(); + if (request.restoreAllShards()) { + if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) { + throw new IllegalStateException( + "cannot restore index [" + + index + + "] because an open index " + + "with same name already exists in the cluster. Close the existing index" + ); + } + updatedIndexMetadata = IndexMetadata.builder(currentIndexMetadata) + .state(IndexMetadata.State.OPEN) + .version(1 + currentIndexMetadata.getVersion()) + .mappingVersion(1 + currentIndexMetadata.getMappingVersion()) + .settingsVersion(1 + currentIndexMetadata.getSettingsVersion()) + .aliasesVersion(1 + currentIndexMetadata.getAliasesVersion()) + .build(); + } else { + activeInitializingShards = currentState.routingTable() + .index(index) + .shards() + .values() + .stream() + .map(IndexShardRoutingTable::primaryShard) + .filter(shardRouting -> shardRouting.unassigned() == false) + .collect(Collectors.toMap(ShardRouting::shardId, Function.identity())); } - IndexMetadata updatedIndexMetadata = IndexMetadata.builder(currentIndexMetadata) - .state(IndexMetadata.State.OPEN) - .version(1 + currentIndexMetadata.getVersion()) - .mappingVersion(1 + currentIndexMetadata.getMappingVersion()) - .settingsVersion(1 + currentIndexMetadata.getSettingsVersion()) - .aliasesVersion(1 + currentIndexMetadata.getAliasesVersion()) - .build(); IndexId indexId = new IndexId(index, updatedIndexMetadata.getIndexUUID()); @@ -257,7 +286,7 @@ public ClusterState execute(ClusterState currentState) { updatedIndexMetadata.getCreationVersion(), indexId ); - rtBuilder.addAsRemoteStoreRestore(updatedIndexMetadata, recoverySource); + rtBuilder.addAsRemoteStoreRestore(updatedIndexMetadata, recoverySource, activeInitializingShards); blocks.updateBlocks(updatedIndexMetadata); mdBuilder.put(updatedIndexMetadata, true); indicesToBeRestored.add(index); @@ -401,7 +430,6 @@ public ClusterManagerTaskThrottler.ThrottlingKey getClusterManagerThrottlingKey( @Override public ClusterState execute(ClusterState currentState) { - RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY); // Check if the snapshot to restore is currently being deleted SnapshotDeletionsInProgress deletionsInProgress = currentState.custom( SnapshotDeletionsInProgress.TYPE, @@ -422,7 +450,9 @@ public ClusterState execute(ClusterState currentState) { ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks()); RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable()); final Map shards; + final boolean isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(request.storageType().toString()); Set aliases = new HashSet<>(); + long totalRestorableRemoteIndexesSize = 0; if (indices.isEmpty() == false) { // We have some indices to restore @@ -433,17 +463,14 @@ public ClusterState execute(ClusterState currentState) { String index = indexEntry.getValue(); boolean partial = checkPartial(index); + IndexId snapshotIndexId = repositoryData.resolveIndexId(index); IndexMetadata snapshotIndexMetadata = updateIndexSettings( metadata.index(index), request.indexSettings(), request.ignoreIndexSettings() ); - if (IndexModule.Type.REMOTE_SNAPSHOT.match(request.storageType().toString())) { - snapshotIndexMetadata = addSnapshotToIndexSettings( - snapshotIndexMetadata, - snapshot, - repositoryData.resolveIndexId(index) - ); + if (isRemoteSnapshot) { + snapshotIndexMetadata = addSnapshotToIndexSettings(snapshotIndexMetadata, snapshot, snapshotIndexId); } final boolean isSearchableSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match( snapshotIndexMetadata.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) @@ -469,7 +496,7 @@ public ClusterState execute(ClusterState currentState) { restoreUUID, snapshot, snapshotInfo.version(), - repositoryData.resolveIndexId(index), + snapshotIndexId, isSearchableSnapshot, isRemoteStoreShallowCopy, request.getSourceRemoteStoreRepository() @@ -588,6 +615,14 @@ public ClusterState execute(ClusterState currentState) { } for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) { + if (isRemoteSnapshot) { + IndexShardSnapshotStatus.Copy shardStatus = repository.getShardSnapshotStatus( + snapshotInfo.snapshotId(), + snapshotIndexId, + new ShardId(metadata.index(index).getIndex(), shard) + ).asCopy(); + totalRestorableRemoteIndexesSize += shardStatus.getTotalSize(); + } if (!ignoreShards.contains(shard)) { shardsBuilder.put( new ShardId(renamedIndex, shard), @@ -624,6 +659,9 @@ public ClusterState execute(ClusterState currentState) { } checkAliasNameConflicts(indices, aliases); + if (isRemoteSnapshot) { + validateSearchableSnapshotRestorable(totalRestorableRemoteIndexesSize); + } Map updatedDataStreams = new HashMap<>(currentState.metadata().dataStreams()); updatedDataStreams.putAll( @@ -823,6 +861,45 @@ private IndexMetadata updateIndexSettings( return builder.settings(settingsBuilder).build(); } + private void validateSearchableSnapshotRestorable(long totalRestorableRemoteIndexesSize) { + ClusterInfo clusterInfo = clusterInfoSupplier.get(); + double remoteDataToFileCacheRatio = DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.get(clusterService.getSettings()); + Map nodeFileCacheStats = clusterInfo.getNodeFileCacheStats(); + if (nodeFileCacheStats.isEmpty() || remoteDataToFileCacheRatio <= 0.01f) { + return; + } + + long totalNodeFileCacheSize = clusterInfo.getNodeFileCacheStats() + .values() + .stream() + .map(fileCacheStats -> fileCacheStats.getTotal().getBytes()) + .mapToLong(Long::longValue) + .sum(); + + Predicate isRemoteSnapshotShard = shardRouting -> shardRouting.primary() + && indicesService.indexService(shardRouting.index()).getIndexSettings().isRemoteSnapshot(); + + ShardsIterator shardsIterator = clusterService.state() + .routingTable() + .allShardsSatisfyingPredicate(isRemoteSnapshotShard); + + long totalRestoredRemoteIndexesSize = shardsIterator.getShardRoutings() + .stream() + .map(clusterInfo::getShardSize) + .mapToLong(Long::longValue) + .sum(); + + if (totalRestoredRemoteIndexesSize + totalRestorableRemoteIndexesSize > remoteDataToFileCacheRatio + * totalNodeFileCacheSize) { + throw new SnapshotRestoreException( + snapshot, + "Size of the indexes to be restored exceeds the file cache bounds. Increase the file cache capacity on the cluster nodes using " + + NODE_SEARCH_CACHE_SIZE_SETTING.getKey() + + " setting." + ); + } + } + @Override public void onFailure(String source, Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e); diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java index 0fcd3761c98ba..d8f9b8e1b984f 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotShardsService.java @@ -47,7 +47,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index cd9c2c4ce325a..2772f061b4c4a 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -82,7 +82,7 @@ import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java index 28a09fd80b408..041a5d32b027c 100644 --- a/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java +++ b/server/src/main/java/org/opensearch/tasks/RawTaskStatus.java @@ -36,8 +36,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -76,7 +76,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { try (InputStream stream = status.streamInput()) { - return builder.rawValue(stream, XContentHelper.xContentType(status)); + return builder.rawValue(stream, MediaTypeRegistry.xContentType(status)); } } diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java index 5b512af56e195..343d4571593a7 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java @@ -11,7 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchShardTask; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/main/java/org/opensearch/tasks/TaskManager.java b/server/src/main/java/org/opensearch/tasks/TaskManager.java index bcd3004188bb6..ec59a13dd1999 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskManager.java +++ b/server/src/main/java/org/opensearch/tasks/TaskManager.java @@ -50,7 +50,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; diff --git a/server/src/main/java/org/opensearch/tasks/TaskResult.java b/server/src/main/java/org/opensearch/tasks/TaskResult.java index b6006636da2e9..34efbe3ec572b 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResult.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResult.java @@ -42,10 +42,10 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.InstantiatingObjectParser; import org.opensearch.common.xcontent.ObjectParserHelper; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; @@ -238,7 +238,7 @@ public int hashCode() { } private static BytesReference toXContent(Exception error) throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { builder.startObject(); OpenSearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, error); builder.endObject(); diff --git a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java index 1feb115cb585a..01be4eaaaf732 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskResultsService.java @@ -54,9 +54,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.Streams; import org.opensearch.threadpool.ThreadPool; @@ -169,7 +169,7 @@ private int getTaskResultMappingVersion(IndexMetadata metadata) { private void doStoreResult(TaskResult taskResult, ActionListener listener) { IndexRequestBuilder index = client.prepareIndex(TASK_INDEX).setId(taskResult.getTask().getTaskId().toString()); - try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { taskResult.toXContent(builder, ToXContent.EMPTY_PARAMS); index.setSource(builder); } catch (IOException e) { diff --git a/server/src/main/java/org/opensearch/transport/InboundAggregator.java b/server/src/main/java/org/opensearch/transport/InboundAggregator.java index 60d45c8b3ad2b..060ccec3563e2 100644 --- a/server/src/main/java/org/opensearch/transport/InboundAggregator.java +++ b/server/src/main/java/org/opensearch/transport/InboundAggregator.java @@ -32,8 +32,8 @@ package org.opensearch.transport; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.bytes.CompositeBytesReference; diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java index 4a4f928860819..9103692e358e4 100644 --- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java +++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.util.PageCacheRecycler; diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java index 75676307f4026..8733cb5fe7e8e 100644 --- a/server/src/main/java/org/opensearch/transport/TcpTransport.java +++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java @@ -40,11 +40,11 @@ import org.opensearch.action.support.ThreadedActionListener; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Booleans; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.component.AbstractLifecycleComponent; -import org.opensearch.common.component.Lifecycle; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.metrics.MeanMetric; @@ -57,14 +57,14 @@ import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.PortsRange; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.common.Strings; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.node.Node; import org.opensearch.core.rest.RestStatus; diff --git a/server/src/main/java/org/opensearch/transport/Transport.java b/server/src/main/java/org/opensearch/transport/Transport.java index fbf6b3ef350d8..3bf855f847685 100644 --- a/server/src/main/java/org/opensearch/transport/Transport.java +++ b/server/src/main/java/org/opensearch/transport/Transport.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.collect.MapBuilder; -import org.opensearch.common.component.LifecycleComponent; +import org.opensearch.common.lifecycle.LifecycleComponent; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java b/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java index e48267d943739..5b542e4080641 100644 --- a/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java +++ b/server/src/main/java/org/opensearch/transport/TransportKeepAlive.java @@ -37,7 +37,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.common.AsyncBiFunction; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.component.Lifecycle; +import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index c2163ae1d490a..3b59c99c03d3b 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -43,7 +43,7 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Streamables; @@ -169,6 +169,7 @@ public void close() {} * over the {@link StreamOutput} and {@link StreamInput} wire */ Streamables.registerStreamables(); + /** Registers OpenSearch server specific exceptions (exceptions outside of core library) */ OpenSearchServerException.registerExceptions(); } diff --git a/server/src/main/java/org/opensearch/transport/TransportSettings.java b/server/src/main/java/org/opensearch/transport/TransportSettings.java index b4d72ab40409b..05887f6f6d131 100644 --- a/server/src/main/java/org/opensearch/transport/TransportSettings.java +++ b/server/src/main/java/org/opensearch/transport/TransportSettings.java @@ -35,7 +35,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import java.util.Arrays; diff --git a/server/src/main/java/org/opensearch/transport/TransportStats.java b/server/src/main/java/org/opensearch/transport/TransportStats.java index 29544754a5729..e3c4773f4a472 100644 --- a/server/src/main/java/org/opensearch/transport/TransportStats.java +++ b/server/src/main/java/org/opensearch/transport/TransportStats.java @@ -35,7 +35,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index 2656810af0379..629fe9ebd4b99 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -64,8 +64,8 @@ import org.opensearch.core.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -73,7 +73,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.CancellableThreadsTests; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentType; @@ -852,7 +852,7 @@ public void testIds() { ids.put(130, org.opensearch.action.NoShardAvailableActionException.class); ids.put(131, org.opensearch.action.UnavailableShardsException.class); ids.put(132, org.opensearch.index.engine.FlushFailedEngineException.class); - ids.put(133, org.opensearch.common.breaker.CircuitBreakingException.class); + ids.put(133, org.opensearch.core.common.breaker.CircuitBreakingException.class); ids.put(134, org.opensearch.transport.NodeNotConnectedException.class); ids.put(135, org.opensearch.index.mapper.StrictDynamicMappingException.class); ids.put(136, org.opensearch.action.support.replication.TransportReplicationAction.RetryOnReplicaException.class); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index cbf7032b50ca5..32f27f3a60e42 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -42,8 +42,8 @@ import org.opensearch.cluster.coordination.PendingClusterStateStats; import org.opensearch.cluster.coordination.PublishClusterStateStats; import org.opensearch.http.HttpStats; -import org.opensearch.indices.breaker.AllCircuitBreakerStats; -import org.opensearch.indices.breaker.CircuitBreakerStats; +import org.opensearch.core.indices.breaker.AllCircuitBreakerStats; +import org.opensearch.core.indices.breaker.CircuitBreakerStats; import org.opensearch.ingest.IngestStats; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.monitor.jvm.JvmStats; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java index 96f2365412e7b..485aae433aa16 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/ResourceAwareTasksTests.java @@ -672,8 +672,8 @@ private void assertTasksRequestFinishedSuccessfully(NodesResponse nodesResponse, } private void assertMemoryUsageWithinLimits(long actual, long expected) { - // 5% buffer up to 200 KB to account for classloading overhead. - long maxOverhead = Math.min(200000, expected * 5 / 100); + // 5% buffer up to 500 KB to account for classloading overhead. + long maxOverhead = Math.min(500000, expected * 5 / 100); assertThat(actual, lessThanOrEqualTo(expected + maxOverhead)); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java index abbaf3a3f7b96..e0676cc9ddbdd 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java @@ -56,7 +56,7 @@ import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.common.lease.Releasable; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.tasks.TaskCancellationService; import org.opensearch.tasks.TaskManager; import org.opensearch.tasks.TaskResourceTrackingService; diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 6885278b08413..c463f7a8f4b8c 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -56,9 +56,9 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.tasks.Task; @@ -761,7 +761,7 @@ public void testTasksToXContentGrouping() throws Exception { } private Map serialize(ListTasksResponse response, boolean byParents) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject(); if (byParents) { DiscoveryNodes nodes = testNodes[0].clusterService.state().nodes(); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java index 81d7074977253..2edfa23286658 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequestTests.java @@ -38,6 +38,7 @@ private RestoreRemoteStoreRequest randomState(RestoreRemoteStoreRequest instance } instance.waitForCompletion(randomBoolean()); + instance.restoreAllShards(randomBoolean()); if (randomBoolean()) { instance.masterNodeTimeout(randomTimeValue()); @@ -76,6 +77,7 @@ public void testSource() throws IOException { RestoreRemoteStoreRequest processed = new RestoreRemoteStoreRequest(); processed.masterNodeTimeout(original.masterNodeTimeout()); processed.waitForCompletion(original.waitForCompletion()); + processed.restoreAllShards(original.restoreAllShards()); processed.source(map); assertEquals(original, processed); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index 6ba7961ccecf3..c9b57d0c41b45 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -46,10 +46,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.network.NetworkModule; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.admin.cluster.RestClusterRerouteAction; @@ -212,7 +212,7 @@ private ClusterRerouteRequest roundTripThroughRestRequest(ClusterRerouteRequest private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException { Map params = new HashMap<>(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); boolean hasBody = false; if (randomBoolean()) { builder.prettyPrint(); @@ -244,7 +244,7 @@ private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOExcep FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry()); requestBuilder.withParams(params); if (hasBody) { - requestBuilder.withContent(BytesReference.bytes(builder), XContentType.fromMediaType(builder.contentType())); + requestBuilder.withContent(BytesReference.bytes(builder), builder.contentType()); } return requestBuilder.build(); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index 5c20b3b262730..82b2cfb2e3e51 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -112,6 +112,10 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { instance.snapshotUuid(randomBoolean() ? null : randomAlphaOfLength(10)); } + instance.storageType( + randomBoolean() ? RestoreSnapshotRequest.StorageType.LOCAL : RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT + ); + if (randomBoolean()) { instance.setSourceRemoteStoreRepository(randomAlphaOfLengthBetween(5, 10)); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java index 2eb6fa504baf2..cfdd776e60832 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -56,13 +56,13 @@ public void testSerialization() throws IOException { new StoredScriptSource("foo", "bar", Collections.emptyMap()) ); - assertEquals(XContentType.JSON, storedScriptRequest.xContentType()); + assertEquals(XContentType.JSON, storedScriptRequest.mediaType()); try (BytesStreamOutput output = new BytesStreamOutput()) { storedScriptRequest.writeTo(output); try (StreamInput in = output.bytes().streamInput()) { PutStoredScriptRequest serialized = new PutStoredScriptRequest(in); - assertEquals(XContentType.JSON, serialized.xContentType()); + assertEquals(XContentType.JSON, serialized.mediaType()); assertEquals(storedScriptRequest.id(), serialized.id()); assertEquals(storedScriptRequest.context(), serialized.context()); } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java index 265f02304ce00..2c4462b964d65 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/AliasActionsTests.java @@ -37,9 +37,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -195,7 +195,7 @@ public void testParseAdd() throws IOException { if (filter == null || filter.isEmpty()) { assertNull(action.filter()); } else { - assertEquals(Strings.toString(XContentFactory.contentBuilder(XContentType.JSON).map(filter)), action.filter()); + assertEquals(Strings.toString(MediaTypeRegistry.contentBuilder(XContentType.JSON).map(filter)), action.filter()); } assertEquals(Objects.toString(searchRouting, null), action.searchRouting()); assertEquals(Objects.toString(indexRouting, null), action.indexRouting()); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java index 030ee6649bba5..f370528e6f770 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheActionTests.java @@ -17,7 +17,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java index e4114a622602c..c8a24fd0482cf 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -38,9 +38,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -101,7 +101,7 @@ public void testMappingKeyedByType() throws IOException { CreateIndexRequest request1 = new CreateIndexRequest("foo"); CreateIndexRequest request2 = new CreateIndexRequest("bar"); { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject("properties") .startObject("field1") @@ -117,7 +117,7 @@ public void testMappingKeyedByType() throws IOException { .endObject() .endObject(); request1.mapping(builder); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject(MapperService.SINGLE_MAPPING_NAME) .startObject("properties") @@ -140,7 +140,7 @@ public void testMappingKeyedByType() throws IOException { } public void testSettingsType() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject().startArray("settings").endArray().endObject(); CreateIndexRequest parsedCreateIndexRequest = new CreateIndexRequest(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java index 5e4e2346a6e5b..c83263e4e22c6 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java @@ -35,7 +35,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java index db61cac534f89..b4512595a1dfe 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/ConditionTests.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.rollover; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.EqualsHashCodeTestUtils; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java index 4b23a68f8b383..f65d39583336b 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -40,9 +40,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; @@ -179,7 +180,7 @@ public void testSerialize() throws Exception { public void testUnknownFields() throws IOException { final RolloverRequest request = new RolloverRequest(); XContentType xContentType = randomFrom(XContentType.values()); - final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + final XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); builder.startObject(); { builder.startObject("conditions"); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java index 233e6166d1caf..2406952af6a3b 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.rollover; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 65d7c0fc62629..76ddb9facd192 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -59,8 +59,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.set.Sets; import org.opensearch.index.cache.query.QueryCacheStats; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java index 3d21af584ea04..05922a6c7419b 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/ResizeRequestTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.RandomCreateIndexGenerator; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java index ef49820192e9b..848df5f8e4979 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -52,7 +52,7 @@ import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.shard.DocsStats; import org.opensearch.index.store.StoreStats; import org.opensearch.snapshots.EmptySnapshotsInfoService; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java index dbca883506b0d..f069ff3bf7ff2 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java @@ -35,6 +35,7 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; @@ -76,7 +77,7 @@ public void testMappingKeyedByType() throws IOException { PutIndexTemplateRequest request1 = new PutIndexTemplateRequest("foo"); PutIndexTemplateRequest request2 = new PutIndexTemplateRequest("bar"); { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject("properties") .startObject("field1") @@ -92,7 +93,7 @@ public void testMappingKeyedByType() throws IOException { .endObject() .endObject(); request1.mapping(builder); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject() .startObject("properties") .startObject("field1") diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java index 40d0df61cbb9f..0111c8d6e3132 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkProcessorTests.java @@ -38,8 +38,8 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.index.IndexRequest; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java index c674be8dbba38..2f2c9b2be9e50 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestTests.java @@ -39,14 +39,14 @@ import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Requests; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.script.Script; import org.opensearch.test.OpenSearchTestCase; @@ -340,7 +340,7 @@ public void testSmileIsSupported() throws IOException { XContentType xContentType = XContentType.SMILE; BytesReference data; try (BytesStreamOutput out = new BytesStreamOutput()) { - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.startObject("index"); builder.field("_index", "index"); @@ -349,7 +349,7 @@ public void testSmileIsSupported() throws IOException { builder.endObject(); } out.write(xContentType.xContent().streamSeparator()); - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.field("field", "value"); builder.endObject(); @@ -375,7 +375,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException XContentType xContentType = XContentType.SMILE; BytesReference data; try (BytesStreamOutput out = new BytesStreamOutput()) { - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.startObject("update"); builder.field("_index", "index"); @@ -386,7 +386,7 @@ public void testToValidateUpsertRequestAndCASInBulkRequest() throws IOException builder.endObject(); } out.write(xContentType.xContent().streamSeparator()); - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType, out)) { builder.startObject(); builder.startObject("doc").endObject(); Map values = new HashMap<>(); diff --git a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java index 866e13c7faf0e..14f2a5c94bcd0 100644 --- a/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java +++ b/server/src/test/java/org/opensearch/action/explain/ExplainResponseTests.java @@ -37,9 +37,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.get.GetResult; @@ -114,7 +114,7 @@ public void testToXContent() throws IOException { ); ExplainResponse response = new ExplainResponse(index, id, exist, explanation, getResult); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); response.toXContent(builder, ToXContent.EMPTY_PARAMS); String generatedResponse = BytesReference.bytes(builder).utf8ToString().replaceAll("\\s+", ""); diff --git a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index 382425ce4e57c..d76ed86ed490e 100644 --- a/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/opensearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -34,9 +34,9 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.AbstractSerializingTestCase; @@ -125,7 +125,7 @@ protected Predicate getRandomFieldsExcludeFilter() { public void testToXContent() throws IOException { FieldCapabilitiesResponse response = createSimpleResponse(); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); response.toXContent(builder, ToXContent.EMPTY_PARAMS); String generatedResponse = BytesReference.bytes(builder).utf8ToString(); diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java index 5498d685dbb6c..cc1e963bc110d 100644 --- a/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java +++ b/server/src/test/java/org/opensearch/action/get/MultiGetRequestTests.java @@ -126,7 +126,7 @@ public void testXContentSerialization() throws IOException { MultiGetRequest expected = createTestInstance(); XContentType xContentType = randomFrom(XContentType.values()); BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false); - try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) { + try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) { MultiGetRequest actual = new MultiGetRequest(); actual.add(null, null, null, null, parser, true); assertThat(parser.nextToken(), nullValue()); diff --git a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java index 71ada43bef116..c2a5f2def31ab 100644 --- a/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java +++ b/server/src/test/java/org/opensearch/action/get/MultiGetResponseTests.java @@ -33,7 +33,6 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.get.GetResult; @@ -53,7 +52,7 @@ public void testFromXContent() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); BytesReference shuffled = toShuffledXContent(expected, xContentType, ToXContent.EMPTY_PARAMS, false); MultiGetResponse parsed; - try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) { + try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) { parsed = MultiGetResponse.fromXContent(parser); assertNull(parser.nextToken()); } diff --git a/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java b/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java index bda9a49bf797b..c4d3b17de1a5a 100644 --- a/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java +++ b/server/src/test/java/org/opensearch/action/index/IndexRequestTests.java @@ -39,7 +39,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.VersionType; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java index a86257fd741c0..336ec67546dc5 100644 --- a/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java +++ b/server/src/test/java/org/opensearch/action/ingest/PutPipelineRequestTests.java @@ -49,14 +49,14 @@ public class PutPipelineRequestTests extends OpenSearchTestCase { public void testSerializationWithXContent() throws IOException { PutPipelineRequest request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON); - assertEquals(XContentType.JSON, request.getXContentType()); + assertEquals(XContentType.JSON, request.getMediaType()); BytesStreamOutput output = new BytesStreamOutput(); request.writeTo(output); StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); PutPipelineRequest serialized = new PutPipelineRequest(in); - assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals(XContentType.JSON, serialized.getMediaType()); assertEquals("{}", serialized.getSource().utf8ToString()); } diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java index 61dd2ac8c14ae..2e50c6225577d 100644 --- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java @@ -64,7 +64,7 @@ import org.opensearch.index.translog.Translog; import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndices; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; diff --git a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java index db371eb9467ba..afb342967bd69 100644 --- a/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/DfsQueryPhaseTests.java @@ -38,8 +38,8 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.opensearch.action.OriginalIndices; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.concurrent.AtomicArray; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java index 24a018d05a16a..bf18c22f9b7ee 100644 --- a/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/opensearch/action/search/FetchSearchPhaseTests.java @@ -37,8 +37,8 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.opensearch.action.OriginalIndices; import org.opensearch.common.UUIDs; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.index.shard.ShardId; diff --git a/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java b/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java index 27d04cba204fb..794554c1ec013 100644 --- a/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java +++ b/server/src/test/java/org/opensearch/action/search/QueryPhaseResultConsumerTests.java @@ -36,8 +36,8 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHits; import org.opensearch.action.OriginalIndices; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java index 93f9f69e68de0..dc946c7963a17 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java @@ -46,9 +46,9 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.action.OriginalIndices; import org.opensearch.common.UUIDs; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; diff --git a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java index 1b67d805fe814..d0a1307d33235 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -41,8 +41,8 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.GroupShardsIterator; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.OpenSearchExecutors; diff --git a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java index 0e25091d845b5..fafd34602c310 100644 --- a/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java +++ b/server/src/test/java/org/opensearch/action/support/IndicesOptionsTests.java @@ -38,10 +38,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent.MapParams; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -422,7 +422,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException { final boolean allowNoIndices = randomBoolean(); BytesReference xContentBytes; - try (XContentBuilder builder = XContentFactory.contentBuilder(type)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) { builder.startObject(); builder.field("expand_wildcards", "all"); builder.field("ignore_unavailable", ignoreUnavailable); @@ -441,7 +441,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException { assertTrue(fromXContentOptions.expandWildcardsHidden()); assertTrue(fromXContentOptions.expandWildcardsOpen()); - try (XContentBuilder builder = XContentFactory.contentBuilder(type)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) { builder.startObject(); builder.field("expand_wildcards", "none"); builder.field("ignore_unavailable", ignoreUnavailable); @@ -461,7 +461,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException { } private BytesReference toXContentBytes(IndicesOptions indicesOptions, XContentType type) throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(type)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(type)) { builder.startObject(); indicesOptions.toXContent(builder, new MapParams(Collections.emptyMap())); builder.endObject(); diff --git a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java index 330d43e791555..9c02bacbadda2 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/BroadcastReplicationTests.java @@ -58,8 +58,8 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.indices.breaker.CircuitBreakerService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.core.rest.RestStatus; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java index 78081a8f83ce3..21d14fb31059d 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java @@ -49,8 +49,8 @@ import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ConcurrentCollections; diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java index 5e32193415bd8..46af25c49f262 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationResponseTests.java @@ -35,8 +35,8 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.support.replication.ReplicationResponse.ShardInfo; import org.opensearch.common.Strings; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java index 089bcf77afbae..5af358cae18a8 100644 --- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java @@ -89,7 +89,7 @@ import org.opensearch.index.shard.ShardNotInPrimaryModeException; import org.opensearch.indices.IndexClosedException; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.indices.cluster.ClusterStateChanges; import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java index 511c1555f1159..a284269dc4151 100644 --- a/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/NodeConnectionsServiceTests.java @@ -45,8 +45,8 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.UUIDs; -import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.component.LifecycleListener; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java index ea615c7cfc09a..f0459240c2bbe 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java @@ -43,7 +43,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java index 6b869ffed7d23..53f1d71947f7c 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/RoutingTableTests.java @@ -49,10 +49,13 @@ import org.junit.Before; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.function.Predicate; +import static org.mockito.Mockito.mock; import static org.opensearch.cluster.routing.ShardRoutingState.UNASSIGNED; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -228,6 +231,49 @@ public void testShardsMatchingPredicateCount() { assertThat(clusterState.routingTable().shardsMatchingPredicateCount(predicate), is(2)); } + public void testAllShardsMatchingPredicate() { + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .build(); + ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build()) + .build(); + clusterState = ClusterState.builder(clusterState) + .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))) + .build(); + clusterState = allocation.reroute(clusterState, "reroute"); + + Predicate predicate = s -> s.state() == ShardRoutingState.UNASSIGNED && s.unassignedInfo().isDelayed(); + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(predicate).size(), is(0)); + + // starting primaries + clusterState = startInitializingShardsAndReroute(allocation, clusterState); + // starting replicas + clusterState = startInitializingShardsAndReroute(allocation, clusterState); + // remove node2 and reroute + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); + // make sure both replicas are marked as delayed (i.e. not reallocated) + clusterState = allocation.disassociateDeadNodes(clusterState, true, "reroute"); + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(predicate).size(), is(2)); + + // Verifies true against all shards on the node (active/inactive) + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(shard -> true).size(), is(4)); + // Verifies false against all shards on the node (active/inactive) + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(shard -> false).size(), is(0)); + // Verifies against all primary shards on the node + assertThat(clusterState.routingTable().allShardsSatisfyingPredicate(ShardRouting::primary).size(), is(2)); + // Verifies a predicate which tests for inactive replicas + assertThat( + clusterState.routingTable() + .allShardsSatisfyingPredicate(shardRouting -> !shardRouting.primary() && !shardRouting.active()) + .size(), + is(2) + ); + } + public void testActivePrimaryShardsGrouped() { assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], true).size(), is(0)); assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], false).size(), is(0)); @@ -502,13 +548,40 @@ public void testAddAsRemoteStoreRestore() { Version.CURRENT, new IndexId(TEST_INDEX_1, "1") ); - final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore(indexMetadata, remoteStoreRecoverySource) - .build(); + final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore( + indexMetadata, + remoteStoreRecoverySource, + new HashMap<>() + ).build(); assertTrue(routingTable.hasIndex(TEST_INDEX_1)); assertEquals(this.numberOfShards, routingTable.allShards(TEST_INDEX_1).size()); assertEquals(this.numberOfShards, routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size()); } + public void testAddAsRemoteStoreRestoreWithActiveShards() { + final IndexMetadata indexMetadata = createIndexMetadata(TEST_INDEX_1).state(IndexMetadata.State.OPEN).build(); + final RemoteStoreRecoverySource remoteStoreRecoverySource = new RemoteStoreRecoverySource( + "restore_uuid", + Version.CURRENT, + new IndexId(TEST_INDEX_1, "1") + ); + Map activeInitializingShards = new HashMap<>(); + for (int i = 0; i < randomIntBetween(1, this.numberOfShards); i++) { + activeInitializingShards.put(new ShardId(indexMetadata.getIndex(), i), mock(ShardRouting.class)); + } + final RoutingTable routingTable = new RoutingTable.Builder().addAsRemoteStoreRestore( + indexMetadata, + remoteStoreRecoverySource, + activeInitializingShards + ).build(); + assertTrue(routingTable.hasIndex(TEST_INDEX_1)); + assertEquals(this.numberOfShards, routingTable.allShards(TEST_INDEX_1).size()); + assertEquals( + this.numberOfShards - activeInitializingShards.size(), + routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size() + ); + } + /** reverse engineer the in sync aid based on the given indexRoutingTable **/ public static IndexMetadata updateActiveAllocations(IndexRoutingTable indexRoutingTable, IndexMetadata indexMetadata) { IndexMetadata.Builder imdBuilder = IndexMetadata.builder(indexMetadata); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java index d23b079e35ef9..22f7b9ecd5206 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdSettingsTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.test.OpenSearchTestCase; import java.util.Locale; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 2180a14f5bf30..bde8a45359814 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -69,6 +69,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.EmptySnapshotsInfoService; @@ -405,6 +406,7 @@ public void testFileCacheRemoteShardsDecisions() { DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings); Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(remoteIndexSettings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0)) + .persistentSettings(Settings.builder().put(FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.getKey(), 5).build()) .build(); RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java index 3c27748daa87d..d4804b18bd160 100644 --- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java @@ -86,6 +86,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import com.carrotsearch.randomizedtesting.annotations.Timeout; + import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.anyOf; @@ -863,6 +865,7 @@ public ClusterManagerTaskThrottler.ThrottlingKey getClusterManagerThrottlingKey( AtomicInteger throttledTask3 = new AtomicInteger(); AtomicInteger succeededTask1 = new AtomicInteger(); AtomicInteger succeededTask2 = new AtomicInteger(); + AtomicInteger succeededTask3 = new AtomicInteger(); AtomicInteger timedOutTask3 = new AtomicInteger(); final ClusterStateTaskListener listener = new ClusterStateTaskListener() { @@ -880,6 +883,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS succeededTask1.incrementAndGet(); } else if (source.equals(task2)) { succeededTask2.incrementAndGet(); + } else if (source.equals(task3)) { + succeededTask3.incrementAndGet(); } latch.countDown(); } @@ -955,7 +960,7 @@ public void run() { assertEquals(numberOfTask1, throttledTask1.get() + succeededTask1.get()); assertEquals(numberOfTask2, succeededTask2.get()); assertEquals(0, throttledTask2.get()); - assertEquals(numberOfTask3, throttledTask3.get() + timedOutTask3.get()); + assertEquals(numberOfTask3, throttledTask3.get() + timedOutTask3.get() + succeededTask3.get()); masterService.close(); } @@ -1378,6 +1383,76 @@ public void testDeprecatedMasterServiceUpdateTaskThreadName() { assertThrows(AssertionError.class, () -> MasterService.assertClusterManagerUpdateThread()); } + @Timeout(millis = 5_000) + public void testTaskTimeout() throws InterruptedException { + try (ClusterManagerService clusterManagerService = createClusterManagerService(true)) { + final AtomicInteger failureCount = new AtomicInteger(); + final AtomicInteger successCount = new AtomicInteger(); + final CountDownLatch taskStartLatch = new CountDownLatch(1); + final CountDownLatch blockingTaskLatch = new CountDownLatch(1); + final CountDownLatch timeoutLatch = new CountDownLatch(1); + final ClusterStateTaskListener blockingListener = new ClusterStateTaskListener() { + @Override + public void onFailure(String source, Exception e) { + fail("Unexpected failure"); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + successCount.incrementAndGet(); + taskStartLatch.countDown(); + try { + blockingTaskLatch.await(); + } catch (InterruptedException e) { + fail("Interrupted"); + } + } + }; + final ClusterStateTaskListener timeoutListener = new ClusterStateTaskListener() { + @Override + public void onFailure(String source, Exception e) { + assertEquals("timeout", source); + failureCount.incrementAndGet(); + timeoutLatch.countDown(); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + fail("Unexpected success"); + } + }; + + final ClusterStateTaskExecutor executor = (currentState, tasks) -> ClusterStateTaskExecutor.ClusterTasksResult.builder() + .successes(tasks) + .build(currentState); + + // start a task and wait for it to start and block on the clusterStateProcessed callback + clusterManagerService.submitStateUpdateTask( + "success", + new Object(), + ClusterStateTaskConfig.build(randomFrom(Priority.values())), + executor, + blockingListener + ); + taskStartLatch.await(); + + // start a second task that is guaranteed to timeout as the first task is still running + clusterManagerService.submitStateUpdateTask( + "timeout", + new Object(), + ClusterStateTaskConfig.build(randomFrom(Priority.values()), TimeValue.timeValueMillis(1L)), + executor, + timeoutListener + ); + + // wait for the timeout to happen, then unblock and assert one success and one failure + timeoutLatch.await(); + blockingTaskLatch.countDown(); + assertEquals(1, failureCount.get()); + assertEquals(1, successCount.get()); + } + } + /** * Returns the cluster state that the cluster-manager service uses (and that is provided by the discovery layer) */ diff --git a/server/src/test/java/org/opensearch/common/UUIDTests.java b/server/src/test/java/org/opensearch/common/UUIDTests.java index a71b68bfe859e..02595d19781ab 100644 --- a/server/src/test/java/org/opensearch/common/UUIDTests.java +++ b/server/src/test/java/org/opensearch/common/UUIDTests.java @@ -45,7 +45,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.TestUtil; import org.opensearch.common.lucene.Lucene; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java b/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java index 920684f06f83c..ede5658f0decd 100644 --- a/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java +++ b/server/src/test/java/org/opensearch/common/bytes/RecyclingBytesStreamOutputTests.java @@ -36,7 +36,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java index 417a79da68ae1..4968f7740b7c9 100644 --- a/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/common/geo/builders/AbstractShapeBuilderTestCase.java @@ -37,9 +37,9 @@ import org.opensearch.core.common.io.stream.NamedWriteable; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.Writeable.Reader; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -86,7 +86,7 @@ public static void afterClass() throws Exception { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { SB testShape = createTestShapeBuilder(); - XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { contentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java b/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java index 20f202149b17d..92ed8fac5aae0 100644 --- a/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java +++ b/server/src/test/java/org/opensearch/common/io/stream/ReleasableBytesStreamOutputTests.java @@ -35,7 +35,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java index ef0bedac72541..f7be2c4876e6f 100644 --- a/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/LuceneTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; @@ -565,12 +566,13 @@ public void testWrapAllDocsLive() throws Exception { } try (DirectoryReader unwrapped = DirectoryReader.open(writer)) { DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped); + StoredFields storedFields = reader.storedFields(); assertThat(reader.numDocs(), equalTo(liveDocs.size())); IndexSearcher searcher = new IndexSearcher(reader); Set actualDocs = new HashSet<>(); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - actualDocs.add(reader.document(scoreDoc.doc).get("id")); + actualDocs.add(storedFields.document(scoreDoc.doc).get("id")); } assertThat(actualDocs, equalTo(liveDocs)); } @@ -609,13 +611,14 @@ public void testWrapLiveDocsNotExposeAbortedDocuments() throws Exception { } try (DirectoryReader unwrapped = DirectoryReader.open(writer)) { DirectoryReader reader = Lucene.wrapAllDocsLive(unwrapped); + StoredFields storedFields = reader.storedFields(); assertThat(reader.maxDoc(), equalTo(numDocs + abortedDocs)); assertThat(reader.numDocs(), equalTo(liveDocs.size())); IndexSearcher searcher = new IndexSearcher(reader); List actualDocs = new ArrayList<>(); TopDocs topDocs = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - actualDocs.add(reader.document(scoreDoc.doc).get("id")); + actualDocs.add(storedFields.document(scoreDoc.doc).get("id")); } assertThat(actualDocs, equalTo(liveDocs)); } diff --git a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java index 60f39208d008a..9423d3c17c98a 100644 --- a/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java +++ b/server/src/test/java/org/opensearch/common/lucene/index/FreqTermsEnumTests.java @@ -43,6 +43,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.Query; @@ -149,9 +150,10 @@ public void setUp() throws Exception { // now go over each doc, build the relevant references and filter reader = DirectoryReader.open(iw); + StoredFields storedFields = reader.storedFields(); List filterTerms = new ArrayList<>(); for (int docId = 0; docId < reader.maxDoc(); docId++) { - Document doc = reader.document(docId); + Document doc = storedFields.document(docId); addFreqs(doc, referenceAll); if (!deletedIds.contains(doc.getField("id").stringValue())) { addFreqs(doc, referenceNotDeleted); diff --git a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java index 48a8e5799993d..f795df2f48b22 100644 --- a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java +++ b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java @@ -32,7 +32,7 @@ package org.opensearch.common.network; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -45,7 +45,7 @@ import org.opensearch.http.HttpServerTransport; import org.opensearch.http.HttpStats; import org.opensearch.http.NullDispatcher; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java b/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java index 2c7251818e2bc..94fb6b010309d 100644 --- a/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java +++ b/server/src/test/java/org/opensearch/common/settings/MemorySizeSettingsTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.settings; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.indices.IndexingMemoryController; import org.opensearch.indices.IndicesQueryCache; diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java index 3d5a5090cdc82..27f9c027e2641 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java @@ -51,8 +51,8 @@ import org.opensearch.common.settings.Setting.MinTimeValueParser; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Setting.RegexValidator; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexSettings; import org.opensearch.monitor.jvm.JvmInfo; diff --git a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java index 0c6352d118be2..1fd8658a09e67 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingsTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingsTests.java @@ -38,8 +38,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.settings.SecureString; import org.opensearch.core.xcontent.ToXContent; diff --git a/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java b/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java index 804a53d687c95..33cfcfcd79d87 100644 --- a/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/WriteableSettingTests.java @@ -13,8 +13,8 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/common/util/BigArraysTests.java b/server/src/test/java/org/opensearch/common/util/BigArraysTests.java index c973a76623452..22aff6c55d39a 100644 --- a/server/src/test/java/org/opensearch/common/util/BigArraysTests.java +++ b/server/src/test/java/org/opensearch/common/util/BigArraysTests.java @@ -33,15 +33,15 @@ package org.opensearch.common.util; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.util.BigArray; import org.opensearch.core.common.util.ByteArray; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java b/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java index bc64a4980947c..98837091d9550 100644 --- a/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java +++ b/server/src/test/java/org/opensearch/common/util/BinarySearcherTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.lease.Releasables; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; diff --git a/server/src/test/java/org/opensearch/common/util/BitArrayTests.java b/server/src/test/java/org/opensearch/common/util/BitArrayTests.java index 16e89b81c695c..e88e2b20c6155 100644 --- a/server/src/test/java/org/opensearch/common/util/BitArrayTests.java +++ b/server/src/test/java/org/opensearch/common/util/BitArrayTests.java @@ -32,12 +32,12 @@ package org.opensearch.common.util; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java b/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java index 8b719283ed71d..905fbfe4e244b 100644 --- a/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java +++ b/server/src/test/java/org/opensearch/common/util/BytesRefHashTests.java @@ -36,7 +36,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.tests.util.TestUtil; import org.opensearch.common.settings.Settings; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/common/util/LongHashTests.java b/server/src/test/java/org/opensearch/common/util/LongHashTests.java index 295497b9f188c..3716c7ad277b3 100644 --- a/server/src/test/java/org/opensearch/common/util/LongHashTests.java +++ b/server/src/test/java/org/opensearch/common/util/LongHashTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.util; import org.opensearch.common.settings.Settings; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java b/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java index fd0f1b5bad4a7..e0b0182216c27 100644 --- a/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java +++ b/server/src/test/java/org/opensearch/common/util/LongLongHashTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.util; import org.opensearch.common.settings.Settings; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; diff --git a/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java b/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java index 73a6fc5d54883..dc4743189f4cf 100644 --- a/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java +++ b/server/src/test/java/org/opensearch/common/util/LongObjectHashMapTests.java @@ -33,7 +33,7 @@ package org.opensearch.common.util; import org.opensearch.common.settings.Settings; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java b/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java index d771fdbef38d6..20e8916565daf 100644 --- a/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java +++ b/server/src/test/java/org/opensearch/common/util/concurrent/AbstractLifecycleRunnableTests.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.SuppressLoggerChecks; -import org.opensearch.common.component.Lifecycle; +import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.test.OpenSearchTestCase; import org.mockito.InOrder; diff --git a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java index 274b6477e3fbd..00ab96dab6a5c 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java +++ b/server/src/test/java/org/opensearch/common/xcontent/BaseXContentTestCase.java @@ -49,6 +49,7 @@ import org.opensearch.common.unit.DistanceUnit; import org.opensearch.common.util.CollectionUtils; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedObjectNotFoundException; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -865,7 +866,7 @@ public void testBasics() throws IOException { generator.writeEndObject(); } byte[] data = os.toByteArray(); - assertEquals(xcontentType(), XContentFactory.xContentType(data)); + assertEquals(xcontentType(), MediaTypeRegistry.xContent(data)); } public void testMissingEndObject() throws IOException { diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java index 866def49af991..1b7c535a550b0 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/XContentFactoryTests.java @@ -37,6 +37,7 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; @@ -64,14 +65,14 @@ public void testGuessCbor() throws IOException { } private void testGuessType(XContentType type) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(type); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(type); builder.startObject(); builder.field("field1", "value1"); builder.endObject(); final BytesReference bytes; if (type == XContentType.JSON && randomBoolean()) { - final int length = randomIntBetween(0, 8 * XContentFactory.GUESS_HEADER_LENGTH); + final int length = randomIntBetween(0, 8 * MediaTypeRegistry.GUESS_HEADER_LENGTH); final String content = Strings.toString(builder); final StringBuilder sb = new StringBuilder(length + content.length()); final char[] chars = new char[length]; @@ -82,24 +83,24 @@ private void testGuessType(XContentType type) throws IOException { bytes = BytesReference.bytes(builder); } - assertThat(XContentHelper.xContentType(bytes), equalTo(type)); - assertThat(XContentFactory.xContentType(bytes.streamInput()), equalTo(type)); + assertThat(MediaTypeRegistry.xContentType(bytes), equalTo(type)); + assertThat(MediaTypeRegistry.xContentType(bytes.streamInput()), equalTo(type)); // CBOR is binary, cannot use String if (type != XContentType.CBOR && type != XContentType.SMILE) { - assertThat(XContentFactory.xContentType(Strings.toString(builder)), equalTo(type)); + assertThat(MediaTypeRegistry.xContentType(Strings.toString(builder)), equalTo(type)); } } public void testCBORBasedOnMajorObjectDetection() { // for this {"f "=> 5} perl encoder for example generates: byte[] bytes = new byte[] { (byte) 0xA1, (byte) 0x43, (byte) 0x66, (byte) 6f, (byte) 6f, (byte) 0x5 }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR)); // assertThat(((Number) XContentHelper.convertToMap(bytes, true).v2().get("foo")).intValue(), equalTo(5)); // this if for {"foo" : 5} in python CBOR bytes = new byte[] { (byte) 0xA1, (byte) 0x63, (byte) 0x66, (byte) 0x6f, (byte) 0x6f, (byte) 0x5 }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR)); assertThat(((Number) XContentHelper.convertToMap(new BytesArray(bytes), true).v2().get("foo")).intValue(), equalTo(5)); // also make sure major type check doesn't collide with SMILE and JSON, just in case @@ -111,36 +112,36 @@ public void testCBORBasedOnMajorObjectDetection() { public void testCBORBasedOnMagicHeaderDetection() { byte[] bytes = new byte[] { (byte) 0xd9, (byte) 0xd9, (byte) 0xf7 }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.CBOR)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.CBOR)); } public void testEmptyStream() throws Exception { ByteArrayInputStream is = new ByteArrayInputStream(new byte[0]); - assertNull(XContentFactory.xContentType(is)); + assertNull(MediaTypeRegistry.xContentType(is)); is = new ByteArrayInputStream(new byte[] { (byte) 1 }); - assertNull(XContentFactory.xContentType(is)); + assertNull(MediaTypeRegistry.xContentType(is)); } public void testInvalidStream() throws Exception { byte[] bytes = new byte[] { (byte) '"' }; - assertNull(XContentFactory.xContentType(bytes)); + assertNull(MediaTypeRegistry.mediaTypeFromBytes(bytes, 0, bytes.length)); bytes = new byte[] { (byte) 'x' }; - assertNull(XContentFactory.xContentType(bytes)); + assertNull(MediaTypeRegistry.mediaTypeFromBytes(bytes, 0, bytes.length)); } public void testJsonFromBytesOptionallyPrecededByUtf8Bom() throws Exception { byte[] bytes = new byte[] { (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); bytes = new byte[] { (byte) 0x20, (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); bytes = new byte[] { (byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); bytes = new byte[] { (byte) 0xef, (byte) 0xbb, (byte) 0xbf, (byte) 0x20, (byte) '{', (byte) '}' }; - assertThat(XContentFactory.xContentType(bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(bytes), equalTo(XContentType.JSON)); } } diff --git a/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java index 53607f432bbd5..8c53d7edebca8 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/XContentTypeTests.java @@ -44,81 +44,80 @@ public class XContentTypeTests extends OpenSearchTestCase { public void testFromJson() throws Exception { String mediaType = "application/json"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromNdJson() throws Exception { String mediaType = "application/x-ndjson"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromJsonUppercase() throws Exception { String mediaType = "application/json".toUpperCase(Locale.ROOT); XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromYaml() throws Exception { String mediaType = "application/yaml"; XContentType expectedXContentType = XContentType.YAML; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + "; charset=UTF-8"), equalTo(expectedXContentType)); } public void testFromSmile() throws Exception { String mediaType = "application/smile"; XContentType expectedXContentType = XContentType.SMILE; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromCbor() throws Exception { String mediaType = "application/cbor"; XContentType expectedXContentType = XContentType.CBOR; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromWildcard() throws Exception { String mediaType = "application/*"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromWildcardUppercase() throws Exception { String mediaType = "APPLICATION/*"; XContentType expectedXContentType = XContentType.JSON; - assertThat(XContentType.fromMediaType(mediaType), equalTo(expectedXContentType)); - assertThat(XContentType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType), equalTo(expectedXContentType)); + assertThat(MediaType.fromMediaType(mediaType + ";"), equalTo(expectedXContentType)); } public void testFromRubbish() throws Exception { - assertThat(XContentType.fromMediaType((String) null), nullValue()); - assertThat(XContentType.fromMediaType((MediaType) null), nullValue()); - assertThat(XContentType.fromMediaType(""), nullValue()); - assertThat(XContentType.fromMediaType("text/plain"), nullValue()); - assertThat(XContentType.fromMediaType("gobbly;goop"), nullValue()); + assertThat(MediaType.fromMediaType(null), nullValue()); + assertThat(MediaType.fromMediaType(""), nullValue()); + assertThat(MediaType.fromMediaType("text/plain"), nullValue()); + assertThat(MediaType.fromMediaType("gobbly;goop"), nullValue()); } public void testVersionedMediaType() throws Exception { - assertThat(XContentType.fromMediaType("application/vnd.opensearch+json;compatible-with=7"), equalTo(XContentType.JSON)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+yaml;compatible-with=7"), equalTo(XContentType.YAML)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+cbor;compatible-with=7"), equalTo(XContentType.CBOR)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+smile;compatible-with=7"), equalTo(XContentType.SMILE)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+json;compatible-with=7"), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+yaml;compatible-with=7"), equalTo(XContentType.YAML)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+cbor;compatible-with=7"), equalTo(XContentType.CBOR)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+smile;compatible-with=7"), equalTo(XContentType.SMILE)); - assertThat(XContentType.fromMediaType("application/vnd.opensearch+json ;compatible-with=7"), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType("application/vnd.opensearch+json ;compatible-with=7"), equalTo(XContentType.JSON)); String mthv = "application/vnd.opensearch+json ;compatible-with=7;charset=utf-8"; - assertThat(XContentType.fromMediaType(mthv), equalTo(XContentType.JSON)); - assertThat(XContentType.fromMediaType(mthv.toUpperCase(Locale.ROOT)), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType(mthv), equalTo(XContentType.JSON)); + assertThat(MediaType.fromMediaType(mthv.toUpperCase(Locale.ROOT)), equalTo(XContentType.JSON)); } } diff --git a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java index 6c97297ce43e7..dc98000d93899 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/builder/XContentBuilderTests.java @@ -39,13 +39,13 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentOpenSearchExtension; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentGenerator; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentGenerator; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; import java.io.ByteArrayOutputStream; @@ -68,7 +68,7 @@ public class XContentBuilderTests extends OpenSearchTestCase { public void testPrettyWithLfAtEnd() throws Exception { ByteArrayOutputStream os = new ByteArrayOutputStream(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); + XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os); generator.usePrettyPrint(); generator.usePrintLineFeedAtEnd(); @@ -87,7 +87,7 @@ public void testPrettyWithLfAtEnd() throws Exception { public void testReuseJsonGenerator() throws Exception { ByteArrayOutputStream os = new ByteArrayOutputStream(); - XContentGenerator generator = XContentFactory.xContent(XContentType.JSON).createGenerator(os); + XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os); generator.writeStartObject(); generator.writeStringField("test", "value"); generator.writeEndObject(); @@ -107,14 +107,14 @@ public void testReuseJsonGenerator() throws Exception { public void testRaw() throws IOException { { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); xContentBuilder.endObject(); assertThat(Strings.toString(xContentBuilder), equalTo("{\"foo\":{\"test\":\"value\"}}")); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -122,7 +122,7 @@ public void testRaw() throws IOException { assertThat(Strings.toString(xContentBuilder), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}")); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -130,7 +130,7 @@ public void testRaw() throws IOException { assertThat(Strings.toString(xContentBuilder), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}")); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -142,7 +142,7 @@ public void testRaw() throws IOException { ); } { - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); xContentBuilder.startObject(); xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}").streamInput()); @@ -157,17 +157,17 @@ public void testRaw() throws IOException { } public void testSimpleGenerator() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", "value").endObject(); assertThat(Strings.toString(builder), equalTo("{\"test\":\"value\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", "value").endObject(); assertThat(Strings.toString(builder), equalTo("{\"test\":\"value\"}")); } public void testOverloadedList() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", Arrays.asList("1", "2")).endObject(); assertThat(Strings.toString(builder), equalTo("{\"test\":[\"1\",\"2\"]}")); } @@ -175,7 +175,7 @@ public void testOverloadedList() throws Exception { public void testWritingBinaryToStream() throws Exception { BytesStreamOutput bos = new BytesStreamOutput(); - XContentGenerator gen = XContentFactory.xContent(XContentType.JSON).createGenerator(bos); + XContentGenerator gen = XContentType.JSON.xContent().createGenerator(bos); gen.writeStartObject(); gen.writeStringField("name", "something"); gen.flush(); @@ -189,7 +189,7 @@ public void testWritingBinaryToStream() throws Exception { } public void testByteConversion() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test_name", (Byte) (byte) 120).endObject(); assertThat(BytesReference.bytes(builder).utf8ToString(), equalTo("{\"test_name\":120}")); } @@ -199,21 +199,21 @@ public void testDateTypesConversion() throws Exception { String expectedDate = XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(date.getTime()); Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"), Locale.ROOT); String expectedCalendar = XContentOpenSearchExtension.DEFAULT_DATE_PRINTER.print(calendar.getTimeInMillis()); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().timeField("date", date).endObject(); assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("calendar", calendar).endObject(); assertThat(Strings.toString(builder), equalTo("{\"calendar\":\"" + expectedCalendar + "\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); Map map = new HashMap<>(); map.put("date", date); builder.map(map); assertThat(Strings.toString(builder), equalTo("{\"date\":\"" + expectedDate + "\"}")); - builder = XContentFactory.contentBuilder(XContentType.JSON); + builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); map = new HashMap<>(); map.put("calendar", calendar); builder.map(map); @@ -221,7 +221,7 @@ public void testDateTypesConversion() throws Exception { } public void testCopyCurrentStructure() throws Exception { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject().field("test", "test field").startObject("filter").startObject("terms"); // up to 20k random terms @@ -248,7 +248,7 @@ public void testCopyCurrentStructure() throws Exception { } } else if (token == XContentParser.Token.START_OBJECT) { if ("filter".equals(currentFieldName)) { - filterBuilder = XContentFactory.contentBuilder(parser.contentType()); + filterBuilder = MediaTypeRegistry.contentBuilder(parser.contentType()); filterBuilder.copyCurrentStructure(parser); } } @@ -288,10 +288,10 @@ public void testHandlingOfPath_absolute() throws IOException { } private void checkPathSerialization(Path path) throws IOException { - XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); pathBuilder.startObject().field("file", path).endObject(); - XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); stringBuilder.startObject().field("file", path.toString()).endObject(); assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder))); @@ -301,10 +301,10 @@ public void testHandlingOfPath_StringName() throws IOException { Path path = PathUtils.get("path"); String name = new String("file"); - XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); pathBuilder.startObject().field(name, path).endObject(); - XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); stringBuilder.startObject().field(name, path.toString()).endObject(); assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder))); @@ -313,36 +313,36 @@ public void testHandlingOfPath_StringName() throws IOException { public void testHandlingOfCollectionOfPaths() throws IOException { Path path = PathUtils.get("path"); - XContentBuilder pathBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder pathBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); pathBuilder.startObject().field("file", Arrays.asList(path)).endObject(); - XContentBuilder stringBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder stringBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); stringBuilder.startObject().field("file", Arrays.asList(path.toString())).endObject(); assertThat(Strings.toString(pathBuilder), equalTo(Strings.toString(stringBuilder))); } public void testIndentIsPlatformIndependent() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint(); builder.startObject().field("test", "foo").startObject("foo").field("foobar", "boom").endObject().endObject(); String string = Strings.toString(builder); assertEquals("{\n" + " \"test\" : \"foo\",\n" + " \"foo\" : {\n" + " \"foobar\" : \"boom\"\n" + " }\n" + "}", string); - builder = XContentFactory.contentBuilder(XContentType.YAML).prettyPrint(); + builder = MediaTypeRegistry.contentBuilder(XContentType.YAML).prettyPrint(); builder.startObject().field("test", "foo").startObject("foo").field("foobar", "boom").endObject().endObject(); string = Strings.toString(builder); assertEquals("---\n" + "test: \"foo\"\n" + "foo:\n" + " foobar: \"boom\"\n", string); } public void testRenderGeoPoint() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint(); builder.startObject().field("foo").value(new GeoPoint(1, 2)).endObject(); String string = Strings.toString(builder); assertEquals("{\n" + " \"foo\" : {\n" + " \"lat\" : 1.0,\n" + " \"lon\" : 2.0\n" + " }\n" + "}", string.trim()); } public void testWriteMapWithNullKeys() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); try { builder.map(Collections.singletonMap(null, "test")); fail("write map should have failed"); @@ -352,7 +352,7 @@ public void testWriteMapWithNullKeys() throws IOException { } public void testWriteMapValueWithNullKeys() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); try { builder.map(Collections.singletonMap(null, "test")); fail("write map should have failed"); @@ -362,7 +362,7 @@ public void testWriteMapValueWithNullKeys() throws IOException { } public void testWriteFieldMapWithNullKeys() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); try { builder.startObject(); builder.field("map", Collections.singletonMap(null, "test")); @@ -374,7 +374,7 @@ public void testWriteFieldMapWithNullKeys() throws IOException { public void testMissingEndObject() throws IOException { IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { builder.startObject(); builder.field("foo", true); } @@ -385,7 +385,7 @@ public void testMissingEndObject() throws IOException { public void testMissingEndArray() throws IOException { IllegalStateException e = expectThrows(IllegalStateException.class, () -> { - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { builder.startObject(); builder.startArray("foo"); builder.value(0); diff --git a/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java b/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java index e304798a0ff0c..89add6ea78722 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/cbor/JsonVsCborTests.java @@ -33,7 +33,6 @@ package org.opensearch.common.xcontent.cbor; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentGenerator; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -48,10 +47,10 @@ public class JsonVsCborTests extends OpenSearchTestCase { public void testCompareParsingTokens() throws IOException { BytesStreamOutput xsonOs = new BytesStreamOutput(); - XContentGenerator xsonGen = XContentFactory.xContent(XContentType.CBOR).createGenerator(xsonOs); + XContentGenerator xsonGen = XContentType.CBOR.xContent().createGenerator(xsonOs); BytesStreamOutput jsonOs = new BytesStreamOutput(); - XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs); + XContentGenerator jsonGen = XContentType.JSON.xContent().createGenerator(jsonOs); xsonGen.writeStartObject(); jsonGen.writeStartObject(); diff --git a/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java b/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java index 4f26bdc7e9c26..a96031f2a1dad 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java +++ b/server/src/test/java/org/opensearch/common/xcontent/smile/JsonVsSmileTests.java @@ -33,7 +33,6 @@ package org.opensearch.common.xcontent.smile; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentGenerator; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -48,10 +47,10 @@ public class JsonVsSmileTests extends OpenSearchTestCase { public void testCompareParsingTokens() throws IOException { BytesStreamOutput xsonOs = new BytesStreamOutput(); - XContentGenerator xsonGen = XContentFactory.xContent(XContentType.SMILE).createGenerator(xsonOs); + XContentGenerator xsonGen = XContentType.SMILE.xContent().createGenerator(xsonOs); BytesStreamOutput jsonOs = new BytesStreamOutput(); - XContentGenerator jsonGen = XContentFactory.xContent(XContentType.JSON).createGenerator(jsonOs); + XContentGenerator jsonGen = XContentType.JSON.xContent().createGenerator(jsonOs); xsonGen.writeStartObject(); jsonGen.writeStartObject(); diff --git a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index c318dd89f573c..0a8df4406b5cd 100644 --- a/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/server/src/test/java/org/opensearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -39,7 +39,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.support.AbstractFilteringTestCase; @@ -89,7 +88,7 @@ static void assertXContentBuilderAsString(final XContentBuilder expected, final } static void assertXContentBuilderAsBytes(final XContentBuilder expected, final XContentBuilder actual) { - XContent xContent = XContentFactory.xContent(actual.contentType()); + XContent xContent = actual.contentType().xContent(); try ( XContentParser jsonParser = xContent.createParser( NamedXContentRegistry.EMPTY, diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java index 0f831ff28af79..880eaeca99bb0 100644 --- a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java +++ b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java @@ -125,7 +125,7 @@ public void testVectorHighlighterPrefixQuery() throws Exception { assertThat(fragment, nullValue()); prefixQuery = new PrefixQuery(new Term("content", "ba"), PrefixQuery.SCORING_BOOLEAN_REWRITE); - Query rewriteQuery = prefixQuery.rewrite(reader); + Query rewriteQuery = prefixQuery.rewrite(searcher); fragment = highlighter.getBestFragment(highlighter.getFieldQuery(rewriteQuery), reader, topDocs.scoreDocs[0].doc, "content", 30); assertThat(fragment, notNullValue()); diff --git a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java index 61aac8247647a..acf94483c8116 100644 --- a/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java +++ b/server/src/test/java/org/opensearch/discovery/FileBasedSeedHostsProviderTests.java @@ -41,7 +41,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java index 95f84a70986e1..07491cc0d6435 100644 --- a/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java +++ b/server/src/test/java/org/opensearch/discovery/SeedHostsResolverTests.java @@ -47,7 +47,7 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.common.util.concurrent.FutureUtils; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; diff --git a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java index 382698a300591..a55d89ed37cd7 100644 --- a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java +++ b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java @@ -46,8 +46,8 @@ import org.opensearch.common.CheckedRunnable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.core.index.Index; diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 41b841aec7be8..9f1050351c7b5 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -66,7 +66,7 @@ import org.opensearch.extensions.settings.RegisterCustomSettingsRequest; import org.opensearch.extensions.ExtensionsSettings.Extension; import org.opensearch.identity.IdentityService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.plugins.ExtensionAwarePlugin; import org.opensearch.rest.RestController; import org.opensearch.test.FeatureFlagSetter; diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java index e46e05cd2dbec..bc216bd3bbbb8 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java @@ -25,7 +25,7 @@ import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.AcknowledgedResponse; import org.opensearch.extensions.rest.RestSendToExtensionActionTests; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; import org.opensearch.test.transport.MockTransportService; diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 030df865c4e4a..1faa3bd9aec55 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -34,7 +34,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.extensions.ExtensionsSettings; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.RestRequest; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index 8695e409de0bc..b59513fc0045d 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -46,7 +46,7 @@ import org.opensearch.extensions.action.ExtensionAction; import org.opensearch.extensions.action.ExtensionTransportAction; import org.opensearch.identity.IdentityService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestHandler.Route; import org.opensearch.rest.RestRequest.Method; diff --git a/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java b/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java index bf9234a7f129d..22d07d7f18920 100644 --- a/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java +++ b/server/src/test/java/org/opensearch/extensions/settings/RegisterCustomSettingsTests.java @@ -16,8 +16,8 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 830a8a9ad8ab7..714a2389e64e1 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -58,7 +58,7 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.TestEnvironment; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.node.Node; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; diff --git a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java index 211c13e00e6db..14ed610942e15 100644 --- a/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/PersistedClusterStateServiceTests.java @@ -62,7 +62,7 @@ import org.opensearch.env.NodeMetadata; import org.opensearch.gateway.PersistedClusterStateService.Writer; import org.opensearch.core.index.Index; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.MockLogAppender; import org.opensearch.test.junit.annotations.TestLogging; diff --git a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java index b36b3110b2123..fbf0e8cd42c72 100644 --- a/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java +++ b/server/src/test/java/org/opensearch/http/AbstractHttpServerTransportTests.java @@ -46,7 +46,7 @@ import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; diff --git a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java index 1ffe0538edaad..fb545e8db582e 100644 --- a/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java +++ b/server/src/test/java/org/opensearch/http/DefaultRestChannelTests.java @@ -47,7 +47,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.lease.Releasable; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 95d0ae706b286..2b89558e394fc 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -56,7 +56,7 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.SetOnce.AlreadySetException; import org.opensearch.common.UUIDs; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; @@ -99,8 +99,8 @@ import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesQueryCache; import org.opensearch.indices.analysis.AnalysisModule; -import org.opensearch.indices.breaker.CircuitBreakerService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index e42e9b4970081..f91905fea9561 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -41,7 +41,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.translog.Translog; diff --git a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java index b1dba0c22c075..387997892ee30 100644 --- a/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java +++ b/server/src/test/java/org/opensearch/index/MergePolicySettingsTests.java @@ -33,8 +33,8 @@ import org.apache.lucene.index.NoMergePolicy; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 612bebf74bc70..d58d166415446 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -111,7 +111,7 @@ import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver; import org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.concurrent.AbstractRunnable; @@ -151,7 +151,7 @@ import org.opensearch.index.translog.TranslogDeletionPolicyFactory; import org.opensearch.index.translog.TranslogException; import org.opensearch.index.translog.listener.TranslogEventListener; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.VersionUtils; import org.opensearch.threadpool.ThreadPool; @@ -2541,6 +2541,7 @@ class OpAndVersion { final Term uidTerm = newUid(doc); engine.index(indexForDoc(doc)); final BiFunction searcherFactory = engine::acquireSearcher; + for (int i = 0; i < thread.length; i++) { thread[i] = new Thread(() -> { startGun.countDown(); @@ -2549,10 +2550,12 @@ class OpAndVersion { } catch (InterruptedException e) { throw new AssertionError(e); } + for (int op = 0; op < opsPerThread; op++) { try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) { + FieldsVisitor visitor = new FieldsVisitor(true); - get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); + get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor); List values = new ArrayList<>(Strings.commaDelimitedListToSet(visitor.source().utf8ToString())); String removed = op % 3 == 0 && values.size() > 0 ? values.remove(0) : null; String added = "v_" + idGenerator.incrementAndGet(); @@ -2608,7 +2611,7 @@ class OpAndVersion { try (Engine.GetResult get = engine.get(new Engine.Get(true, false, doc.id(), uidTerm), searcherFactory)) { FieldsVisitor visitor = new FieldsVisitor(true); - get.docIdAndVersion().reader.document(get.docIdAndVersion().docId, visitor); + get.docIdAndVersion().reader.storedFields().document(get.docIdAndVersion().docId, visitor); List values = Arrays.asList(Strings.commaDelimitedListToStringArray(visitor.source().utf8ToString())); assertThat(currentValues, equalTo(new HashSet<>(values))); } diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java index 98f1a416731e4..d635b38e811c4 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationReaderManagerTests.java @@ -46,43 +46,4 @@ public void testCreateNRTreaderManager() throws IOException { } } } - - public void testUpdateSegmentsWhileRefreshing() throws IOException, InterruptedException { - try (final Store store = createStore()) { - store.createEmpty(Version.LATEST); - final DirectoryReader reader = DirectoryReader.open(store.directory()); - NRTReplicationReaderManager readerManager = new NRTReplicationReaderManager( - OpenSearchDirectoryReader.wrap(reader, shardId), - (files) -> {}, - (files) -> {} - ); - - final SegmentInfos infos_2 = readerManager.getSegmentInfos().clone(); - infos_2.changed(); - - Thread refreshThread = new Thread(() -> { - try { - readerManager.maybeRefresh(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - Thread updateThread = new Thread(() -> { - try { - readerManager.updateSegments(infos_2); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - refreshThread.start(); - updateThread.start(); - refreshThread.join(); - updateThread.join(); - try (final OpenSearchDirectoryReader acquire = readerManager.acquire()) { - final StandardDirectoryReader standardReader = NRTReplicationReaderManager.unwrapStandardReader(acquire); - assertEquals(infos_2.version, standardReader.getSegmentInfos().version); - } - assertEquals(infos_2, readerManager.getSegmentInfos()); - } - } } diff --git a/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java index b9fe69c282471..5fe6c2d08cdc7 100644 --- a/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NoOpEngineTests.java @@ -44,7 +44,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; diff --git a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java index a8279ed908779..fed521e2d5ed9 100644 --- a/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java +++ b/server/src/test/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicyTests.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.ShuffleForcedMergePolicy; import org.apache.lucene.index.StandardDirectoryReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.MatchAllDocsQuery; @@ -89,8 +90,9 @@ public void testPruneAll() throws IOException { writer.forceMerge(1); writer.commit(); try (DirectoryReader reader = DirectoryReader.open(writer)) { + StoredFields storedFields = reader.storedFields(); for (int i = 0; i < reader.maxDoc(); i++) { - Document document = reader.document(i); + Document document = storedFields.document(i); assertEquals(1, document.getFields().size()); assertEquals("source", document.getFields().get(0).name()); } @@ -157,11 +159,12 @@ public void testPruneSome() throws IOException { writer.forceMerge(1); writer.commit(); try (DirectoryReader reader = DirectoryReader.open(writer)) { + StoredFields storedFields = reader.storedFields(); assertEquals(1, reader.leaves().size()); NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source"); assertNotNull(extra_source); for (int i = 0; i < reader.maxDoc(); i++) { - Document document = reader.document(i); + Document document = storedFields.document(i); Set collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet()); assertTrue(collect.contains("source")); assertTrue(collect.contains("even")); @@ -197,11 +200,12 @@ public void testPruneNone() throws IOException { writer.forceMerge(1); writer.commit(); try (DirectoryReader reader = DirectoryReader.open(writer)) { + StoredFields storedFields = reader.storedFields(); assertEquals(1, reader.leaves().size()); NumericDocValues extra_source = reader.leaves().get(0).reader().getNumericDocValues("extra_source"); assertNotNull(extra_source); for (int i = 0; i < reader.maxDoc(); i++) { - Document document = reader.document(i); + Document document = storedFields.document(i); Set collect = document.getFields().stream().map(IndexableField::name).collect(Collectors.toSet()); assertTrue(collect.contains("source")); assertTrue(collect.contains("extra_source")); diff --git a/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java b/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java index 48ab9d60727b7..babf0782cdec0 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java +++ b/server/src/test/java/org/opensearch/index/fielddata/FieldDataCacheTests.java @@ -49,7 +49,7 @@ import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.opensearch.index.mapper.TextFieldMapper; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.FieldMaskingReader; diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index d360a2a767e8a..510dd3456924b 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -40,6 +40,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexOrDocValuesQuery; @@ -259,7 +260,10 @@ public void testRangeQuery() throws IOException { LongPoint.newRangeQuery("field", instant1, instant2), SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) ); - assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new MultiReader())); + assertEquals( + expected, + ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(new IndexSearcher(new MultiReader())) + ); instant1 = nowInMillis; instant2 = instant1 + 100; diff --git a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java index 365d3738f4022..0e04e57a290b6 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DynamicMappingTests.java @@ -367,9 +367,7 @@ private void doTestDefaultFloatingPointMappings(DocumentMapper mapper, XContentB .field("quux", "3.2") // float detected through numeric detection .endObject() ); - ParsedDocument parsedDocument = mapper.parse( - new SourceToParse("index", "id", source, XContentType.fromMediaType(builder.contentType())) - ); + ParsedDocument parsedDocument = mapper.parse(new SourceToParse("index", "id", source, builder.contentType())); Mapping update = parsedDocument.dynamicMappingsUpdate(); assertNotNull(update); assertThat(((FieldMapper) update.root().getMapper("foo")).fieldType().typeName(), equalTo("float")); diff --git a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java index 518e5e880f4f8..9e467dca17409 100644 --- a/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java @@ -38,7 +38,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -75,7 +75,7 @@ public void testNoFormat() throws Exception { ) ); - assertThat(XContentFactory.xContentType(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON)); + assertThat(MediaTypeRegistry.xContent(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON)); documentMapper = parser.parse("type", new CompressedXContent(mapping)); doc = documentMapper.parse( @@ -87,7 +87,7 @@ public void testNoFormat() throws Exception { ) ); - assertThat(XContentHelper.xContentType(doc.source()), equalTo(XContentType.SMILE)); + assertThat(MediaTypeRegistry.xContentType(doc.source()), equalTo(XContentType.SMILE)); } public void testIncludes() throws Exception { diff --git a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java index 92efd19732ef0..54afde41daa5b 100644 --- a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java @@ -37,8 +37,8 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -181,7 +181,7 @@ public void testIllegalArguments() { // https://github.com/elastic/elasticsearch/issues/7240 public void testEmptyBooleanQuery() throws Exception { - XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); contentBuilder.startObject().startObject("bool").endObject().endObject(); try (XContentParser xParser = createParser(contentBuilder)) { Query parsedQuery = parseQuery(xParser).toQuery(createShardContext()); diff --git a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java index cb75889a0aa24..5e6226561582e 100644 --- a/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/InnerHitBuilderTests.java @@ -34,9 +34,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -133,7 +133,7 @@ public void testSerializationOrder() throws Exception { public void testFromAndToXContent() throws Exception { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { InnerHitBuilder innerHit = randomInnerHits(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS); // fields is printed out as an object but parsed into a List where order matters, we disable shuffling XContentBuilder shuffled = shuffleXContent(builder, "fields"); diff --git a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java index e6ec132a10474..2d950f0994976 100644 --- a/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -268,7 +268,7 @@ private static Fields generateFields(String[] fieldNames, String text) throws IO for (String fieldName : fieldNames) { index.addField(fieldName, text, new WhitespaceAnalyzer()); } - return index.createSearcher().getIndexReader().getTermVectors(0); + return index.createSearcher().getIndexReader().termVectors().get(0); } @Override diff --git a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java index bc514208c803d..be87f7ca15907 100644 --- a/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SpanMultiTermQueryBuilderTests.java @@ -284,8 +284,9 @@ public void testTermExpansionExceptionOnSpanFailure() throws Exception { BooleanQuery.setMaxClauseCount(1); try { QueryBuilder queryBuilder = new SpanMultiTermQueryBuilder(QueryBuilders.prefixQuery("body", "bar")); - Query query = queryBuilder.toQuery(createShardContext(new IndexSearcher(reader))); - RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(reader)); + IndexSearcher searcher = new IndexSearcher(reader); + Query query = queryBuilder.toQuery(createShardContext(searcher)); + RuntimeException exc = expectThrows(RuntimeException.class, () -> query.rewrite(searcher)); assertThat(exc.getMessage(), containsString("maxClauseCount")); } finally { BooleanQuery.setMaxClauseCount(origBoolMaxClauseCount); diff --git a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java index 39160b1ac49bc..45172c946ab36 100644 --- a/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java +++ b/server/src/test/java/org/opensearch/index/search/OpenSearchToParentBlockJoinQueryTests.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -112,7 +113,7 @@ public void testRewrite() throws IOException { ScoreMode.Avg, "nested" ); - Query rewritten = q.rewrite(new MultiReader()); + Query rewritten = q.rewrite(new IndexSearcher(new MultiReader())); assertEquals(expected, rewritten); } } diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java index d56cbda2fe6e4..03a6fc3df824d 100644 --- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java @@ -1795,12 +1795,8 @@ public void testSegmentReplicationCheckpointTracking() { assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds))); assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable)); assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync)); - // get insync ids, filter out the primary. - final Set inSyncAllocationIds = tracker.getReplicationGroup() - .getInSyncAllocationIds() - .stream() - .filter(id -> tracker.shardAllocationId.equals(id) == false) - .collect(Collectors.toSet()); + + initializingIds.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED)); final ReplicationCheckpoint initialCheckpoint = new ReplicationCheckpoint( tracker.shardId(), @@ -1831,8 +1827,10 @@ public void testSegmentReplicationCheckpointTracking() { tracker.setLatestReplicationCheckpoint(secondCheckpoint); tracker.setLatestReplicationCheckpoint(thirdCheckpoint); + final Set expectedIds = ids(initializingIds); + Set groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + assertEquals(expectedIds.size(), groupStats.size()); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(3, shardStat.getCheckpointsBehindCount()); assertEquals(100L, shardStat.getBytesBehindCount()); @@ -1840,7 +1838,7 @@ public void testSegmentReplicationCheckpointTracking() { // simulate replicas moved up to date. final Map checkpoints = tracker.checkpoints; - for (String id : inSyncAllocationIds) { + for (String id : expectedIds) { final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id); assertEquals(3, checkpointState.checkpointTimers.size()); tracker.updateVisibleCheckpointForShard(id, initialCheckpoint); @@ -1848,13 +1846,13 @@ public void testSegmentReplicationCheckpointTracking() { } groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + assertEquals(expectedIds.size(), groupStats.size()); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(2, shardStat.getCheckpointsBehindCount()); assertEquals(99L, shardStat.getBytesBehindCount()); } - for (String id : inSyncAllocationIds) { + for (String id : expectedIds) { final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id); assertEquals(2, checkpointState.checkpointTimers.size()); tracker.updateVisibleCheckpointForShard(id, thirdCheckpoint); @@ -1862,7 +1860,7 @@ public void testSegmentReplicationCheckpointTracking() { } groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + assertEquals(expectedIds.size(), groupStats.size()); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(0, shardStat.getCheckpointsBehindCount()); assertEquals(0L, shardStat.getBytesBehindCount()); @@ -1883,19 +1881,24 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() { AllocationId primaryId = activeAllocationIds.iterator().next(); IndexShardRoutingTable routingTable = routingTable(initializingIds, primaryId); final ReplicationTracker tracker = newTracker(primaryId, settings); - tracker.updateFromClusterManager(initialClusterStateVersion, ids(activeAllocationIds), routingTable); tracker.activatePrimaryMode(NO_OPS_PERFORMED); - assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds))); - assertThat(tracker.getReplicationGroup().getRoutingTable(), equalTo(routingTable)); - assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync)); - // get insync ids, filter out the primary. - final Set inSyncAllocationIds = tracker.getReplicationGroup() - .getInSyncAllocationIds() - .stream() - .filter(id -> tracker.shardAllocationId.equals(id) == false) - .collect(Collectors.toSet()); + initializingIds.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED)); + + assertEquals(tracker.getReplicationGroup().getRoutingTable(), routingTable); + assertEquals( + "All active & initializing ids are now marked in-sync", + Sets.union(ids(activeAllocationIds), ids(initializingIds)), + tracker.getReplicationGroup().getInSyncAllocationIds() + ); + + assertEquals( + "Active ids are in-sync but still unavailable", + tracker.getReplicationGroup().getUnavailableInSyncShards(), + Sets.difference(ids(activeAllocationIds), Set.of(primaryId.getId())) + ); + assertTrue(activeAllocationIds.stream().allMatch(a -> tracker.getTrackedLocalCheckpointForShard(a.getId()).inSync)); final ReplicationCheckpoint initialCheckpoint = new ReplicationCheckpoint( tracker.shardId(), @@ -1907,15 +1910,20 @@ public void testSegmentReplicationCheckpointTrackingInvalidAllocationIDs() { ); tracker.setLatestReplicationCheckpoint(initialCheckpoint); + // we expect that the only returned ids from getSegmentReplicationStats will be the initializing ids we marked with + // markAsTrackingAndInSyncQuietly. + // This is because the ids marked active initially are still unavailable (don't have an associated routing entry). + final Set expectedIds = ids(initializingIds); Set groupStats = tracker.getSegmentReplicationStats(); - assertEquals(inSyncAllocationIds.size(), groupStats.size()); + final Set actualIds = groupStats.stream().map(SegmentReplicationShardStats::getAllocationId).collect(Collectors.toSet()); + assertEquals(expectedIds, actualIds); for (SegmentReplicationShardStats shardStat : groupStats) { assertEquals(1, shardStat.getCheckpointsBehindCount()); } // simulate replicas moved up to date. final Map checkpoints = tracker.checkpoints; - for (String id : inSyncAllocationIds) { + for (String id : expectedIds) { final ReplicationTracker.CheckpointState checkpointState = checkpoints.get(id); assertEquals(1, checkpointState.checkpointTimers.size()); tracker.updateVisibleCheckpointForShard(id, initialCheckpoint); diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 915a15da6cb1d..1340ff1868a11 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -141,7 +141,7 @@ import org.opensearch.index.translog.TranslogStats; import org.opensearch.index.translog.listener.TranslogEventListener; import org.opensearch.indices.IndicesQueryCache; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.recovery.RecoveryTarget; diff --git a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java index 282f4c09b982d..b51ceb2250e8f 100644 --- a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java @@ -47,7 +47,7 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java index f812312004b21..f3cffa4721ee5 100644 --- a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java @@ -36,6 +36,7 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; @@ -70,7 +71,7 @@ import org.opensearch.index.store.Store; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogConfig; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.DummyShardLock; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.IndexSettingsModule; @@ -369,9 +370,8 @@ public void testLotsOfThreads() throws Exception { try (Engine.GetResult getResult = engine.get(get, engine::acquireSearcher)) { assertTrue("document not found", getResult.exists()); assertEquals(iteration, getResult.version()); - org.apache.lucene.document.Document document = getResult.docIdAndVersion().reader.document( - getResult.docIdAndVersion().docId - ); + StoredFields storedFields = getResult.docIdAndVersion().reader.storedFields(); + org.apache.lucene.document.Document document = storedFields.document(getResult.docIdAndVersion().docId); assertThat(document.getValues("test"), arrayContaining(testFieldValue)); } } catch (Exception t) { diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index d988e34ef18dc..9107606326150 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -120,6 +120,12 @@ public void testReplicationCheckpointNotNullForSegRep() throws IOException { closeShards(indexShard); } + public void testNRTReplicasDoNotAcceptRefreshListeners() throws IOException { + final IndexShard indexShard = newStartedShard(false, settings, new NRTReplicationEngineFactory()); + indexShard.addRefreshListener(mock(Translog.Location.class), Assert::assertFalse); + closeShards(indexShard); + } + public void testSegmentInfosAndReplicationCheckpointTuple() throws Exception { try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { shards.startAll(); @@ -184,7 +190,6 @@ public void testIsSegmentReplicationAllowed_WrongEngineType() throws IOException * reader close operation on replica shard deletes the segment files copied in current round of segment replication. * It does this by blocking the finalizeReplication on replica shard and performing close operation on acquired * searcher that triggers the reader close operation. - * @throws Exception */ public void testSegmentReplication_With_ReaderClosedConcurrently() throws Exception { String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; @@ -234,7 +239,6 @@ public void testSegmentReplication_With_ReaderClosedConcurrently() throws Except /** * Similar to test above, this test shows the issue where an engine close operation during active segment replication * can result in Lucene CorruptIndexException. - * @throws Exception */ public void testSegmentReplication_With_EngineClosedConcurrently() throws Exception { String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; @@ -283,7 +287,6 @@ public void testSegmentReplication_With_EngineClosedConcurrently() throws Except /** * Verifies that commits on replica engine resulting from engine or reader close does not cleanup the temporary * replication files from ongoing round of segment replication - * @throws Exception */ public void testTemporaryFilesNotCleanup() throws Exception { String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; @@ -884,7 +887,6 @@ public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitRefresh() throws testNRTReplicaWithRemoteStorePromotedAsPrimary(true, false); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8817") public void testNRTReplicaWithRemoteStorePromotedAsPrimaryCommitCommit() throws Exception { testNRTReplicaWithRemoteStorePromotedAsPrimary(true, true); } diff --git a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java index 04dcea210640c..940d9a4ead5f9 100644 --- a/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ShardSplittingQueryTests.java @@ -326,7 +326,7 @@ void assertSplit(Directory dir, IndexMetadata metadata, int targetShardId, boole } assertEquals(shard_id.docID(), doc); long shardID = shard_id.nextValue(); - BytesRef id = reader.document(doc).getBinaryValue("_id"); + BytesRef id = reader.storedFields().document(doc).getBinaryValue("_id"); String actualId = Uid.decodeId(id.bytes, id.offset, id.length); assertNotEquals(ctx.reader() + " docID: " + doc + " actualID: " + actualId, shardID, targetShardId); } diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java index 0f957517d48e1..566a53c2508c8 100644 --- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java +++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/FileInfoTests.java @@ -35,10 +35,10 @@ import org.apache.lucene.util.Version; import org.opensearch.OpenSearchParseException; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -73,7 +73,7 @@ public void testToFromXContent() throws IOException { ); ByteSizeValue size = new ByteSizeValue(Math.abs(randomLong())); BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON).prettyPrint(); BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS); byte[] xcontent = BytesReference.toBytes(BytesReference.bytes(shuffleXContent(builder))); @@ -126,7 +126,7 @@ public void testInvalidFieldsInFromXContent() throws IOException { fail("shouldn't be here"); } - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject(); builder.field(FileInfo.NAME, name); builder.field(FileInfo.PHYSICAL_NAME, physicalName); diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java index ccb89ec3d1547..eb2006124662a 100644 --- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java +++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java @@ -10,9 +10,9 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -177,7 +177,7 @@ public void testFromXContentInvalid() throws IOException { fail("shouldn't be here"); } - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); builder.startObject(); builder.field(RemoteStoreShardShallowCopySnapshot.VERSION, version); builder.field(RemoteStoreShardShallowCopySnapshot.NAME, snapshot); diff --git a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java index ab0046c9554f5..350fc384458ab 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/file/OnDemandBlockSnapshotIndexInputTests.java @@ -18,7 +18,7 @@ import org.apache.lucene.util.Constants; import org.apache.lucene.util.Version; import org.junit.Before; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; import org.opensearch.index.store.StoreFileMetadata; import org.opensearch.index.store.remote.utils.BlobFetchRequest; diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java index eec9666cc06ba..7a912ecf98292 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheCleanerTests.java @@ -20,8 +20,8 @@ import org.junit.Before; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java index 02b6a48b6f48e..19bfc38056af1 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheTests.java @@ -14,9 +14,9 @@ import org.opensearch.common.breaker.TestCircuitBreaker; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.store.remote.directory.RemoteSnapshotDirectoryFactory; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.index.store.remote.utils.cache.CacheUsage; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java index 962b3a58c0658..ddf61de292eed 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTests.java @@ -25,8 +25,8 @@ import org.junit.After; import org.junit.Before; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter; import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheFactory; diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java index 788391f0e42c0..cd10b7bde3a73 100644 --- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java @@ -63,8 +63,8 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.concurrent.ReleasableLock; diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java index d26379eaefa5c..349f78c3c2d7b 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java @@ -30,8 +30,8 @@ import org.opensearch.common.bytes.ReleasableBytesReference; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.common.util.io.IOUtils; diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java index b7091f3f4f8a6..3d622d6bdf8b8 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java @@ -44,6 +44,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; @@ -205,11 +206,12 @@ public void testReadMetadataNoFile() throws IOException { null ); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); List bmList = new LinkedList<>(); latchedActionListener.onResponse(bmList); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); assertNull(translogTransferManager.readMetadata()); } @@ -225,12 +227,13 @@ public void testReadMetadataSingleFile() throws IOException { TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2); String mdFilename = tm.getFileName(); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); List bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(mdFilename, 1)); latchedActionListener.onResponse(bmList); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); TranslogTransferMetadata metadata = createTransferSnapshot().getTranslogTransferMetadata(); when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenReturn( @@ -252,12 +255,13 @@ public void testReadMetadataReadException() throws IOException { String mdFilename = tm.getFileName(); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); List bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(mdFilename, 1)); latchedActionListener.onResponse(bmList); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); when(transferService.downloadBlob(any(BlobPath.class), eq(mdFilename))).thenThrow(new IOException("Something went wrong")); @@ -283,10 +287,11 @@ public void testReadMetadataListException() throws IOException { ); doAnswer(invocation -> { - LatchedActionListener> latchedActionListener = invocation.getArgument(2); + LatchedActionListener> latchedActionListener = invocation.getArgument(3); latchedActionListener.onFailure(new IOException("Issue while listing")); return null; - }).when(transferService).listAllInSortedOrder(any(BlobPath.class), anyInt(), any(ActionListener.class)); + }).when(transferService) + .listAllInSortedOrder(any(BlobPath.class), eq(TranslogTransferMetadata.METADATA_PREFIX), anyInt(), any(ActionListener.class)); when(transferService.downloadBlob(any(BlobPath.class), any(String.class))).thenThrow(new IOException("Something went wrong")); @@ -416,7 +421,7 @@ public void testDeleteStaleTranslogMetadata() { String tm2 = new TranslogTransferMetadata(1, 2, 1, 2).getFileName(); String tm3 = new TranslogTransferMetadata(2, 3, 1, 2).getFileName(); doAnswer(invocation -> { - ActionListener> actionListener = invocation.getArgument(3); + ActionListener> actionListener = invocation.getArgument(4); List bmList = new LinkedList<>(); bmList.add(new PlainBlobMetadata(tm1, 1)); bmList.add(new PlainBlobMetadata(tm2, 1)); @@ -424,12 +429,19 @@ public void testDeleteStaleTranslogMetadata() { actionListener.onResponse(bmList); return null; }).when(transferService) - .listAllInSortedOrderAsync(eq(ThreadPool.Names.REMOTE_PURGE), any(BlobPath.class), anyInt(), any(ActionListener.class)); + .listAllInSortedOrderAsync( + eq(ThreadPool.Names.REMOTE_PURGE), + any(BlobPath.class), + anyString(), + anyInt(), + any(ActionListener.class) + ); List files = List.of(tm2, tm3); translogTransferManager.deleteStaleTranslogMetadataFilesAsync(() -> { verify(transferService).listAllInSortedOrderAsync( eq(ThreadPool.Names.REMOTE_PURGE), any(BlobPath.class), + eq(TranslogTransferMetadata.METADATA_PREFIX), eq(Integer.MAX_VALUE), any() ); diff --git a/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java b/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java index 62984f43744c8..19ed7122741aa 100644 --- a/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java +++ b/server/src/test/java/org/opensearch/indices/IndexingMemoryControllerTests.java @@ -36,8 +36,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.SetOnce; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.EngineConfig; diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java index 3508020cb23d7..2a03ddc6f46e4 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java @@ -51,7 +51,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.util.io.IOUtils; @@ -353,7 +353,7 @@ public BytesReference get() { IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1); assertEquals(1, topDocs.totalHits.value); - Document document = reader.document(topDocs.scoreDocs[0].doc); + Document document = reader.storedFields().document(topDocs.scoreDocs[0].doc); out.writeString(document.get("value")); loadedFromCache = false; return out.bytes(); diff --git a/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java b/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java index 93c3b0f48315d..7b1aee370ddb7 100644 --- a/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java +++ b/server/src/test/java/org/opensearch/indices/breaker/BreakerSettingsTests.java @@ -32,7 +32,7 @@ package org.opensearch.indices.breaker; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index 7bdff59e7c334..7ed072306e2cb 100644 --- a/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -33,13 +33,14 @@ package org.opensearch.indices.breaker; import org.opensearch.common.breaker.ChildMemoryCircuitBreaker; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.search.aggregations.MultiBucketConsumerService; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java b/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java index a2705a0730aaf..fad619f40222b 100644 --- a/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java +++ b/server/src/test/java/org/opensearch/indices/memory/breaker/CircuitBreakerUnitTests.java @@ -32,7 +32,7 @@ package org.opensearch.indices.memory.breaker; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.indices.breaker.BreakerSettings; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java index 3b289114f5ca1..84a53ae22a6bc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java @@ -403,4 +403,38 @@ public void testCancelForMissingIds() throws IOException { assertEquals(0, replications.cachedCopyStateSize()); closeShards(replica_2); } + + public void testPrepareForReplicationAlreadyReplicating() throws IOException { + OngoingSegmentReplications replications = new OngoingSegmentReplications(mockIndicesService, recoverySettings); + final String replicaAllocationId = replica.routingEntry().allocationId().getId(); + final CheckpointInfoRequest request = new CheckpointInfoRequest(1L, replicaAllocationId, primaryDiscoveryNode, testCheckpoint); + + final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class)); + + final SegmentReplicationSourceHandler handler = replications.getHandlers().get(replicaAllocationId); + assertEquals(handler.getCopyState(), copyState); + assertEquals(1, copyState.refCount()); + + ReplicationCheckpoint secondCheckpoint = new ReplicationCheckpoint( + testCheckpoint.getShardId(), + testCheckpoint.getPrimaryTerm(), + testCheckpoint.getSegmentsGen(), + testCheckpoint.getSegmentInfosVersion() + 1, + testCheckpoint.getCodec() + ); + + final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest( + 1L, + replicaAllocationId, + primaryDiscoveryNode, + secondCheckpoint + ); + + final CopyState secondCopyState = replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class)); + final SegmentReplicationSourceHandler secondHandler = replications.getHandlers().get(replicaAllocationId); + assertEquals(secondHandler.getCopyState(), secondCopyState); + assertEquals("New copy state is incref'd", 1, secondCopyState.refCount()); + assertEquals("Old copy state is cleaned up", 0, copyState.refCount()); + + } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 4643615d45d7e..94e57f4a0d3e4 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -32,6 +32,7 @@ import org.opensearch.indices.recovery.ForceSyncRequest; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; +import org.opensearch.indices.replication.common.CopyState; import org.opensearch.indices.replication.common.ReplicationCollection; import org.opensearch.indices.replication.common.ReplicationFailedException; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; @@ -49,6 +50,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.atLeastOnce; @@ -70,10 +72,7 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase { private IndexShard replicaShard; private IndexShard primaryShard; private ReplicationCheckpoint checkpoint; - private SegmentReplicationSource replicationSource; private SegmentReplicationTargetService sut; - - private ReplicationCheckpoint initialCheckpoint; private ReplicationCheckpoint aheadCheckpoint; private ReplicationCheckpoint newPrimaryCheckpoint; @@ -83,11 +82,10 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase { private DiscoveryNode localNode; private IndicesService indicesService; - private ClusterService clusterService; private SegmentReplicationState state; - private static long TRANSPORT_TIMEOUT = 30000;// 30sec + private static final long TRANSPORT_TIMEOUT = 30000;// 30sec @Override public void setUp() throws Exception { @@ -107,9 +105,6 @@ public void setUp() throws Exception { 0L, replicaShard.getLatestReplicationCheckpoint().getCodec() ); - SegmentReplicationSourceFactory replicationSourceFactory = mock(SegmentReplicationSourceFactory.class); - replicationSource = mock(SegmentReplicationSource.class); - when(replicationSourceFactory.get(replicaShard)).thenReturn(replicationSource); testThreadPool = new TestThreadPool("test", Settings.EMPTY); localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); @@ -126,7 +121,7 @@ public void setUp() throws Exception { transportService.acceptIncomingRequests(); indicesService = mock(IndicesService.class); - clusterService = mock(ClusterService.class); + ClusterService clusterService = mock(ClusterService.class); ClusterState clusterState = mock(ClusterState.class); RoutingTable mockRoutingTable = mock(RoutingTable.class); when(clusterService.state()).thenReturn(clusterState); @@ -135,7 +130,7 @@ public void setUp() throws Exception { when(clusterState.nodes()).thenReturn(DiscoveryNodes.builder().add(localNode).build()); sut = prepareForReplication(primaryShard, replicaShard, transportService, indicesService, clusterService); - initialCheckpoint = replicaShard.getLatestReplicationCheckpoint(); + ReplicationCheckpoint initialCheckpoint = replicaShard.getLatestReplicationCheckpoint(); aheadCheckpoint = new ReplicationCheckpoint( initialCheckpoint.getShardId(), initialCheckpoint.getPrimaryTerm(), @@ -242,7 +237,46 @@ public void testAlreadyOnNewCheckpoint() { } public void testShardAlreadyReplicating() { - sut.startReplication(replicaShard, mock(SegmentReplicationTargetService.SegmentReplicationListener.class)); + CountDownLatch blockGetCheckpointMetadata = new CountDownLatch(1); + SegmentReplicationSource source = new TestReplicationSource() { + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + try { + blockGetCheckpointMetadata.await(); + final CopyState copyState = new CopyState( + ReplicationCheckpoint.empty(primaryShard.shardId(), primaryShard.getLatestReplicationCheckpoint().getCodec()), + primaryShard + ); + listener.onResponse( + new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes()) + ); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + IndexShard indexShard, + ActionListener listener + ) { + listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList())); + } + }; + final SegmentReplicationTarget target = spy( + new SegmentReplicationTarget(replicaShard, source, mock(SegmentReplicationTargetService.SegmentReplicationListener.class)) + ); + // Start first round of segment replication. + sut.startReplication(target); + + // Start second round of segment replication, this should fail to start as first round is still in-progress sut.startReplication(replicaShard, new SegmentReplicationTargetService.SegmentReplicationListener() { @Override public void onReplicationDone(SegmentReplicationState state) { @@ -255,6 +289,7 @@ public void onReplicationFailure(SegmentReplicationState state, ReplicationFaile assertFalse(sendShardFailure); } }); + blockGetCheckpointMetadata.countDown(); } public void testOnNewCheckpointFromNewPrimaryCancelOngoingReplication() throws InterruptedException { diff --git a/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java b/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java index fdaf6d145da64..82e6e1c9ff450 100644 --- a/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java +++ b/server/src/test/java/org/opensearch/ingest/IngestMetadataTests.java @@ -36,9 +36,9 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -68,7 +68,7 @@ public void testFromXContent() throws IOException { map.put(pipeline.getId(), pipeline); map.put(pipeline2.getId(), pipeline2); IngestMetadata ingestMetadata = new IngestMetadata(map); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.prettyPrint(); builder.startObject(); ingestMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java index 51ebd1b1f7ad5..aed08c2d1875a 100644 --- a/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java +++ b/server/src/test/java/org/opensearch/ingest/PipelineConfigurationTests.java @@ -58,13 +58,13 @@ public void testSerialization() throws IOException { new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON ); - assertEquals(XContentType.JSON, configuration.getXContentType()); + assertEquals(XContentType.JSON, configuration.getMediaType()); BytesStreamOutput out = new BytesStreamOutput(); configuration.writeTo(out); StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes); PipelineConfiguration serialized = PipelineConfiguration.readFrom(in); - assertEquals(XContentType.JSON, serialized.getXContentType()); + assertEquals(XContentType.JSON, serialized.getMediaType()); assertEquals("{}", serialized.getConfig().utf8ToString()); } @@ -83,8 +83,8 @@ public void testParser() throws IOException { XContentParser xContentParser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput()); PipelineConfiguration parsed = parser.parse(xContentParser, null); - assertEquals(xContentType, parsed.getXContentType()); - assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType())); + assertEquals(xContentType, parsed.getMediaType()); + assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getMediaType())); assertEquals("1", parsed.getId()); } diff --git a/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java index bf9aac344e3df..6844742759883 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/BlendedTermQueryTests.java @@ -112,7 +112,7 @@ public void testDismaxQuery() throws IOException { query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "generator"), 0.1f), BooleanClause.Occur.SHOULD); TopDocs search = searcher.search(query.build(), 10); ScoreDoc[] scoreDocs = search.scoreDocs; - assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue()); + assertEquals(Integer.toString(0), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue()); } { BooleanQuery.Builder query = new BooleanQuery.Builder(); @@ -134,7 +134,7 @@ public void testDismaxQuery() throws IOException { query.add(gen, BooleanClause.Occur.SHOULD); TopDocs search = searcher.search(query.build(), 4); ScoreDoc[] scoreDocs = search.scoreDocs; - assertEquals(Integer.toString(1), reader.document(scoreDocs[0].doc).getField("id").stringValue()); + assertEquals(Integer.toString(1), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue()); } { @@ -269,7 +269,7 @@ public void testMinTTF() throws IOException { Query query = BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "foo"), 0.1f); TopDocs search = searcher.search(query, 10); ScoreDoc[] scoreDocs = search.scoreDocs; - assertEquals(Integer.toString(0), reader.document(scoreDocs[0].doc).getField("id").stringValue()); + assertEquals(Integer.toString(0), reader.storedFields().document(scoreDocs[0].doc).getField("id").stringValue()); } reader.close(); w.close(); diff --git a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java index a75a1d5b2fdb7..701ba5a756b40 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/MinDocQueryTests.java @@ -61,10 +61,11 @@ public void testBasics() { public void testRewrite() throws Exception { IndexReader reader = new MultiReader(); + IndexSearcher searcher = new IndexSearcher(reader); MinDocQuery query = new MinDocQuery(42); - Query rewritten = query.rewrite(reader); + Query rewritten = query.rewrite(searcher); QueryUtils.checkUnequal(query, rewritten); - Query rewritten2 = rewritten.rewrite(reader); + Query rewritten2 = rewritten.rewrite(searcher); assertSame(rewritten, rewritten2); } diff --git a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java index 727377c9a5d53..81f4290b6588e 100644 --- a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java +++ b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java @@ -56,19 +56,11 @@ public class SpanMatchNoDocsQueryTests extends OpenSearchTestCase { public void testSimple() throws Exception { - Directory dir = newDirectory(); - IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig()); - IndexReader ir = DirectoryReader.open(iw); - SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason"); assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")"); - Query rewrite = query.rewrite(ir); + Query rewrite = query.rewrite((IndexSearcher) null); assertTrue(rewrite instanceof SpanMatchNoDocsQuery); assertEquals(rewrite.toString(), "SpanMatchNoDocsQuery(\"a good reason\")"); - - iw.close(); - ir.close(); - dir.close(); } public void testQuery() throws Exception { @@ -101,7 +93,7 @@ public void testQuery() throws Exception { assertEquals(searcher.count(orQuery), 1); hits = searcher.search(orQuery, 1000).scoreDocs; assertEquals(1, hits.length); - Query rewrite = orQuery.rewrite(ir); + Query rewrite = orQuery.rewrite(searcher); assertEquals(rewrite, orQuery); SpanNearQuery nearQuery = new SpanNearQuery( @@ -112,7 +104,7 @@ public void testQuery() throws Exception { assertEquals(searcher.count(nearQuery), 0); hits = searcher.search(nearQuery, 1000).scoreDocs; assertEquals(0, hits.length); - rewrite = nearQuery.rewrite(ir); + rewrite = nearQuery.rewrite(searcher); assertEquals(rewrite, nearQuery); iw.close(); diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java index 216594f24e2ea..06a8e27b079ee 100644 --- a/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java +++ b/server/src/test/java/org/opensearch/monitor/fs/FsProbeTests.java @@ -33,12 +33,12 @@ package org.opensearch.monitor.fs; import org.apache.lucene.util.Constants; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.NodeEnvironment.NodePath; import org.opensearch.index.store.remote.filecache.FileCache; diff --git a/server/src/test/java/org/opensearch/monitor/jvm/JvmGcMonitorServiceTests.java b/server/src/test/java/org/opensearch/monitor/jvm/JvmGcMonitorServiceTests.java index 3ea53bd0ea811..6c0bdfc6cede1 100644 --- a/server/src/test/java/org/opensearch/monitor/jvm/JvmGcMonitorServiceTests.java +++ b/server/src/test/java/org/opensearch/monitor/jvm/JvmGcMonitorServiceTests.java @@ -33,7 +33,7 @@ package org.opensearch.monitor.jvm; import org.apache.logging.log4j.Logger; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/monitor/jvm/JvmStatsTests.java b/server/src/test/java/org/opensearch/monitor/jvm/JvmStatsTests.java index 6a7d3afdc5dad..e6f45ca47a2fe 100644 --- a/server/src/test/java/org/opensearch/monitor/jvm/JvmStatsTests.java +++ b/server/src/test/java/org/opensearch/monitor/jvm/JvmStatsTests.java @@ -32,7 +32,7 @@ package org.opensearch.monitor.jvm; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.common.Strings; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/node/NodeTests.java b/server/src/test/java/org/opensearch/node/NodeTests.java index ae8028c143498..6dfecd8a692f1 100644 --- a/server/src/test/java/org/opensearch/node/NodeTests.java +++ b/server/src/test/java/org/opensearch/node/NodeTests.java @@ -37,13 +37,13 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.SetOnce; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; import org.opensearch.common.transport.BoundTransportAddress; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexService; @@ -51,7 +51,7 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.indices.breaker.BreakerSettings; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.monitor.fs.FsInfo; import org.opensearch.monitor.fs.FsProbe; import org.opensearch.plugins.CircuitBreakerPlugin; diff --git a/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java b/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java index 7b17d4d3b0471..347fef773fc8a 100644 --- a/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java +++ b/server/src/test/java/org/opensearch/nodesinfo/NodeInfoStreamingTests.java @@ -42,7 +42,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.http.HttpInfo; diff --git a/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java b/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java index 7fae4d5bb23e3..56e2ae56fdc71 100644 --- a/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java +++ b/server/src/test/java/org/opensearch/persistent/PersistentTasksCustomMetadataTests.java @@ -54,7 +54,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.persistent.PersistentTasksCustomMetadata.Assignment; @@ -204,7 +203,7 @@ public void testSerializationContext() throws Exception { BytesReference shuffled = toShuffledXContent(testInstance, xContentType, params, false); PersistentTasksCustomMetadata newInstance; - try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) { + try (XContentParser parser = createParser(xContentType.xContent(), shuffled)) { newInstance = doParseInstance(parser); } assertNotSame(newInstance, testInstance); diff --git a/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java index 3991f6721a350..63f753f698597 100644 --- a/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java +++ b/server/src/test/java/org/opensearch/persistent/TestPersistentTasksPlugin.java @@ -53,7 +53,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; -import org.opensearch.common.component.Lifecycle; +import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.inject.Inject; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index 6cbbb4ffcb7c7..085a64b439bbe 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -51,8 +51,8 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; -import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.component.LifecycleListener; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java index 28513f279f8ad..893631ce4b564 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -32,7 +32,6 @@ package org.opensearch.repositories.blobstore; -import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.Version; import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; @@ -46,7 +45,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -91,7 +90,6 @@ /** * Tests for the {@link BlobStoreRepository} and its subclasses. */ -@LuceneTestCase.SuppressFileSystems("ExtrasFS") public class BlobStoreRepositoryTests extends OpenSearchSingleNodeTestCase { static final String REPO_TYPE = "fsLike"; diff --git a/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java index 3049aa161f10b..752bd5f5ede82 100644 --- a/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java @@ -59,7 +59,7 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; diff --git a/server/src/test/java/org/opensearch/rest/RestControllerTests.java b/server/src/test/java/org/opensearch/rest/RestControllerTests.java index f8ea59040d22b..ea0ce54913a8d 100644 --- a/server/src/test/java/org/opensearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/opensearch/rest/RestControllerTests.java @@ -33,15 +33,15 @@ package org.opensearch.rest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java index b181d571c14fb..b1ddea9329885 100644 --- a/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java +++ b/server/src/test/java/org/opensearch/rest/RestHttpResponseHeadersTests.java @@ -39,7 +39,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.rest.RestStatus; import org.opensearch.identity.IdentityService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestChannel; diff --git a/server/src/test/java/org/opensearch/rest/RestRequestTests.java b/server/src/test/java/org/opensearch/rest/RestRequestTests.java index 15066362555ef..97350824dd1e4 100644 --- a/server/src/test/java/org/opensearch/rest/RestRequestTests.java +++ b/server/src/test/java/org/opensearch/rest/RestRequestTests.java @@ -203,12 +203,12 @@ public void testContentTypeParsing() { Map> map = new HashMap<>(); map.put("Content-Type", Collections.singletonList(xContentType.mediaType())); RestRequest restRequest = contentRestRequest("", Collections.emptyMap(), map); - assertEquals(xContentType, restRequest.getXContentType()); + assertEquals(xContentType, restRequest.getMediaType()); map = new HashMap<>(); map.put("Content-Type", Collections.singletonList(xContentType.mediaTypeWithoutParameters())); restRequest = contentRestRequest("", Collections.emptyMap(), map); - assertEquals(xContentType, restRequest.getXContentType()); + assertEquals(xContentType, restRequest.getMediaType()); } } @@ -221,7 +221,7 @@ public void testPlainTextSupport() { Collections.singletonList(randomFrom("text/plain", "text/plain; charset=utf-8", "text/plain;charset=utf-8")) ) ); - assertNull(restRequest.getXContentType()); + assertNull(restRequest.getMediaType()); } public void testMalformedContentTypeHeader() { @@ -237,7 +237,7 @@ public void testMalformedContentTypeHeader() { public void testNoContentTypeHeader() { RestRequest contentRestRequest = contentRestRequest("", Collections.emptyMap(), Collections.emptyMap()); - assertNull(contentRestRequest.getXContentType()); + assertNull(contentRestRequest.getMediaType()); } public void testMultipleContentTypeHeaders() { diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java index 9739419406851..1fbc628ed7f2c 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -33,8 +33,8 @@ package org.opensearch.rest.action.admin.indices; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestResponse; import org.opensearch.test.OpenSearchTestCase; @@ -60,7 +60,7 @@ public class RestGetAliasesActionTests extends OpenSearchTestCase { // }' public void testBareRequest() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata foobarAliasMetadata = AliasMetadata.builder("foobar").build(); final AliasMetadata fooAliasMetadata = AliasMetadata.builder("foo").build(); @@ -72,7 +72,7 @@ public void testBareRequest() throws Exception { } public void testSimpleAliasWildcardMatchingNothing() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse( true, @@ -86,7 +86,7 @@ public void testSimpleAliasWildcardMatchingNothing() throws Exception { } public void testMultipleAliasWildcardsSomeMatching() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata aliasMetadata = AliasMetadata.builder("foobar").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); @@ -102,7 +102,7 @@ public void testMultipleAliasWildcardsSomeMatching() throws Exception { } public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse( true, @@ -116,7 +116,7 @@ public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { } public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata aliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); @@ -132,7 +132,7 @@ public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { } public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final AliasMetadata aliasMetadata = AliasMetadata.builder("foo").build(); openMapBuilder.put("index", Arrays.asList(aliasMetadata)); @@ -153,7 +153,7 @@ public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws E } public void testAliasWildcardsExcludeExplicitMissing() throws Exception { - final XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + final XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); final Map> openMapBuilder = new HashMap<>(); final RestResponse restResponse = RestGetAliasesAction.buildRestResponse( true, diff --git a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java index 094b4d8b8c12d..2f15090e49f75 100644 --- a/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/admin/indices/RestValidateQueryActionTests.java @@ -45,7 +45,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.identity.IdentityService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.search.AbstractSearchTestCase; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java index 44d8f84eef524..339e6870efb92 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationActionTests.java @@ -15,7 +15,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Randomness; import org.opensearch.common.Table; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.index.Index; import org.opensearch.index.SegmentReplicationPerGroupStats; diff --git a/server/src/test/java/org/opensearch/script/ScriptCacheTests.java b/server/src/test/java/org/opensearch/script/ScriptCacheTests.java index 6395b8b315196..a7d573de5f5e3 100644 --- a/server/src/test/java/org/opensearch/script/ScriptCacheTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptCacheTests.java @@ -31,7 +31,7 @@ package org.opensearch.script; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.rest.RestStatus; diff --git a/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java b/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java index bcab5dc7b776a..83e7a3712a9ad 100644 --- a/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptMetadataTests.java @@ -36,6 +36,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -130,13 +131,13 @@ public void testGetScript() throws Exception { .endObject() .endObject() .endObject(); - XContentType xContentType = XContentType.fromMediaType(sourceBuilder.contentType()); - builder.storeScript("source_template", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), xContentType)); + MediaType mediaType = sourceBuilder.contentType(); + builder.storeScript("source_template", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), mediaType)); sourceBuilder = XContentFactory.jsonBuilder(); - xContentType = XContentType.fromMediaType(sourceBuilder.contentType()); + mediaType = sourceBuilder.contentType(); sourceBuilder.startObject().startObject("script").field("lang", "_lang").field("source", "_source").endObject().endObject(); - builder.storeScript("script", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), xContentType)); + builder.storeScript("script", StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), mediaType)); ScriptMetadata scriptMetadata = builder.build(); assertEquals("_source", scriptMetadata.getStoredScript("script").getSource()); @@ -303,7 +304,7 @@ private ScriptMetadata randomScriptMetadata(XContentType sourceContentType, int .endObject(); builder.storeScript( randomAlphaOfLength(i + 1), - StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), XContentType.fromMediaType(sourceBuilder.contentType())) + StoredScriptSource.parse(BytesReference.bytes(sourceBuilder), sourceBuilder.contentType()) ); } return builder.build(); diff --git a/server/src/test/java/org/opensearch/script/ScriptServiceTests.java b/server/src/test/java/org/opensearch/script/ScriptServiceTests.java index 489b267f586e4..a4288eeab4524 100644 --- a/server/src/test/java/org/opensearch/script/ScriptServiceTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptServiceTests.java @@ -36,7 +36,7 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.ClusterSettings; diff --git a/server/src/test/java/org/opensearch/script/ScriptTests.java b/server/src/test/java/org/opensearch/script/ScriptTests.java index 47b4557c3e5e1..bb74bd2e5f6b0 100644 --- a/server/src/test/java/org/opensearch/script/ScriptTests.java +++ b/server/src/test/java/org/opensearch/script/ScriptTests.java @@ -37,6 +37,7 @@ import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -58,7 +59,7 @@ public class ScriptTests extends OpenSearchTestCase { public void testScriptParsing() throws IOException { Script expectedScript = createScript(); - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS); try (XContentParser parser = createParser(builder)) { Script actualScript = Script.parse(parser); @@ -103,7 +104,7 @@ private Script createScript() throws IOException { public void testParse() throws IOException { Script expectedScript = createScript(); - try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values()))) { expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS); try (XContentParser xParser = createParser(builder)) { Settings settings = Settings.fromXContent(xParser); diff --git a/server/src/test/java/org/opensearch/script/StoredScriptTests.java b/server/src/test/java/org/opensearch/script/StoredScriptTests.java index c16c4f93e6582..96c2b19bf204f 100644 --- a/server/src/test/java/org/opensearch/script/StoredScriptTests.java +++ b/server/src/test/java/org/opensearch/script/StoredScriptTests.java @@ -37,8 +37,8 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.AbstractSerializingTestCase; @@ -73,7 +73,7 @@ public void testInvalidDelete() { public void testSourceParsing() throws Exception { // simple script value string - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().startObject("script").field("lang", "lang").field("source", "code").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -83,7 +83,7 @@ public void testSourceParsing() throws Exception { } // complex template using script as the field name - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject() .startObject("script") .field("lang", "mustache") @@ -94,7 +94,7 @@ public void testSourceParsing() throws Exception { .endObject(); String code; - try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) { + try (XContentBuilder cb = MediaTypeRegistry.contentBuilder(builder.contentType())) { code = Strings.toString(cb.startObject().field("query", "code").endObject()); } @@ -109,7 +109,7 @@ public void testSourceParsing() throws Exception { } // complex script with script object - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "lang").field("source", "code").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -119,7 +119,7 @@ public void testSourceParsing() throws Exception { } // complex script using "code" backcompat - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -130,7 +130,7 @@ public void testSourceParsing() throws Exception { assertWarnings("Deprecated field [code] used, expected [source] instead"); // complex script with script object and empty options - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject() .field("script") .startObject() @@ -149,7 +149,7 @@ public void testSourceParsing() throws Exception { } // complex script with embedded template - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { Strings.toString( builder.startObject() .field("script") @@ -165,7 +165,7 @@ public void testSourceParsing() throws Exception { ); String code; - try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) { + try (XContentBuilder cb = MediaTypeRegistry.contentBuilder(builder.contentType())) { code = Strings.toString(cb.startObject().field("query", "code").endObject()); } @@ -182,7 +182,7 @@ public void testSourceParsing() throws Exception { public void testSourceParsingErrors() throws Exception { // check for missing lang parameter when parsing a script - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("source", "code").endObject().endObject(); IllegalArgumentException iae = expectThrows( @@ -193,7 +193,7 @@ public void testSourceParsingErrors() throws Exception { } // check for missing source parameter when parsing a script - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "lang").endObject().endObject(); IllegalArgumentException iae = expectThrows( @@ -204,7 +204,7 @@ public void testSourceParsingErrors() throws Exception { } // check for illegal options parameter when parsing a script - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject() .field("script") .startObject() @@ -224,7 +224,7 @@ public void testSourceParsingErrors() throws Exception { } // check for unsupported template context - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("template", "code").endObject(); ParsingException pEx = expectThrows( ParsingException.class, @@ -238,7 +238,7 @@ public void testSourceParsingErrors() throws Exception { } public void testEmptyTemplateDeprecations() throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); @@ -248,7 +248,7 @@ public void testEmptyTemplateDeprecations() throws IOException { assertWarnings("empty templates should no longer be used"); } - try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)) { builder.startObject().field("script").startObject().field("lang", "mustache").field("source", "").endObject().endObject(); StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java index 8d8f2856d7703..958548ed49d67 100644 --- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java @@ -65,7 +65,7 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; import org.opensearch.core.index.shard.ShardId; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.LegacyReaderContext; import org.opensearch.search.internal.PitReaderContext; diff --git a/server/src/test/java/org/opensearch/search/NestedIdentityTests.java b/server/src/test/java/org/opensearch/search/NestedIdentityTests.java index 341570247ffa9..17b1da2ba3a8c 100644 --- a/server/src/test/java/org/opensearch/search/NestedIdentityTests.java +++ b/server/src/test/java/org/opensearch/search/NestedIdentityTests.java @@ -35,9 +35,9 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -66,7 +66,7 @@ public static NestedIdentity createTestItem(int depth) { public void testFromXContent() throws IOException { NestedIdentity nestedIdentity = createTestItem(randomInt(3)); XContentType xcontentType = randomFrom(XContentType.values()); - XContentBuilder builder = XContentFactory.contentBuilder(xcontentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xcontentType); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java index 312ebfcfae5e8..6ae719cf00e7a 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java @@ -38,7 +38,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.index.IndexService; @@ -47,7 +47,7 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; import org.opensearch.test.OpenSearchSingleNodeTestCase; diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java index 1f29a17deeaa8..3a527ec7455d3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java @@ -37,10 +37,10 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -239,7 +239,7 @@ public void testInvalidType() throws Exception { public void testRewriteAggregation() throws Exception { XContentType xContentType = randomFrom(XContentType.values()); BytesReference bytesReference; - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) { builder.startObject(); { builder.startObject("terms"); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/BucketsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/BucketsAggregatorTests.java index 55f8d11e2d934..5f33777562721 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/BucketsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/BucketsAggregatorTests.java @@ -40,9 +40,9 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java index 3a8829d5a6433..136210a636224 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java @@ -32,8 +32,8 @@ package org.opensearch.search.aggregations.bucket; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.BoolQueryBuilder; @@ -106,7 +106,7 @@ public void testFiltersSortedByKey() { } public void testOtherBucket() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").startObject().startObject("term").field("field", "foo").endObject().endObject().endArray(); builder.endObject(); @@ -116,7 +116,7 @@ public void testOtherBucket() throws IOException { // The other bucket is disabled by default assertFalse(filters.otherBucket()); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").startObject().startObject("term").field("field", "foo").endObject().endObject().endArray(); builder.field("other_bucket_key", "some_key"); @@ -128,7 +128,7 @@ public void testOtherBucket() throws IOException { // but setting a key enables it automatically assertTrue(filters.otherBucket()); - builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").startObject().startObject("term").field("field", "foo").endObject().endObject().endArray(); builder.field("other_bucket", false); @@ -207,7 +207,7 @@ public void testRewritePreservesOtherBucket() throws IOException { public void testEmptyFilters() throws IOException { { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startArray("filters").endArray(); // unkeyed array builder.endObject(); @@ -221,7 +221,7 @@ public void testEmptyFilters() throws IOException { } { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); builder.startObject("filters").endObject(); // keyed object builder.endObject(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogramTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogramTests.java index 72b11c1657fb5..ea0e45687c583 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogramTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/InternalVariableWidthHistogramTests.java @@ -33,11 +33,11 @@ package org.opensearch.search.aggregations.bucket.histogram; import org.apache.lucene.tests.util.TestUtil; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.script.ScriptService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.InternalAggregation; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java index 427d0b89aa688..a7a65419b73b1 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/sampler/BestDocsDeferringCollectorTests.java @@ -48,7 +48,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.BucketCollector; import org.opensearch.search.aggregations.LeafBucketCollector; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/BytesKeyedBucketOrdsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/BytesKeyedBucketOrdsTests.java index f73146038a5f1..c593ed4147e7e 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/BytesKeyedBucketOrdsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/BytesKeyedBucketOrdsTests.java @@ -36,7 +36,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrdsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrdsTests.java index d0a6500ecf9e7..bdecae8f7dfa3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrdsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/LongKeyedBucketOrdsTests.java @@ -35,7 +35,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java index a2792114e9529..6c27335502e34 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregatorTests.java @@ -41,7 +41,7 @@ import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.Script; import org.opensearch.script.ScriptEngine; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 4ce66e0495f83..6a407469e5273 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -52,7 +52,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.network.InetAddresses; import org.opensearch.common.settings.Settings; @@ -74,7 +74,7 @@ import org.opensearch.index.mapper.Uid; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.Script; import org.opensearch.script.ScriptEngine; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java index 5968605fc0c34..cca51a8fc3837 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusSparseTests.java @@ -32,13 +32,13 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.BitMixer; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.hamcrest.CoreMatchers; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java index 6f5f7494331a5..52fe314eaf079 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/HyperLogLogPlusPlusTests.java @@ -32,12 +32,12 @@ package org.opensearch.search.aggregations.metrics; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.BitMixer; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.util.HashSet; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java index a95533bb3c33c..e5abe380091dc 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalCardinalityTests.java @@ -37,7 +37,7 @@ import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.lease.Releasables; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.aggregations.ParsedAggregation; import org.opensearch.test.InternalAggregationTestCase; import org.junit.After; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java index f0273b46ef97e..9deff7839ffda 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/MinAggregatorTests.java @@ -69,7 +69,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.Script; import org.opensearch.script.ScriptEngine; diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index 6a4ecf01577d8..c0b5743bbfb9a 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -44,16 +44,16 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.script.MockScriptEngine; import org.opensearch.script.Script; import org.opensearch.script.ScriptEngine; diff --git a/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java b/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java index 1423fd6771674..d0b3d84bc78f9 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java @@ -37,9 +37,9 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.fielddata.AbstractSortedSetDocValues; @@ -218,7 +218,7 @@ public void testRegexExclude() throws IOException { // Serializes/deserializes an IncludeExclude statement with a single clause private IncludeExclude serialize(IncludeExclude incExc, ParseField field) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -264,7 +264,7 @@ public void testRegexIncludeAndExclude() throws IOException { // Serializes/deserializes the IncludeExclude statement with include AND // exclude clauses private IncludeExclude serializeMixedRegex(IncludeExclude incExc) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java index 461812077eba9..51ff64321acfa 100644 --- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java @@ -39,9 +39,9 @@ import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -72,7 +72,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase { public void testFromXContent() throws IOException { SearchSourceBuilder testSearchSourceBuilder = createSearchSourceBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -425,7 +425,7 @@ public void testToXContent() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = BytesReference.bytes(builder); Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); @@ -434,7 +434,7 @@ public void testToXContent() throws IOException { { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(RandomQueryBuilder.createQuery(random())); - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference source = BytesReference.bytes(builder); Map sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); @@ -448,7 +448,7 @@ public void testToXContentWithPointInTime() throws IOException { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); TimeValue keepAlive = randomBoolean() ? TimeValue.timeValueHours(1) : null; searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder("id").setKeepAlive(keepAlive)); - XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType); searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); BytesReference bytes = BytesReference.bytes(builder); Map sourceAsMap = XContentHelper.convertToMap(bytes, false, xContentType).v2(); diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index b1dfd836f7461..6d31986cabdc4 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -41,10 +41,10 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -137,7 +137,7 @@ public void testEqualsAndHashcode() throws IOException { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { HighlightBuilder highlightBuilder = randomHighlighterBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -547,7 +547,7 @@ public void testOrderSerialization() throws Exception { } protected static XContentBuilder toXContent(HighlightBuilder highlight, XContentType contentType) throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(contentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(contentType); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java index a66c7de6ced74..3fb2718f5ccc3 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/HighlightFieldTests.java @@ -36,9 +36,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.text.Text; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.xcontent.json.JsonXContent; @@ -82,7 +82,7 @@ public void testReplaceUnicodeControlCharacters() { public void testFromXContent() throws IOException { HighlightField highlightField = createTestItem(); XContentType xcontentType = randomFrom(XContentType.values()); - XContentBuilder builder = XContentFactory.contentBuilder(xcontentType); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(xcontentType); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java index f3907355ac6ec..823fc6b463906 100644 --- a/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ContextIndexSearcherTests.java @@ -40,7 +40,6 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; @@ -466,12 +465,12 @@ public String toString(String field) { } @Override - public Query rewrite(IndexReader reader) throws IOException { - Query queryRewritten = query.rewrite(reader); + public Query rewrite(IndexSearcher searcher) throws IOException { + Query queryRewritten = query.rewrite(searcher); if (query != queryRewritten) { return new CreateScorerOnceQuery(queryRewritten); } - return super.rewrite(reader); + return super.rewrite(searcher); } @Override diff --git a/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java b/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java index 1f632251cf588..4913ee92aa79b 100644 --- a/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java +++ b/server/src/test/java/org/opensearch/search/internal/ShardSearchRequestTests.java @@ -44,6 +44,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -220,7 +221,7 @@ public QueryBuilder aliasFilter(IndexMetadata indexMetadata, String... aliasName return ShardSearchRequest.parseAliasFilter(bytes -> { try ( InputStream inputStream = bytes.streamInput(); - XContentParser parser = XContentFactory.xContentType(inputStream) + XContentParser parser = MediaTypeRegistry.xContentType(inputStream) .xContent() .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream) ) { diff --git a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java index 0155e288a96fd..85aacfbd63ee2 100644 --- a/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java +++ b/server/src/test/java/org/opensearch/search/lookup/LeafFieldsLookupTests.java @@ -36,6 +36,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.index.mapper.MappedFieldType; @@ -43,11 +44,11 @@ import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; +import java.io.IOException; import java.util.Collections; import java.util.List; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -88,12 +89,12 @@ public void setUp() throws Exception { ); LeafReader leafReader = mock(LeafReader.class); - doAnswer(invocation -> { - Object[] args = invocation.getArguments(); - StoredFieldVisitor visitor = (StoredFieldVisitor) args[1]; - visitor.doubleField(mockFieldInfo, 2.718); - return null; - }).when(leafReader).document(anyInt(), any(StoredFieldVisitor.class)); + doAnswer(invocation -> new StoredFields() { + @Override + public void document(int docID, StoredFieldVisitor visitor) throws IOException { + visitor.doubleField(mockFieldInfo, 2.718); + } + }).when(leafReader).storedFields(); fieldsLookup = new LeafFieldsLookup(mapperService, leafReader); } diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java index d0f4a974fc305..9fc0be38d66a5 100644 --- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java +++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java @@ -37,8 +37,8 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.lucene.search.TopDocsAndMaxScore; import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java b/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java index ae5a07478e814..70b876b41ba08 100644 --- a/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java +++ b/server/src/test/java/org/opensearch/search/profile/ProfileResultTests.java @@ -56,7 +56,7 @@ public class ProfileResultTests extends OpenSearchTestCase { - public static ProfileResult createTestItem(int depth) { + public static ProfileResult createTestItem(int depth, boolean concurrentSegmentSearchEnabled) { String type = randomAlphaOfLengthBetween(5, 10); String description = randomAlphaOfLengthBetween(5, 10); int breakdownsSize = randomIntBetween(0, 5); @@ -77,13 +77,28 @@ public static ProfileResult createTestItem(int depth) { int childrenSize = depth > 0 ? randomIntBetween(0, 1) : 0; List children = new ArrayList<>(childrenSize); for (int i = 0; i < childrenSize; i++) { - children.add(createTestItem(depth - 1)); + children.add(createTestItem(depth - 1, concurrentSegmentSearchEnabled)); + } + if (concurrentSegmentSearchEnabled) { + return new ProfileResult( + type, + description, + breakdown, + debug, + randomNonNegativeLong(), + children, + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong() + ); + } else { + return new ProfileResult(type, description, breakdown, debug, randomNonNegativeLong(), children); } - return new ProfileResult(type, description, breakdown, debug, randomNonNegativeLong(), children); } public void testFromXContent() throws IOException { - doFromXContentTestWithRandomFields(false); + doFromXContentTestWithRandomFields(false, false); + doFromXContentTestWithRandomFields(false, true); } /** @@ -91,11 +106,12 @@ public void testFromXContent() throws IOException { * back to be forward compatible with additions to the xContent */ public void testFromXContentWithRandomFields() throws IOException { - doFromXContentTestWithRandomFields(true); + doFromXContentTestWithRandomFields(true, false); + doFromXContentTestWithRandomFields(true, true); } - private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { - ProfileResult profileResult = createTestItem(2); + private void doFromXContentTestWithRandomFields(boolean addRandomFields, boolean concurrentSegmentSearchEnabled) throws IOException { + ProfileResult profileResult = createTestItem(2, concurrentSegmentSearchEnabled); XContentType xContentType = randomFrom(XContentType.values()); boolean humanReadable = randomBoolean(); BytesReference originalBytes = toShuffledXContent(profileResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); @@ -116,6 +132,9 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws assertNull(parser.nextToken()); } assertEquals(profileResult.getTime(), parsed.getTime()); + assertEquals(profileResult.getMaxSliceTime(), parsed.getMaxSliceTime()); + assertEquals(profileResult.getMinSliceTime(), parsed.getMinSliceTime()); + assertEquals(profileResult.getAvgSliceTime(), parsed.getAvgSliceTime()); assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType); } @@ -238,5 +257,72 @@ public void testToXContent() throws IOException { + "}", Strings.toString(builder) ); + + result = new ProfileResult("profileName", "some description", Map.of("key1", 1234L), Map.of(), 1234L, List.of(), 321L, 123L, 222L); + builder = XContentFactory.jsonBuilder().prettyPrint(); + result.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals( + "{\n" + + " \"type\" : \"profileName\",\n" + + " \"description\" : \"some description\",\n" + + " \"time_in_nanos\" : 1234,\n" + + " \"max_slice_time_in_nanos\" : 321,\n" + + " \"min_slice_time_in_nanos\" : 123,\n" + + " \"avg_slice_time_in_nanos\" : 222,\n" + + " \"breakdown\" : {\n" + + " \"key1\" : 1234\n" + + " }\n" + + "}", + Strings.toString(builder) + ); + + result = new ProfileResult( + "profileName", + "some description", + Map.of("key1", 1234567890L), + Map.of(), + 1234567890L, + List.of(), + 87654321L, + 12345678L, + 54637281L + ); + builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true); + result.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals( + "{\n" + + " \"type\" : \"profileName\",\n" + + " \"description\" : \"some description\",\n" + + " \"time\" : \"1.2s\",\n" + + " \"max_slice_time\" : \"87.6ms\",\n" + + " \"min_slice_time\" : \"12.3ms\",\n" + + " \"avg_slice_time\" : \"54.6ms\",\n" + + " \"time_in_nanos\" : 1234567890,\n" + + " \"max_slice_time_in_nanos\" : 87654321,\n" + + " \"min_slice_time_in_nanos\" : 12345678,\n" + + " \"avg_slice_time_in_nanos\" : 54637281,\n" + + " \"breakdown\" : {\n" + + " \"key1\" : 1234567890\n" + + " }\n" + + "}", + Strings.toString(builder) + ); + + } + + public void testRemoveStartTimeFields() { + Map breakdown = new HashMap<>(); + breakdown.put("initialize_start_time", 123456L); + breakdown.put("initialize_count", 1L); + breakdown.put("initialize", 654321L); + Map modifiedBreakdown = new LinkedHashMap<>(breakdown); + assertEquals(3, modifiedBreakdown.size()); + assertEquals(123456L, (long) modifiedBreakdown.get("initialize_start_time")); + assertEquals(1L, (long) modifiedBreakdown.get("initialize_count")); + assertEquals(654321L, (long) modifiedBreakdown.get("initialize")); + ProfileResult.removeStartTimeFields(modifiedBreakdown); + assertFalse(modifiedBreakdown.containsKey("initialize_start_time")); + assertTrue(modifiedBreakdown.containsKey("initialize_count")); + assertTrue(modifiedBreakdown.containsKey("initialize")); } } diff --git a/server/src/test/java/org/opensearch/search/profile/TimerTests.java b/server/src/test/java/org/opensearch/search/profile/TimerTests.java index deed451c21933..5997292eb8f56 100644 --- a/server/src/test/java/org/opensearch/search/profile/TimerTests.java +++ b/server/src/test/java/org/opensearch/search/profile/TimerTests.java @@ -71,10 +71,14 @@ long nanoTime() { return time += 42; } }; - for (int i = 1; i < 100000; ++i) { + t.start(); + t.stop(); + long timerStartTime = t.getEarliestTimerStartTime(); + for (int i = 2; i < 100000; ++i) { t.start(); t.stop(); assertEquals(i, t.getCount()); + assertEquals(timerStartTime, t.getEarliestTimerStartTime()); // Make sure the cumulated timing is 42 times the number of calls as expected assertEquals(i * 42L, t.getApproximateTiming()); } diff --git a/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java b/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java index 33c95725dcd13..75fb846909aa1 100644 --- a/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java +++ b/server/src/test/java/org/opensearch/search/profile/aggregation/AggregationProfileShardResultTests.java @@ -57,7 +57,7 @@ public static AggregationProfileShardResult createTestItem(int depth) { int size = randomIntBetween(0, 5); List aggProfileResults = new ArrayList<>(size); for (int i = 0; i < size; i++) { - aggProfileResults.add(ProfileResultTests.createTestItem(1)); + aggProfileResults.add(ProfileResultTests.createTestItem(depth, false)); } return new AggregationProfileShardResult(aggProfileResults); } diff --git a/server/src/test/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfilerTests.java b/server/src/test/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfilerTests.java new file mode 100644 index 0000000000000..e36b65f0a7b69 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/profile/aggregation/ConcurrentAggregationProfilerTests.java @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.profile.aggregation; + +import org.opensearch.search.profile.ProfileResult; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class ConcurrentAggregationProfilerTests extends OpenSearchTestCase { + + public static List createConcurrentSearchProfileTree() { + List tree = new ArrayList<>(); + // Aggregation + tree.add( + new ProfileResult( + "NumericTermsAggregator", + "test_scoped_agg", + new LinkedHashMap<>(), + new HashMap<>(), + 10847417L, + List.of( + new ProfileResult( + "GlobalOrdinalsStringTermsAggregator", + "test_terms", + new LinkedHashMap<>(), + new HashMap<>(), + 3359835L, + List.of(), + 1490667L, + 1180123L, + 1240676L + ) + ), + 94582L, + 18667L, + 211749L + ) + ); + tree.add( + new ProfileResult( + "NumericTermsAggregator", + "test_scoped_agg", + new LinkedHashMap<>(), + new HashMap<>(), + 10776655L, + List.of( + new ProfileResult( + "GlobalOrdinalsStringTermsAggregator", + "test_terms", + new LinkedHashMap<>(), + new HashMap<>(), + 3359567L, + List.of(), + 1390554L, + 1180321L, + 1298776L + ) + ), + 94560L, + 11237L, + 236440L + ) + ); + // Global Aggregation + tree.add( + new ProfileResult( + "GlobalAggregator", + "test_global_agg", + new LinkedHashMap<>(), + new HashMap<>(), + 19631335L, + List.of(), + 563002L, + 142210L, + 1216631L + ) + ); + tree.add( + new ProfileResult( + "GlobalAggregator", + "test_global_agg", + new LinkedHashMap<>(), + new HashMap<>(), + 19634567L, + List.of(), + 563333L, + 146783L, + 1496600L + ) + ); + return tree; + } + + public void testBuildTimeStatsBreakdownMap() { + List tree = createConcurrentSearchProfileTree(); + Map breakdown = new HashMap<>(); + Map timeStatsMap = new HashMap<>(); + timeStatsMap.put("max_initialize", 30L); + timeStatsMap.put("min_initialize", 10L); + timeStatsMap.put("avg_initialize", 60L); + ConcurrentAggregationProfiler.buildBreakdownMap(tree.size(), breakdown, timeStatsMap, "initialize"); + assertTrue(breakdown.containsKey("max_initialize")); + assertTrue(breakdown.containsKey("min_initialize")); + assertTrue(breakdown.containsKey("avg_initialize")); + assertEquals(30L, (long) breakdown.get("max_initialize")); + assertEquals(10L, (long) breakdown.get("min_initialize")); + assertEquals(15L, (long) breakdown.get("avg_initialize")); + } + + public void testBuildCountStatsBreakdownMap() { + List tree = createConcurrentSearchProfileTree(); + Map breakdown = new HashMap<>(); + Map countStatsMap = new HashMap<>(); + countStatsMap.put("max_collect_count", 3L); + countStatsMap.put("min_collect_count", 1L); + countStatsMap.put("avg_collect_count", 6L); + ConcurrentAggregationProfiler.buildBreakdownMap(tree.size(), breakdown, countStatsMap, "collect_count"); + assertTrue(breakdown.containsKey("max_collect_count")); + assertTrue(breakdown.containsKey("min_collect_count")); + assertTrue(breakdown.containsKey("avg_collect_count")); + assertEquals(3L, (long) breakdown.get("max_collect_count")); + assertEquals(1L, (long) breakdown.get("min_collect_count")); + assertEquals(1L, (long) breakdown.get("avg_collect_count")); + } + + public void testBuildBreakdownStatsMap() { + Map statsMap = new HashMap<>(); + ConcurrentAggregationProfiler.buildBreakdownStatsMap( + statsMap, + new ProfileResult("NumericTermsAggregator", "desc", Map.of("initialize", 100L), Map.of(), 130L, List.of()), + "initialize" + ); + assertTrue(statsMap.containsKey("max_initialize")); + assertTrue(statsMap.containsKey("min_initialize")); + assertTrue(statsMap.containsKey("avg_initialize")); + assertEquals(100L, (long) statsMap.get("max_initialize")); + assertEquals(100L, (long) statsMap.get("min_initialize")); + assertEquals(100L, (long) statsMap.get("avg_initialize")); + ConcurrentAggregationProfiler.buildBreakdownStatsMap( + statsMap, + new ProfileResult("NumericTermsAggregator", "desc", Map.of("initialize", 50L), Map.of(), 120L, List.of()), + "initialize" + ); + assertEquals(100L, (long) statsMap.get("max_initialize")); + assertEquals(50L, (long) statsMap.get("min_initialize")); + assertEquals(150L, (long) statsMap.get("avg_initialize")); + } + + public void testGetSliceLevelAggregationMap() { + List tree = createConcurrentSearchProfileTree(); + Map> aggregationMap = ConcurrentAggregationProfiler.getSliceLevelAggregationMap(tree); + assertEquals(2, aggregationMap.size()); + assertTrue(aggregationMap.containsKey("test_scoped_agg")); + assertTrue(aggregationMap.containsKey("test_global_agg")); + assertEquals(2, aggregationMap.get("test_scoped_agg").size()); + assertEquals(2, aggregationMap.get("test_global_agg").size()); + for (int slice_id : new int[] { 0, 1 }) { + assertEquals(1, aggregationMap.get("test_scoped_agg").get(slice_id).getProfiledChildren().size()); + assertEquals( + "test_terms", + aggregationMap.get("test_scoped_agg").get(slice_id).getProfiledChildren().get(0).getLuceneDescription() + ); + assertEquals(0, aggregationMap.get("test_global_agg").get(slice_id).getProfiledChildren().size()); + } + } +} diff --git a/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java b/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java index e703396f5cf02..b371015621647 100644 --- a/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java +++ b/server/src/test/java/org/opensearch/search/profile/query/QueryProfileShardResultTests.java @@ -54,7 +54,7 @@ public static QueryProfileShardResult createTestItem() { int size = randomIntBetween(0, 5); List queryProfileResults = new ArrayList<>(size); for (int i = 0; i < size; i++) { - queryProfileResults.add(ProfileResultTests.createTestItem(1)); + queryProfileResults.add(ProfileResultTests.createTestItem(1, false)); } CollectorResult profileCollector = CollectorResultTests.createTestItem(2, false); long rewriteTime = randomNonNegativeLong(); diff --git a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java index aa7bcf4d63e3d..199ab7d76ec99 100644 --- a/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/rescore/QueryRescorerBuilderTests.java @@ -40,11 +40,11 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedObjectNotFoundException; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -130,7 +130,7 @@ private RescorerBuilder copy(RescorerBuilder original) throws IOException public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { RescorerBuilder rescoreBuilder = randomRescoreBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java index 47579e31dc7c2..811d3d2784595 100644 --- a/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/searchafter/SearchAfterBuilderTests.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.text.Text; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; @@ -190,7 +191,7 @@ public void testEqualsAndHashcode() throws Exception { public void testFromXContent() throws Exception { for (int runs = 0; runs < 20; runs++) { SearchAfterBuilder searchAfterBuilder = randomJsonSearchFromBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } @@ -216,7 +217,7 @@ public void testFromXContentIllegalType() throws Exception { if (type == XContentType.JSON || type == XContentType.YAML) { continue; } - XContentBuilder xContent = XContentFactory.contentBuilder(type); + XContentBuilder xContent = MediaTypeRegistry.contentBuilder(type); xContent.startObject().startArray("search_after").value(new BigDecimal("9223372036854776003.3")).endArray().endObject(); try (XContentParser parser = createParser(xContent)) { parser.nextToken(); diff --git a/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java b/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java index 86dc27d73d21f..4d2a441a180a6 100644 --- a/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java +++ b/server/src/test/java/org/opensearch/search/slice/DocValuesSliceQueryTests.java @@ -110,7 +110,7 @@ public void setScorer(Scorable scorer) throws IOException {} @Override public void collect(int doc) throws IOException { - Document d = context.reader().document(doc, Collections.singleton("uuid")); + Document d = context.reader().storedFields().document(doc, Collections.singleton("uuid")); String uuid = d.get("uuid"); assertThat(keys.contains(uuid), equalTo(true)); keys.remove(uuid); diff --git a/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java index 28d96c916bc41..fbbc667cbcb42 100644 --- a/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/slice/SliceBuilderTests.java @@ -56,9 +56,9 @@ import org.opensearch.common.Nullable; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexSettings; @@ -208,7 +208,7 @@ public void testEqualsAndHashcode() throws Exception { public void testFromXContent() throws Exception { SliceBuilder sliceBuilder = randomSliceBuilder(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java b/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java index 84a65b3c8a7cb..8efa4eeef80dd 100644 --- a/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java +++ b/server/src/test/java/org/opensearch/search/slice/TermsSliceQueryTests.java @@ -120,7 +120,7 @@ public void setScorer(Scorable scorer) throws IOException {} @Override public void collect(int doc) throws IOException { - Document d = context.reader().document(doc, Collections.singleton("uuid")); + Document d = context.reader().storedFields().document(doc, Collections.singleton("uuid")); String uuid = d.get("uuid"); assertThat(keys.contains(uuid), equalTo(true)); keys.remove(uuid); diff --git a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java index e8d713d15b337..02f32faa6ffd9 100644 --- a/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/opensearch/search/sort/AbstractSortTestCase.java @@ -40,10 +40,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.env.Environment; @@ -131,7 +131,7 @@ public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { T testItem = createTestItem(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/sort/BucketedSortTestCase.java b/server/src/test/java/org/opensearch/search/sort/BucketedSortTestCase.java index 98c04eaf9415a..78cfda8e70660 100644 --- a/server/src/test/java/org/opensearch/search/sort/BucketedSortTestCase.java +++ b/server/src/test/java/org/opensearch/search/sort/BucketedSortTestCase.java @@ -40,7 +40,7 @@ import org.opensearch.common.util.MockBigArrays; import org.opensearch.common.util.MockPageCacheRecycler; import org.opensearch.common.lease.Releasable; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java b/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java index bf37e822bfc3c..f8f2564dacbcb 100644 --- a/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/sort/NestedSortBuilderTests.java @@ -34,10 +34,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.ConstantScoreQueryBuilder; @@ -83,7 +83,7 @@ protected NamedXContentRegistry xContentRegistry() { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { NestedSortBuilder testItem = createRandomNestedSort(3); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); XContentBuilder shuffled = shuffleXContent(builder); try (XContentParser parser = createParser(shuffled)) { diff --git a/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java index eee13ec56d841..1ac97fded27ce 100644 --- a/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/opensearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -39,10 +39,10 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.index.Index; @@ -146,7 +146,7 @@ public void testEqualsAndHashcode() { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { SB suggestionBuilder = randomTestBuilder(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { xContentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java b/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java index f8996c528cf32..f2d1acc35f356 100644 --- a/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/SuggestBuilderTests.java @@ -34,10 +34,10 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.SearchModule; @@ -82,7 +82,7 @@ public static void afterClass() { public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) { SuggestBuilder suggestBuilder = randomSuggestBuilder(); - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { xContentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java index 8484861999c3f..d0e42f8b91e2a 100644 --- a/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/phrase/DirectCandidateGeneratorTests.java @@ -47,9 +47,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; @@ -140,7 +140,7 @@ private static DirectCandidateGeneratorBuilder mutate(DirectCandidateGeneratorBu public void testFromXContent() throws IOException { for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) { DirectCandidateGeneratorBuilder generator = randomCandidateGenerator(); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java b/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java index a4adf03cff316..9b807ee35e4bf 100644 --- a/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java +++ b/server/src/test/java/org/opensearch/search/suggest/phrase/SmoothingModelTestCase.java @@ -45,9 +45,9 @@ import org.apache.lucene.store.ByteBuffersDirectory; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.lucene.BytesRefs; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.SearchModule; @@ -101,7 +101,7 @@ public static void afterClass() throws Exception { */ public void testFromXContent() throws IOException { SmoothingModel testModel = createTestModel(); - XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder contentBuilder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { contentBuilder.prettyPrint(); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index a121a190096b4..32e0edf7c8f11 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.mockito.Mockito; import org.opensearch.ExceptionsHelper; import org.opensearch.Version; import org.opensearch.action.ActionListener; @@ -105,6 +106,8 @@ import org.opensearch.client.AdminClient; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterInfo; +import org.opensearch.cluster.ClusterInfoService; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -176,13 +179,15 @@ import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.shard.PrimaryReplicaSyncer; import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory; +import org.opensearch.index.store.remote.filecache.FileCache; import org.opensearch.index.store.remote.filecache.FileCacheCleaner; +import org.opensearch.index.store.remote.filecache.FileCacheStats; import org.opensearch.indices.IndicesModule; import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.indices.SystemIndices; import org.opensearch.indices.analysis.AnalysisModule; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.indices.recovery.PeerRecoverySourceService; @@ -213,6 +218,7 @@ import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.disruption.DisruptableMockTransport; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.RemoteTransportException; import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportInterceptor; import org.opensearch.transport.TransportRequest; @@ -245,6 +251,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static org.mockito.Mockito.when; import static org.opensearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.opensearch.env.Environment.PATH_HOME_SETTING; import static org.opensearch.monitor.StatusInfo.Status.HEALTHY; @@ -260,6 +267,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; +import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING; public class SnapshotResiliencyTests extends OpenSearchTestCase { @@ -413,6 +421,106 @@ public void testSuccessfulSnapshotAndRestore() { assertEquals(0, snapshotInfo.failedShards()); } + public void testSearchableSnapshotOverSubscription() { + setupTestCluster(1, 2, 2); + + String repoName = "repo"; + String snapshotName = "snapshot"; + final String index = "test"; + final int shards = randomIntBetween(1, 10); + final int documents = randomIntBetween(0, 100); + + final TestClusterNodes.TestClusterNode clusterManagerNode = testClusterNodes.currentClusterManager( + testClusterNodes.nodes.values().iterator().next().clusterService.state() + ); + + Map nodeFileCacheStats = new HashMap<>(); + for (TestClusterNodes.TestClusterNode node : testClusterNodes.nodes.values()) { + nodeFileCacheStats.put(node.node.getId(), new FileCacheStats(0, 1, 0, 0, 0, 0, 0)); + } + ClusterInfo clusterInfo = new ClusterInfo(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), nodeFileCacheStats); + testClusterNodes.nodes.values().forEach(node -> when(node.getMockClusterInfoService().getClusterInfo()).thenReturn(clusterInfo)); + + final StepListener createSnapshotResponseListener = new StepListener<>(); + + continueOrDie(createRepoAndIndex(repoName, index, shards), createIndexResponse -> { + final Runnable afterIndexing = () -> client().admin() + .cluster() + .prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true) + .execute(createSnapshotResponseListener); + if (documents == 0) { + afterIndexing.run(); + } else { + final BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < documents; ++i) { + bulkRequest.add(new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i))); + } + final StepListener bulkResponseStepListener = new StepListener<>(); + client().bulk(bulkRequest, bulkResponseStepListener); + continueOrDie(bulkResponseStepListener, bulkResponse -> { + assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); + assertEquals(documents, bulkResponse.getItems().length); + afterIndexing.run(); + }); + } + }); + + final StepListener deleteIndexListener = new StepListener<>(); + + continueOrDie( + createSnapshotResponseListener, + createSnapshotResponse -> client().admin().indices().delete(new DeleteIndexRequest(index), deleteIndexListener) + ); + + final StepListener restoreSnapshotResponseListener = new StepListener<>(); + continueOrDie( + deleteIndexListener, + ignored -> client().admin() + .cluster() + .restoreSnapshot( + new RestoreSnapshotRequest(repoName, snapshotName).waitForCompletion(true) + .storageType(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT), + restoreSnapshotResponseListener + ) + ); + + final AtomicBoolean exceptionVerified = new AtomicBoolean(); + + restoreSnapshotResponseListener.whenComplete(null, restoreSnapshotException -> { + Throwable throwable = restoreSnapshotException; + if (restoreSnapshotException instanceof RemoteTransportException) { + throwable = restoreSnapshotException.getCause(); + } + try { + assertTrue(throwable instanceof SnapshotRestoreException); + assertTrue( + throwable.getMessage() + .contains( + "Size of the indexes to be restored exceeds the file cache bounds. Increase the file cache capacity on the cluster nodes using " + + NODE_SEARCH_CACHE_SIZE_SETTING.getKey() + + " setting." + ) + ); + } catch (SnapshotRestoreException ignored) {} + exceptionVerified.set(true); + }); + + runUntil(exceptionVerified::get, TimeUnit.MINUTES.toMillis(5L)); + assertTrue(exceptionVerified.get()); + SnapshotsInProgress finalSnapshotsInProgress = clusterManagerNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + assertFalse(finalSnapshotsInProgress.entries().stream().anyMatch(entry -> entry.state().completed() == false)); + final Repository repository = clusterManagerNode.repositoriesService.repository(repoName); + Collection snapshotIds = getRepositoryData(repository).getSnapshotIds(); + assertThat(snapshotIds, hasSize(1)); + + final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotIds.iterator().next()); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); + assertThat(snapshotInfo.indices(), containsInAnyOrder(index)); + assertEquals(shards, snapshotInfo.successfulShards()); + assertEquals(0, snapshotInfo.failedShards()); + } + public void testSnapshotWithNodeDisconnects() { final int dataNodes = randomIntBetween(2, 10); final int clusterManagerNodes = randomFrom(1, 3, 5); @@ -1415,6 +1523,11 @@ private void setupTestCluster(int clusterManagerNodes, int dataNodes) { startCluster(); } + private void setupTestCluster(int clusterManagerNodes, int dataNodes, int searchNodes) { + testClusterNodes = new TestClusterNodes(clusterManagerNodes, dataNodes, searchNodes); + startCluster(); + } + private void scheduleSoon(Runnable runnable) { deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), runnable); } @@ -1465,6 +1578,7 @@ private Environment createEnvironment(String nodeName) { ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING.getKey(), ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING.get(Settings.EMPTY) ) + .put(FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING.getKey(), 5) .put(MappingUpdatedAction.INDICES_MAX_IN_FLIGHT_UPDATES_SETTING.getKey(), 1000) // o.w. some tests might block .build() ); @@ -1488,6 +1602,10 @@ private final class TestClusterNodes { private final Set disconnectedNodes = new HashSet<>(); TestClusterNodes(int clusterManagerNodes, int dataNodes) { + this(clusterManagerNodes, dataNodes, 0); + } + + TestClusterNodes(int clusterManagerNodes, int dataNodes, int searchNodes) { for (int i = 0; i < clusterManagerNodes; ++i) { nodes.computeIfAbsent("node" + i, nodeName -> { try { @@ -1506,6 +1624,15 @@ private final class TestClusterNodes { } }); } + for (int i = 0; i < searchNodes; ++i) { + nodes.computeIfAbsent("search-node" + i, nodeName -> { + try { + return newSearchNode(nodeName); + } catch (IOException e) { + throw new AssertionError(e); + } + }); + } } public TestClusterNode nodeById(final String nodeId) { @@ -1524,6 +1651,10 @@ private TestClusterNode newDataNode(String nodeName) throws IOException { return newNode(nodeName, DiscoveryNodeRole.DATA_ROLE); } + private TestClusterNode newSearchNode(String nodeName) throws IOException { + return newNode(nodeName, DiscoveryNodeRole.SEARCH_ROLE); + } + private TestClusterNode newNode(String nodeName, DiscoveryNodeRole role) throws IOException { return new TestClusterNode( new DiscoveryNode( @@ -1667,6 +1798,8 @@ private final class TestClusterNode { private final ThreadPool threadPool; + private final ClusterInfoService clusterInfoService; + private Coordinator coordinator; TestClusterNode(DiscoveryNode node) throws IOException { @@ -1784,6 +1917,7 @@ public void onFailure(final Exception e) { final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); client = new NodeClient(settings, threadPool); + clusterInfoService = Mockito.mock(ClusterInfoService.class); final SetOnce rerouteServiceSetOnce = new SetOnce<>(); final SnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( settings, @@ -1993,8 +2127,9 @@ public void onFailure(final Exception e) { new SystemIndices(emptyMap()), null ), - clusterSettings, - shardLimitValidator + shardLimitValidator, + indicesService, + clusterInfoService::getClusterInfo ); actions.put( PutMappingAction.INSTANCE, @@ -2207,6 +2342,10 @@ protected void assertSnapshotOrGenericThread() { } } + public ClusterInfoService getMockClusterInfoService() { + return clusterInfoService; + } + public void restart() { testClusterNodes.disconnectNode(this); final ClusterState oldState = this.clusterService.state(); diff --git a/server/src/test/java/org/opensearch/tasks/TaskResultTests.java b/server/src/test/java/org/opensearch/tasks/TaskResultTests.java index f27a920d812e9..fac7a0538090a 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskResultTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskResultTests.java @@ -36,9 +36,9 @@ import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; @@ -120,7 +120,7 @@ private XContentBuilder addRandomUnknownFields(XContentBuilder builder) throws I map.put("unknown_field" + i, Collections.singletonMap("inner", randomAlphaOfLength(20))); } } - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType()); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(parser.contentType()); return xContentBuilder.map(map); } } diff --git a/server/src/test/java/org/opensearch/test/NoopDiscovery.java b/server/src/test/java/org/opensearch/test/NoopDiscovery.java index c1127aed95c1c..f7365d7d71693 100644 --- a/server/src/test/java/org/opensearch/test/NoopDiscovery.java +++ b/server/src/test/java/org/opensearch/test/NoopDiscovery.java @@ -33,8 +33,8 @@ import org.opensearch.action.ActionListener; import org.opensearch.cluster.ClusterChangedEvent; -import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.component.LifecycleListener; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.discovery.Discovery; import org.opensearch.discovery.DiscoveryStats; diff --git a/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java b/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java index dfb14be54a5b9..fd99942d27255 100644 --- a/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java @@ -33,7 +33,7 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.CircuitBreakingException; import org.opensearch.common.breaker.TestCircuitBreaker; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.bytes.ReleasableBytesReference; diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java index 33fe928d5c09d..c6b16d70fc7e0 100644 --- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java +++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java @@ -33,9 +33,9 @@ package org.opensearch.transport; import org.opensearch.Version; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.CircuitBreakingException; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.common.breaker.TestCircuitBreaker; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java index c61ac51ba57d4..fe8be400b03c9 100644 --- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java +++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java @@ -36,8 +36,8 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.breaker.CircuitBreaker; -import org.opensearch.common.breaker.NoopCircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; diff --git a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java index 05d375579f3a5..39bc2d9bd2d48 100644 --- a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java +++ b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java @@ -38,7 +38,7 @@ import org.opensearch.Version; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.component.Lifecycle; +import org.opensearch.common.lifecycle.Lifecycle; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.network.NetworkService; import org.opensearch.common.network.NetworkUtils; @@ -46,7 +46,7 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.MockLogAppender; import org.opensearch.test.junit.annotations.TestLogging; diff --git a/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java index 71c69acb75896..b2bdf8e3cb4d9 100644 --- a/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java +++ b/server/src/test/java/org/opensearch/transport/TransportServiceHandshakeTests.java @@ -40,7 +40,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; diff --git a/settings.gradle b/settings.gradle index b7d47cd9b745e..c04b5997d49b1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.enterprise" version "3.13.3" + id "com.gradle.enterprise" version "3.14.1" } buildCache { @@ -53,6 +53,7 @@ List projects = [ 'distribution:archives:no-jdk-linux-ppc64le-tar', 'distribution:archives:linux-tar', 'distribution:archives:no-jdk-linux-tar', + 'distribution:archives:jre-linux-tar', 'distribution:docker', 'distribution:docker:docker-arm64-build-context', 'distribution:docker:docker-arm64-export', diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index b6dc8820a85c5..b052988f88a2b 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -61,7 +61,7 @@ dependencies { api "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" api "org.eclipse.jetty:jetty-server:${versions.jetty}" api "org.eclipse.jetty.websocket:javax-websocket-server-impl:${versions.jetty}" - api 'org.apache.zookeeper:zookeeper:3.8.1' + api 'org.apache.zookeeper:zookeeper:3.8.2' api "org.apache.commons:commons-text:1.10.0" api "commons-net:commons-net:3.9.0" runtimeOnly "com.google.guava:guava:${versions.guava}" diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 98b4774ce1836..72fabc001760f 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -80,7 +80,7 @@ import org.opensearch.gateway.GatewayService; import org.opensearch.gateway.MockGatewayMetaState; import org.opensearch.gateway.PersistedClusterStateService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; import org.opensearch.test.OpenSearchTestCase; diff --git a/test/framework/src/main/java/org/opensearch/common/breaker/TestCircuitBreaker.java b/test/framework/src/main/java/org/opensearch/common/breaker/TestCircuitBreaker.java index bbacdd2ac4eec..9705a56700f72 100644 --- a/test/framework/src/main/java/org/opensearch/common/breaker/TestCircuitBreaker.java +++ b/test/framework/src/main/java/org/opensearch/common/breaker/TestCircuitBreaker.java @@ -32,6 +32,9 @@ package org.opensearch.common.breaker; +import org.opensearch.core.common.breaker.CircuitBreakingException; +import org.opensearch.core.common.breaker.NoopCircuitBreaker; + import java.util.concurrent.atomic.AtomicBoolean; public class TestCircuitBreaker extends NoopCircuitBreaker { diff --git a/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java b/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java index 07fd383dc230a..7b12591365cf4 100644 --- a/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/common/bytes/AbstractBytesReferenceTestCase.java @@ -35,7 +35,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.BytesRefIterator; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.core.common.bytes.AbstractBytesReference; @@ -45,7 +45,7 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.core.common.util.ByteArray; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.OpenSearchTestCase; import java.io.EOFException; diff --git a/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java index 5ef68f0eab757..54d53b931f171 100644 --- a/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/opensearch/common/util/MockBigArrays.java @@ -38,11 +38,11 @@ import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.util.BigArray; import org.opensearch.core.common.util.ByteArray; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import java.util.Collection; import java.util.Collections; diff --git a/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java index 3c20f22e392b8..8d951c603425d 100644 --- a/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java +++ b/test/framework/src/main/java/org/opensearch/index/RandomCreateIndexGenerator.java @@ -35,8 +35,8 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -96,7 +96,7 @@ public static Settings randomIndexSettings() { * Creates a random mapping */ public static XContentBuilder randomMapping() throws IOException { - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(randomFrom(XContentType.values())); builder.startObject(); randomMappingFields(builder, true); diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index 9c6f36c87896c..df2e6da972154 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -49,6 +49,7 @@ import org.apache.lucene.index.LiveIndexWriterConfig; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -118,8 +119,8 @@ import org.opensearch.index.translog.TranslogDeletionPolicy; import org.opensearch.index.translog.TranslogManager; import org.opensearch.index.translog.listener.TranslogEventListener; -import org.opensearch.indices.breaker.CircuitBreakerService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.test.DummyShardLock; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; @@ -1302,6 +1303,7 @@ public static List getDocIds(Engine engine, boolean refresh NumericDocValues primaryTermDocValues = reader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); NumericDocValues versionDocValues = reader.getNumericDocValues(VersionFieldMapper.NAME); Bits liveDocs = reader.getLiveDocs(); + StoredFields storedFields = reader.storedFields(); for (int i = 0; i < reader.maxDoc(); i++) { if (liveDocs == null || liveDocs.get(i)) { if (primaryTermDocValues.advanceExact(i) == false) { @@ -1309,7 +1311,7 @@ public static List getDocIds(Engine engine, boolean refresh continue; } final long primaryTerm = primaryTermDocValues.longValue(); - Document doc = reader.document(i, Sets.newHashSet(IdFieldMapper.NAME, SourceFieldMapper.NAME)); + Document doc = storedFields.document(i, Sets.newHashSet(IdFieldMapper.NAME, SourceFieldMapper.NAME)); BytesRef binaryID = doc.getBinaryValue(IdFieldMapper.NAME); String id = Uid.decodeId(Arrays.copyOfRange(binaryID.bytes, binaryID.offset, binaryID.offset + binaryID.length)); final BytesRef source = doc.getBinaryValue(SourceFieldMapper.NAME); @@ -1463,6 +1465,7 @@ public static void assertAtMostOneLuceneDocumentPerSequenceNumber(IndexSettings for (LeafReaderContext leaf : wrappedReader.leaves()) { NumericDocValues primaryTermDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); NumericDocValues seqNoDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.NAME); + final StoredFields storedFields = leaf.reader().storedFields(); int docId; while ((docId = seqNoDocValues.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { assertTrue(seqNoDocValues.advanceExact(docId)); @@ -1471,7 +1474,7 @@ public static void assertAtMostOneLuceneDocumentPerSequenceNumber(IndexSettings if (primaryTermDocValues.advanceExact(docId)) { if (seqNos.add(seqNo) == false) { final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor(); - leaf.reader().document(docId, idFieldVisitor); + storedFields.document(docId, idFieldVisitor); throw new AssertionError("found multiple documents for seq=" + seqNo + " id=" + idFieldVisitor.getId()); } } diff --git a/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java index 6a715599a8e1a..9e4e59d9a4d15 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/TranslogHandler.java @@ -33,8 +33,8 @@ package org.opensearch.index.engine; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.analysis.AnalysisRegistry; @@ -137,7 +137,13 @@ public Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.O final String indexName = mapperService.index().getName(); final Engine.Index engineIndex = IndexShard.prepareIndex( docMapper(MapperService.SINGLE_MAPPING_NAME), - new SourceToParse(indexName, index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()), + new SourceToParse( + indexName, + index.id(), + index.source(), + MediaTypeRegistry.xContentType(index.source()), + index.routing() + ), index.seqNo(), index.primaryTerm(), index.version(), diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java index 57c09db577247..09e6cb07998c2 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java @@ -53,7 +53,7 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.lookup.SearchLookup; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 93b9742ada0da..7a492dbebd836 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -73,7 +73,7 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.xcontent.XContentType; @@ -112,7 +112,7 @@ import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; import org.opensearch.indices.recovery.AsyncRecoveryTarget; import org.opensearch.indices.recovery.PeerRecoveryTargetService; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java index a9e1a526b1786..3744b06c35a72 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java @@ -40,7 +40,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.snapshots.IndexShardSnapshotStatus; diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java index 0bd6e1872a101..7d3a88fb1aba2 100644 --- a/test/framework/src/main/java/org/opensearch/node/MockNode.java +++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java @@ -49,7 +49,7 @@ import org.opensearch.env.Environment; import org.opensearch.http.HttpServerTransport; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.plugins.Plugin; import org.opensearch.script.MockScriptService; diff --git a/test/framework/src/main/java/org/opensearch/node/RecoverySettingsChunkSizePlugin.java b/test/framework/src/main/java/org/opensearch/node/RecoverySettingsChunkSizePlugin.java index 062675737f277..dabf23ce08263 100644 --- a/test/framework/src/main/java/org/opensearch/node/RecoverySettingsChunkSizePlugin.java +++ b/test/framework/src/main/java/org/opensearch/node/RecoverySettingsChunkSizePlugin.java @@ -34,7 +34,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.plugins.Plugin; diff --git a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index bcbf3b5b9a610..387a15899ce6f 100644 --- a/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/opensearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -40,7 +40,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Streams; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.test.OpenSearchTestCase; diff --git a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java index 808dc50512c58..bad8b8cd87a9a 100644 --- a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java +++ b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java @@ -35,7 +35,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.util.BigArrays; import org.opensearch.indices.IndicesService; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.node.MockNode; import org.opensearch.plugins.Plugin; import org.opensearch.script.ScriptService; diff --git a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java index e7b1dd2bde1ea..5480c4629ebf6 100644 --- a/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/opensearch/search/RandomSearchRequestGenerator.java @@ -353,7 +353,7 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } jsonBuilder.endArray(); jsonBuilder.endObject(); - XContentParser parser = XContentFactory.xContent(XContentType.JSON) + XContentParser parser = XContentType.JSON.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index 60d337599771c..874d60a4097f2 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -63,7 +63,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.TriFunction; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.network.NetworkAddress; @@ -110,8 +110,8 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.core.index.shard.ShardId; import org.opensearch.indices.IndicesModule; -import org.opensearch.indices.breaker.CircuitBreakerService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.plugins.SearchPlugin; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java index 89d705c8fd1e8..b7b5fa091810d 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/BaseAggregationTestCase.java @@ -36,9 +36,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; @@ -67,7 +67,7 @@ public abstract class BaseAggregationTestCase, List> randomStoredFieldValues(Random r List originalValues = randomStoredFieldValues(random, numValues); List expectedParsedValues = new ArrayList<>(numValues); for (Object originalValue : originalValues) { - expectedParsedValues.add(getExpectedParsedValue(XContentType.fromMediaType(mediaType), originalValue)); + expectedParsedValues.add(getExpectedParsedValue(mediaType, originalValue)); } return Tuple.tuple(originalValues, expectedParsedValues); } @@ -154,15 +154,15 @@ private static List randomStoredFieldValues(Random random, int numValues * Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via * {@link org.opensearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans. */ - public static Object getExpectedParsedValue(XContentType xContentType, Object value) { + public static Object getExpectedParsedValue(MediaType mediaType, Object value) { if (value instanceof BytesArray) { - if (xContentType == XContentType.JSON) { + if (mediaType == XContentType.JSON) { // JSON writes base64 format return Base64.getEncoder().encodeToString(((BytesArray) value).toBytesRef().bytes); } } if (value instanceof Float) { - if (xContentType == XContentType.CBOR || xContentType == XContentType.SMILE) { + if (mediaType == XContentType.CBOR || mediaType == XContentType.SMILE) { // with binary content types we pass back the object as is return value; } @@ -205,7 +205,7 @@ public static BytesReference randomSource(Random random, XContentType xContentTy * @param random Random generator */ public static BytesReference randomSource(Random random, XContentType xContentType, int minNumFields) { - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) { builder.startObject(); addFields(random, builder, minNumFields, 0); builder.endObject(); diff --git a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java index 4e44791e77566..0ce63fbe2977e 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/opensearch/test/TestSearchContext.java @@ -160,6 +160,7 @@ public TestSearchContext( this.indexShard = indexShard; this.queryShardContext = queryShardContext; this.searcher = searcher; + this.concurrentSegmentSearchEnabled = searcher != null && (searcher.getExecutor() != null); this.scrollContext = scrollContext; } @@ -686,7 +687,7 @@ public TestSearchContext withCleanQueryResult() { * Add profilers to the query */ public TestSearchContext withProfilers() { - this.profilers = new Profilers(searcher); + this.profilers = new Profilers(searcher, concurrentSegmentSearchEnabled); return this; } } diff --git a/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java index 75a86d3d9a0bb..e1fcd47aa38f5 100644 --- a/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java +++ b/test/framework/src/main/java/org/opensearch/test/XContentTestUtils.java @@ -35,6 +35,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContent; @@ -75,7 +76,7 @@ public static Map convertToMap(ToXContent part) throws IOExcepti } public static BytesReference convertToXContent(Map map, XContentType xContentType) throws IOException { - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(xContentType)) { builder.map(map); return BytesReference.bytes(builder); } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java index 375768ed12cc4..2bbaf8c80dfa4 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/FakeRestRequest.java @@ -35,8 +35,8 @@ import org.opensearch.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; import org.opensearch.http.HttpResponse; @@ -231,10 +231,10 @@ public Builder withParams(Map params) { return this; } - public Builder withContent(BytesReference content, XContentType xContentType) { + public Builder withContent(BytesReference content, MediaType mediaType) { this.content = content; - if (xContentType != null) { - headers.put("Content-Type", Collections.singletonList(xContentType.mediaType())); + if (mediaType != null) { + headers.put("Content-Type", Collections.singletonList(mediaType.mediaType())); } return this; } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java index 512369fdfe8a1..6d3c2eaa972e6 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java @@ -66,6 +66,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; @@ -143,10 +144,10 @@ public abstract class OpenSearchRestTestCase extends OpenSearchTestCase { * Convert the entity from a {@link Response} into a map of maps. */ public static Map entityAsMap(Response response) throws IOException { - XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType()); + MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType()); // EMPTY and THROW are fine here because `.map` doesn't use named x content or deprecation try ( - XContentParser parser = xContentType.xContent() + XContentParser parser = mediaType.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, @@ -161,10 +162,10 @@ public static Map entityAsMap(Response response) throws IOExcept * Convert the entity from a {@link Response} into a list of maps. */ public static List entityAsList(Response response) throws IOException { - XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType()); + MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType()); // EMPTY and THROW are fine here because `.map` doesn't use named x content or deprecation try ( - XContentParser parser = xContentType.xContent() + XContentParser parser = mediaType.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, @@ -1074,7 +1075,7 @@ protected static Map getAsMap(final String endpoint) throws IOEx } protected static Map responseAsMap(Response response) throws IOException { - XContentType entityContentType = XContentType.fromMediaType(response.getEntity().getContentType()); + MediaType entityContentType = MediaType.fromMediaType(response.getEntity().getContentType()); Map responseEntity = XContentHelper.convertToMap( entityContentType.xContent(), response.getEntity().getContent(), diff --git a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java index df2d3790d42bb..31f94039ae28f 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/RestActionTestCase.java @@ -39,7 +39,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.identity.IdentityService; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.client.node.NodeClient; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java index cb460f17cbec0..3045b3fe66edc 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -41,9 +41,10 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.client.NodeSelector; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; @@ -138,20 +139,20 @@ private HttpEntity createEntity(List> bodies, Map bytesRefList = new ArrayList<>(bodies.size()); int totalBytesLength = 0; for (Map body : bodies) { - BytesRef bytesRef = bodyAsBytesRef(body, xContentType); + BytesRef bytesRef = bodyAsBytesRef(body, mediaType); bytesRefList.add(bytesRef); totalBytesLength += bytesRef.length - bytesRef.offset + 1; } @@ -161,20 +162,20 @@ private HttpEntity createEntity(List> bodies, Map headers, XContentType[] supportedContentTypes) { - XContentType xContentType = null; + private MediaType getContentType(Map headers, XContentType[] supportedContentTypes) { + MediaType mediaType = null; String contentType = headers.get("Content-Type"); if (contentType != null) { - xContentType = XContentType.fromMediaType(contentType); + mediaType = MediaType.fromMediaType(contentType); } - if (xContentType != null) { - return xContentType; + if (mediaType != null) { + return mediaType; } if (randomizeContentType) { return RandomizedTest.randomFrom(supportedContentTypes); @@ -182,9 +183,9 @@ private XContentType getContentType(Map headers, XContentType[] return XContentType.JSON; } - private BytesRef bodyAsBytesRef(Map bodyAsMap, XContentType xContentType) throws IOException { + private BytesRef bodyAsBytesRef(Map bodyAsMap, MediaType mediaType) throws IOException { Map finalBodyAsMap = stash.replaceStashedValues(bodyAsMap); - try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) { + try (XContentBuilder builder = MediaTypeRegistry.contentBuilder(mediaType)) { return BytesReference.bytes(builder.map(finalBodyAsMap)).toBytesRef(); } } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java index 5c6190ab860b2..0f14461eb0f86 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ClientYamlTestResponse.java @@ -38,6 +38,7 @@ import org.opensearch.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -58,7 +59,7 @@ public class ClientYamlTestResponse { private final Response response; private final byte[] body; - private final XContentType bodyContentType; + private final MediaType bodyContentType; private ObjectPath parsedResponse; private String bodyAsString; @@ -66,7 +67,7 @@ public ClientYamlTestResponse(Response response) throws IOException { this.response = response; if (response.getEntity() != null) { String contentType = response.getHeader("Content-Type"); - this.bodyContentType = XContentType.fromMediaType(contentType); + this.bodyContentType = MediaType.fromMediaType(contentType); try { byte[] bytes = EntityUtils.toByteArray(response.getEntity()); // skip parsing if we got text back (e.g. if we called _cat apis) diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java index 8b62f0d0a9783..6c1510f8ca60a 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/ObjectPath.java @@ -36,11 +36,11 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; @@ -57,8 +57,8 @@ public class ObjectPath { public static ObjectPath createFromResponse(Response response) throws IOException { byte[] bytes = EntityUtils.toByteArray(response.getEntity()); String contentType = response.getHeader("Content-Type"); - XContentType xContentType = XContentType.fromMediaType(contentType); - return ObjectPath.createFromXContent(xContentType.xContent(), new BytesArray(bytes)); + MediaType mediaType = MediaType.fromMediaType(contentType); + return ObjectPath.createFromXContent(mediaType.xContent(), new BytesArray(bytes)); } public static ObjectPath createFromXContent(XContent xContent, BytesReference input) throws IOException { diff --git a/test/framework/src/main/java/org/opensearch/test/transport/FakeTransport.java b/test/framework/src/main/java/org/opensearch/test/transport/FakeTransport.java index b0aea3d2e3841..4d59afd5f99ed 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/FakeTransport.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/FakeTransport.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.component.AbstractLifecycleComponent; +import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; import org.opensearch.transport.CloseableConnection; diff --git a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java index 87f198fb40ee5..7a1d730ac1b27 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/MockTransportService.java @@ -53,7 +53,7 @@ import org.opensearch.common.util.concurrent.AbstractRunnable; import org.opensearch.common.util.concurrent.RunOnce; import org.opensearch.common.util.io.IOUtils; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.node.Node; import org.opensearch.plugins.Plugin; import org.opensearch.tasks.TaskManager; diff --git a/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java index 8c3b32dbb4ca4..8d66d481dc4aa 100644 --- a/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java +++ b/test/framework/src/main/java/org/opensearch/test/transport/StubbableTransport.java @@ -35,8 +35,8 @@ import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.component.Lifecycle; -import org.opensearch.common.component.LifecycleListener; +import org.opensearch.common.lifecycle.Lifecycle; +import org.opensearch.common.lifecycle.LifecycleListener; import org.opensearch.common.transport.BoundTransportAddress; import org.opensearch.common.transport.TransportAddress; import org.opensearch.tasks.Task; diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java index bfacf9d3f2080..73b2e5dd82f62 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransport.java @@ -40,7 +40,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.breaker.CircuitBreaker; +import org.opensearch.core.common.breaker.CircuitBreaker; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.bytes.CompositeBytesReference; import org.opensearch.common.bytes.ReleasableBytesReference; @@ -53,7 +53,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.nio.BytesChannelContext; import org.opensearch.nio.BytesWriteHandler; import org.opensearch.nio.ChannelFactory; diff --git a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java index e51f96a86bebf..6334788edbbf2 100644 --- a/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java +++ b/test/framework/src/main/java/org/opensearch/transport/nio/MockNioTransportPlugin.java @@ -36,7 +36,7 @@ import org.opensearch.common.network.NetworkService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.PageCacheRecycler; -import org.opensearch.indices.breaker.CircuitBreakerService; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.NetworkPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.threadpool.ThreadPool; diff --git a/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java b/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java index a6482ed09b253..ca07e39719e23 100644 --- a/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java +++ b/test/framework/src/test/java/org/opensearch/test/rest/yaml/ObjectPathTests.java @@ -31,10 +31,9 @@ package org.opensearch.test.rest.yaml; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -56,7 +55,7 @@ public class ObjectPathTests extends OpenSearchTestCase { private static XContentBuilder randomXContentBuilder() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); - return XContentBuilder.builder(XContentFactory.xContent(xContentType)); + return XContentBuilder.builder(xContentType.xContent()); } public void testEvaluateObjectPathEscape() throws Exception { @@ -342,7 +341,7 @@ public void testEvaluateArrayAsRoot() throws Exception { xContentBuilder.endObject(); xContentBuilder.endArray(); ObjectPath objectPath = ObjectPath.createFromXContent( - XContentFactory.xContent(xContentBuilder.contentType()), + xContentBuilder.contentType().xContent(), BytesReference.bytes(xContentBuilder) ); Object object = objectPath.evaluate(""); diff --git a/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java b/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java index 8b0ffb2d0652d..868affc81be37 100644 --- a/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java +++ b/test/framework/src/test/java/org/opensearch/transport/nio/SimpleMockNioTransportTests.java @@ -41,7 +41,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.util.MockPageCacheRecycler; -import org.opensearch.indices.breaker.NoneCircuitBreakerService; +import org.opensearch.core.indices.breaker.NoneCircuitBreakerService; import org.opensearch.transport.AbstractSimpleTransportTestCase; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.ConnectionProfile;