From bf6d35ea8881cf76fb153480e263a0fa689e1e59 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 11 Jun 2024 19:20:39 +0530 Subject: [PATCH] Address further comments on PR Signed-off-by: Shivansh Arora --- .../AbstractRemoteWritableBlobEntity.java | 2 + .../remote/RemoteClusterStateService.java | 4 - .../remote/RemoteGlobalMetadataManager.java | 143 ++++++++++-------- .../remote/model/RemoteClusterBlocks.java | 5 + .../model/RemoteClusterMetadataManifest.java | 6 + .../model/RemoteClusterStateCustoms.java | 5 + .../model/RemoteCoordinationMetadata.java | 5 + .../remote/model/RemoteCustomMetadata.java | 5 + .../remote/model/RemoteDiscoveryNodes.java | 5 + .../remote/model/RemoteGlobalMetadata.java | 6 + .../RemoteHashesOfConsistentSettings.java | 5 + .../remote/model/RemoteIndexMetadata.java | 9 +- .../RemotePersistentSettingsMetadata.java | 5 + .../remote/model/RemoteTemplatesMetadata.java | 5 + .../RemoteTransientSettingsMetadata.java | 5 + .../index/remote/RemoteIndexPathUploader.java | 14 +- .../RemoteIndexMetadataManagerTests.java | 92 ----------- .../model/RemoteIndexMetadataTests.java | 4 +- .../remote/RemoteIndexPathUploaderTests.java | 6 +- 19 files changed, 159 insertions(+), 172 deletions(-) delete mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java diff --git a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java index 632b2b70d61df..23fc9d3ad77cb 100644 --- a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java +++ b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java @@ -42,6 +42,8 @@ public AbstractRemoteWritableBlobEntity( public abstract BlobPathParameters getBlobPathParameters(); + public abstract String getType(); + public String getFullBlobName() { return blobName; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 12747829d319f..112af53dffd41 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -531,7 +531,6 @@ private UploadedMetadataResults writeMetadataInParallel( blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry() ), - remoteGlobalMetadataManager.getPersistentSettingsBlobStore(), listener ) ); @@ -559,7 +558,6 @@ private UploadedMetadataResults writeMetadataInParallel( blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry() ), - remoteGlobalMetadataManager.getCoordinationMetadataBlobStore(), listener ) ); @@ -575,7 +573,6 @@ private UploadedMetadataResults writeMetadataInParallel( blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry() ), - remoteGlobalMetadataManager.getTemplatesMetadataBlobStore(), listener ) ); @@ -627,7 +624,6 @@ private UploadedMetadataResults writeMetadataInParallel( blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry() ), - remoteGlobalMetadataManager.getCustomMetadataBlobStore(), listener ) ); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 0298065698226..a67370cacb0c0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -62,13 +62,7 @@ public class RemoteGlobalMetadataManager { public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; private volatile TimeValue globalMetadataUploadTimeout; - private final RemoteWritableEntityStore globalMetadataBlobStore; - private final RemoteClusterStateBlobStore coordinationMetadataBlobStore; - private final RemoteClusterStateBlobStore transientSettingsBlobStore; - private final RemoteClusterStateBlobStore persistentSettingsBlobStore; - private final RemoteClusterStateBlobStore templatesMetadataBlobStore; - private final RemoteClusterStateBlobStore customMetadataBlobStore; - private final RemoteClusterStateBlobStore hashesOfConsistentSettingsBlobStore; + private Map remoteWritableEntityStores; private final Compressor compressor; private final NamedXContentRegistry namedXContentRegistry; @@ -82,40 +76,56 @@ public class RemoteGlobalMetadataManager { this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); this.compressor = blobStoreRepository.getCompressor(); this.namedXContentRegistry = blobStoreRepository.getNamedXContentRegistry(); - globalMetadataBlobStore = new RemoteClusterStateBlobStore<>( - blobStoreTransferService, - blobStoreRepository, - clusterName, - threadpool, - ThreadPool.Names.REMOTE_STATE_READ + this.remoteWritableEntityStores = new HashMap<>(); + this.remoteWritableEntityStores.put( + RemoteGlobalMetadata.GLOBAL_METADATA, + new RemoteClusterStateBlobStore<>( + blobStoreTransferService, + blobStoreRepository, + clusterName, + threadpool, + ThreadPool.Names.REMOTE_STATE_READ + ) ); - coordinationMetadataBlobStore = new RemoteClusterStateBlobStore<>( - blobStoreTransferService, - blobStoreRepository, - clusterName, - threadpool, - ThreadPool.Names.REMOTE_STATE_READ + this.remoteWritableEntityStores.put( + RemoteCoordinationMetadata.COORDINATION_METADATA, + new RemoteClusterStateBlobStore<>( + blobStoreTransferService, + blobStoreRepository, + clusterName, + threadpool, + ThreadPool.Names.REMOTE_STATE_READ + ) ); - persistentSettingsBlobStore = new RemoteClusterStateBlobStore<>( - blobStoreTransferService, - blobStoreRepository, - clusterName, - threadpool, - ThreadPool.Names.REMOTE_STATE_READ + this.remoteWritableEntityStores.put( + RemotePersistentSettingsMetadata.SETTING_METADATA, + new RemoteClusterStateBlobStore<>( + blobStoreTransferService, + blobStoreRepository, + clusterName, + threadpool, + ThreadPool.Names.REMOTE_STATE_READ + ) ); - templatesMetadataBlobStore = new RemoteClusterStateBlobStore<>( - blobStoreTransferService, - blobStoreRepository, - clusterName, - threadpool, - ThreadPool.Names.REMOTE_STATE_READ + this.remoteWritableEntityStores.put( + RemoteTemplatesMetadata.TEMPLATES_METADATA, + new RemoteClusterStateBlobStore<>( + blobStoreTransferService, + blobStoreRepository, + clusterName, + threadpool, + ThreadPool.Names.REMOTE_STATE_READ + ) ); - customMetadataBlobStore = new RemoteClusterStateBlobStore<>( - blobStoreTransferService, - blobStoreRepository, - clusterName, - threadpool, - ThreadPool.Names.REMOTE_STATE_READ + this.remoteWritableEntityStores.put( + RemoteCustomMetadata.CUSTOM_METADATA, + new RemoteClusterStateBlobStore<>( + blobStoreTransferService, + blobStoreRepository, + clusterName, + threadpool, + ThreadPool.Names.REMOTE_STATE_READ + ) ); clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); } @@ -125,10 +135,20 @@ public class RemoteGlobalMetadataManager { */ CheckedRunnable getAsyncMetadataWriteAction( AbstractRemoteWritableBlobEntity writeEntity, - RemoteWritableEntityStore remoteStore, LatchedActionListener latchedActionListener ) { - return () -> remoteStore.writeAsync(writeEntity, getActionListener(writeEntity, latchedActionListener)); + return (() -> getRemoteWriteableEntityStoreForObject(writeEntity).writeAsync( + writeEntity, + getActionListener(writeEntity, latchedActionListener) + )); + } + + private RemoteWritableEntityStore getRemoteWriteableEntityStoreForObject(AbstractRemoteWritableBlobEntity entity) { + RemoteWritableEntityStore remoteStore = remoteWritableEntityStores.get(entity.getType()); + if (remoteStore == null) { + throw new IllegalArgumentException("Unknown entity type [" + entity.getType() + "]"); + } + return remoteStore; } private ActionListener getActionListener( @@ -185,7 +205,7 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe compressor, namedXContentRegistry ); - return globalMetadataBlobStore.read(remoteGlobalMetadata); + return (Metadata) getRemoteWriteableEntityStoreForObject(remoteGlobalMetadata).read(remoteGlobalMetadata); } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { Metadata.Builder builder = new Metadata.Builder(); if (clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() != null) { @@ -195,7 +215,11 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe compressor, namedXContentRegistry ); - builder.coordinationMetadata(coordinationMetadataBlobStore.read(remoteCoordinationMetadata)); + builder.coordinationMetadata( + (CoordinationMetadata) getRemoteWriteableEntityStoreForObject(remoteCoordinationMetadata).read( + remoteCoordinationMetadata + ) + ); } if (clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() != null) { RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata( @@ -204,7 +228,9 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe compressor, namedXContentRegistry ); - builder.templates(templatesMetadataBlobStore.read(remoteTemplatesMetadata)); + builder.templates( + (TemplatesMetadata) getRemoteWriteableEntityStoreForObject(remoteTemplatesMetadata).read(remoteTemplatesMetadata) + ); } if (clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() != null) { RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata( @@ -213,18 +239,27 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe compressor, namedXContentRegistry ); - builder.persistentSettings(persistentSettingsBlobStore.read(remotePersistentSettingsMetadata)); + builder.persistentSettings( + (Settings) getRemoteWriteableEntityStoreForObject(remotePersistentSettingsMetadata).read( + remotePersistentSettingsMetadata + ) + ); } builder.clusterUUID(clusterMetadataManifest.getClusterUUID()); builder.clusterUUIDCommitted(clusterMetadataManifest.isClusterUUIDCommitted()); builder.version(clusterMetadataManifest.getMetadataVersion()); clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> { try { + RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata( + value.getUploadedFilename(), + key, + clusterUUID, + compressor, + namedXContentRegistry + ); builder.putCustom( key, - customMetadataBlobStore.read( - new RemoteCustomMetadata(value.getUploadedFilename(), key, clusterUUID, compressor, namedXContentRegistry) - ) + (Custom) getRemoteWriteableEntityStoreForObject(remoteCustomMetadata).read(remoteCustomMetadata) ); } catch (IOException e) { throw new IllegalStateException( @@ -283,20 +318,4 @@ private void setGlobalMetadataUploadTimeout(TimeValue newGlobalMetadataUploadTim public TimeValue getGlobalMetadataUploadTimeout() { return this.globalMetadataUploadTimeout; } - - public RemoteClusterStateBlobStore getCoordinationMetadataBlobStore() { - return coordinationMetadataBlobStore; - } - - public RemoteClusterStateBlobStore getPersistentSettingsBlobStore() { - return persistentSettingsBlobStore; - } - - public RemoteClusterStateBlobStore getTemplatesMetadataBlobStore() { - return templatesMetadataBlobStore; - } - - public RemoteClusterStateBlobStore getCustomMetadataBlobStore() { - return customMetadataBlobStore; - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index 937f9dc2c8631..01704df700e20 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -55,6 +55,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), CLUSTER_BLOCKS); } + @Override + public String getType() { + return CLUSTER_BLOCKS; + } + @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/transient/______ diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index f7024c8fde7ed..328601139c150 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -86,6 +86,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(MANIFEST), MANIFEST); } + @Override + public String getType() { + return MANIFEST; + } + @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ @@ -150,4 +155,5 @@ private ChecksumBlobStoreFormat getClusterMetadataManif } throw new IllegalArgumentException("Cluster metadata manifest file is corrupted, don't have valid codec version"); } + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java index 60a21c9b53148..25d6ce5848f9f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -77,6 +77,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), prefix); } + @Override + public String getType() { + return CLUSTER_STATE_CUSTOM; + } + @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java index 95ad7f7724a08..a90721ab59f66 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java @@ -70,6 +70,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of("global-metadata"), COORDINATION_METADATA); } + @Override + public String getType() { + return COORDINATION_METADATA; + } + @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java index 682cac4b39d10..1947fd9e0bb88 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java @@ -85,6 +85,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), prefix); } + @Override + public String getType() { + return CUSTOM_METADATA; + } + @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index 7dc2b6492de7e..970f485abdc49 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -60,6 +60,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), DISCOVERY_NODES); } + @Override + public String getType() { + return DISCOVERY_NODES; + } + @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java index 8e41b155ecb93..09f07de0d5c24 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java @@ -26,6 +26,7 @@ * Wrapper class for uploading/downloading global metadata ({@link Metadata}) to/from remote blob store */ public class RemoteGlobalMetadata extends AbstractRemoteWritableBlobEntity { + public static final String GLOBAL_METADATA = "global_metadata"; public static final ChecksumBlobStoreFormat GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( "metadata", @@ -48,6 +49,11 @@ public BlobPathParameters getBlobPathParameters() { throw new UnsupportedOperationException(); } + @Override + public String getType() { + return GLOBAL_METADATA; + } + @Override public String generateBlobFileName() { throw new UnsupportedOperationException(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java index a70506bcd6846..d343f8633131e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -58,6 +58,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), HASHES_OF_CONSISTENT_SETTINGS); } + @Override + public String getType() { + return HASHES_OF_CONSISTENT_SETTINGS; + } + @Override public String generateBlobFileName() { String blobFileName = String.join( diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java index 0966a7b09fe17..3acbcf545563c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java @@ -38,7 +38,7 @@ public class RemoteIndexMetadata extends AbstractRemoteWritableBlobEntity______ diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java index fe32b95f5e957..fd0526f05d015 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java @@ -72,6 +72,11 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), TRANSIENT_SETTING_METADATA); } + @Override + public String getType() { + return TRANSIENT_SETTING_METADATA; + } + @Override public String generateBlobFileName() { String blobFileName = String.join( diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index f0fa7423ff431..d5617bdfd94a7 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -45,7 +45,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING; import static org.opensearch.index.remote.RemoteIndexPath.COMBINED_PATH; import static org.opensearch.index.remote.RemoteIndexPath.SEGMENT_PATH; import static org.opensearch.index.remote.RemoteIndexPath.TRANSLOG_PATH; @@ -80,7 +80,7 @@ public class RemoteIndexPathUploader extends IndexMetadataUploadListener { private final boolean isRemoteDataAttributePresent; private final boolean isTranslogSegmentRepoSame; private final Supplier repositoriesService; - private volatile TimeValue indexMetadataUploadTimeout; + private volatile TimeValue metadataUploadTimeout; private BlobStoreRepository translogRepository; private BlobStoreRepository segmentRepository; @@ -98,8 +98,8 @@ public RemoteIndexPathUploader( // If the remote data attributes are not present, then there is no effect of translog and segment being same or different or null. isTranslogSegmentRepoSame = isTranslogSegmentRepoSame(); Objects.requireNonNull(clusterSettings); - indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); - clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); + metadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); + clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setMetadataUploadTimeout); } @Override @@ -131,7 +131,7 @@ protected void doOnUpload( logger.trace(new ParameterizedMessage("Remote index path upload started for {}", indexNames)); try { - if (latch.await(indexMetadataUploadTimeout.millis(), TimeUnit.MILLISECONDS) == false) { + if (latch.await(metadataUploadTimeout.millis(), TimeUnit.MILLISECONDS) == false) { RemoteStateTransferException ex = new RemoteStateTransferException( String.format(Locale.ROOT, TIMEOUT_EXCEPTION_MSG, indexNames) ); @@ -289,8 +289,8 @@ private boolean requiresPathUpload(IndexMetadata indexMetadata, IndexMetadata pr return pathType == PathType.HASHED_PREFIX && (Objects.isNull(prevPathType) || prevPathType != PathType.HASHED_PREFIX); } - private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) { - this.indexMetadataUploadTimeout = newIndexMetadataUploadTimeout; + private void setMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) { + this.metadataUploadTimeout = newIndexMetadataUploadTimeout; } /** diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java deleted file mode 100644 index 1096883db29b6..0000000000000 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote; - -import org.opensearch.cluster.ClusterModule; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.network.NetworkModule; -import org.opensearch.common.remote.RemoteWritableEntityStore; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.core.compress.Compressor; -import org.opensearch.core.compress.NoneCompressor; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; -import org.opensearch.gateway.remote.model.RemoteIndexMetadata; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.indices.IndicesModule; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.test.OpenSearchTestCase; -import org.opensearch.threadpool.TestThreadPool; -import org.opensearch.threadpool.ThreadPool; -import org.junit.After; -import org.junit.Before; - -import java.util.function.Function; -import java.util.stream.Stream; - -import static java.util.stream.Collectors.toList; -import static org.mockito.Mockito.when; -import static org.opensearch.threadpool.ThreadPool.Names.REMOTE_STATE_READ; -import static org.mockito.Mockito.mock; - -public class RemoteIndexMetadataManagerTests extends OpenSearchTestCase { - private RemoteIndexMetadataManager remoteIndexMetadataManager; - private BlobStoreRepository blobStoreRepository; - private ClusterSettings clusterSettings; - private BlobStoreTransferService blobStoreTransferService; - private ThreadPool threadPool; - - @Before - public void setup() { - clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - blobStoreRepository = mock(BlobStoreRepository.class); - blobStoreTransferService = mock(BlobStoreTransferService.class); - threadPool = new TestThreadPool("test"); - NamedXContentRegistry xContentRegistry = new NamedXContentRegistry( - Stream.of( - NetworkModule.getNamedXContents().stream(), - IndicesModule.getNamedXContents().stream(), - ClusterModule.getNamedXWriteables().stream() - ).flatMap(Function.identity()).collect(toList()) - ); - Compressor compressor = new NoneCompressor(); - when(blobStoreRepository.getCompressor()).thenReturn(compressor); - when(blobStoreRepository.getNamedXContentRegistry()).thenReturn(xContentRegistry); - remoteIndexMetadataManager = new RemoteIndexMetadataManager( - clusterSettings, - "cluster-name", - blobStoreRepository, - blobStoreTransferService, - threadPool - ); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); - threadPool.shutdown(); - } - - public void testIndexMetadataUploadWaitTimeSetting() { - // verify default value - assertEquals( - RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT, - remoteIndexMetadataManager.getIndexMetadataUploadTimeout() - ); - - // verify update index metadata upload timeout - int indexMetadataUploadTimeout = randomIntBetween(1, 10); - Settings newSettings = Settings.builder() - .put("cluster.remote_store.state.index_metadata.upload_timeout", indexMetadataUploadTimeout + "s") - .build(); - clusterSettings.applySettings(newSettings); - assertEquals(indexMetadataUploadTimeout, remoteIndexMetadataManager.getIndexMetadataUploadTimeout().seconds()); - } -} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java index 663d6655f0aff..7f9c3fdbae91b 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java @@ -40,8 +40,8 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX; import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -137,7 +137,7 @@ public void testBlobPathParameters() { IndexMetadata indexMetadata = getIndexMetadata(); RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); - assertThat(params.getPathTokens(), is(List.of(INDEX_PATH_TOKEN, indexMetadata.getIndexUUID()))); + assertThat(params.getPathTokens(), is(List.of(INDEX, indexMetadata.getIndexUUID()))); assertThat(params.getFilePrefix(), is("metadata")); } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java index 3b64b5a0662bb..e0a75f7296705 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java @@ -46,7 +46,7 @@ import org.mockito.Mockito; -import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING; import static org.opensearch.index.remote.RemoteStoreEnums.PathType.FIXED; import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_INFIX; import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_PREFIX; @@ -277,7 +277,7 @@ public void testInterceptWithLatchAwaitTimeout() throws IOException { Settings settings = Settings.builder() .put(this.settings) - .put(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) + .put(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) .build(); clusterSettings.applySettings(settings); SetOnce exceptionSetOnce = new SetOnce<>(); @@ -307,7 +307,7 @@ public void testInterceptWithInterruptedExceptionDuringLatchAwait() throws Excep remoteIndexPathUploader.start(); Settings settings = Settings.builder() .put(this.settings) - .put(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(1)) + .put(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(1)) .build(); clusterSettings.applySettings(settings); SetOnce exceptionSetOnce = new SetOnce<>();