Skip to content

Commit

Permalink
Handle read ClusterState for V1 manifest
Browse files Browse the repository at this point in the history 
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
@shiv0408
shiv0408 committed Jun 10, 2024
1 parent e8c8413 commit 4def30f
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 106 deletions.
67 changes: 48 additions & 19 deletions server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@

package org.opensearch.gateway.remote;

import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD;
import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1;
import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2;
import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS;
import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE;
import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES;
Expand Down Expand Up @@ -222,7 +226,7 @@ public RemoteUploadDetails writeFullMetadata(ClusterState clusterState, String p
UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel(
clusterState,
new ArrayList<>(clusterState.metadata().indices().values()),
Collections.emptyMap(),
emptyMap(),
clusterState.metadata().customs(),
true,
true,
Expand Down Expand Up @@ -1226,23 +1230,48 @@ private ClusterState readClusterStateInParallel(

public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId, boolean includeEphemeral)
throws IOException {
return readClusterStateInParallel(
ClusterState.builder(new ClusterName(clusterName)).build(),
manifest,
manifest.getClusterUUID(),
localNodeId,
manifest.getIndices(),
manifest.getCustomMetadataMap(),
manifest.getCoordinationMetadata() != null,
manifest.getSettingsMetadata() != null,
manifest.getTransientSettingsMetadata() != null,
manifest.getTemplatesMetadata() != null,
includeEphemeral && manifest.getDiscoveryNodesMetadata() != null,
includeEphemeral && manifest.getClusterBlocksMetadata() != null,
includeEphemeral ? manifest.getIndicesRouting() : Collections.emptyList(),
includeEphemeral && manifest.getHashesOfConsistentSettings() != null,
includeEphemeral ? manifest.getClusterStateCustomMap() : Collections.emptyMap()
);
if (manifest.onOrAfterCodecVersion(CODEC_V2)) {
return readClusterStateInParallel(
ClusterState.builder(new ClusterName(clusterName)).build(),
manifest,
manifest.getClusterUUID(),
localNodeId,
manifest.getIndices(),
manifest.getCustomMetadataMap(),
manifest.getCoordinationMetadata() != null,
manifest.getSettingsMetadata() != null,
manifest.getTransientSettingsMetadata() != null,
manifest.getTemplatesMetadata() != null,
includeEphemeral && manifest.getDiscoveryNodesMetadata() != null,
includeEphemeral && manifest.getClusterBlocksMetadata() != null,
includeEphemeral ? manifest.getIndicesRouting() : emptyList(),
includeEphemeral && manifest.getHashesOfConsistentSettings() != null,
includeEphemeral ? manifest.getClusterStateCustomMap() : emptyMap()
);
} else {
ClusterState clusterState = readClusterStateInParallel(
ClusterState.builder(new ClusterName(clusterName)).build(),
manifest,
manifest.getClusterUUID(),
localNodeId,
manifest.getIndices(),
// for manifest codec V1, we don't have the following objects to read, so not passing anything
emptyMap(),
false,
false,
false,
false,
false,
false,
emptyList(),
false,
emptyMap()
);
Metadata.Builder mb = Metadata.builder(remoteGlobalMetadataManager.getGlobalMetadata(manifest.getClusterUUID(), manifest));
mb.indices(clusterState.metadata().indices());
return ClusterState.builder(clusterState).metadata(mb).build();
}

}

public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId)
Expand Down Expand Up @@ -1413,7 +1442,7 @@ private List<String> createClusterChain(final Map<String, ClusterMetadataManifes
// This can occur only when there are no valid cluster UUIDs
assert validClusterManifests.isEmpty() : "There are no top level cluster UUIDs even when there are valid cluster UUIDs";
logger.info("There is no valid previous cluster UUID. All cluster UUIDs evaluated are: {}", manifestsByClusterUUID.keySet());
return Collections.emptyList();
return emptyList();
}
if (topLevelClusterUUIDs.size() > 1) {
logger.info("Top level cluster UUIDs: {}", topLevelClusterUUIDs);
Expand Down
133 changes: 46 additions & 87 deletions server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,31 +195,58 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe
compressor, namedXContentRegistry);
return globalMetadataBlobStore.read(remoteGlobalMetadata);
} else if (clusterMetadataManifest.hasMetadataAttributesFiles()) {
CoordinationMetadata coordinationMetadata = getCoordinationMetadata(
clusterUUID,
clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()
);
Settings settingsMetadata = getSettingsMetadata(
clusterUUID,
clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()
);
TemplatesMetadata templatesMetadata = getTemplatesMetadata(
clusterUUID,
clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()
);
Metadata.Builder builder = new Metadata.Builder();
builder.coordinationMetadata(coordinationMetadata);
builder.persistentSettings(settingsMetadata);
builder.templates(templatesMetadata);
if (clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() != null) {
RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(
clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename(),
clusterUUID,
compressor,
namedXContentRegistry
);
builder.coordinationMetadata(coordinationMetadataBlobStore.read(remoteCoordinationMetadata));
}
if (clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() != null) {
RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(
clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename(),
clusterUUID,
compressor,
namedXContentRegistry
);
builder.templates(templatesMetadataBlobStore.read(remoteTemplatesMetadata));
}
if (clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() != null) {
RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(
clusterMetadataManifest.getSettingsMetadata().getUploadedFilename(),
clusterUUID,
compressor,
namedXContentRegistry
);
builder.persistentSettings(persistentSettingsBlobStore.read(remotePersistentSettingsMetadata));
}
builder.clusterUUID(clusterMetadataManifest.getClusterUUID());
builder.clusterUUIDCommitted(clusterMetadataManifest.isClusterUUIDCommitted());
builder.version(clusterMetadataManifest.getMetadataVersion());
clusterMetadataManifest.getCustomMetadataMap()
.forEach(
(key, value) -> builder.putCustom(
key,
getCustomsMetadata(clusterUUID, value.getUploadedFilename(), key)
)
(key, value) -> {
try {
builder.putCustom(
key,
customMetadataBlobStore.read(new RemoteCustomMetadata(
value.getUploadedFilename(),
key,
clusterUUID,
compressor,
namedXContentRegistry
))
);
} catch (IOException e) {
throw new IllegalStateException(
String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", value.getUploadedFilename()),
e
);
}
}
);
return builder.build();
} else {
Expand All @@ -233,74 +260,6 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe
}
}

public CoordinationMetadata getCoordinationMetadata(String clusterUUID, String coordinationMetadataFileName) {
try {
// Fetch Coordination metadata
if (coordinationMetadataFileName != null) {
RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(coordinationMetadataFileName, clusterUUID,
compressor, namedXContentRegistry);
return coordinationMetadataBlobStore.read(remoteCoordinationMetadata);
} else {
return CoordinationMetadata.EMPTY_METADATA;
}
} catch (IOException e) {
throw new IllegalStateException(
String.format(Locale.ROOT, "Error while downloading Coordination Metadata - %s", coordinationMetadataFileName),
e
);
}
}

public Settings getSettingsMetadata(String clusterUUID, String settingsMetadataFileName) {
try {
// Fetch Settings metadata
if (settingsMetadataFileName != null) {
RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(settingsMetadataFileName, clusterUUID,
compressor, namedXContentRegistry);
return persistentSettingsBlobStore.read(remotePersistentSettingsMetadata);
} else {
return Settings.EMPTY;
}
} catch (IOException e) {
throw new IllegalStateException(
String.format(Locale.ROOT, "Error while downloading Settings Metadata - %s", settingsMetadataFileName),
e
);
}
}

public TemplatesMetadata getTemplatesMetadata(String clusterUUID, String templatesMetadataFileName) {
try {
// Fetch Templates metadata
if (templatesMetadataFileName != null) {
RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(templatesMetadataFileName, clusterUUID,
compressor, namedXContentRegistry);
return templatesMetadataBlobStore.read(remoteTemplatesMetadata);
} else {
return TemplatesMetadata.EMPTY_METADATA;
}
} catch (IOException e) {
throw new IllegalStateException(
String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", templatesMetadataFileName),
e
);
}
}

public Metadata.Custom getCustomsMetadata(String clusterUUID, String customMetadataFileName, String custom) {
requireNonNull(customMetadataFileName);
try {
// Fetch Custom metadata
RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, compressor, namedXContentRegistry);
return customMetadataBlobStore.read(remoteCustomMetadata);
} catch (IOException e) {
throw new IllegalStateException(
String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName),
e
);
}
}

Map<String, Metadata.Custom> getUpdatedCustoms(ClusterState currentState, ClusterState previousState) {
if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) {
return new HashMap<>();
Expand Down

0 comments on commit 4def30f

Please sign in to comment.