Skip to content

Commit

Permalink
Common interfaces for diff computation
Browse files Browse the repository at this point in the history
Signed-off-by: Bukhtawar Khan <bukhtawa@amazon.com>
  • Loading branch information
Bukhtawar committed Aug 6, 2024
1 parent 387ad51 commit ba3a2a9
Show file tree
Hide file tree
Showing 8 changed files with 26 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ public void writeTo(StreamOutput out) throws IOException {
}
}

private static class RoutingTableDiff implements Diff<RoutingTable> {
private static class RoutingTableDiff implements Diff<RoutingTable>, StringKeyDiffProvider<IndexRoutingTable> {

private final long version;

Expand Down Expand Up @@ -436,6 +436,11 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeLong(version);
indicesRouting.writeTo(out);
}

@Override
public DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> provideDiff() {
return (DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>>) indicesRouting;
}
}

public static Builder builder() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,16 @@
import org.opensearch.core.index.Index;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static org.opensearch.cluster.DiffableUtils.MapDiff;

/**
* Represents a difference between {@link RoutingTable} objects that can be serialized and deserialized.
*/
public class RoutingTableIncrementalDiff implements Diff<RoutingTable> {
public class RoutingTableIncrementalDiff implements Diff<RoutingTable>, StringKeyDiffProvider<IndexRoutingTable> {

private DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> indicesRouting;
private final Diff<Map<String, IndexRoutingTable>> indicesRouting;

private final long version;

Expand Down Expand Up @@ -77,8 +77,9 @@ public void writeTo(StreamOutput out) throws IOException {
indicesRouting.writeTo(out);
}

public DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> getIndicesRouting() {
return indicesRouting;
@Override
public MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> provideDiff() {
return (MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>>) indicesRouting;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.opensearch.cluster.routing.IndexRoutingTable;
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.cluster.routing.RoutingTableIncrementalDiff;
import org.opensearch.cluster.routing.StringKeyDiffProvider;
import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.common.remote.RemoteWritableEntityStore;
Expand Down Expand Up @@ -97,11 +98,12 @@ public List<IndexRoutingTable> getIndicesRouting(RoutingTable routingTable) {
* @param after current routing table
* @return incremental diff of the previous and current routing table
*/
public DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> getIndicesRoutingMapDiff(
@Override
public StringKeyDiffProvider<IndexRoutingTable> getIndicesRoutingMapDiff(
RoutingTable before,
RoutingTable after
) {
return new RoutingTableIncrementalDiff(before, after).getIndicesRouting();
return new RoutingTableIncrementalDiff(before, after);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,15 @@

import org.opensearch.action.LatchedActionListener;
import org.opensearch.cluster.Diff;
import org.opensearch.cluster.DiffableUtils;
import org.opensearch.cluster.routing.IndexRoutingTable;
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.cluster.routing.RoutingTableIncrementalDiff;
import org.opensearch.cluster.routing.StringKeyDiffProvider;
import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
import org.opensearch.gateway.remote.ClusterMetadataManifest;

import java.io.IOException;
import java.util.List;
import java.util.Map;

/**
* Noop impl for RemoteRoutingTableService.
Expand All @@ -32,11 +31,11 @@ public List<IndexRoutingTable> getIndicesRouting(RoutingTable routingTable) {
}

@Override
public DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> getIndicesRoutingMapDiff(
public StringKeyDiffProvider<IndexRoutingTable> getIndicesRoutingMapDiff(
RoutingTable before,
RoutingTable after
) {
return new RoutingTableIncrementalDiff(before, after).getIndicesRouting();
return new RoutingTableIncrementalDiff(before, after);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.opensearch.cluster.DiffableUtils;
import org.opensearch.cluster.routing.IndexRoutingTable;
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.cluster.routing.StringKeyDiffProvider;
import org.opensearch.common.lifecycle.LifecycleComponent;

import org.opensearch.gateway.remote.ClusterMetadataManifest;
Expand Down Expand Up @@ -47,7 +48,7 @@ List<ClusterMetadataManifest.UploadedIndexMetadata> getUpdatedIndexRoutingTableM
List<ClusterMetadataManifest.UploadedIndexMetadata> allIndicesRouting
);

DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> getIndicesRoutingMapDiff(
StringKeyDiffProvider<IndexRoutingTable> getIndicesRoutingMapDiff(
RoutingTable before,
RoutingTable after
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.opensearch.cluster.node.DiscoveryNodes.Builder;
import org.opensearch.cluster.routing.IndexRoutingTable;
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.cluster.routing.RoutingTableIncrementalDiff;
import org.opensearch.cluster.routing.remote.RemoteRoutingTableService;
import org.opensearch.cluster.routing.remote.RemoteRoutingTableServiceFactory;
import org.opensearch.cluster.service.ClusterService;
Expand Down Expand Up @@ -343,7 +342,7 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata(

final List<IndexRoutingTable> indicesRoutingToUpload = new ArrayList<>();
final DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> routingTableIncrementalDiff =
remoteRoutingTableService.getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable());
remoteRoutingTableService.getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()).provideDiff();

routingTableIncrementalDiff.getDiffs().forEach((k, v) -> indicesRoutingToUpload.add(clusterState.getRoutingTable().index(k)));
routingTableIncrementalDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public class RemoteRoutingTableDiff extends AbstractRemoteWritableBlobEntity<Dif

private final Diff<RoutingTable> routingTableDiff;

protected ChecksumWritableBlobStoreFormat<Diff<RoutingTable>> writeableBlobFormat;
private final ChecksumWritableBlobStoreFormat<Diff<RoutingTable>> writeableBlobFormat;

private long term;
private long version;
Expand Down Expand Up @@ -71,19 +71,6 @@ public RemoteRoutingTableDiff(
this.writeableBlobFormat = writeableBlobFormat;
}

/**
* Constructs a new RemoteRoutingTableDiff with the given differences.
*
* @param routingTableDiff a RoutingTableIncrementalDiff object containing the differences of {@link IndexRoutingTable}.
* @param clusterUUID the cluster UUID.
* @param compressor the compressor to be used.
*/
public RemoteRoutingTableDiff(Diff<RoutingTable> routingTableDiff, ChecksumWritableBlobStoreFormat<Diff<RoutingTable>> writeableBlobFormat, String clusterUUID, Compressor compressor) {
super(clusterUUID, compressor);
this.routingTableDiff = routingTableDiff;
this.writeableBlobFormat = writeableBlobFormat;
}

/**
* Constructs a new RemoteIndexRoutingTableDiff with the given blob name, cluster UUID, and compressor.
*
Expand All @@ -94,6 +81,7 @@ public RemoteRoutingTableDiff(Diff<RoutingTable> routingTableDiff, ChecksumWrita
public RemoteRoutingTableDiff(String blobName, String clusterUUID, Compressor compressor) {
super(clusterUUID, compressor);
this.routingTableDiff = null;
this.writeableBlobFormat = null;
this.blobName = blobName;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ private ClusterStateDiffManifest verifyRoutingTableDiffManifest(ClusterState pre
// Create initial and updated IndexRoutingTable maps
DiffableUtils.MapDiff<String, IndexRoutingTable, Map<String, IndexRoutingTable>> routingTableIncrementalDiff = new RoutingTableIncrementalDiff(
previousState.getRoutingTable(),
currentState.getRoutingTable()).getIndicesRouting();
currentState.getRoutingTable()).provideDiff();
ClusterStateDiffManifest manifest = new ClusterStateDiffManifest(
currentState,
previousState,
Expand Down

0 comments on commit ba3a2a9

Please sign in to comment.