Introduce mapping version to index metadata #33147
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,6 +24,7 @@ | |
| import com.carrotsearch.hppc.cursors.ObjectCursor; | ||
| import com.carrotsearch.hppc.cursors.ObjectObjectCursor; | ||
|
|
||
| import org.elasticsearch.Assertions; | ||
| import org.elasticsearch.Version; | ||
| import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; | ||
| import org.elasticsearch.action.support.ActiveShardCount; | ||
|
|
@@ -291,6 +292,7 @@ public Iterator<Setting<Integer>> settings() { | |
|
|
||
| public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations"; | ||
| static final String KEY_VERSION = "version"; | ||
| static final String KEY_MAPPING_VERSION = "mapping_version"; | ||
| static final String KEY_ROUTING_NUM_SHARDS = "routing_num_shards"; | ||
| static final String KEY_SETTINGS = "settings"; | ||
| static final String KEY_STATE = "state"; | ||
|
|
@@ -309,6 +311,9 @@ public Iterator<Setting<Integer>> settings() { | |
|
|
||
| private final Index index; | ||
| private final long version; | ||
|
|
||
| private final long mappingVersion; | ||
|
|
||
| private final long[] primaryTerms; | ||
|
|
||
| private final State state; | ||
|
|
@@ -336,7 +341,7 @@ public Iterator<Setting<Integer>> settings() { | |
| private final ActiveShardCount waitForActiveShards; | ||
| private final ImmutableOpenMap<String, RolloverInfo> rolloverInfos; | ||
|
|
||
| private IndexMetaData(Index index, long version, long mappingVersion, long[] primaryTerms, State state, int numberOfShards, int numberOfReplicas, Settings settings, | ||
| ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases, | ||
| ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> inSyncAllocationIds, | ||
| DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters, | ||
|
|
@@ -345,6 +350,8 @@ private IndexMetaData(Index index, long version, long[] primaryTerms, State stat | |
|
|
||
| this.index = index; | ||
| this.version = version; | ||
| assert mappingVersion >= 0 : mappingVersion; | ||
| this.mappingVersion = mappingVersion; | ||
|
There was a problem hiding this comment. can we assert that mappingVersion is positive here? |
||
| this.primaryTerms = primaryTerms; | ||
| assert primaryTerms.length == numberOfShards; | ||
| this.state = state; | ||
|
|
@@ -394,6 +401,9 @@ public long getVersion() { | |
| return this.version; | ||
| } | ||
|
|
||
| public long getMappingVersion() { | ||
| return mappingVersion; | ||
| } | ||
|
|
||
| /** | ||
| * The term of the current selected primary. This is a non-negative number incremented when | ||
|
|
@@ -644,6 +654,7 @@ private static class IndexMetaDataDiff implements Diff<IndexMetaData> { | |
| private final String index; | ||
| private final int routingNumShards; | ||
| private final long version; | ||
| private final long mappingVersion; | ||
| private final long[] primaryTerms; | ||
| private final State state; | ||
| private final Settings settings; | ||
|
|
@@ -656,6 +667,7 @@ private static class IndexMetaDataDiff implements Diff<IndexMetaData> { | |
| IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) { | ||
| index = after.index.getName(); | ||
| version = after.version; | ||
| mappingVersion = after.mappingVersion; | ||
| routingNumShards = after.routingNumShards; | ||
| state = after.state; | ||
| settings = after.settings; | ||
|
|
@@ -672,6 +684,11 @@ private static class IndexMetaDataDiff implements Diff<IndexMetaData> { | |
| index = in.readString(); | ||
| routingNumShards = in.readInt(); | ||
| version = in.readLong(); | ||
| if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { | ||
|
There was a problem hiding this comment. I assume when this is backported the version will be changed to 6.5.0? There was a problem hiding this comment. If we set the version to 6.5.0 now then the BWC tests will fail. |
||
| mappingVersion = in.readVLong(); | ||
| } else { | ||
| mappingVersion = 1; | ||
| } | ||
| state = State.fromId(in.readByte()); | ||
| settings = Settings.readSettingsFromStream(in); | ||
| primaryTerms = in.readVLongArray(); | ||
|
|
@@ -707,6 +724,9 @@ public void writeTo(StreamOutput out) throws IOException { | |
| out.writeString(index); | ||
| out.writeInt(routingNumShards); | ||
| out.writeLong(version); | ||
| if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { | ||
| out.writeVLong(mappingVersion); | ||
| } | ||
| out.writeByte(state.id); | ||
| Settings.writeSettingsToStream(settings, out); | ||
| out.writeVLongArray(primaryTerms); | ||
|
|
@@ -723,6 +743,7 @@ public void writeTo(StreamOutput out) throws IOException { | |
| public IndexMetaData apply(IndexMetaData part) { | ||
| Builder builder = builder(index); | ||
| builder.version(version); | ||
| builder.mappingVersion(mappingVersion); | ||
| builder.setRoutingNumShards(routingNumShards); | ||
| builder.state(state); | ||
| builder.settings(settings); | ||
|
|
@@ -739,6 +760,11 @@ public IndexMetaData apply(IndexMetaData part) { | |
| public static IndexMetaData readFrom(StreamInput in) throws IOException { | ||
| Builder builder = new Builder(in.readString()); | ||
| builder.version(in.readLong()); | ||
| if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { | ||
| builder.mappingVersion(in.readVLong()); | ||
| } else { | ||
| builder.mappingVersion(1); | ||
| } | ||
| builder.setRoutingNumShards(in.readInt()); | ||
| builder.state(State.fromId(in.readByte())); | ||
| builder.settings(readSettingsFromStream(in)); | ||
|
|
@@ -778,6 +804,9 @@ public static IndexMetaData readFrom(StreamInput in) throws IOException { | |
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeString(index.getName()); // uuid will come as part of settings | ||
| out.writeLong(version); | ||
| if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { | ||
| out.writeVLong(mappingVersion); | ||
| } | ||
| out.writeInt(routingNumShards); | ||
| out.writeByte(state.id()); | ||
| writeSettingsToStream(settings, out); | ||
|
|
@@ -821,6 +850,7 @@ public static class Builder { | |
| private String index; | ||
| private State state = State.OPEN; | ||
| private long version = 1; | ||
| private long mappingVersion = 1; | ||
| private long[] primaryTerms = null; | ||
| private Settings settings = Settings.Builder.EMPTY_SETTINGS; | ||
| private final ImmutableOpenMap.Builder<String, MappingMetaData> mappings; | ||
|
|
@@ -843,6 +873,7 @@ public Builder(IndexMetaData indexMetaData) { | |
| this.index = indexMetaData.getIndex().getName(); | ||
| this.state = indexMetaData.state; | ||
| this.version = indexMetaData.version; | ||
| this.mappingVersion = indexMetaData.mappingVersion; | ||
| this.settings = indexMetaData.getSettings(); | ||
| this.primaryTerms = indexMetaData.primaryTerms.clone(); | ||
| this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings); | ||
|
|
@@ -1009,6 +1040,15 @@ public Builder version(long version) { | |
| return this; | ||
| } | ||
|
|
||
| public long mappingVersion() { | ||
| return mappingVersion; | ||
| } | ||
|
|
||
| public Builder mappingVersion(final long mappingVersion) { | ||
| this.mappingVersion = mappingVersion; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * returns the primary term for the given shard. | ||
| * See {@link IndexMetaData#primaryTerm(int)} for more information. | ||
|
|
@@ -1136,7 +1176,7 @@ public IndexMetaData build() { | |
|
|
||
| final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE); | ||
|
|
||
| return new IndexMetaData(new Index(index, uuid), version, mappingVersion, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(), | ||
| tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters, | ||
| indexCreatedVersion, indexUpgradedVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards, rolloverInfos.build()); | ||
| } | ||
|
|
@@ -1145,6 +1185,7 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build | |
| builder.startObject(indexMetaData.getIndex().getName()); | ||
|
|
||
| builder.field(KEY_VERSION, indexMetaData.getVersion()); | ||
| builder.field(KEY_MAPPING_VERSION, indexMetaData.getMappingVersion()); | ||
| builder.field(KEY_ROUTING_NUM_SHARDS, indexMetaData.getRoutingNumShards()); | ||
| builder.field(KEY_STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH)); | ||
|
|
||
|
|
@@ -1218,6 +1259,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti | |
| if (token != XContentParser.Token.START_OBJECT) { | ||
| throw new IllegalArgumentException("expected object but got a " + token); | ||
| } | ||
| boolean mappingVersion = false; | ||
| while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { | ||
| if (token == XContentParser.Token.FIELD_NAME) { | ||
| currentFieldName = parser.currentName(); | ||
|
|
@@ -1316,6 +1358,9 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti | |
| builder.state(State.fromString(parser.text())); | ||
| } else if (KEY_VERSION.equals(currentFieldName)) { | ||
| builder.version(parser.longValue()); | ||
| } else if (KEY_MAPPING_VERSION.equals(currentFieldName)) { | ||
| mappingVersion = true; | ||
| builder.mappingVersion(parser.longValue()); | ||
|
There was a problem hiding this comment. can we somehow make sure that if the index is 7.0 that the version is present? I really want to make sure we are not missing it. There was a problem hiding this comment. Let me know what you think of that one @s1monw. |
||
| } else if (KEY_ROUTING_NUM_SHARDS.equals(currentFieldName)) { | ||
| builder.setRoutingNumShards(parser.intValue()); | ||
| } else { | ||
|
|
@@ -1325,6 +1370,9 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti | |
| throw new IllegalArgumentException("Unexpected token " + token); | ||
| } | ||
| } | ||
| if (Assertions.ENABLED && Version.indexCreated(builder.settings).onOrAfter(Version.V_7_0_0_alpha1)) { | ||
| assert mappingVersion : "mapping version should be present for indices created on or after 7.0.0"; | ||
| } | ||
| return builder.build(); | ||
| } | ||
| } | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I wonder whether
MappingMetaDatawould be a better place to maintain the mapping version. The reason it feels like a better place to me is that this version is about an instance ofMappingMetaData.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I considered this but it ends up that we construct mapping metadata in quite a few places where having the mapping version is not natural. We do not hold on to the mapping metadata in the mapping service, so in some of these places it means we would have to hold on to the mapping version in the mapper service. But then we would have multiple places in the system carrying the mapping version. Because of this I considered the index metadata to be better. Additionally, like that we have the builder for the index metadata. Let me know if you feel strongly about this, or see a good way to have it on the mapping metadata.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't feel strongly about this. I was just wondering about this. Your explanation is a good reason why to add the version to IndexMetadata instead of MappingMetaData.