diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TabletMergeability.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TabletMergeability.java new file mode 100644 index 00000000000..341930ce353 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TabletMergeability.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.client.admin; + +import java.io.Serializable; +import java.time.Duration; +import java.util.Objects; + +import com.google.common.base.Preconditions; + +public class TabletMergeability implements Serializable { + private static final long serialVersionUID = 1L; + + public static final TabletMergeability NEVER = new TabletMergeability(Duration.ofNanos(-1)); + public static final TabletMergeability NOW = new TabletMergeability(Duration.ofNanos(0)); + + private final Duration delay; + + private TabletMergeability(Duration delay) { + this.delay = Objects.requireNonNull(delay); + } + + public boolean isNever() { + return this.delay.isNegative(); + } + + public boolean isNow() { + return this.delay.isZero(); + } + + public Duration getDelay() { + return delay; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + TabletMergeability that = (TabletMergeability) o; + return Objects.equals(delay, that.delay); + } + + @Override + public int hashCode() { + return Objects.hashCode(delay); + } + + @Override + public String toString() { + if (isNow()) { + return "TabletMergeability=NOW"; + } else if (isNever()) { + return "TabletMergeability=NEVER"; + } + return "TabletMergeability=AFTER:" + delay.toNanos(); + } + + public static TabletMergeability from(Duration delay) { + Preconditions.checkArgument(delay.toNanos() >= -1, + "Duration of delay must be -1, 0, or a positive offset."); + return new TabletMergeability(delay); + } + + public static TabletMergeability after(Duration delay) { + Preconditions.checkArgument(delay.toNanos() > 0, "Duration of delay must be greater than 0."); + return new TabletMergeability(delay); + } + +} diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TabletMergeabilityUtil.java b/core/src/main/java/org/apache/accumulo/core/metadata/TabletMergeabilityUtil.java new file mode 100644 index 00000000000..ffdad80c3e7 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletMergeabilityUtil.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.metadata; + +import java.time.Duration; + +import org.apache.accumulo.core.client.admin.TabletMergeability; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.util.time.SteadyTime; + +public class TabletMergeabilityUtil { + + public static Value toValue(TabletMergeability tabletMergeability) { + return new Value(Long.toString(tabletMergeability.getDelay().toNanos())); + } + + public static TabletMergeability fromValue(Value value) { + return TabletMergeability.from(Duration.ofNanos(Long.parseLong(value.toString()))); + } + + public boolean isMergeable(TabletMergeability mergeability, SteadyTime currentTime) { + if (mergeability.isNever()) { + return false; + } + return currentTime.getDuration().compareTo(mergeability.getDelay()) >= 0; + } + +} diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java index 7135b5a978f..61355131ed0 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.client.ConditionalWriter; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.TableId; @@ -392,6 +393,8 @@ interface TabletUpdates { T putCloned(); + T putTabletMergeability(TabletMergeability tabletMergeability); + /** * By default the server lock is automatically added to mutations unless this method is set to * false. diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java index 5360c98274b..885fd326239 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java @@ -158,6 +158,10 @@ public static class TabletColumnFamily { public static final String REQUESTED_QUAL = "requestToHost"; public static final ColumnFQ REQUESTED_COLUMN = new ColumnFQ(NAME, new Text(REQUESTED_QUAL)); + public static final String MERGEABILITY_QUAL = "mergeability"; + public static final ColumnFQ MERGEABILITY_COLUMN = + new ColumnFQ(NAME, new Text(MERGEABILITY_QUAL)); + public static Value encodePrevEndRow(Text per) { if (per == null) { return new Value(new byte[] {0}); diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java index 795ebfafed4..b5b5774adc9 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java @@ -25,6 +25,7 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.SELECTED_QUAL; import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_QUAL; import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.AVAILABILITY_QUAL; +import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.MERGEABILITY_QUAL; import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_QUAL; import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.REQUESTED_QUAL; @@ -41,6 +42,7 @@ import java.util.Set; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil; import org.apache.accumulo.core.data.ByteSequence; @@ -58,6 +60,7 @@ import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SuspendingTServer; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.TabletMergeabilityUtil; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CompactedColumnFamily; @@ -123,6 +126,7 @@ public class TabletMetadata { private final Set compacted; private final Set userCompactionsRequested; private final UnSplittableMetadata unSplittableMetadata; + private final TabletMergeability mergeability; private final Supplier fileSize; private TabletMetadata(Builder tmBuilder) { @@ -155,6 +159,7 @@ private TabletMetadata(Builder tmBuilder) { this.compacted = tmBuilder.compacted.build(); this.userCompactionsRequested = tmBuilder.userCompactionsRequested.build(); this.unSplittableMetadata = tmBuilder.unSplittableMetadata; + this.mergeability = Objects.requireNonNull(tmBuilder.mergeability); this.fileSize = Suppliers.memoize(() -> { // This code was using a java stream. While profiling SplitMillionIT, the stream was showing // up as hot when scanning 1 million tablets. Converted to a for loop to improve performance. @@ -198,7 +203,8 @@ public enum ColumnType { SELECTED, COMPACTED, USER_COMPACTION_REQUESTED, - UNSPLITTABLE + UNSPLITTABLE, + MERGEABILITY } public static class Location { @@ -439,6 +445,11 @@ public UnSplittableMetadata getUnSplittable() { return unSplittableMetadata; } + public TabletMergeability getTabletMergeability() { + ensureFetched(ColumnType.MERGEABILITY); + return mergeability; + } + @Override public String toString() { return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("tableId", tableId) @@ -453,7 +464,8 @@ public String toString() { .append("operationId", operationId).append("selectedFiles", selectedFiles) .append("futureAndCurrentLocationSet", futureAndCurrentLocationSet) .append("userCompactionsRequested", userCompactionsRequested) - .append("unSplittableMetadata", unSplittableMetadata).toString(); + .append("unSplittableMetadata", unSplittableMetadata).append("mergeability", mergeability) + .toString(); } public List> getKeyValues() { @@ -527,6 +539,9 @@ public static > TabletMetadata convertRow(Iterator case REQUESTED_QUAL: tmBuilder.onDemandHostingRequested(true); break; + case MERGEABILITY_QUAL: + tmBuilder.mergeability(TabletMergeabilityUtil.fromValue(kv.getValue())); + break; default: throw new IllegalStateException("Unexpected TabletColumnFamily qualifier: " + qual); } @@ -689,7 +704,7 @@ static class Builder { private final ImmutableSet.Builder compacted = ImmutableSet.builder(); private final ImmutableSet.Builder userCompactionsRequested = ImmutableSet.builder(); private UnSplittableMetadata unSplittableMetadata; - // private Supplier fileSize; + private TabletMergeability mergeability = TabletMergeability.NEVER; void table(TableId tableId) { this.tableId = tableId; @@ -799,6 +814,10 @@ void unSplittableMetadata(UnSplittableMetadata unSplittableMetadata) { this.unSplittableMetadata = unSplittableMetadata; } + void mergeability(TabletMergeability mergeability) { + this.mergeability = mergeability; + } + void keyValue(Entry kv) { if (this.keyValues == null) { this.keyValues = ImmutableList.builder(); diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java index 44f1915e0ea..047ef7a51f4 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java @@ -30,6 +30,7 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOADED; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGEABILITY; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGED; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; @@ -48,6 +49,7 @@ import java.util.TreeMap; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; @@ -312,6 +314,13 @@ public TabletMetadataBuilder putCloned() { return this; } + @Override + public TabletMetadataBuilder putTabletMergeability(TabletMergeability tabletMergeability) { + fetched.add(MERGEABILITY); + internalBuilder.putTabletMergeability(tabletMergeability); + return this; + } + @Override public TabletMetadataBuilder automaticallyPutServerLock(boolean b) { throw new UnsupportedOperationException(); diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java index 6052c73a799..577120f8435 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; @@ -35,6 +36,7 @@ import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SuspendingTServer; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.TabletMergeabilityUtil; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CompactedColumnFamily; @@ -390,6 +392,13 @@ public T automaticallyPutServerLock(boolean b) { return getThis(); } + @Override + public T putTabletMergeability(TabletMergeability tabletMergeability) { + TabletColumnFamily.MERGEABILITY_COLUMN.put(mutation, + TabletMergeabilityUtil.toValue(tabletMergeability)); + return getThis(); + } + public void setCloseAfterMutate(AutoCloseable closeable) { this.closeAfterMutate = closeable; } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java index a3914ea0ed0..04115dfaeac 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java @@ -77,6 +77,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SplitColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.metadata.schema.filters.TabletMetadataFilter; @@ -394,6 +395,9 @@ public Options fetch(ColumnType... colsToFetch) { case UNSPLITTABLE: qualifiers.add(SplitColumnFamily.UNSPLITTABLE_COLUMN); break; + case MERGEABILITY: + qualifiers.add(TabletColumnFamily.MERGEABILITY_COLUMN); + break; default: throw new IllegalArgumentException("Unknown col type " + colToFetch); } diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index 9f4ba14def1..928173190a5 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -44,6 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.lang.reflect.Constructor; +import java.time.Duration; import java.util.AbstractMap; import java.util.EnumSet; import java.util.LinkedHashSet; @@ -59,6 +60,7 @@ import java.util.stream.Stream; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; @@ -638,12 +640,13 @@ public void testBuilder() { FateId compactFateId1 = FateId.from(type, UUID.randomUUID()); FateId compactFateId2 = FateId.from(type, UUID.randomUUID()); - TabletMetadata tm = TabletMetadata.builder(extent) - .putTabletAvailability(TabletAvailability.UNHOSTED).putLocation(Location.future(ser1)) - .putFile(sf1, dfv1).putFile(sf2, dfv2).putBulkFile(rf1, loadedFateId1) - .putBulkFile(rf2, loadedFateId2).putFlushId(27).putDirName("dir1").putScan(sf3).putScan(sf4) - .putCompacted(compactFateId1).putCompacted(compactFateId2).putCloned() - .build(ECOMP, HOSTING_REQUESTED, MERGED, USER_COMPACTION_REQUESTED, UNSPLITTABLE); + TabletMetadata tm = + TabletMetadata.builder(extent).putTabletAvailability(TabletAvailability.UNHOSTED) + .putLocation(Location.future(ser1)).putFile(sf1, dfv1).putFile(sf2, dfv2) + .putBulkFile(rf1, loadedFateId1).putBulkFile(rf2, loadedFateId2).putFlushId(27) + .putDirName("dir1").putScan(sf3).putScan(sf4).putCompacted(compactFateId1) + .putCompacted(compactFateId2).putCloned().putTabletMergeability(TabletMergeability.NOW) + .build(ECOMP, HOSTING_REQUESTED, MERGED, USER_COMPACTION_REQUESTED, UNSPLITTABLE); assertEquals(extent, tm.getExtent()); assertEquals(TabletAvailability.UNHOSTED, tm.getTabletAvailability()); @@ -662,6 +665,7 @@ public void testBuilder() { assertFalse(tm.hasMerged()); assertNull(tm.getUnSplittable()); assertEquals("OK", tm.getCloned()); + assertEquals(TabletMergeability.NOW, tm.getTabletMergeability()); assertThrows(IllegalStateException.class, tm::getOperationId); assertThrows(IllegalStateException.class, tm::getSuspend); assertThrows(IllegalStateException.class, tm::getTime); @@ -688,6 +692,7 @@ public void testBuilder() { assertThrows(IllegalStateException.class, tm2::hasMerged); assertThrows(IllegalStateException.class, tm2::getUserCompactionsRequested); assertThrows(IllegalStateException.class, tm2::getUnSplittable); + assertThrows(IllegalStateException.class, tm2::getTabletAvailability); var ecid1 = ExternalCompactionId.generate(UUID.randomUUID()); CompactionMetadata ecm = @@ -707,7 +712,8 @@ public void testBuilder() { .putSuspension(ser1, SteadyTime.from(45L, TimeUnit.MILLISECONDS)) .putTime(new MetadataTime(479, TimeType.LOGICAL)).putWal(le1).putWal(le2) .setHostingRequested().putSelectedFiles(selFiles).setMerged() - .putUserCompactionRequested(selFilesFateId).setUnSplittable(unsplittableMeta).build(); + .putUserCompactionRequested(selFilesFateId).setUnSplittable(unsplittableMeta) + .putTabletMergeability(TabletMergeability.after(Duration.ofDays(3))).build(); assertEquals(Set.of(ecid1), tm3.getExternalCompactions().keySet()); assertEquals(Set.of(sf1, sf2), tm3.getExternalCompactions().get(ecid1).getJobFiles()); @@ -724,6 +730,7 @@ public void testBuilder() { assertTrue(tm3.hasMerged()); assertTrue(tm3.getUserCompactionsRequested().contains(selFilesFateId)); assertEquals(unsplittableMeta, tm3.getUnSplittable()); + assertEquals(Duration.ofDays(3), tm3.getTabletMergeability().getDelay()); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java index f91828361f0..50348d47f3c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java +++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.metadata.AccumuloTable; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SuspendingTServer; +import org.apache.accumulo.core.metadata.TabletMergeabilityUtil; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; @@ -97,6 +98,7 @@ public class MetadataConstraints implements Constraint { TabletColumnFamily.REQUESTED_COLUMN, ServerColumnFamily.SELECTED_COLUMN, SplitColumnFamily.UNSPLITTABLE_COLUMN, + TabletColumnFamily.MERGEABILITY_COLUMN, Upgrade12to13.COMPACT_COL); @SuppressWarnings("deprecation") @@ -297,6 +299,8 @@ public String getViolationDescription(short violationCode) { return "Invalid unsplittable column"; case 4005: return "Malformed availability value"; + case 4006: + return "Malformed mergeability value"; } return null; @@ -376,6 +380,13 @@ private void validateTabletFamily(ArrayList violations, ColumnUpdate colu addViolation(violations, 4005); } break; + case (TabletColumnFamily.MERGEABILITY_QUAL): + try { + TabletMergeabilityUtil.fromValue(new Value(columnUpdate.getValue())); + } catch (IllegalArgumentException e) { + addViolation(violations, 4006); + } + break; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java index b315f1a58c3..175e6d9c666 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; @@ -93,7 +94,8 @@ private Map createEntries() { KeyExtent keyExtent = new KeyExtent(tableId, endRow, prevEndRow); var builder = TabletMetadata.builder(keyExtent).putDirName(dirName) .putTime(new MetadataTime(0, TimeType.LOGICAL)) - .putTabletAvailability(TabletAvailability.HOSTED).putPrevEndRow(prevEndRow); + .putTabletAvailability(TabletAvailability.HOSTED) + .putTabletMergeability(TabletMergeability.NEVER).putPrevEndRow(prevEndRow); for (String file : files) { builder.putFile(new ReferencedTabletFile(new Path(file)).insert(), new DataFileValue(0, 0)); } diff --git a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java index dfcf970330c..5cf74f220ee 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java @@ -26,12 +26,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.lang.reflect.Method; +import java.time.Duration; import java.util.Base64; import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; @@ -44,6 +46,7 @@ import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SuspendingTServer; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.TabletMergeabilityUtil; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CompactedColumnFamily; @@ -639,6 +642,35 @@ public void testDirectoryColumn() { assertEquals((short) 3102, violations.get(0)); } + @Test + public void testMergeabilityColumn() { + MetadataConstraints mc = new MetadataConstraints(); + Mutation m; + List violations; + + m = new Mutation(new Text("0;foo")); + TabletColumnFamily.MERGEABILITY_COLUMN.put(m, new Value("-2")); + assertViolation(mc, m, (short) 4006); + + m = new Mutation(new Text("0;foo")); + TabletColumnFamily.MERGEABILITY_COLUMN.put(m, + TabletMergeabilityUtil.toValue(TabletMergeability.NEVER)); + violations = mc.check(createEnv(), m); + assertTrue(violations.isEmpty()); + + m = new Mutation(new Text("0;foo")); + TabletColumnFamily.MERGEABILITY_COLUMN.put(m, + TabletMergeabilityUtil.toValue(TabletMergeability.NOW)); + violations = mc.check(createEnv(), m); + assertTrue(violations.isEmpty()); + + m = new Mutation(new Text("0;foo")); + TabletColumnFamily.MERGEABILITY_COLUMN.put(m, + TabletMergeabilityUtil.toValue(TabletMergeability.after(Duration.ofDays(3)))); + violations = mc.check(createEnv(), m); + assertTrue(violations.isEmpty()); + } + // Encode a row how it would appear in Json private static String encodeRowForMetadata(String row) { try { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java index 32303ade9d0..897af5a2131 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.InitialTableState; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.clientImpl.Namespaces; import org.apache.accumulo.core.clientImpl.TableOperationsImpl; @@ -254,7 +255,7 @@ public void executeFateOperation(TInfo tinfo, TCredentials c, TFateId opid, Fate manager.fate(type).seedTransaction(op.toString(), fateId, new TraceRepo<>(new CreateTable(c.getPrincipal(), tableName, timeType, options, splitsPath, splitCount, splitsDirsPath, initialTableState, - initialTabletAvailability, namespaceId)), + initialTabletAvailability, namespaceId, TabletMergeability.NEVER)), autoCleanup, goalMessage); break; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/TableInfo.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/TableInfo.java index 5294c3ef04b..eb987cb1ce4 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/TableInfo.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/TableInfo.java @@ -23,6 +23,7 @@ import org.apache.accumulo.core.client.admin.InitialTableState; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; @@ -51,6 +52,8 @@ public class TableInfo implements Serializable { private TabletAvailability initialTabletAvailability; + private TabletMergeability initialTabletMergeability; + public TabletAvailability getInitialTabletAvailability() { return initialTabletAvailability; } @@ -133,4 +136,11 @@ public void setInitialSplitSize(int initialSplitSize) { this.initialSplitSize = initialSplitSize; } + public TabletMergeability getInitialTabletMergeability() { + return initialTabletMergeability; + } + + public void setInitialTabletMergeability(TabletMergeability initialTabletMergeability) { + this.initialTabletMergeability = initialTabletMergeability; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/create/CreateTable.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/create/CreateTable.java index 3f5a379c8a4..53b0515c856 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/create/CreateTable.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/create/CreateTable.java @@ -23,6 +23,7 @@ import org.apache.accumulo.core.client.admin.InitialTableState; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.clientImpl.thrift.TableOperation; import org.apache.accumulo.core.data.NamespaceId; @@ -47,7 +48,8 @@ public class CreateTable extends ManagerRepo { public CreateTable(String user, String tableName, TimeType timeType, Map props, Path splitPath, int splitCount, Path splitDirsPath, InitialTableState initialTableState, - TabletAvailability initialTabletAvailability, NamespaceId namespaceId) { + TabletAvailability initialTabletAvailability, NamespaceId namespaceId, + TabletMergeability initialTabletMergeability) { tableInfo = new TableInfo(); tableInfo.setTableName(tableName); tableInfo.setTimeType(timeType); @@ -59,6 +61,7 @@ public CreateTable(String user, String tableName, TimeType timeType, Map s tabletMutator.putDirName(dirName); tabletMutator.putTime(new MetadataTime(0, tableInfo.getTimeType())); tabletMutator.putTabletAvailability(tableInfo.getInitialTabletAvailability()); + tabletMutator.putTabletMergeability(tableInfo.getInitialTabletMergeability()); tabletMutator.mutate(); prevSplit = split; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java index cdd54ac1439..9295714c6e2 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java @@ -187,6 +187,20 @@ public Repo call(FateId fateId, Manager manager) throws Exception { DeleteRows.getMergeTabletAvailability(range, tabletAvailabilities)); tabletMutator.putPrevEndRow(firstTabletMeta.getPrevEndRow()); + // TODO: How should we determine this value this after merging? + // Do we just keep the value that's already in the last tablet in the range? + // For now the lastTabletMetadata is the simplest to use + // + // All the tablets being merged could have different values + // even if they are all currently mergeable. Right now merging is + // always user generated but eventually will be possible to be + // system generated auto merge because of this column. + // + // Another option could be to track if the user started the merge and if + // they did then treat it like a user split and mark this as NEVER , + // and if the system triggered the merge we set it as SYSTEM + tabletMutator.putTabletMergeability(lastTabletMeta.getTabletMergeability()); + // scan entries are related to a hosted tablet, this tablet is not hosted so can safely // delete these lastTabletMeta.getScans().forEach(tabletMutator::deleteScan); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java index 60073e987a2..c614ce4be2e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/FindSplits.java @@ -48,7 +48,7 @@ public class FindSplits extends ManagerRepo { private final SplitInfo splitInfo; public FindSplits(KeyExtent extent) { - this.splitInfo = new SplitInfo(extent, new TreeSet<>()); + this.splitInfo = new SplitInfo(extent, new TreeSet<>(), true); } @Override @@ -156,7 +156,7 @@ public Repo call(FateId fateId, Manager manager) throws Exception { return null; } - return new PreSplit(extent, splits); + return new PreSplit(extent, splits, true); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java index 6d89878f955..41d8d039ea7 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java @@ -50,12 +50,16 @@ public class PreSplit extends ManagerRepo { private final SplitInfo splitInfo; - public PreSplit(KeyExtent expectedExtent, SortedSet splits) { + public PreSplit(KeyExtent expectedExtent, SortedSet splits, boolean systemCreated) { Objects.requireNonNull(expectedExtent); Objects.requireNonNull(splits); Preconditions.checkArgument(!splits.isEmpty()); Preconditions.checkArgument(!expectedExtent.isRootTablet()); - this.splitInfo = new SplitInfo(expectedExtent, splits); + this.splitInfo = new SplitInfo(expectedExtent, splits, systemCreated); + } + + public PreSplit(KeyExtent expectedExtent, SortedSet splits) { + this(expectedExtent, splits, false); } @Override diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/SplitInfo.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/SplitInfo.java index 7d97e6a34e1..14bac1fe7ef 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/SplitInfo.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/SplitInfo.java @@ -36,12 +36,14 @@ public class SplitInfo implements Serializable { private final byte[] prevEndRow; private final byte[] endRow; private final byte[][] splits; + private final boolean systemCreated; - public SplitInfo(KeyExtent extent, SortedSet splits) { + public SplitInfo(KeyExtent extent, SortedSet splits, boolean systemCreated) { this.tableId = extent.tableId(); this.prevEndRow = extent.prevEndRow() == null ? null : TextUtil.getBytes(extent.prevEndRow()); this.endRow = extent.endRow() == null ? null : TextUtil.getBytes(extent.endRow()); this.splits = new byte[splits.size()][]; + this.systemCreated = systemCreated; int index = 0; for (var split : splits) { @@ -85,4 +87,7 @@ SortedSet getTablets() { return tablets; } + boolean isSystemCreated() { + return systemCreated; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java index ce48d480b14..a3c435940c9 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java @@ -27,6 +27,7 @@ import java.util.SortedSet; import java.util.TreeMap; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.FateId; @@ -218,7 +219,8 @@ private void addNewTablets(FateId fateId, Manager manager, TabletMetadata tablet .debug("{} copying compacted marker to new child tablet {}", fateId, compactedFateId)); mutator.putTabletAvailability(tabletMetadata.getTabletAvailability()); - + mutator.putTabletMergeability( + splitInfo.isSystemCreated() ? TabletMergeability.NOW : TabletMergeability.NEVER); tabletMetadata.getLoaded().forEach((k, v) -> mutator.putBulkFile(k.getTabletFile(), v)); newTabletsFiles.get(newExtent).forEach(mutator::putFile); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java index 76a475105e6..b8c7eaed1c6 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java @@ -32,6 +32,7 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOADED; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGEABILITY; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGED; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; @@ -56,6 +57,7 @@ import java.util.function.Consumer; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; @@ -110,7 +112,7 @@ public class MergeTabletsTest { private static final Set COLUMNS_HANDLED_BY_MERGE = EnumSet.of(TIME, LOGS, FILES, PREV_ROW, OPID, LOCATION, ECOMP, SELECTED, LOADED, USER_COMPACTION_REQUESTED, MERGED, LAST, SCANS, DIR, CLONED, FLUSH_ID, FLUSH_NONCE, - SUSPEND, AVAILABILITY, HOSTING_REQUESTED, COMPACTED, UNSPLITTABLE); + SUSPEND, AVAILABILITY, HOSTING_REQUESTED, COMPACTED, UNSPLITTABLE, MERGEABILITY); /** * The purpose of this test is to catch new tablet metadata columns that were added w/o @@ -150,15 +152,18 @@ public void testManyColumns() throws Exception { var availability = TabletAvailability.HOSTED; var lastLocation = TabletMetadata.Location.last("1.2.3.4:1234", "123456789"); var suspendingTServer = SuspendingTServer.fromValue(new Value("1.2.3.4:5|56")); - - var tablet1 = TabletMetadata.builder(ke1).putOperation(opid).putDirName("td1") - .putFile(file3, dfv3).putTime(MetadataTime.parse("L3")) - .putTabletAvailability(TabletAvailability.HOSTED).build(LOCATION, LOGS, FILES, ECOMP, - MERGED, COMPACTED, SELECTED, USER_COMPACTION_REQUESTED, LOADED, CLONED); - var tablet2 = TabletMetadata.builder(ke2).putOperation(opid).putDirName("td2") - .putFile(file4, dfv4).putTime(MetadataTime.parse("L2")) - .putTabletAvailability(TabletAvailability.HOSTED).build(LOCATION, LOGS, FILES, ECOMP, - MERGED, COMPACTED, SELECTED, USER_COMPACTION_REQUESTED, LOADED, CLONED); + var mergeability = TabletMergeability.NOW; + + var tablet1 = + TabletMetadata.builder(ke1).putOperation(opid).putDirName("td1").putFile(file3, dfv3) + .putTime(MetadataTime.parse("L3")).putTabletAvailability(TabletAvailability.HOSTED) + .putTabletMergeability(mergeability).build(LOCATION, LOGS, FILES, ECOMP, MERGED, + COMPACTED, SELECTED, USER_COMPACTION_REQUESTED, LOADED, CLONED); + var tablet2 = + TabletMetadata.builder(ke2).putOperation(opid).putDirName("td2").putFile(file4, dfv4) + .putTime(MetadataTime.parse("L2")).putTabletAvailability(TabletAvailability.HOSTED) + .putTabletMergeability(mergeability).build(LOCATION, LOGS, FILES, ECOMP, MERGED, + COMPACTED, SELECTED, USER_COMPACTION_REQUESTED, LOADED, CLONED); var tabletFiles = Map.of(file1, dfv1, file2, dfv2); @@ -193,6 +198,7 @@ public void testManyColumns() throws Exception { EasyMock.expect(lastTabletMeta.getSuspend()).andReturn(suspendingTServer).atLeastOnce(); EasyMock.expect(lastTabletMeta.getLast()).andReturn(lastLocation).atLeastOnce(); EasyMock.expect(lastTabletMeta.getUnSplittable()).andReturn(unsplittableMeta).atLeastOnce(); + EasyMock.expect(lastTabletMeta.getTabletMergeability()).andReturn(mergeability).atLeastOnce(); EasyMock.replay(lastTabletMeta, compactions); @@ -228,6 +234,8 @@ public void testManyColumns() throws Exception { EasyMock.expect(tabletMutator.deleteSuspension()).andReturn(tabletMutator); EasyMock.expect(tabletMutator.deleteLocation(lastLocation)).andReturn(tabletMutator); EasyMock.expect(tabletMutator.deleteUnSplittable()).andReturn(tabletMutator); + EasyMock.expect(tabletMutator.putTabletMergeability(TabletMergeability.NOW)) + .andReturn(tabletMutator).once(); }); @@ -376,17 +384,17 @@ public void testTime() throws Exception { .putTime(MetadataTime.parse(times[0])).putTabletAvailability(TabletAvailability.HOSTED) .build(LOCATION, LOGS, FILES, ECOMP, MERGED, COMPACTED, SELECTED, USER_COMPACTION_REQUESTED, LOADED, CLONED, SCANS, HOSTING_REQUESTED, SUSPEND, LAST, - UNSPLITTABLE); + UNSPLITTABLE, MERGEABILITY); var tablet2 = TabletMetadata.builder(ke2).putOperation(opid).putDirName("td2") .putTime(MetadataTime.parse(times[1])).putTabletAvailability(TabletAvailability.HOSTED) .build(LOCATION, LOGS, FILES, ECOMP, MERGED, COMPACTED, SELECTED, USER_COMPACTION_REQUESTED, LOADED, CLONED, SCANS, HOSTING_REQUESTED, SUSPEND, LAST, - UNSPLITTABLE); + UNSPLITTABLE, MERGEABILITY); var tablet3 = TabletMetadata.builder(ke3).putOperation(opid).putDirName("td3") .putTime(MetadataTime.parse(times[2])).putTabletAvailability(TabletAvailability.HOSTED) .build(LOCATION, LOGS, FILES, ECOMP, MERGED, COMPACTED, SELECTED, USER_COMPACTION_REQUESTED, LOADED, CLONED, SCANS, HOSTING_REQUESTED, SUSPEND, LAST, - UNSPLITTABLE); + UNSPLITTABLE, MERGEABILITY); testMerge(List.of(tablet1, tablet2, tablet3), tableId, null, null, tabletMutator -> { EasyMock.expect(tabletMutator.putTime(MetadataTime.parse("L30"))).andReturn(tabletMutator) @@ -396,6 +404,9 @@ public void testTime() throws Exception { EasyMock.expect(tabletMutator.putPrevEndRow(ke1.prevEndRow())).andReturn(tabletMutator) .once(); EasyMock.expect(tabletMutator.setMerged()).andReturn(tabletMutator).once(); + // Current default if not set is NEVER + EasyMock.expect(tabletMutator.putTabletMergeability(TabletMergeability.NEVER)) + .andReturn(tabletMutator).once(); }); } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java index a317f8375a9..53852a42222 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java @@ -33,6 +33,7 @@ import java.util.UUID; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; @@ -83,13 +84,13 @@ Splitter.FileInfo newFileInfo(String start, String end) { * developer has determined that split code can handle that column OR has opened an issue about * handling it. */ - private static final Set COLUMNS_HANDLED_BY_SPLIT = - EnumSet.of(ColumnType.TIME, ColumnType.LOGS, ColumnType.FILES, ColumnType.PREV_ROW, - ColumnType.OPID, ColumnType.LOCATION, ColumnType.ECOMP, ColumnType.SELECTED, - ColumnType.LOADED, ColumnType.USER_COMPACTION_REQUESTED, ColumnType.MERGED, - ColumnType.LAST, ColumnType.SCANS, ColumnType.DIR, ColumnType.CLONED, ColumnType.FLUSH_ID, - ColumnType.FLUSH_NONCE, ColumnType.SUSPEND, ColumnType.AVAILABILITY, - ColumnType.HOSTING_REQUESTED, ColumnType.COMPACTED, ColumnType.UNSPLITTABLE); + private static final Set COLUMNS_HANDLED_BY_SPLIT = EnumSet.of(ColumnType.TIME, + ColumnType.LOGS, ColumnType.FILES, ColumnType.PREV_ROW, ColumnType.OPID, ColumnType.LOCATION, + ColumnType.ECOMP, ColumnType.SELECTED, ColumnType.LOADED, + ColumnType.USER_COMPACTION_REQUESTED, ColumnType.MERGED, ColumnType.LAST, ColumnType.SCANS, + ColumnType.DIR, ColumnType.CLONED, ColumnType.FLUSH_ID, ColumnType.FLUSH_NONCE, + ColumnType.SUSPEND, ColumnType.AVAILABILITY, ColumnType.HOSTING_REQUESTED, + ColumnType.COMPACTED, ColumnType.UNSPLITTABLE, ColumnType.MERGEABILITY); /** * The purpose of this test is to catch new tablet metadata columns that were added w/o @@ -294,6 +295,9 @@ public void testManyColumns() throws Exception { EasyMock.expect(tablet1Mutator.putFile(file1, new DataFileValue(333, 33, 20))) .andReturn(tablet1Mutator); EasyMock.expect(tablet1Mutator.putFile(file2, dfv2)).andReturn(tablet1Mutator); + // SplitInfo marked as system generated so should be set to NOW + EasyMock.expect(tablet1Mutator.putTabletMergeability(TabletMergeability.NOW)) + .andReturn(tablet1Mutator); tablet1Mutator.submit(EasyMock.anyObject()); EasyMock.expectLastCall().once(); EasyMock.expect(tabletsMutator.mutateTablet(newExtent1)).andReturn(tablet1Mutator); @@ -310,6 +314,9 @@ public void testManyColumns() throws Exception { EasyMock.expect(tablet2Mutator.putCompacted(ucfid1)).andReturn(tablet2Mutator); EasyMock.expect(tablet2Mutator.putCompacted(ucfid3)).andReturn(tablet2Mutator); EasyMock.expect(tablet2Mutator.putTabletAvailability(availability)).andReturn(tablet2Mutator); + // SplitInfo marked as system generated so should be set to NOW + EasyMock.expect(tablet2Mutator.putTabletMergeability(TabletMergeability.NOW)) + .andReturn(tablet2Mutator); EasyMock.expect(tablet2Mutator.putBulkFile(loaded1.getTabletFile(), flid1)) .andReturn(tablet2Mutator); EasyMock.expect(tablet2Mutator.putBulkFile(loaded2.getTabletFile(), flid2)) @@ -367,7 +374,7 @@ public void testManyColumns() throws Exception { // the original tablet SortedSet splits = new TreeSet<>(List.of(newExtent1.endRow(), newExtent2.endRow())); UpdateTablets updateTablets = - new UpdateTablets(new SplitInfo(origExtent, splits), List.of(dir1, dir2)); + new UpdateTablets(new SplitInfo(origExtent, splits, true), List.of(dir1, dir2)); updateTablets.call(fateId, manager); EasyMock.verify(manager, context, ample, tabletMeta, splitter, tabletsMutator, tablet1Mutator, @@ -446,7 +453,7 @@ private static void testError(KeyExtent origExtent, TabletMetadata tm1, FateId f // the original tablet SortedSet splits = new TreeSet<>(List.of(new Text("c"))); UpdateTablets updateTablets = - new UpdateTablets(new SplitInfo(origExtent, splits), List.of("d1")); + new UpdateTablets(new SplitInfo(origExtent, splits, true), List.of("d1")); updateTablets.call(fateId, manager); EasyMock.verify(manager, context, ample); diff --git a/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleIT.java b/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleIT.java index 2c8359bf4ef..45593012357 100644 --- a/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleIT.java @@ -119,6 +119,7 @@ public void testCreateMetadata() throws Exception { assertNotNull(tm.getExtent()); assertNotNull(tm.getTabletAvailability()); assertNotNull(tm.getTime()); + assertNotNull(tm.getTabletMergeability()); count.incrementAndGet(); }); } diff --git a/test/src/main/java/org/apache/accumulo/test/ample/metadata/TestAmple.java b/test/src/main/java/org/apache/accumulo/test/ample/metadata/TestAmple.java index 0b1c2b9e84c..7ca024d2020 100644 --- a/test/src/main/java/org/apache/accumulo/test/ample/metadata/TestAmple.java +++ b/test/src/main/java/org/apache/accumulo/test/ample/metadata/TestAmple.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.TabletAvailability; import org.apache.accumulo.core.client.admin.TabletInformation; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.SiteConfiguration; @@ -149,6 +150,7 @@ public void createMetadata(TableId tableId) { tabletMutator.putDirName(dirName); tabletMutator.putTime(new MetadataTime(0, TimeType.MILLIS)); tabletMutator.putTabletAvailability(TabletAvailability.HOSTED); + tabletMutator.putTabletMergeability(TabletMergeability.NEVER); tabletMutator.mutate(); } catch (Exception e) { throw new IllegalStateException(e); diff --git a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT.java b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT.java index a0cf45ee796..3b524d90bcb 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT.java @@ -198,7 +198,7 @@ public void testSplitOffline() throws Exception { assertEquals(opid, testAmple.readTablet(extent).getOperationId()); var eoRepo = new AllocateDirsAndEnsureOnline( - new SplitInfo(extent, new TreeSet<>(List.of(new Text("sp1"))))); + new SplitInfo(extent, new TreeSet<>(List.of(new Text("sp1"))), true)); // The repo should delete the opid and throw an exception assertThrows(ThriftTableOperationException.class, () -> eoRepo.call(fateId, manager)); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AddSplitIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AddSplitIT.java index 76c62a5460c..b22abcc2a7f 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AddSplitIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AddSplitIT.java @@ -18,6 +18,8 @@ */ package org.apache.accumulo.test.functional; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.time.Duration; import java.util.Collection; import java.util.Iterator; @@ -28,9 +30,12 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.harness.AccumuloClusterHarness; import org.apache.hadoop.io.Text; @@ -87,6 +92,13 @@ public void addSplitTest() throws Exception { } verifyData(c, tableName, 2L); + + TableId id = TableId.of(c.tableOperations().tableIdMap().get(tableName)); + try (TabletsMetadata tm = getServerContext().getAmple().readTablets().forTable(id).build()) { + // Default for user created tablets should be mergeability set to NEVER + tm.stream().forEach( + tablet -> assertEquals(TabletMergeability.NEVER, tablet.getTabletMergeability())); + } } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java index 5a4dd306cce..38094b58762 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java @@ -42,6 +42,7 @@ import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; @@ -284,10 +285,12 @@ private void testCommonSystemTableConfig(ClientContext client, TableId tableId, assertEquals(maxVersions, tableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() + "majc.vers.opt.maxVersions")); - // Verify all tablets are HOSTED + // Verify all tablets are HOSTED and Mergeablity is NEVER try (var tablets = client.getAmple().readTablets().forTable(tableId).build()) { assertTrue( tablets.stream().allMatch(tm -> tm.getTabletAvailability() == TabletAvailability.HOSTED)); + assertTrue(tablets.stream() + .allMatch(tm -> tm.getTabletMergeability().equals(TabletMergeability.NEVER))); } } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java index 720e56f93f7..84076f5f3dc 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java @@ -57,6 +57,7 @@ import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.client.admin.NewTableConfiguration; +import org.apache.accumulo.core.client.admin.TabletMergeability; import org.apache.accumulo.core.client.rfile.RFile; import org.apache.accumulo.core.client.rfile.RFileWriter; import org.apache.accumulo.core.conf.Property; @@ -66,6 +67,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.metadata.AccumuloTable; +import org.apache.accumulo.core.metadata.TabletMergeabilityUtil; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.security.Authorizations; @@ -250,6 +252,7 @@ public void tabletShouldSplit() throws Exception { KeyExtent extent = new KeyExtent(id, null, null); s.setRange(extent.toMetaRange()); TabletColumnFamily.PREV_ROW_COLUMN.fetch(s); + TabletColumnFamily.MERGEABILITY_COLUMN.fetch(s); int count = 0; int shortened = 0; for (Entry entry : s) { @@ -257,6 +260,14 @@ public void tabletShouldSplit() throws Exception { if (extent.endRow() != null && extent.endRow().toString().length() < 14) { shortened++; } + if (TabletColumnFamily.MERGEABILITY_COLUMN.getColumnQualifier() + .equals(entry.getKey().getColumnQualifier())) { + // Default tablet should be set to NEVER, all newly generated system splits should be + // set to NOW + var mergeability = + extent.endRow() == null ? TabletMergeability.NEVER : TabletMergeability.NOW; + assertEquals(mergeability, TabletMergeabilityUtil.fromValue(entry.getValue())); + } count++; }