diff --git a/connectors/flink/src/main/scala/io/delta/flink/internal/KernelSnapshotWrapper.java b/connectors/flink/src/main/scala/io/delta/flink/internal/KernelSnapshotWrapper.java index ba1342132b..a08405b184 100644 --- a/connectors/flink/src/main/scala/io/delta/flink/internal/KernelSnapshotWrapper.java +++ b/connectors/flink/src/main/scala/io/delta/flink/internal/KernelSnapshotWrapper.java @@ -140,7 +140,7 @@ private Metadata convertMetadata() { ); // Convert the partition columns from a ColumnVector to a List - ColumnVector partitionsVec = kernelMetadata.getPartitionColumns().getElements(); + ColumnVector partitionsVec = kernelMetadata.getPartitionColumnsArrayValue().getElements(); ArrayList partitionColumns = new ArrayList(partitionsVec.getSize()); for(int i = 0; i < partitionsVec.getSize(); i++) { partitionColumns.add(partitionsVec.getString(i)); @@ -186,12 +186,12 @@ private Metadata convertMetadata() { return new Metadata( kernelMetadata.getId(), - kernelMetadata.getName().orElse(null), - kernelMetadata.getDescription().orElse(null), + kernelMetadata.getName(), + kernelMetadata.getDescription(), format, partitionColumns, kernelMetadata.getConfiguration(), - kernelMetadata.getCreatedTime(), + Optional.ofNullable(kernelMetadata.getCreatedTime()), schema ); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java b/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java new file mode 100644 index 0000000000..23e32a0f5c --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java @@ -0,0 +1,27 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.annotation; + +import java.lang.annotation.*; + +/** + * Annotation to indicate that a field, method parameter, method return value, or local variable may + * be null. + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE}) +public @interface Nullable {} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/engine/coordinatedcommits/actions/AbstractMetadata.java b/kernel/kernel-api/src/main/java/io/delta/kernel/engine/coordinatedcommits/actions/AbstractMetadata.java index 0bc152115c..7c02c8575f 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/engine/coordinatedcommits/actions/AbstractMetadata.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/engine/coordinatedcommits/actions/AbstractMetadata.java @@ -17,9 +17,9 @@ package io.delta.kernel.engine.coordinatedcommits.actions; import io.delta.kernel.annotation.Evolving; +import io.delta.kernel.annotation.Nullable; import java.util.List; import java.util.Map; -import java.util.Optional; /** * Interface for metadata actions in Delta. The metadata defines the metadata of the table. @@ -33,9 +33,11 @@ public interface AbstractMetadata { String getId(); /** User-specified table identifier. */ + @Nullable String getName(); /** User-specified table description. */ + @Nullable String getDescription(); /** The table provider format. */ @@ -54,5 +56,6 @@ public interface AbstractMetadata { Map getConfiguration(); /** Timestamp for the creation of this metadata. */ - Optional getCreatedTime(); + @Nullable + Long getCreatedTime(); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/ScanImpl.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/ScanImpl.java index 96e93cd02e..040a957354 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/ScanImpl.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/ScanImpl.java @@ -77,7 +77,8 @@ public ScanImpl( this.dataPath = dataPath; this.partitionColToStructFieldMap = () -> { - Set partitionColNames = metadata.getPartitionColNames(); + final Set partitionColNames = + PartitionUtils.arrayValueToLowerCaseSet(metadata.getPartitionColumnsArrayValue()); return metadata.getSchema().fields().stream() .filter(field -> partitionColNames.contains(field.getName().toLowerCase(Locale.ROOT))) .collect(toMap(field -> field.getName().toLowerCase(Locale.ROOT), identity())); @@ -156,7 +157,8 @@ public Row getScanState(Engine engine) { // Compute the physical data read schema, basically the list of columns to read // from a Parquet data file. It should exclude partition columns and include // row_index metadata columns (in case DVs are present) - List partitionColumns = VectorUtils.toJavaList(metadata.getPartitionColumns()); + List partitionColumns = + VectorUtils.toJavaList(metadata.getPartitionColumnsArrayValue()); StructType physicalDataReadSchema = PartitionUtils.physicalSchemaWithoutPartitionColumns( readSchema, /* logical read schema */ @@ -185,7 +187,8 @@ private Optional> splitFilters(Optional return filter.map( predicate -> PartitionUtils.splitMetadataAndDataPredicates( - predicate, metadata.getPartitionColNames())); + predicate, + PartitionUtils.arrayValueToLowerCaseSet(metadata.getPartitionColumnsArrayValue()))); } private Optional getDataFilters() { diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java index 7b90d2be1d..70dc0e02ab 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java @@ -112,7 +112,7 @@ public Row getTransactionState(Engine engine) { @Override public List getPartitionColumns(Engine engine) { - return VectorUtils.toJavaList(metadata.getPartitionColumns()); + return VectorUtils.toJavaList(metadata.getPartitionColumnsArrayValue()); } @Override @@ -303,7 +303,7 @@ private boolean isReadyForCheckpoint(long newVersion) { private Map getOperationParameters() { if (isNewTable) { - List partitionCols = VectorUtils.toJavaList(metadata.getPartitionColumns()); + List partitionCols = VectorUtils.toJavaList(metadata.getPartitionColumnsArrayValue()); String partitionBy = partitionCols.stream() .map(col -> "\"" + col + "\"") diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/Metadata.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/Metadata.java index dac74b4f8a..ff711654a7 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/Metadata.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/Metadata.java @@ -16,19 +16,46 @@ package io.delta.kernel.internal.actions; import static io.delta.kernel.internal.util.InternalUtils.requireNonNull; -import static io.delta.kernel.internal.util.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; +import io.delta.kernel.annotation.Nullable; import io.delta.kernel.data.*; +import io.delta.kernel.engine.coordinatedcommits.actions.AbstractMetadata; import io.delta.kernel.internal.data.GenericRow; import io.delta.kernel.internal.lang.Lazy; import io.delta.kernel.internal.types.DataTypeJsonSerDe; +import io.delta.kernel.internal.util.PartitionUtils; import io.delta.kernel.internal.util.VectorUtils; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Collectors; -public class Metadata { +public class Metadata implements AbstractMetadata { + + /////////////////// + // Static Fields // + /////////////////// + + public static final StructType FULL_SCHEMA = + new StructType() + .add("id", StringType.STRING, false /* nullable */) + .add("name", StringType.STRING, true /* nullable */) + .add("description", StringType.STRING, true /* nullable */) + .add("format", Format.FULL_SCHEMA, false /* nullable */) + .add("schemaString", StringType.STRING, false /* nullable */) + .add( + "partitionColumns", + new ArrayType(StringType.STRING, false /* contains null */), + false /* nullable */) + .add("createdTime", LongType.LONG, true /* contains null */) + .add( + "configuration", + new MapType(StringType.STRING, StringType.STRING, false), + false /* nullable */); + + //////////////////// + // Static Methods // + //////////////////// public static Metadata fromColumnVector(ColumnVector vector, int rowId) { if (vector.isNullAt(rowId)) { @@ -37,7 +64,8 @@ public static Metadata fromColumnVector(ColumnVector vector, int rowId) { final String schemaJson = requireNonNull(vector.getChild(4), rowId, "schemaString").getString(rowId); - StructType schema = DataTypeJsonSerDe.deserializeStructType(schemaJson); + + final StructType schema = DataTypeJsonSerDe.deserializeStructType(schemaJson); return new Metadata( requireNonNull(vector.getChild(0), rowId, "id").getString(rowId), @@ -54,22 +82,11 @@ public static Metadata fromColumnVector(ColumnVector vector, int rowId) { vector.getChild(7).getMap(rowId)); } - public static final StructType FULL_SCHEMA = - new StructType() - .add("id", StringType.STRING, false /* nullable */) - .add("name", StringType.STRING, true /* nullable */) - .add("description", StringType.STRING, true /* nullable */) - .add("format", Format.FULL_SCHEMA, false /* nullable */) - .add("schemaString", StringType.STRING, false /* nullable */) - .add( - "partitionColumns", - new ArrayType(StringType.STRING, false /* contains null */), - false /* nullable */) - .add("createdTime", LongType.LONG, true /* contains null */) - .add( - "configuration", - new MapType(StringType.STRING, StringType.STRING, false), - false /* nullable */); + ///////////////////////////// + // Member Fields / Methods // + ///////////////////////////// + + // Constructor Params - anything Optional is not required as per the protocol spec. // private final String id; private final Optional name; @@ -80,11 +97,11 @@ public static Metadata fromColumnVector(ColumnVector vector, int rowId) { private final ArrayValue partitionColumns; private final Optional createdTime; private final MapValue configurationMapValue; + + // Private member fields // + private final Lazy> configuration; - // Partition column names in lower case. - private final Lazy> partitionColNames; - // Logical data schema excluding partition columns - private final Lazy dataSchema; + private final Lazy dataSchema; // Logical data schema excluding partition columns public Metadata( String id, @@ -97,141 +114,94 @@ public Metadata( Optional createdTime, MapValue configurationMapValue) { this.id = requireNonNull(id, "id is null"); - this.name = name; + this.name = requireNonNull(name, "name is null"); this.description = requireNonNull(description, "description is null"); this.format = requireNonNull(format, "format is null"); this.schemaString = requireNonNull(schemaString, "schemaString is null"); - this.schema = schema; + ; + this.schema = requireNonNull(schema, "schema is null"); this.partitionColumns = requireNonNull(partitionColumns, "partitionColumns is null"); this.createdTime = createdTime; this.configurationMapValue = requireNonNull(configurationMapValue, "configuration is null"); this.configuration = new Lazy<>(() -> VectorUtils.toJavaMap(configurationMapValue)); - this.partitionColNames = new Lazy<>(this::loadPartitionColNames); - this.dataSchema = - new Lazy<>( - () -> - new StructType( - schema.fields().stream() - .filter( - field -> - !partitionColNames - .get() - .contains(field.getName().toLowerCase(Locale.ROOT))) - .collect(Collectors.toList()))); + this.dataSchema = new Lazy<>(this::loadDataSchema); } - public Metadata withNewConfiguration(Map configuration) { - Map newConfiguration = new HashMap<>(getConfiguration()); - newConfiguration.putAll(configuration); - return new Metadata( - this.id, - this.name, - this.description, - this.format, - this.schemaString, - this.schema, - this.partitionColumns, - this.createdTime, - VectorUtils.stringStringMapValue(newConfiguration)); + /////////////////////////// + // AbstractMetadata APIs // + /////////////////////////// + + @Override + public String getId() { + return id; } - public Metadata withNewSchema(StructType schema) { - return new Metadata( - this.id, - this.name, - this.description, - this.format, - schema.toJson(), - schema, - this.partitionColumns, - this.createdTime, - this.configurationMapValue); + @Override + @Nullable + public String getName() { + return name.orElse(null); } @Override - public String toString() { - List partitionColumnsStr = VectorUtils.toJavaList(partitionColumns); - StringBuilder sb = new StringBuilder(); - sb.append("List("); - for (String partitionColumn : partitionColumnsStr) { - sb.append(partitionColumn).append(", "); - } - if (sb.substring(sb.length() - 2).equals(", ")) { - sb.setLength(sb.length() - 2); // Remove the last comma and space - } - sb.append(")"); - return "Metadata{" - + "id='" - + id - + '\'' - + ", name=" - + name - + ", description=" - + description - + ", format=" - + format - + ", schemaString='" - + schemaString - + '\'' - + ", partitionColumns=" - + sb - + ", createdTime=" - + createdTime - + ", configuration=" - + configuration.get() - + '}'; + @Nullable + public String getDescription() { + return description.orElse(null); } + @Override + public String getProvider() { + return format.getProvider(); + } + + @Override + public Map getFormatOptions() { + return format.getOptions(); + } + + @Override public String getSchemaString() { return schemaString; } - public StructType getSchema() { - return schema; + @Override + public List getPartitionColumns() { + return VectorUtils.toJavaList(partitionColumns); } - public ArrayValue getPartitionColumns() { - return partitionColumns; + @Override + public Map getConfiguration() { + return Collections.unmodifiableMap(configuration.get()); } - /** Set of lowercase partition column names */ - public Set getPartitionColNames() { - return partitionColNames.get(); + public Long getCreatedTime() { + return createdTime.orElse(null); } - /** The logical data schema which excludes partition columns */ - public StructType getDataSchema() { - return dataSchema.get(); - } + //////////////////////////////// + // Other Kernel Metadata APIs // + //////////////////////////////// - public String getId() { - return id; + public StructType getSchema() { + return schema; } - public Optional getName() { - return name; + public ArrayValue getPartitionColumnsArrayValue() { + return partitionColumns; } - public Optional getDescription() { - return description; + /** The logical data schema which excludes partition columns */ + public StructType getDataSchema() { + return dataSchema.get(); } public Format getFormat() { return format; } - public Optional getCreatedTime() { - return createdTime; - } - public MapValue getConfigurationMapValue() { return configurationMapValue; } - public Map getConfiguration() { - return Collections.unmodifiableMap(configuration.get()); - } - /** * Filter out the key-value pair matches exactly with the old properties. * @@ -267,19 +237,80 @@ public Row toRow() { return new GenericRow(Metadata.FULL_SCHEMA, metadataMap); } - /** Helper method to load the partition column names. */ - private Set loadPartitionColNames() { - ColumnVector partitionColNameVector = partitionColumns.getElements(); - Set partitionColumnNames = new HashSet<>(); - for (int i = 0; i < partitionColumns.getSize(); i++) { - checkArgument( - !partitionColNameVector.isNullAt(i), "Expected a non-null partition column name"); - String partitionColName = partitionColNameVector.getString(i); - checkArgument( - partitionColName != null && !partitionColName.isEmpty(), - "Expected non-null and non-empty partition column name"); - partitionColumnNames.add(partitionColName.toLowerCase(Locale.ROOT)); + public Metadata withNewConfiguration(Map configuration) { + Map newConfiguration = new HashMap<>(getConfiguration()); + newConfiguration.putAll(configuration); + return new Metadata( + this.id, + this.name, + this.description, + this.format, + this.schemaString, + this.schema, + this.partitionColumns, + this.createdTime, + VectorUtils.stringStringMapValue(newConfiguration)); + } + + public Metadata withNewSchema(StructType schema) { + return new Metadata( + this.id, + this.name, + this.description, + this.format, + schema.toJson(), + schema, + this.partitionColumns, + this.createdTime, + this.configurationMapValue); + } + + @Override + public String toString() { + List partitionColumnsStr = VectorUtils.toJavaList(partitionColumns); + StringBuilder sb = new StringBuilder(); + sb.append("List("); + for (String partitionColumn : partitionColumnsStr) { + sb.append(partitionColumn).append(", "); + } + if (sb.substring(sb.length() - 2).equals(", ")) { + sb.setLength(sb.length() - 2); // Remove the last comma and space } - return Collections.unmodifiableSet(partitionColumnNames); + sb.append(")"); + return "Metadata{" + + "id='" + + id + + '\'' + + ", name=" + + name + + ", description=" + + description + + ", format=" + + format + + ", schemaString='" + + schemaString + + '\'' + + ", partitionColumns=" + + sb + + ", createdTime=" + + createdTime + + ", configuration=" + + configuration.get() + + '}'; + } + + //////////////////// + // Helper methods // + //////////////////// + + private StructType loadDataSchema() { + final Set partColNamesLowerCase = + PartitionUtils.arrayValueToLowerCaseSet(partitionColumns); + + return new StructType( + schema.fields().stream() + .filter( + field -> !partColNamesLowerCase.contains(field.getName().toLowerCase(Locale.ROOT))) + .collect(Collectors.toList())); } } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/ScanStateRow.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/ScanStateRow.java index 20f0f975ab..dfcb86e35f 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/ScanStateRow.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/ScanStateRow.java @@ -61,7 +61,8 @@ public static ScanStateRow of( valueMap.put(COL_NAME_TO_ORDINAL.get("physicalSchemaString"), readSchemaPhysicalJson); valueMap.put( COL_NAME_TO_ORDINAL.get("physicalDataReadSchemaString"), readPhysicalDataSchemaJson); - valueMap.put(COL_NAME_TO_ORDINAL.get("partitionColumns"), metadata.getPartitionColumns()); + valueMap.put( + COL_NAME_TO_ORDINAL.get("partitionColumns"), metadata.getPartitionColumnsArrayValue()); valueMap.put(COL_NAME_TO_ORDINAL.get("minReaderVersion"), protocol.getMinReaderVersion()); valueMap.put(COL_NAME_TO_ORDINAL.get("minWriterVersion"), protocol.getMinWriterVersion()); valueMap.put(COL_NAME_TO_ORDINAL.get("tablePath"), tablePath); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/TransactionStateRow.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/TransactionStateRow.java index 11c50a2784..4fab7f3403 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/TransactionStateRow.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/data/TransactionStateRow.java @@ -46,7 +46,8 @@ public class TransactionStateRow extends GenericRow { public static TransactionStateRow of(Metadata metadata, String tablePath) { HashMap valueMap = new HashMap<>(); valueMap.put(COL_NAME_TO_ORDINAL.get("logicalSchemaString"), metadata.getSchemaString()); - valueMap.put(COL_NAME_TO_ORDINAL.get("partitionColumns"), metadata.getPartitionColumns()); + valueMap.put( + COL_NAME_TO_ORDINAL.get("partitionColumns"), metadata.getPartitionColumnsArrayValue()); valueMap.put(COL_NAME_TO_ORDINAL.get("configuration"), metadata.getConfigurationMapValue()); valueMap.put(COL_NAME_TO_ORDINAL.get("tablePath"), tablePath); return new TransactionStateRow(valueMap); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java index ac4de23b8b..9de842f31b 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/CoordinatedCommitsUtils.java @@ -16,66 +16,12 @@ package io.delta.kernel.internal.util; import io.delta.kernel.engine.coordinatedcommits.actions.AbstractCommitInfo; -import io.delta.kernel.engine.coordinatedcommits.actions.AbstractMetadata; import io.delta.kernel.engine.coordinatedcommits.actions.AbstractProtocol; import io.delta.kernel.internal.actions.CommitInfo; -import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.Protocol; import java.util.*; public class CoordinatedCommitsUtils { - public static AbstractMetadata convertMetadataToAbstractMetadata(Metadata metadata) { - return new AbstractMetadata() { - @Override - public String getId() { - return metadata.getId(); - } - - @Override - public String getName() { - return metadata.getName().orElse(null); - } - - @Override - public String getDescription() { - return metadata.getDescription().orElse(null); - } - - @Override - public String getProvider() { - return metadata.getFormat().getProvider(); - } - - @Override - public Map getFormatOptions() { - // Assuming Format class has a method to get format options - return metadata.getFormat().getOptions(); - } - - @Override - public String getSchemaString() { - // Assuming Metadata class has a method to get schema string - return metadata.getSchemaString(); - } - - @Override - public List getPartitionColumns() { - // Assuming Metadata class has a method to get partition columns - return VectorUtils.toJavaList(metadata.getPartitionColumns()); - } - - @Override - public Map getConfiguration() { - return metadata.getConfiguration(); - } - - @Override - public Optional getCreatedTime() { - return metadata.getCreatedTime(); - } - }; - } - public static AbstractProtocol convertProtocolToAbstractProtocol(Protocol protocol) { return new AbstractProtocol() { @Override diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java index 5533babeba..c84acd392a 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/PartitionUtils.java @@ -26,6 +26,7 @@ import io.delta.kernel.data.*; import io.delta.kernel.engine.Engine; import io.delta.kernel.engine.ExpressionHandler; +import io.delta.kernel.exceptions.KernelException; import io.delta.kernel.expressions.*; import io.delta.kernel.internal.InternalScanFileUtils; import io.delta.kernel.internal.fs.Path; @@ -46,6 +47,28 @@ public class PartitionUtils { private PartitionUtils() {} + /** + * Converts an {@link ArrayValue} of partition columns name to a {@link Set} of lowercase strings. + * + * @param partitionColumnsVector an {@code ArrayValue} containing the partition column names. + * @return a {@code Set} with each partition column name in lowercase. + */ + public static Set arrayValueToLowerCaseSet(ArrayValue partitionColumnsVector) { + final Set result = + VectorUtils.toJavaList(partitionColumnsVector).stream() + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); + + if (result.size() != partitionColumnsVector.getSize()) { + throw new KernelException( + String.format( + "partitionColumnsVector contains duplicate partition column names: %s", + partitionColumnsVector)); + } + + return result; + } + /** * Utility method to remove the given columns (as {@code columnsToRemove}) from the given {@code * physicalSchema}. diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/coordinatedcommits/StorageKernelAPIAdapter.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/coordinatedcommits/StorageKernelAPIAdapter.java index 53e379741a..157e6b209c 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/coordinatedcommits/StorageKernelAPIAdapter.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/coordinatedcommits/StorageKernelAPIAdapter.java @@ -121,7 +121,7 @@ public Map getConfiguration() { @Override public Long getCreatedTime() { - return metadata.getCreatedTime().orElse(null); + return metadata.getCreatedTime(); } }; } diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/InCommitTimestampSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/InCommitTimestampSuite.scala index ac7b302c2c..b989698e4e 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/InCommitTimestampSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/InCommitTimestampSuite.scala @@ -349,7 +349,7 @@ class InCommitTimestampSuite extends DeltaTableWriteSuiteBase { "delta.enableInCommitTimestamps=true, " + "delta.inCommitTimestampEnablementVersion=1}}", metadata.getId, - metadata.getCreatedTime.get, + metadata.getCreatedTime, inCommitTimestamp.toString)) } } diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsSuite.scala index 2709fef5a3..779ac8de92 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsSuite.scala @@ -91,7 +91,7 @@ class CoordinatedCommitsSuite extends DeltaTableWriteSuiteBase tableConf = handler.registerTable( logPath.toString, version - 1L, - CoordinatedCommitsUtils.convertMetadataToAbstractMetadata(getEmptyMetadata), + getEmptyMetadata, CoordinatedCommitsUtils.convertProtocolToAbstractProtocol(getProtocol(1, 1))) val tableConfString = if (tableConfToOverwrite != null) { tableConfToOverwrite diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsTestUtils.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsTestUtils.scala index 6137956477..4f039c120c 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsTestUtils.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/internal/coordinatedcommits/CoordinatedCommitsTestUtils.scala @@ -43,11 +43,11 @@ trait CoordinatedCommitsTestUtils { def getEmptyMetadata: Metadata = { new Metadata( util.UUID.randomUUID().toString, - Optional.empty(), - Optional.empty(), + Optional.empty(), /* name */ + Optional.empty(), /* description */ new Format(), "", - null, + new StructType(), stringArrayValue(Collections.emptyList()), Optional.empty(), VectorUtils.stringStringMapValue(Collections.emptyMap()) @@ -122,9 +122,9 @@ trait CoordinatedCommitsTestUtils { val newMetadata = oldMetadata.withNewConfiguration(newMetadataConfiguration.asJava) new UpdatedActions( CoordinatedCommitsUtils.convertCommitInfoToAbstractCommitInfo(commitInfo), - CoordinatedCommitsUtils.convertMetadataToAbstractMetadata(newMetadata), + newMetadata, CoordinatedCommitsUtils.convertProtocolToAbstractProtocol(getProtocol(3, 7)), - CoordinatedCommitsUtils.convertMetadataToAbstractMetadata(oldMetadata), + oldMetadata, CoordinatedCommitsUtils.convertProtocolToAbstractProtocol(getProtocol(3, 7))) }