Skip to content

Commit

Permalink
Hide TableLayouts from engine
Browse files Browse the repository at this point in the history 
This change hides table layouts from the engine as a first-class
concept. We keep the SPI as is for backward compatibility for now.

When predicates are pushed into a table scan by PickLayout (now
PushPredicateIntoTableScan) or AddExchanges, we now replace the
table handle associated with the table scan with a new one that
contains the reference to the ConnectorTableLayoutHandle under
the covers.
  • Loading branch information
@martint
martint committed Mar 9, 2019
1 parent 0a767d3 commit deff7b1
Show file tree
Hide file tree
Showing 56 changed files with 298 additions and 533 deletions.
8 changes: 2 additions & 6 deletions presto-benchmark/src/main/java/io/prestosql/benchmark/AbstractOperatorBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@
import io.prestosql.metadata.QualifiedObjectName;
import io.prestosql.metadata.Split;
import io.prestosql.metadata.TableHandle;
import io.prestosql.metadata.TableLayoutHandle;
import io.prestosql.metadata.TableLayoutResult;
import io.prestosql.operator.Driver;
import io.prestosql.operator.DriverContext;
import io.prestosql.operator.FilterAndProjectOperator;
Expand All @@ -48,7 +46,6 @@
import io.prestosql.spi.QueryId;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ConnectorPageSource;
import io.prestosql.spi.connector.Constraint;
import io.prestosql.spi.memory.MemoryPoolId;
import io.prestosql.spi.type.Type;
import io.prestosql.spiller.SpillSpaceTracker;
Expand Down Expand Up @@ -170,8 +167,7 @@ protected final OperatorFactory createTableScanOperator(int operatorId, PlanNode
List<ColumnHandle> columnHandles = columnHandlesBuilder.build();

// get the split for this table
Optional<TableLayoutResult> layout = metadata.getLayout(session, tableHandle, Constraint.alwaysTrue(), Optional.empty());
Split split = getLocalQuerySplit(session, layout.get().getLayout().getHandle());
Split split = getLocalQuerySplit(session, tableHandle);

return new OperatorFactory()
{
Expand All @@ -196,7 +192,7 @@ public OperatorFactory duplicate()
};
}

private Split getLocalQuerySplit(Session session, TableLayoutHandle handle)
private Split getLocalQuerySplit(Session session, TableHandle handle)
{
SplitSource splitSource = localQueryRunner.getSplitManager().getSplits(session, handle, UNGROUPED_SCHEDULING);
List<Split> splits = new ArrayList<>();
Expand Down
11 changes: 7 additions & 4 deletions presto-hive/src/test/java/io/prestosql/plugin/hive/TestHiveIntegrationSmokeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@
import io.prestosql.metadata.Metadata;
import io.prestosql.metadata.QualifiedObjectName;
import io.prestosql.metadata.TableHandle;
import io.prestosql.metadata.TableLayout;
import io.prestosql.metadata.TableMetadata;
import io.prestosql.plugin.hive.HiveSessionProperties.InsertExistingPartitionsBehavior;
import io.prestosql.spi.connector.CatalogSchemaTableName;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.connector.ConnectorTableLayoutHandle;
import io.prestosql.spi.connector.Constraint;
import io.prestosql.spi.security.Identity;
import io.prestosql.spi.security.SelectedRole;
Expand Down Expand Up @@ -1672,10 +1672,13 @@ private Object getHiveTableProperty(String tableName, Function<HiveTableLayoutHa
Optional<TableHandle> tableHandle = metadata.getTableHandle(transactionSession, new QualifiedObjectName(catalog, TPCH_SCHEMA, tableName));
assertTrue(tableHandle.isPresent());

TableLayout layout = metadata.getLayout(transactionSession, tableHandle.get(), Constraint.alwaysTrue(), Optional.empty())
ConnectorTableLayoutHandle connectorLayout = metadata.getLayout(transactionSession, tableHandle.get(), Constraint.alwaysTrue(), Optional.empty())
.get()
.getLayout();
return propertyGetter.apply((HiveTableLayoutHandle) layout.getHandle().getConnectorHandle());
.getNewTableHandle()
.getLayout()
.get();

return propertyGetter.apply((HiveTableLayoutHandle) connectorLayout);
});
}

Expand Down
18 changes: 9 additions & 9 deletions presto-main/src/main/java/io/prestosql/metadata/Metadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,23 +75,23 @@ public interface Metadata

Optional<TableLayoutResult> getLayout(Session session, TableHandle tableHandle, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns);

TableLayout getLayout(Session session, TableLayoutHandle handle);
TableLayout getLayout(Session session, TableHandle handle);

/**
* Return a table layout handle whose partitioning is converted to the provided partitioning handle,
* but otherwise identical to the provided table layout handle.
* The provided table layout handle must be one that the connector can transparently convert to from
* the original partitioning handle associated with the provided table layout handle,
* Return a table handle whose partitioning is converted to the provided partitioning handle,
* but otherwise identical to the provided table handle.
* The provided table handle must be one that the connector can transparently convert to from
* the original partitioning handle associated with the provided table handle,
* as promised by {@link #getCommonPartitioning}.
*/
TableLayoutHandle makeCompatiblePartitioning(Session session, TableLayoutHandle tableLayoutHandle, PartitioningHandle partitioningHandle);
TableHandle makeCompatiblePartitioning(Session session, TableHandle table, PartitioningHandle partitioningHandle);

/**
* Return a partitioning handle which the connector can transparently convert both {@code left} and {@code right} into.
*/
Optional<PartitioningHandle> getCommonPartitioning(Session session, PartitioningHandle left, PartitioningHandle right);

Optional<Object> getInfo(Session session, TableLayoutHandle handle);
Optional<Object> getInfo(Session session, TableHandle handle);

/**
* Return the metadata for the specified table handle.
Expand Down Expand Up @@ -241,14 +241,14 @@ public interface Metadata
/**
* @return whether delete without table scan is supported
*/
boolean supportsMetadataDelete(Session session, TableHandle tableHandle, TableLayoutHandle tableLayoutHandle);
boolean supportsMetadataDelete(Session session, TableHandle tableHandle);

/**
* Delete the provide table layout
*
* @return number of rows deleted, or empty for unknown
*/
OptionalLong metadataDelete(Session session, TableHandle tableHandle, TableLayoutHandle tableLayoutHandle);
OptionalLong metadataDelete(Session session, TableHandle tableHandle);

/**
* Begin delete query
Expand Down
64 changes: 44 additions & 20 deletions presto-main/src/main/java/io/prestosql/metadata/MetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static io.prestosql.metadata.QualifiedObjectName.convertFromSchemaTableName;
import static io.prestosql.metadata.TableLayout.fromConnectorLayout;
import static io.prestosql.metadata.ViewDefinition.ViewColumn;
import static io.prestosql.spi.StandardErrorCode.INVALID_VIEW;
import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
Expand Down Expand Up @@ -325,9 +324,14 @@ public Optional<TableHandle> getTableHandle(Session session, QualifiedObjectName
ConnectorId connectorId = catalogMetadata.getConnectorId(session, table);
ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId);

ConnectorTableHandle tableHandle = metadata.getTableHandle(session.toConnectorSession(connectorId), table.asSchemaTableName());
ConnectorSession connectorSession = session.toConnectorSession(connectorId);
ConnectorTableHandle tableHandle = metadata.getTableHandle(connectorSession, table.asSchemaTableName());
if (tableHandle != null) {
return Optional.of(new TableHandle(connectorId, tableHandle));
return Optional.of(new TableHandle(
connectorId,
tableHandle,
catalogMetadata.getTransactionHandleFor(connectorId),
Optional.empty()));
}
}
return Optional.empty();
Expand All @@ -346,7 +350,11 @@ public Optional<TableHandle> getTableHandleForStatisticsCollection(Session sessi

ConnectorTableHandle tableHandle = metadata.getTableHandleForStatisticsCollection(session.toConnectorSession(connectorId), table.asSchemaTableName(), analyzeProperties);
if (tableHandle != null) {
return Optional.of(new TableHandle(connectorId, tableHandle));
return Optional.of(new TableHandle(
connectorId,
tableHandle,
catalogMetadata.getTransactionHandleFor(connectorId),
Optional.empty()));
}
}
return Optional.empty();
Expand Down Expand Up @@ -394,30 +402,39 @@ public Optional<TableLayoutResult> getLayout(Session session, TableHandle table,
throw new PrestoException(NOT_SUPPORTED, format("Connector returned multiple layouts for table %s", table));
}

return Optional.of(new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layouts.get(0).getTableLayout()), layouts.get(0).getUnenforcedConstraint()));
ConnectorTableLayout tableLayout = layouts.get(0).getTableLayout();
return Optional.of(new TableLayoutResult(
new TableHandle(connectorId, connectorTable, transaction, Optional.of(tableLayout.getHandle())),
new TableLayout(connectorId, transaction, tableLayout),
layouts.get(0).getUnenforcedConstraint()));
}

@Override
public TableLayout getLayout(Session session, TableLayoutHandle handle)
public TableLayout getLayout(Session session, TableHandle handle)
{
ConnectorId connectorId = handle.getConnectorId();
CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId);
ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId);
ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId);
return fromConnectorLayout(connectorId, transaction, metadata.getTableLayout(session.toConnectorSession(connectorId), handle.getConnectorHandle()));
ConnectorSession connectorSession = session.toConnectorSession(connectorId);

return handle.getLayout()
.map(layout -> new TableLayout(connectorId, handle.getTransaction(), metadata.getTableLayout(connectorSession, layout)))
.orElseGet(() -> getLayout(session, handle, Constraint.alwaysTrue(), Optional.empty())
.get()
.getLayout());
}

@Override
public TableLayoutHandle makeCompatiblePartitioning(Session session, TableLayoutHandle tableLayoutHandle, PartitioningHandle partitioningHandle)
public TableHandle makeCompatiblePartitioning(Session session, TableHandle tableHandle, PartitioningHandle partitioningHandle)
{
checkArgument(partitioningHandle.getConnectorId().isPresent(), "Expect partitioning handle from connector, got system partitioning handle");
ConnectorId connectorId = partitioningHandle.getConnectorId().get();
checkArgument(connectorId.equals(tableLayoutHandle.getConnectorId()), "ConnectorId of tableLayoutHandle and partitioningHandle does not match");
checkArgument(connectorId.equals(tableHandle.getConnectorId()), "ConnectorId of tableHandle and partitioningHandle does not match");
CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId);
ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId);
ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId);
ConnectorTableLayoutHandle newTableLayoutHandle = metadata.makeCompatiblePartitioning(session.toConnectorSession(connectorId), tableLayoutHandle.getConnectorHandle(), partitioningHandle.getConnectorHandle());
return new TableLayoutHandle(connectorId, transaction, newTableLayoutHandle);
ConnectorTableLayoutHandle newTableLayoutHandle = metadata.makeCompatiblePartitioning(session.toConnectorSession(connectorId), tableHandle.getLayout().get(), partitioningHandle.getConnectorHandle());
return new TableHandle(connectorId, tableHandle.getConnectorHandle(), transaction, Optional.of(newTableLayoutHandle));
}

@Override
Expand All @@ -439,12 +456,19 @@ public Optional<PartitioningHandle> getCommonPartitioning(Session session, Parti
}

@Override
public Optional<Object> getInfo(Session session, TableLayoutHandle handle)
public Optional<Object> getInfo(Session session, TableHandle handle)
{
ConnectorId connectorId = handle.getConnectorId();
ConnectorMetadata metadata = getMetadata(session, connectorId);
ConnectorTableLayout tableLayout = metadata.getTableLayout(session.toConnectorSession(connectorId), handle.getConnectorHandle());
return metadata.getInfo(tableLayout.getHandle());

ConnectorTableLayoutHandle layoutHandle = handle.getLayout()
.orElseGet(() -> getLayout(session, handle, Constraint.alwaysTrue(), Optional.empty())
.get()
.getNewTableHandle()
.getLayout()
.get());

return metadata.getInfo(layoutHandle);
}

@Override
Expand Down Expand Up @@ -785,22 +809,22 @@ public ColumnHandle getUpdateRowIdColumnHandle(Session session, TableHandle tabl
}

@Override
public boolean supportsMetadataDelete(Session session, TableHandle tableHandle, TableLayoutHandle tableLayoutHandle)
public boolean supportsMetadataDelete(Session session, TableHandle tableHandle)
{
ConnectorId connectorId = tableHandle.getConnectorId();
ConnectorMetadata metadata = getMetadata(session, connectorId);
return metadata.supportsMetadataDelete(
session.toConnectorSession(connectorId),
tableHandle.getConnectorHandle(),
tableLayoutHandle.getConnectorHandle());
tableHandle.getLayout().get());
}

@Override
public OptionalLong metadataDelete(Session session, TableHandle tableHandle, TableLayoutHandle tableLayoutHandle)
public OptionalLong metadataDelete(Session session, TableHandle tableHandle)
{
ConnectorId connectorId = tableHandle.getConnectorId();
ConnectorMetadata metadata = getMetadataForWrite(session, connectorId);
return metadata.metadataDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableLayoutHandle.getConnectorHandle());
return metadata.metadataDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableHandle.getLayout().get());
}

@Override
Expand All @@ -809,7 +833,7 @@ public TableHandle beginDelete(Session session, TableHandle tableHandle)
ConnectorId connectorId = tableHandle.getConnectorId();
ConnectorMetadata metadata = getMetadataForWrite(session, connectorId);
ConnectorTableHandle newHandle = metadata.beginDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle());
return new TableHandle(tableHandle.getConnectorId(), newHandle);
return new TableHandle(tableHandle.getConnectorId(), newHandle, tableHandle.getTransaction(), tableHandle.getLayout());
}

@Override
Expand Down
35 changes: 19 additions & 16 deletions presto-main/src/main/java/io/prestosql/metadata/TableHandle.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,34 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import io.prestosql.connector.ConnectorId;
import io.prestosql.spi.connector.ConnectorTableHandle;
import io.prestosql.spi.connector.ConnectorTableLayoutHandle;
import io.prestosql.spi.connector.ConnectorTransactionHandle;

import java.util.Objects;
import java.util.Optional;

import static java.util.Objects.requireNonNull;

public final class TableHandle
{
private final ConnectorId connectorId;
private final ConnectorTableHandle connectorHandle;
private final ConnectorTransactionHandle transaction;

// Table layouts are deprecated, but we keep this here to hide the notion of layouts
// from the engine. TODO: it should be removed once table layouts are finally deleted
private final Optional<ConnectorTableLayoutHandle> layout;

@JsonCreator
public TableHandle(
@JsonProperty("connectorId") ConnectorId connectorId,
@JsonProperty("connectorHandle") ConnectorTableHandle connectorHandle)
@JsonProperty("connectorHandle") ConnectorTableHandle connectorHandle,
@JsonProperty("transaction") ConnectorTransactionHandle transaction,
@JsonProperty("layout") Optional<ConnectorTableLayoutHandle> layout)
{
this.connectorId = requireNonNull(connectorId, "connectorId is null");
this.connectorHandle = requireNonNull(connectorHandle, "connectorHandle is null");
this.transaction = requireNonNull(transaction, "transaction is null");
this.layout = requireNonNull(layout, "layout is null");
}

@JsonProperty
Expand All @@ -48,24 +59,16 @@ public ConnectorTableHandle getConnectorHandle()
return connectorHandle;
}

@Override
public int hashCode()
@JsonProperty
public Optional<ConnectorTableLayoutHandle> getLayout()
{
return Objects.hash(connectorId, connectorHandle);
return layout;
}

@Override
public boolean equals(Object obj)
@JsonProperty
public ConnectorTransactionHandle getTransaction()
{
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
final TableHandle other = (TableHandle) obj;
return Objects.equals(this.connectorId, other.connectorId) &&
Objects.equals(this.connectorHandle, other.connectorHandle);
return transaction;
}

@Override
Expand Down
25 changes: 9 additions & 16 deletions presto-main/src/main/java/io/prestosql/metadata/TableLayout.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,18 @@

public class TableLayout
{
private final TableLayoutHandle handle;
private final ConnectorTableLayout layout;
private final ConnectorId connectorId;
private final ConnectorTransactionHandle transaction;

public TableLayout(TableLayoutHandle handle, ConnectorTableLayout layout)
public TableLayout(ConnectorId connectorId, ConnectorTransactionHandle transaction, ConnectorTableLayout layout)
{
requireNonNull(handle, "handle is null");
requireNonNull(connectorId, "connectorId is null");
requireNonNull(transaction, "transaction is null");
requireNonNull(layout, "layout is null");

this.handle = handle;
this.connectorId = connectorId;
this.transaction = transaction;
this.layout = layout;
}

Expand All @@ -59,18 +62,13 @@ public List<LocalProperty<ColumnHandle>> getLocalProperties()
return layout.getLocalProperties();
}

public TableLayoutHandle getHandle()
{
return handle;
}

public Optional<TablePartitioning> getTablePartitioning()
{
return layout.getTablePartitioning()
.map(nodePartitioning -> new TablePartitioning(
new PartitioningHandle(
Optional.of(handle.getConnectorId()),
Optional.of(handle.getTransactionHandle()),
Optional.of(connectorId),
Optional.of(transaction),
nodePartitioning.getPartitioningHandle()),
nodePartitioning.getPartitioningColumns()));
}
Expand All @@ -85,11 +83,6 @@ public Optional<DiscretePredicates> getDiscretePredicates()
return layout.getDiscretePredicates();
}

public static TableLayout fromConnectorLayout(ConnectorId connectorId, ConnectorTransactionHandle transactionHandle, ConnectorTableLayout layout)
{
return new TableLayout(new TableLayoutHandle(connectorId, transactionHandle, layout.getHandle()), layout);
}

public static class TablePartitioning
{
private final PartitioningHandle partitioningHandle;
Expand Down
Loading

0 comments on commit deff7b1

Please sign in to comment.