From aa18f3236db1ac5957cfb11ccdadaa8a5df0f599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Sun, 6 Oct 2024 15:45:16 +0200 Subject: [PATCH] Add the Faker connector --- core/trino-server/src/main/provisio/trino.xml | 6 + plugin/trino-faker/pom.xml | 171 +++++ .../io/trino/plugin/faker/ColumnInfo.java | 80 ++ .../trino/plugin/faker/FakerColumnHandle.java | 35 + .../io/trino/plugin/faker/FakerConfig.java | 55 ++ .../io/trino/plugin/faker/FakerConnector.java | 150 ++++ .../plugin/faker/FakerConnectorFactory.java | 63 ++ .../io/trino/plugin/faker/FakerFunctions.java | 42 ++ .../plugin/faker/FakerInsertTableHandle.java | 28 + .../io/trino/plugin/faker/FakerMetadata.java | 452 +++++++++++ .../io/trino/plugin/faker/FakerModule.java | 53 ++ .../plugin/faker/FakerOutputTableHandle.java | 29 + .../plugin/faker/FakerPageSinkProvider.java | 69 ++ .../trino/plugin/faker/FakerPageSource.java | 713 ++++++++++++++++++ .../plugin/faker/FakerPageSourceProvider.java | 67 ++ .../io/trino/plugin/faker/FakerPlugin.java | 40 + .../io/trino/plugin/faker/FakerSplit.java | 21 + .../trino/plugin/faker/FakerSplitManager.java | 40 + .../trino/plugin/faker/FakerTableHandle.java | 43 ++ .../plugin/faker/FakerTransactionHandle.java | 23 + .../io/trino/plugin/faker/SchemaInfo.java | 50 ++ .../java/io/trino/plugin/faker/TableInfo.java | 119 +++ .../trino/plugin/faker/FakerQueryRunner.java | 85 +++ .../trino/plugin/faker/TestFakerQueries.java | 607 +++++++++++++++ pom.xml | 7 + .../EnvMultinodeAllConnectors.java | 1 + .../multinode-all/faker.properties | 1 + .../etc/catalog/faker.properties | 1 + .../trino-server-dev/etc/config.properties | 1 + 29 files changed, 3052 insertions(+) create mode 100644 plugin/trino-faker/pom.xml create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConfig.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerFunctions.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerInsertTableHandle.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerOutputTableHandle.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSinkProvider.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPlugin.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplit.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplitManager.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTransactionHandle.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/SchemaInfo.java create mode 100644 plugin/trino-faker/src/main/java/io/trino/plugin/faker/TableInfo.java create mode 100644 plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java create mode 100644 plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/faker.properties create mode 100644 testing/trino-server-dev/etc/catalog/faker.properties diff --git a/core/trino-server/src/main/provisio/trino.xml b/core/trino-server/src/main/provisio/trino.xml index 5fd5c6e2415db6..d49c39d3696573 100644 --- a/core/trino-server/src/main/provisio/trino.xml +++ b/core/trino-server/src/main/provisio/trino.xml @@ -108,6 +108,12 @@ + + + + + + diff --git a/plugin/trino-faker/pom.xml b/plugin/trino-faker/pom.xml new file mode 100644 index 00000000000000..330bf75d540ed2 --- /dev/null +++ b/plugin/trino-faker/pom.xml @@ -0,0 +1,171 @@ + + + 4.0.0 + + + io.trino + trino-root + 461-SNAPSHOT + ../../pom.xml + + + trino-faker + trino-plugin + Trino - Faker connector + + + + + com.google.errorprone + error_prone_annotations + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + io.airlift + bootstrap + + + + io.airlift + configuration + + + + io.trino + trino-main + + + + io.trino + trino-plugin-toolkit + + + + jakarta.inject + jakarta.inject-api + + + + jakarta.validation + jakarta.validation-api + + + + net.datafaker + datafaker + 2.3.1 + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.airlift + slice + provided + + + + io.opentelemetry + opentelemetry-api + provided + + + + io.opentelemetry + opentelemetry-context + provided + + + + io.trino + trino-spi + provided + + + + org.openjdk.jol + jol-core + provided + + + + io.airlift + units + runtime + + + + io.airlift + junit-extensions + test + + + + io.airlift + testing + test + + + + io.trino + trino-memory + test + + + + io.trino + trino-testing + test + + + + io.trino + trino-tpch + test + + + + org.assertj + assertj-core + test + + + + org.codehaus.plexus + plexus-utils + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + io.airlift + log + test-jar + + + + io.airlift + log-manager + test-jar + + + diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java new file mode 100644 index 00000000000000..68de0d63ae65ac --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/ColumnInfo.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.type.Type; + +import java.util.Map; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class ColumnInfo +{ + public static final String NULL_PROBABILITY_PROPERTY = "null_probability"; + public static final String GENERATOR_PROPERTY = "generator"; + + private final ColumnHandle handle; + private final String name; + private final Type type; + private ColumnMetadata metadata; + + public ColumnInfo(ColumnHandle handle, String name, Type type, Map properties, Optional comment) + { + this(handle, ColumnMetadata.builder() + .setName(name) + .setType(type) + .setProperties(properties) + .setComment(comment) + .build()); + } + + public ColumnInfo(ColumnHandle handle, ColumnMetadata metadata) + { + this.handle = requireNonNull(handle, "handle is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.name = metadata.getName(); + this.type = metadata.getType(); + } + + public ColumnHandle getHandle() + { + return handle; + } + + public String getName() + { + return name; + } + + public ColumnMetadata getMetadata() + { + return metadata; + } + + @Override + public String toString() + { + return name + "::" + type; + } + + public ColumnInfo withComment(Optional comment) + { + return new ColumnInfo(handle, ColumnMetadata.builderFrom(metadata) + .setComment(comment) + .build()); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java new file mode 100644 index 00000000000000..e6ae47dec501f9 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerColumnHandle.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.type.Type; + +import static java.util.Objects.requireNonNull; + +public record FakerColumnHandle( + int columnIndex, + String name, + Type type, + double nullProbability, + String generator) + implements ColumnHandle +{ + public FakerColumnHandle + { + requireNonNull(name, "name is null"); + requireNonNull(type, "type is null"); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConfig.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConfig.java new file mode 100644 index 00000000000000..256310f557543c --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; + +public class FakerConfig +{ + private double nullProbability = 0.5; + private long defaultLimit = 1000L; + + @Max(1) + @Min(0) + public double getNullProbability() + { + return nullProbability; + } + + @Config("faker.null-probability") + @ConfigDescription("Default null probability for any column in any table that allows them") + public FakerConfig setNullProbability(double value) + { + this.nullProbability = value; + return this; + } + + @Min(1) + public long getDefaultLimit() + { + return defaultLimit; + } + + @Config("faker.default-limit") + @ConfigDescription("Default number of rows for each table, when the LIMIT clause is not specified in the query") + public FakerConfig setDefaultLimit(long value) + { + this.defaultLimit = value; + return this; + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java new file mode 100644 index 00000000000000..ea1d181fd166b1 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnector.java @@ -0,0 +1,150 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import com.google.common.collect.ImmutableList; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorCapabilities; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorPageSinkProvider; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.session.PropertyMetadata; +import io.trino.spi.transaction.IsolationLevel; +import jakarta.inject.Inject; + +import java.util.List; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkState; +import static io.trino.spi.connector.ConnectorCapabilities.NOT_NULL_COLUMN_CONSTRAINT; +import static io.trino.spi.session.PropertyMetadata.doubleProperty; +import static io.trino.spi.session.PropertyMetadata.longProperty; +import static io.trino.spi.session.PropertyMetadata.stringProperty; +import static java.util.Objects.requireNonNull; + +public class FakerConnector + implements Connector +{ + private final FakerMetadata metadata; + private final FakerSplitManager splitManager; + private final FakerPageSourceProvider pageSourceProvider; + private final FakerPageSinkProvider pageSinkProvider; + + @Inject + public FakerConnector( + FakerMetadata metadata, + FakerSplitManager splitManager, + FakerPageSourceProvider pageSourceProvider, + FakerPageSinkProvider pageSinkProvider) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.splitManager = requireNonNull(splitManager, "splitManager is null"); + this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit) + { + return FakerTransactionHandle.INSTANCE; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transaction) + { + return metadata; + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return splitManager; + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + return pageSourceProvider; + } + + @Override + public ConnectorPageSinkProvider getPageSinkProvider() + { + return pageSinkProvider; + } + + @Override + public Set getCapabilities() + { + return Set.of(NOT_NULL_COLUMN_CONSTRAINT); + } + + @Override + public List> getSchemaProperties() + { + return ImmutableList.of( + doubleProperty( + SchemaInfo.NULL_PROBABILITY_PROPERTY, + "Default probability of null values in any column that allows them, in any table of this schema", + null, + nullProbability -> checkState(0 <= nullProbability && nullProbability <= 1, "null_probability value must be between 0 and 1, inclusive"), + false), + longProperty( + SchemaInfo.DEFAULT_LIMIT_PROPERTY, + "Default limit of rows returned from any table in this schema, if not specified in the query", + null, + defaultLimit -> checkState(1 <= defaultLimit, "default_limit value must be equal or greater than 1"), + false)); + } + + @Override + public List> getTableProperties() + { + return ImmutableList.of( + doubleProperty( + TableInfo.NULL_PROBABILITY_PROPERTY, + "Default probability of null values in any column in this table that allows them", + null, + nullProbability -> checkState(0 <= nullProbability && nullProbability <= 1, "null_probability value must be between 0 and 1, inclusive"), + false), + longProperty( + TableInfo.DEFAULT_LIMIT_PROPERTY, + "Default limit of rows returned from this table if not specified in the query", + null, + defaultLimit -> checkState(1 <= defaultLimit, "default_limit value must be equal or greater than 1"), + false)); + } + + @Override + public List> getColumnProperties() + { + return ImmutableList.of( + doubleProperty( + ColumnInfo.NULL_PROBABILITY_PROPERTY, + "Default probability of null values in this column, if it allows them", + null, + nullProbability -> checkState(0 <= nullProbability && nullProbability <= 1, "null_probability value must be between 0 and 1, inclusive"), + false), + stringProperty( + ColumnInfo.GENERATOR_PROPERTY, + "Name of the Faker library generator used to generate data for this column", + null, + pageSourceProvider::validateGenerator, + false)); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java new file mode 100644 index 00000000000000..f5d9f5e13f0062 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.ConnectorFactory; + +import java.util.Map; + +import static io.trino.plugin.base.Versions.checkStrictSpiVersionMatch; +import static java.util.Objects.requireNonNull; + +public class FakerConnectorFactory + implements ConnectorFactory +{ + private final String name; + + public FakerConnectorFactory(String name) + { + this.name = name; + } + + @Override + public String getName() + { + return name; + } + + @Override + public Connector create(String catalogName, Map requiredConfig, ConnectorContext context) + { + requireNonNull(requiredConfig, "requiredConfig is null"); + requireNonNull(context, "context is null"); + checkStrictSpiVersionMatch(context, this); + + Bootstrap app = new Bootstrap( + new FakerModule( + context.getNodeManager(), + context.getTypeManager())); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(requiredConfig) + .initialize(); + + return injector.getInstance(FakerConnector.class); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerFunctions.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerFunctions.java new file mode 100644 index 00000000000000..4e63b4d77aebfb --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerFunctions.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import io.airlift.slice.Slice; +import io.trino.spi.function.Description; +import io.trino.spi.function.ScalarFunction; +import io.trino.spi.function.SqlType; +import net.datafaker.Faker; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.spi.type.StandardTypes.VARCHAR; + +public final class FakerFunctions +{ + private final Faker faker; + + public FakerFunctions() + { + faker = new Faker(); + } + + @ScalarFunction + @Description("Generate a random string based on the Faker expression") + @SqlType(VARCHAR) + public Slice randomExpression(@SqlType(VARCHAR) Slice expression) + { + return utf8Slice(faker.expression(expression.toStringUtf8())); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerInsertTableHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerInsertTableHandle.java new file mode 100644 index 00000000000000..e4108932d5f773 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerInsertTableHandle.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import io.trino.spi.connector.ConnectorInsertTableHandle; + +import static java.util.Objects.requireNonNull; + +public record FakerInsertTableHandle(FakerTableHandle tableHandle) + implements ConnectorInsertTableHandle +{ + public FakerInsertTableHandle + { + requireNonNull(tableHandle, "tableHandle is null"); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java new file mode 100644 index 00000000000000..a48ee25a8a4a35 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerMetadata.java @@ -0,0 +1,452 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import io.airlift.slice.Slice; +import io.trino.spi.StandardErrorCode; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorOutputMetadata; +import io.trino.spi.connector.ConnectorOutputTableHandle; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTableLayout; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.ConnectorTableVersion; +import io.trino.spi.connector.Constraint; +import io.trino.spi.connector.ConstraintApplicationResult; +import io.trino.spi.connector.LimitApplicationResult; +import io.trino.spi.connector.RetryMode; +import io.trino.spi.connector.SaveMode; +import io.trino.spi.connector.SchemaNotFoundException; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.SchemaTablePrefix; +import io.trino.spi.connector.TableColumnsMetadata; +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.security.TrinoPrincipal; +import io.trino.spi.statistics.ComputedStatistics; +import io.trino.spi.type.CharType; +import io.trino.spi.type.VarbinaryType; +import io.trino.spi.type.VarcharType; +import jakarta.inject.Inject; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; +import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY; +import static io.trino.spi.StandardErrorCode.INVALID_TABLE_FUNCTION_INVOCATION; +import static io.trino.spi.StandardErrorCode.NOT_FOUND; +import static io.trino.spi.connector.RetryMode.NO_RETRIES; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class FakerMetadata + implements ConnectorMetadata +{ + public static final String SCHEMA_NAME = "default"; + + @GuardedBy("this") + private final List schemas = new ArrayList<>(); + private final double nullProbability; + private final long defaultLimit; + + private final AtomicLong nextTableId = new AtomicLong(); + @GuardedBy("this") + private final Map tableIds = new HashMap<>(); + @GuardedBy("this") + private final Map tables = new HashMap<>(); + + @Inject + public FakerMetadata(FakerConfig config) + { + this.schemas.add(new SchemaInfo(SCHEMA_NAME)); + this.nullProbability = config.getNullProbability(); + this.defaultLimit = config.getDefaultLimit(); + } + + @Override + public synchronized List listSchemaNames(ConnectorSession connectorSession) + { + return schemas.stream() + .map(SchemaInfo::getName) + .collect(toImmutableList()); + } + + @Override + public synchronized void createSchema(ConnectorSession session, String schemaName, Map properties, TrinoPrincipal owner) + { + if (schemas.stream().anyMatch(schema -> schema.getName().equals(schemaName))) { + throw new TrinoException(ALREADY_EXISTS, format("Schema [%s] already exists", schemaName)); + } + schemas.add(new SchemaInfo(schemaName, properties)); + } + + @Override + public synchronized void dropSchema(ConnectorSession session, String schemaName, boolean cascade) + { + verify(schemas.remove(getSchema(schemaName))); + } + + private synchronized SchemaInfo getSchema(String name) + { + Optional schema = schemas.stream() + .filter(schemaInfo -> schemaInfo.getName().equals(name)) + .findAny(); + if (schema.isEmpty()) { + throw new TrinoException(NOT_FOUND, format("Schema [%s] does not exist", name)); + } + return schema.get(); + } + + @Override + public synchronized ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName, Optional startVersion, Optional endVersion) + { + if (startVersion.isPresent() || endVersion.isPresent()) { + throw new TrinoException(StandardErrorCode.NOT_SUPPORTED, "This connector does not support versioned tables"); + } + SchemaInfo schema = getSchema(tableName.getSchemaName()); + Long id = tableIds.get(tableName); + if (id == null) { + return null; + } + long schemaLimit = (long) schema.getProperties().getOrDefault(SchemaInfo.DEFAULT_LIMIT_PROPERTY, defaultLimit); + long tableLimit = (long) tables.get(id).getProperties().getOrDefault(TableInfo.DEFAULT_LIMIT_PROPERTY, schemaLimit); + return new FakerTableHandle(id, tableName, TupleDomain.all(), tableLimit); + } + + @Override + public synchronized ConnectorTableMetadata getTableMetadata( + ConnectorSession connectorSession, + ConnectorTableHandle connectorTableHandle) + { + FakerTableHandle tableHandle = (FakerTableHandle) connectorTableHandle; + if (tableHandle.id() == null) { + throw new TrinoException(INVALID_TABLE_FUNCTION_INVOCATION, "Table functions are not supported"); + } + return tables.get(tableHandle.id()).getMetadata(); + } + + @Override + public synchronized List listTables(ConnectorSession session, Optional schemaName) + { + return tables.values().stream() + .filter(table -> schemaName.map(table.getSchemaName()::contentEquals).orElse(true)) + .map(TableInfo::getSchemaTableName) + .collect(toImmutableList()); + } + + @Override + public synchronized Map getColumnHandles( + ConnectorSession connectorSession, + ConnectorTableHandle connectorTableHandle) + { + FakerTableHandle tableHandle = (FakerTableHandle) connectorTableHandle; + return tables.get(tableHandle.id()) + .getColumns().stream() + .collect(toImmutableMap(ColumnInfo::getName, ColumnInfo::getHandle)); + } + + @Override + public synchronized ColumnMetadata getColumnMetadata( + ConnectorSession connectorSession, + ConnectorTableHandle connectorTableHandle, + ColumnHandle columnHandle) + { + FakerTableHandle tableHandle = (FakerTableHandle) connectorTableHandle; + if (tableHandle.id() == null) { + throw new TrinoException(INVALID_TABLE_FUNCTION_INVOCATION, "Table functions are not supported"); + } + return tables.get(tableHandle.id()) + .getColumn(columnHandle) + .getMetadata(); + } + + @Override + public synchronized Iterator streamTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + return tables.values().stream() + .filter(table -> prefix.matches(table.getSchemaTableName())) + .map(table -> TableColumnsMetadata.forTable( + table.getSchemaTableName(), + table.getMetadata().getColumns())) + .iterator(); + } + + @Override + public synchronized void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) + { + FakerTableHandle handle = (FakerTableHandle) tableHandle; + TableInfo info = tables.remove(handle.id()); + if (info != null) { + tableIds.remove(info.getSchemaTableName()); + } + } + + @Override + public synchronized void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTableName) + { + checkSchemaExists(newTableName.getSchemaName()); + checkTableNotExists(newTableName); + + FakerTableHandle handle = (FakerTableHandle) tableHandle; + long tableId = handle.id(); + + TableInfo oldInfo = tables.get(tableId); + tables.put(tableId, new TableInfo( + tableId, + newTableName.getSchemaName(), + newTableName.getTableName(), + oldInfo.getColumns(), + oldInfo.getProperties(), + oldInfo.getComment())); + + tableIds.remove(oldInfo.getSchemaTableName()); + tableIds.put(newTableName, tableId); + } + + @Override + public synchronized void setTableProperties(ConnectorSession session, ConnectorTableHandle tableHandle, Map> properties) + { + FakerTableHandle handle = (FakerTableHandle) tableHandle; + long tableId = handle.id(); + + TableInfo oldInfo = tables.get(tableId); + Map newProperties = Stream.concat( + oldInfo.getProperties().entrySet().stream() + .filter(entry -> !properties.containsKey(entry.getKey())), + properties.entrySet().stream() + .filter(entry -> entry.getValue().isPresent())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + tables.put(tableId, oldInfo.withProperties(newProperties)); + } + + @Override + public synchronized void setTableComment(ConnectorSession session, ConnectorTableHandle tableHandle, Optional comment) + { + FakerTableHandle handle = (FakerTableHandle) tableHandle; + long tableId = handle.id(); + + TableInfo oldInfo = requireNonNull(tables.get(tableId), "tableInfo is null"); + tables.put(tableId, oldInfo.withComment(comment)); + } + + @Override + public synchronized void setColumnComment(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle column, Optional comment) + { + FakerTableHandle handle = (FakerTableHandle) tableHandle; + long tableId = handle.id(); + + TableInfo oldInfo = tables.get(tableId); + List columns = oldInfo.getColumns().stream() + .map(columnInfo -> { + if (columnInfo.getHandle().equals(column)) { + return columnInfo.withComment(comment); + } + return columnInfo; + }) + .collect(toImmutableList()); + tables.put(tableId, oldInfo.withColumns(columns)); + } + + @Override + public synchronized void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, SaveMode saveMode) + { + if (saveMode == SaveMode.REPLACE) { + throw new TrinoException(StandardErrorCode.NOT_SUPPORTED, "This connector does not support replacing tables"); + } + ConnectorOutputTableHandle outputTableHandle = beginCreateTable(session, tableMetadata, Optional.empty(), NO_RETRIES, false); + finishCreateTable(session, outputTableHandle, ImmutableList.of(), ImmutableList.of()); + } + + @Override + public synchronized FakerOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout, RetryMode retryMode, boolean replace) + { + if (replace) { + throw new TrinoException(StandardErrorCode.NOT_SUPPORTED, "This connector does not support replacing tables"); + } + SchemaInfo schema = getSchema(tableMetadata.getTable().getSchemaName()); + checkTableNotExists(tableMetadata.getTable()); + long tableId = nextTableId.getAndIncrement(); + + double schemaNullProbability = (double) schema.getProperties().getOrDefault(SchemaInfo.NULL_PROBABILITY_PROPERTY, nullProbability); + double tableNullProbability = (double) tableMetadata.getProperties().getOrDefault(TableInfo.NULL_PROBABILITY_PROPERTY, schemaNullProbability); + + ImmutableList.Builder columns = ImmutableList.builder(); + for (int i = 0; i < tableMetadata.getColumns().size(); i++) { + ColumnMetadata column = tableMetadata.getColumns().get(i); + if (column.getProperties().containsKey(ColumnInfo.GENERATOR_PROPERTY) + && !(column.getType() instanceof CharType || column.getType() instanceof VarcharType || column.getType() instanceof VarbinaryType)) { + throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `generator` property can only be set for CHAR, VARCHAR or VARBINARY columns"); + } + double nullProbability = 0; + if (column.isNullable()) { + nullProbability = (double) column.getProperties().getOrDefault(ColumnInfo.NULL_PROBABILITY_PROPERTY, tableNullProbability); + } + columns.add(new ColumnInfo( + new FakerColumnHandle( + i, + column.getName(), + column.getType(), + nullProbability, + (String) column.getProperties().get(ColumnInfo.GENERATOR_PROPERTY)), + column)); + } + + tableIds.put(tableMetadata.getTable(), tableId); + tables.put(tableId, new TableInfo( + tableId, + tableMetadata.getTable().getSchemaName(), + tableMetadata.getTable().getTableName(), + columns.build(), + tableMetadata.getProperties(), + tableMetadata.getComment())); + + return new FakerOutputTableHandle(tableId, ImmutableSet.copyOf(tableIds.values())); + } + + private synchronized void checkSchemaExists(String schemaName) + { + if (schemas.stream().noneMatch(schema -> schema.getName().equals(schemaName))) { + throw new SchemaNotFoundException(schemaName); + } + } + + private synchronized void checkTableNotExists(SchemaTableName tableName) + { + if (tableIds.containsKey(tableName)) { + throw new TrinoException(ALREADY_EXISTS, format("Table [%s] already exists", tableName)); + } + } + + @Override + public synchronized Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) + { + requireNonNull(tableHandle, "tableHandle is null"); + FakerOutputTableHandle fakerOutputHandle = (FakerOutputTableHandle) tableHandle; + + long tableId = fakerOutputHandle.table(); + + TableInfo info = tables.get(tableId); + requireNonNull(info, "info is null"); + + // TODO ensure fragments is empty? + + tables.put(tableId, new TableInfo(tableId, info.getSchemaName(), info.getTableName(), info.getColumns(), info.getProperties(), info.getComment())); + return Optional.empty(); + } + + @Override + public synchronized Map getSchemaProperties(ConnectorSession session, String schemaName) + { + Optional schema = schemas.stream() + .filter(s -> s.getName().equals(schemaName)) + .findAny(); + if (schema.isEmpty()) { + throw new TrinoException(NOT_FOUND, format("Schema [%s] does not exist", schemaName)); + } + + return schema.get().getProperties(); + } + + @Override + public Optional> applyLimit( + ConnectorSession session, + ConnectorTableHandle table, + long limit) + { + FakerTableHandle fakerTable = (FakerTableHandle) table; + if (fakerTable.limit() == limit) { + return Optional.empty(); + } + return Optional.of(new LimitApplicationResult<>( + fakerTable.withLimit(limit), + false, + true)); + } + + @Override + public Optional> applyFilter( + ConnectorSession session, + ConnectorTableHandle table, + Constraint constraint) + { + FakerTableHandle fakerTable = (FakerTableHandle) table; + + TupleDomain summary = constraint.getSummary(); + if (summary.isAll()) { + return Optional.empty(); + } + // the only reason not to use isNone is so the linter doesn't complain about not checking an Optional + if (summary.getDomains().isEmpty()) { + throw new IllegalArgumentException("summary cannot be none"); + } + + TupleDomain currentConstraint = fakerTable.constraint(); + if (currentConstraint.getDomains().isEmpty()) { + throw new IllegalArgumentException("currentConstraint is none but should be all!"); + } + + // push down everything, unsupported constraints will throw an exception during data generation + boolean anyUpdated = false; + for (Map.Entry entry : summary.getDomains().get().entrySet()) { + FakerColumnHandle column = (FakerColumnHandle) entry.getKey(); + Domain domain = entry.getValue(); + + if (currentConstraint.getDomains().get().containsKey(column)) { + Domain currentDomain = currentConstraint.getDomains().get().get(column); + // it is important to avoid processing same constraint multiple times + // so that planner doesn't get stuck in a loop + if (currentDomain.equals(domain)) { + continue; + } + // TODO write test cases for this, it doesn't seem to work with IS NULL + currentDomain.union(domain); + } + else { + Map domains = new HashMap<>(currentConstraint.getDomains().get()); + domains.put(column, domain); + currentConstraint = TupleDomain.withColumnDomains(domains); + } + anyUpdated = true; + } + if (!anyUpdated) { + return Optional.empty(); + } + + return Optional.of(new ConstraintApplicationResult<>( + fakerTable.withConstraint(currentConstraint), + TupleDomain.all(), + constraint.getExpression(), + true)); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java new file mode 100644 index 00000000000000..aab73558e9d46e --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; +import io.trino.spi.NodeManager; +import io.trino.spi.type.TypeManager; +import jakarta.inject.Inject; + +import static io.airlift.configuration.ConfigBinder.configBinder; +import static java.util.Objects.requireNonNull; + +public class FakerModule + implements Module +{ + private final NodeManager nodeManager; + private final TypeManager typeManager; + + @Inject + public FakerModule(NodeManager nodeManager, TypeManager typeManager) + { + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + public void configure(Binder binder) + { + binder.bind(NodeManager.class).toInstance(nodeManager); + binder.bind(TypeManager.class).toInstance(typeManager); + + binder.bind(FakerConnector.class).in(Scopes.SINGLETON); + binder.bind(FakerMetadata.class).in(Scopes.SINGLETON); + binder.bind(FakerSplitManager.class).in(Scopes.SINGLETON); + binder.bind(FakerPageSourceProvider.class).in(Scopes.SINGLETON); + binder.bind(FakerPageSinkProvider.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(FakerConfig.class); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerOutputTableHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerOutputTableHandle.java new file mode 100644 index 00000000000000..9fc1b1acda818b --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerOutputTableHandle.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.trino.spi.connector.ConnectorOutputTableHandle; + +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public record FakerOutputTableHandle(long table, Set activeTableIds) + implements ConnectorOutputTableHandle +{ + public FakerOutputTableHandle + { + requireNonNull(activeTableIds, "activeTableIds is null"); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSinkProvider.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSinkProvider.java new file mode 100644 index 00000000000000..f1ec8d0613d2a2 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSinkProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.trino.spi.Page; +import io.trino.spi.connector.ConnectorInsertTableHandle; +import io.trino.spi.connector.ConnectorOutputTableHandle; +import io.trino.spi.connector.ConnectorPageSink; +import io.trino.spi.connector.ConnectorPageSinkId; +import io.trino.spi.connector.ConnectorPageSinkProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTransactionHandle; + +import java.util.Collection; +import java.util.concurrent.CompletableFuture; + +import static java.util.concurrent.CompletableFuture.completedFuture; + +public class FakerPageSinkProvider + implements ConnectorPageSinkProvider +{ + @Override + public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle outputTableHandle, ConnectorPageSinkId pageSinkId) + { + return new PageSink(); + } + + @Override + public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle insertTableHandle, ConnectorPageSinkId pageSinkId) + { + return new PageSink(); + } + + private static class PageSink + implements ConnectorPageSink + { + @Override + public CompletableFuture appendPage(Page page) + { + throw new UnsupportedOperationException("The faker connector does not support writes"); + } + + @Override + public CompletableFuture> finish() + { + return completedFuture(ImmutableList.of()); + } + + @Override + public void abort() + { + throw new UnsupportedOperationException("The faker connector does not support writes"); + } + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java new file mode 100644 index 00000000000000..92fa1ffb88e9e6 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSource.java @@ -0,0 +1,713 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.trino.spi.Page; +import io.trino.spi.PageBuilder; +import io.trino.spi.TrinoException; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.Range; +import io.trino.spi.type.CharType; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.Decimals; +import io.trino.spi.type.Int128; +import io.trino.spi.type.Int128Math; +import io.trino.spi.type.LongTimeWithTimeZone; +import io.trino.spi.type.LongTimestamp; +import io.trino.spi.type.LongTimestampWithTimeZone; +import io.trino.spi.type.TimeType; +import io.trino.spi.type.TimeWithTimeZoneType; +import io.trino.spi.type.TimeZoneKey; +import io.trino.spi.type.TimestampType; +import io.trino.spi.type.TimestampWithTimeZoneType; +import io.trino.spi.type.Type; +import io.trino.spi.type.VarbinaryType; +import io.trino.spi.type.VarcharType; +import net.datafaker.Faker; + +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Random; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.spi.StandardErrorCode.INVALID_ROW_FILTER; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateTimeEncoding.packDateTimeWithZone; +import static io.trino.spi.type.DateTimeEncoding.packTimeWithTimeZone; +import static io.trino.spi.type.DateTimeEncoding.unpackMillisUtc; +import static io.trino.spi.type.DateTimeEncoding.unpackOffsetMinutes; +import static io.trino.spi.type.DateTimeEncoding.unpackTimeNanos; +import static io.trino.spi.type.DateTimeEncoding.unpackZoneKey; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.LongTimestampWithTimeZone.fromEpochMillisAndFraction; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_DAY; +import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_DAY; +import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_MICROSECOND; +import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_MILLISECOND; +import static io.trino.spi.type.Timestamps.roundDiv; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.UuidType.UUID; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.type.IntervalDayTimeType.INTERVAL_DAY_TIME; +import static io.trino.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; +import static io.trino.type.IpAddressType.IPADDRESS; +import static java.lang.Float.floatToRawIntBits; +import static java.lang.Float.intBitsToFloat; +import static java.lang.Math.toIntExact; +import static java.lang.System.arraycopy; +import static java.util.Objects.requireNonNull; + +class FakerPageSource + implements ConnectorPageSource +{ + static final long[] POWERS_OF_TEN = { + 1L, + 10L, + 100L, + 1_000L, + 10_000L, + 100_000L, + 1_000_000L, + 10_000_000L, + 100_000_000L, + 1_000_000_000L, + 10_000_000_000L, + 100_000_000_000L, + 1_000_000_000_000L, + 10_000_000_000_000L, + 100_000_000_000_000L, + 1_000_000_000_000_000L, + 10_000_000_000_000_000L, + 100_000_000_000_000_000L, + 1_000_000_000_000_000_000L + }; + + private static final int ROWS_PER_REQUEST = 4096; + + private final Random random; + private final Faker faker; + private final FakerTableHandle table; + private final List generators; + private long completedBytes; + private long completedRows; + + private final PageBuilder pageBuilder; + + private boolean closed; + + FakerPageSource(Faker faker, Random random, List columns, FakerTableHandle table) + { + this.faker = requireNonNull(faker, "faker is null"); + this.random = requireNonNull(random, "random is null"); + this.table = requireNonNull(table, "table is null"); + List types = requireNonNull(columns, "columns is null") + .stream() + .map(FakerColumnHandle::type) + .collect(toImmutableList()); + this.generators = columns + .stream() + .map(column -> constraintedValueGenerator( + column, + table.constraint().getDomains().get().getOrDefault(column, Domain.all(column.type())))) + .collect(toImmutableList()); + // TODO create generators for every column + this.pageBuilder = new PageBuilder(types); + } + + @Override + public long getCompletedBytes() + { + return completedBytes; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + @Override + public boolean isFinished() + { + return closed && pageBuilder.isEmpty(); + } + + @Override + public Page getNextPage() + { + if (!closed) { + int positions = (int) Math.min(table.limit() - completedRows, ROWS_PER_REQUEST); + if (positions <= 0) { + closed = true; + } + else { + pageBuilder.declarePositions(positions); + for (int column = 0; column < generators.size(); column++) { + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(column); + Generator generator = generators.get(column); + for (int i = 0; i < positions; i++) { + generator.accept(blockBuilder); + } + } + completedRows += positions; + } + } + + // only return a page if the buffer is full, or we are finishing + if ((closed && !pageBuilder.isEmpty()) || pageBuilder.isFull()) { + Page page = pageBuilder.build(); + pageBuilder.reset(); + completedBytes += page.getSizeInBytes(); + return page; + } + + return null; + } + + @Override + public long getMemoryUsage() + { + return pageBuilder.getSizeInBytes(); + } + + @Override + public void close() + { + closed = true; + } + + private Generator constraintedValueGenerator(FakerColumnHandle handle, Domain domain) + { + if (domain.isSingleValue()) { + ObjectWriter singleValueWriter = objectWriter(handle.type()); + return (blockBuilder) -> singleValueWriter.accept(blockBuilder, domain.getSingleValue()); + } + if (domain.getValues().isDiscreteSet()) { + List values = domain.getValues().getDiscreteSet(); + if (domain.getValues().getDiscreteValues().isInclusive()) { + ObjectWriter singleValueWriter = objectWriter(handle.type()); + return (blockBuilder) -> singleValueWriter.accept(blockBuilder, values.get(random.nextInt(values.size()))); + } + throw new TrinoException(INVALID_ROW_FILTER, "Generating random values for an exclusive discrete set is not supported"); + } + if (domain.getValues().getRanges().getRangeCount() > 1) { + // this would require calculating weights for each range to retain uniform distribution + throw new TrinoException(INVALID_ROW_FILTER, "Generating random values from more than one range is not supported"); + } + Generator generator = randomValueGenerator(handle, domain.getValues().getRanges().getSpan()); + if (handle.nullProbability() == 0) { + return generator; + } + return (blockBuilder) -> { + if (random.nextDouble() <= handle.nullProbability()) { + blockBuilder.appendNull(); + } + else { + generator.accept(blockBuilder); + } + }; + } + + private Generator randomValueGenerator(FakerColumnHandle handle, Range range) + { + if (handle.generator() != null) { + if (!range.isAll()) { + throw new TrinoException(INVALID_ROW_FILTER, "Predicates for columns with a generator expression are not supported"); + } + return (blockBuilder) -> VARCHAR.writeSlice(blockBuilder, Slices.utf8Slice(faker.expression(handle.generator()))); + } + Type type = handle.type(); + // check every type in order defined in StandardTypes + if (BIGINT.equals(type)) { + return (blockBuilder) -> BIGINT.writeLong(blockBuilder, generateLong(range, 1)); + } + if (INTEGER.equals(type)) { + return (blockBuilder) -> INTEGER.writeLong(blockBuilder, generateInt(range)); + } + if (SMALLINT.equals(type)) { + return (blockBuilder) -> SMALLINT.writeLong(blockBuilder, generateShort(range)); + } + if (TINYINT.equals(type)) { + return (blockBuilder) -> TINYINT.writeLong(blockBuilder, generateTiny(range)); + } + if (BOOLEAN.equals(type)) { + if (!range.isAll()) { + throw new TrinoException(INVALID_ROW_FILTER, "Range or not a single value predicates for boolean columns are not supported"); + } + return (blockBuilder) -> BOOLEAN.writeBoolean(blockBuilder, random.nextBoolean()); + } + if (DATE.equals(type)) { + return (blockBuilder) -> DATE.writeLong(blockBuilder, generateInt(range)); + } + if (type instanceof DecimalType decimalType) { + return decimalGenerator(range, decimalType); + } + if (REAL.equals(type)) { + return (blockBuilder) -> REAL.writeLong(blockBuilder, floatToRawIntBits(generateFloat(range))); + } + if (DOUBLE.equals(type)) { + return (blockBuilder) -> DOUBLE.writeDouble(blockBuilder, generateDouble(range)); + } + // not supported: HYPER_LOG_LOG, QDIGEST, TDIGEST, P4_HYPER_LOG_LOG + if (INTERVAL_DAY_TIME.equals(type)) { + return (blockBuilder) -> INTERVAL_DAY_TIME.writeLong(blockBuilder, generateLong(range, 1)); + } + if (INTERVAL_YEAR_MONTH.equals(type)) { + return (blockBuilder) -> INTERVAL_YEAR_MONTH.writeLong(blockBuilder, generateInt(range)); + } + if (type instanceof TimestampType) { + return timestampGenerator(range, (TimestampType) type); + } + if (type instanceof TimestampWithTimeZoneType) { + return timestampWithTimeZoneGenerator(range, (TimestampWithTimeZoneType) type); + } + if (type instanceof TimeType timeType) { + long factor = POWERS_OF_TEN[12 - timeType.getPrecision()]; + return (blockBuilder) -> timeType.writeLong(blockBuilder, generateLongDefaults(range, factor, 0, PICOSECONDS_PER_DAY)); + } + if (type instanceof TimeWithTimeZoneType) { + return timeWithTimeZoneGenerator(range, (TimeWithTimeZoneType) type); + } + if (type instanceof VarbinaryType varType) { + if (!range.isAll()) { + throw new TrinoException(INVALID_ROW_FILTER, "Predicates for varbinary columns are not supported"); + } + return (blockBuilder) -> varType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().sentence(3 + random.nextInt(38)))); + } + if (type instanceof VarcharType varcharType) { + if (!range.isAll()) { + throw new TrinoException(INVALID_ROW_FILTER, "Predicates for varchar columns are not supported"); + } + if (varcharType.getLength().isPresent()) { + int length = varcharType.getLength().get(); + return (blockBuilder) -> varcharType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().maxLengthSentence(random.nextInt(length)))); + } + return (blockBuilder) -> varcharType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().sentence(3 + random.nextInt(38)))); + } + if (type instanceof CharType charType) { + if (!range.isAll()) { + throw new TrinoException(INVALID_ROW_FILTER, "Predicates for char columns are not supported"); + } + return (blockBuilder) -> charType.writeSlice(blockBuilder, Slices.utf8Slice(faker.lorem().maxLengthSentence(charType.getLength()))); + } + // not supported: ROW, ARRAY, MAP, JSON + if (IPADDRESS.equals(type)) { + return generateIpV4(range); + } + // not supported: GEOMETRY + if (UUID.equals(type)) { + return generateUUID(range); + } + + throw new IllegalArgumentException("Unsupported type " + type); + } + + private ObjectWriter objectWriter(Type type) + { + // check every type in order defined in StandardTypes + if (BIGINT.equals(type)) { + return (blockBuilder, value) -> BIGINT.writeLong(blockBuilder, (Long) value); + } + if (INTEGER.equals(type)) { + return (blockBuilder, value) -> INTEGER.writeLong(blockBuilder, (Long) value); + } + if (SMALLINT.equals(type)) { + return (blockBuilder, value) -> SMALLINT.writeLong(blockBuilder, (Long) value); + } + if (TINYINT.equals(type)) { + return (blockBuilder, value) -> TINYINT.writeLong(blockBuilder, (Long) value); + } + if (BOOLEAN.equals(type)) { + return (blockBuilder, value) -> BOOLEAN.writeBoolean(blockBuilder, (Boolean) value); + } + if (DATE.equals(type)) { + return (blockBuilder, value) -> DATE.writeLong(blockBuilder, (Long) value); + } + if (type instanceof DecimalType decimalType) { + if (decimalType.isShort()) { + return (blockBuilder, value) -> decimalType.writeLong(blockBuilder, (Long) value); + } + else { + return decimalType::writeObject; + } + } + if (REAL.equals(type) || DOUBLE.equals(type)) { + return (blockBuilder, value) -> REAL.writeDouble(blockBuilder, (Double) value); + } + // not supported: HYPER_LOG_LOG, QDIGEST, TDIGEST, P4_HYPER_LOG_LOG + if (INTERVAL_DAY_TIME.equals(type)) { + return (blockBuilder, value) -> INTERVAL_DAY_TIME.writeLong(blockBuilder, (Long) value); + } + if (INTERVAL_YEAR_MONTH.equals(type)) { + return (blockBuilder, value) -> INTERVAL_YEAR_MONTH.writeLong(blockBuilder, (Long) value); + } + if (type instanceof TimestampType tzType) { + if (tzType.isShort()) { + return (blockBuilder, value) -> tzType.writeLong(blockBuilder, (Long) value); + } + else { + return tzType::writeObject; + } + } + if (type instanceof TimestampWithTimeZoneType tzType) { + if (tzType.isShort()) { + return (blockBuilder, value) -> tzType.writeLong(blockBuilder, (Long) value); + } + else { + return tzType::writeObject; + } + } + if (type instanceof TimeType timeType) { + return (blockBuilder, value) -> timeType.writeLong(blockBuilder, (Long) value); + } + if (type instanceof TimeWithTimeZoneType tzType) { + if (tzType.isShort()) { + return (blockBuilder, value) -> tzType.writeLong(blockBuilder, (Long) value); + } + else { + return tzType::writeObject; + } + } + if (type instanceof VarbinaryType varType) { + return (blockBuilder, value) -> varType.writeSlice(blockBuilder, (Slice) value); + } + if (type instanceof VarcharType varType) { + return (blockBuilder, value) -> varType.writeSlice(blockBuilder, (Slice) value); + } + if (type instanceof CharType charType) { + return (blockBuilder, value) -> charType.writeSlice(blockBuilder, (Slice) value); + } + // not supported: ROW, ARRAY, MAP, JSON + if (IPADDRESS.equals(type)) { + return (blockBuilder, value) -> IPADDRESS.writeSlice(blockBuilder, (Slice) value); + } + // not supported: GEOMETRY + if (UUID.equals(type)) { + return (blockBuilder, value) -> UUID.writeSlice(blockBuilder, (Slice) value); + } + + throw new IllegalArgumentException("Unsupported type " + type); + } + + private long generateLong(Range range, long factor) + { + return generateLongDefaults(range, factor, Long.MIN_VALUE, Long.MAX_VALUE); + } + + private long generateLongDefaults(Range range, long factor, long min, long max) + { + return faker.number().numberBetween( + roundDiv((long) range.getLowValue().orElse(min), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + // TODO does the inclusion only apply to positive numbers? + roundDiv((long) range.getHighValue().orElse(max), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)) * factor; + } + + private int generateInt(Range range) + { + return (int) faker.number().numberBetween( + (long) range.getLowValue().orElse((long) Integer.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + (long) range.getHighValue().orElse((long) Integer.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); + } + + private short generateShort(Range range) + { + return (short) faker.number().numberBetween( + (long) range.getLowValue().orElse((long) Short.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + (long) range.getHighValue().orElse((long) Short.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); + } + + private byte generateTiny(Range range) + { + return (byte) faker.number().numberBetween( + (long) range.getLowValue().orElse((long) Byte.MIN_VALUE) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + (long) range.getHighValue().orElse((long) Byte.MAX_VALUE) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)); + } + + private float generateFloat(Range range) + { + // TODO normalize ranges in applyFilter, so they always have bounds + float minValue = range.getLowValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MIN_VALUE); + if (!range.isLowUnbounded() && !range.isLowInclusive()) { + minValue = Math.nextUp(minValue); + } + float maxValue = range.getHighValue().map(v -> intBitsToFloat(toIntExact((long) v))).orElse(Float.MAX_VALUE); + if (!range.isHighUnbounded() && !range.isHighInclusive()) { + maxValue = Math.nextDown(maxValue); + } + return minValue + (maxValue - minValue) * random.nextFloat(); + } + + private double generateDouble(Range range) + { + double minValue = (double) range.getLowValue().orElse(Double.MIN_VALUE); + if (!range.isLowUnbounded() && !range.isLowInclusive()) { + minValue = Math.nextUp(minValue); + } + double maxValue = (double) range.getHighValue().orElse(Double.MAX_VALUE); + if (!range.isHighUnbounded() && !range.isHighInclusive()) { + maxValue = Math.nextDown(maxValue); + } + return minValue + (maxValue - minValue) * random.nextDouble(); + } + + private Generator decimalGenerator(Range range, DecimalType decimalType) + { + if (decimalType.isShort()) { + long min = -999999999999999999L / POWERS_OF_TEN[18 - decimalType.getPrecision()]; + long max = 999999999999999999L / POWERS_OF_TEN[18 - decimalType.getPrecision()]; + return (blockBuilder) -> decimalType.writeLong(blockBuilder, generateLongDefaults(range, 1, min, max)); + } + Int128 low = (Int128) range.getLowValue().orElse(Decimals.MIN_UNSCALED_DECIMAL); + Int128 high = (Int128) range.getHighValue().orElse(Decimals.MAX_UNSCALED_DECIMAL); + if (!range.isLowUnbounded() && !range.isLowInclusive()) { + long[] result = new long[2]; + Int128Math.add(low.getHigh(), low.getLow(), 0, 1, result, 0); + low = Int128.valueOf(result); + } + if (!range.isHighUnbounded() && range.isHighInclusive()) { + long[] result = new long[2]; + Int128Math.add(high.getHigh(), high.getLow(), 0, 1, result, 0); + high = Int128.valueOf(result); + } + + BigInteger currentRange = BigInteger.valueOf(Long.MAX_VALUE); + BigInteger desiredRange = high.toBigInteger().subtract(low.toBigInteger()); + Int128 finalLow = low; + return (blockBuilder) -> decimalType.writeObject(blockBuilder, Int128.valueOf( + new BigInteger(63, random).multiply(desiredRange).divide(currentRange).add(finalLow.toBigInteger()))); + } + + private Generator timestampGenerator(Range range, TimestampType tzType) + { + if (tzType.isShort()) { + long factor = POWERS_OF_TEN[6 - tzType.getPrecision()]; + return (blockBuilder) -> tzType.writeLong(blockBuilder, generateLong(range, factor)); + } + LongTimestamp low = (LongTimestamp) range.getLowValue() + .orElse(new LongTimestamp(Long.MIN_VALUE, 0)); + LongTimestamp high = (LongTimestamp) range.getHighValue() + .orElse(new LongTimestamp(Long.MAX_VALUE, PICOSECONDS_PER_MICROSECOND - 1)); + int factor; + if (tzType.getPrecision() <= 6) { + factor = (int) POWERS_OF_TEN[6 - tzType.getPrecision()]; + low = new LongTimestamp( + roundDiv(low.getEpochMicros(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + 0); + high = new LongTimestamp( + roundDiv(high.getEpochMicros(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0), + 0); + } + else { + factor = (int) POWERS_OF_TEN[12 - tzType.getPrecision()]; + int lowPicosOfMicro = roundDiv(low.getPicosOfMicro(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + low = new LongTimestamp( + low.getEpochMicros() - (lowPicosOfMicro < 0 ? 1 : 0), + (lowPicosOfMicro + factor) % factor); + int highPicosOfMicro = roundDiv(high.getPicosOfMicro(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + high = new LongTimestamp( + high.getEpochMicros() + (highPicosOfMicro > factor ? 1 : 0), + highPicosOfMicro % factor); + } + LongTimestamp finalLow = low; + LongTimestamp finalHigh = high; + return (blockBuilder) -> { + long epochMicros = faker.number().numberBetween(finalLow.getEpochMicros(), finalHigh.getEpochMicros()); + if (tzType.getPrecision() <= 6) { + epochMicros *= factor; + tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros * factor, 0)); + return; + } + int picosOfMicro; + if (epochMicros == finalLow.getEpochMicros()) { + picosOfMicro = faker.number().numberBetween( + finalLow.getPicosOfMicro(), + finalLow.getEpochMicros() == finalHigh.getEpochMicros() ? + finalHigh.getPicosOfMicro() + : (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); + } + else if (epochMicros == finalHigh.getEpochMicros()) { + picosOfMicro = faker.number().numberBetween(0, finalHigh.getPicosOfMicro()); + } + else { + picosOfMicro = faker.number().numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 6] - 1); + } + tzType.writeObject(blockBuilder, new LongTimestamp(epochMicros, picosOfMicro * factor)); + }; + } + + private Generator timestampWithTimeZoneGenerator(Range range, TimestampWithTimeZoneType tzType) + { + if (tzType.isShort()) { + TimeZoneKey defaultTZ = range.getLowValue() + .map(v -> unpackZoneKey((long) v)) + .orElse(range.getHighValue() + .map(v -> unpackZoneKey((long) v)) + .orElse(TimeZoneKey.UTC_KEY)); + long factor = POWERS_OF_TEN[3 - tzType.getPrecision()]; + return (blockBuilder) -> { + long millis = faker.number().numberBetween( + roundDiv(unpackMillisUtc((long) range.getLowValue().orElse(Long.MIN_VALUE)), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0), + roundDiv(unpackMillisUtc((long) range.getHighValue().orElse(Long.MAX_VALUE)), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0)) * factor; + tzType.writeLong(blockBuilder, packDateTimeWithZone(millis, defaultTZ)); + }; + } + short defaultTZ = range.getLowValue() + .map(v -> ((LongTimestampWithTimeZone) v).getTimeZoneKey()) + .orElse(range.getHighValue() + .map(v -> ((LongTimestampWithTimeZone) v).getTimeZoneKey()) + .orElse(TimeZoneKey.UTC_KEY.getKey())); + LongTimestampWithTimeZone low = (LongTimestampWithTimeZone) range.getLowValue() + .orElse(fromEpochMillisAndFraction(Long.MIN_VALUE >> 12, 0, defaultTZ)); + LongTimestampWithTimeZone high = (LongTimestampWithTimeZone) range.getHighValue() + .orElse(fromEpochMillisAndFraction(Long.MAX_VALUE >> 12, PICOSECONDS_PER_MILLISECOND - 1, defaultTZ)); + if (low.getTimeZoneKey() != high.getTimeZoneKey()) { + throw new TrinoException(INVALID_ROW_FILTER, "Range boundaries for timestamp with time zone columns must have the same time zone"); + } + int factor = (int) POWERS_OF_TEN[12 - tzType.getPrecision()]; + int lowPicosOfMilli = roundDiv(low.getPicosOfMilli(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + low = fromEpochMillisAndFraction( + low.getEpochMillis() - (lowPicosOfMilli < 0 ? 1 : 0), + (lowPicosOfMilli + factor) % factor, + low.getTimeZoneKey()); + int highPicosOfMilli = roundDiv(high.getPicosOfMilli(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + high = fromEpochMillisAndFraction( + high.getEpochMillis() + (highPicosOfMilli > factor ? 1 : 0), + highPicosOfMilli % factor, + high.getTimeZoneKey()); + LongTimestampWithTimeZone finalLow = low; + LongTimestampWithTimeZone finalHigh = high; + return (blockBuilder) -> { + long millis = faker.number().numberBetween(finalLow.getEpochMillis(), finalHigh.getEpochMillis()); + int picosOfMilli; + if (millis == finalLow.getEpochMillis()) { + picosOfMilli = faker.number().numberBetween( + finalLow.getPicosOfMilli(), + finalLow.getEpochMillis() == finalHigh.getEpochMillis() ? + finalHigh.getPicosOfMilli() + : (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1); + } + else if (millis == finalHigh.getEpochMillis()) { + picosOfMilli = faker.number().numberBetween(0, finalHigh.getPicosOfMilli()); + } + else { + picosOfMilli = faker.number().numberBetween(0, (int) POWERS_OF_TEN[tzType.getPrecision() - 3] - 1); + } + tzType.writeObject(blockBuilder, fromEpochMillisAndFraction(millis, picosOfMilli * factor, defaultTZ)); + }; + } + + private Generator timeWithTimeZoneGenerator(Range range, TimeWithTimeZoneType timeType) + { + if (timeType.isShort()) { + int offsetMinutes = range.getLowValue() + .map(v -> unpackOffsetMinutes((long) v)) + .orElse(range.getHighValue() + .map(v -> unpackOffsetMinutes((long) v)) + .orElse(0)); + long factor = POWERS_OF_TEN[9 - timeType.getPrecision()]; + long low = roundDiv(range.getLowValue().map(v -> unpackTimeNanos((long) v)).orElse(0L), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + long high = roundDiv(range.getHighValue().map(v -> unpackTimeNanos((long) v)).orElse(NANOSECONDS_PER_DAY), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + return (blockBuilder) -> { + long nanos = faker.number().numberBetween(low, high) * factor; + timeType.writeLong(blockBuilder, packTimeWithTimeZone(nanos, offsetMinutes)); + }; + } + int offsetMinutes = range.getLowValue() + .map(v -> ((LongTimeWithTimeZone) v).getOffsetMinutes()) + .orElse(range.getHighValue() + .map(v -> ((LongTimeWithTimeZone) v).getOffsetMinutes()) + .orElse(0)); + LongTimeWithTimeZone low = (LongTimeWithTimeZone) range.getLowValue() + .orElse(new LongTimeWithTimeZone(0, offsetMinutes)); + LongTimeWithTimeZone high = (LongTimeWithTimeZone) range.getHighValue() + .orElse(new LongTimeWithTimeZone(PICOSECONDS_PER_DAY, offsetMinutes)); + if (low.getOffsetMinutes() != high.getOffsetMinutes()) { + throw new TrinoException(INVALID_ROW_FILTER, "Range boundaries for time with time zone columns must have the same time zone"); + } + int factor = (int) POWERS_OF_TEN[12 - timeType.getPrecision()]; + long longLow = roundDiv(low.getPicoseconds(), factor) + (!range.isLowUnbounded() && !range.isLowInclusive() ? 1 : 0); + long longHigh = roundDiv(high.getPicoseconds(), factor) + (!range.isHighUnbounded() && range.isHighInclusive() ? 1 : 0); + return (blockBuilder) -> { + long picoseconds = faker.number().numberBetween(longLow, longHigh) * factor; + timeType.writeObject(blockBuilder, new LongTimeWithTimeZone(picoseconds, offsetMinutes)); + }; + } + + private Generator generateIpV4(Range range) + { + if (!range.isAll()) { + throw new TrinoException(INVALID_ROW_FILTER, "Predicates for ipaddress columns are not supported"); + } + return (blockBuilder) -> { + byte[] address; + try { + address = Inet4Address.getByAddress(new byte[] { + (byte) (random.nextInt(254) + 2), + (byte) (random.nextInt(254) + 2), + (byte) (random.nextInt(254) + 2), + (byte) (random.nextInt(254) + 2)}).getAddress(); + } + catch (UnknownHostException e) { + // ignore + blockBuilder.appendNull(); + return; + } + + byte[] bytes = new byte[16]; + bytes[10] = (byte) 0xff; + bytes[11] = (byte) 0xff; + arraycopy(address, 0, bytes, 12, 4); + + IPADDRESS.writeSlice(blockBuilder, Slices.wrappedBuffer(bytes, 0, 16)); + }; + } + + private Generator generateUUID(Range range) + { + if (!range.isAll()) { + throw new TrinoException(INVALID_ROW_FILTER, "Predicates for uuid columns are not supported"); + } + return (blockBuilder) -> { + java.util.UUID uuid = java.util.UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.wrap(new byte[16]); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + UUID.writeSlice(blockBuilder, Slices.wrappedBuffer(bb.array(), 0, 16)); + }; + } + + @FunctionalInterface + private interface ObjectWriter + { + void accept(BlockBuilder blockBuilder, Object value); + } + + @FunctionalInterface + private interface Generator + { + void accept(BlockBuilder blockBuilder); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java new file mode 100644 index 00000000000000..fad2ff6958da68 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPageSourceProvider.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.DynamicFilter; +import jakarta.inject.Inject; +import net.datafaker.Faker; + +import java.util.List; +import java.util.Random; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +public class FakerPageSourceProvider + implements ConnectorPageSourceProvider +{ + private final Random random; + private final Faker faker; + + @Inject + public FakerPageSourceProvider() + { + random = new Random(); + faker = new Faker(random); + } + + @Override + public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorSplit split, + ConnectorTableHandle table, + List columns, + DynamicFilter dynamicFilter) + { + List handles = columns + .stream() + .map(FakerColumnHandle.class::cast) + .collect(toImmutableList()); + + FakerTableHandle fakerTable = (FakerTableHandle) table; + + return new FakerPageSource(faker, random, handles, fakerTable); + } + + public void validateGenerator(String generator) + { + faker.expression(generator); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPlugin.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPlugin.java new file mode 100644 index 00000000000000..c3a0fa96a884bf --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerPlugin.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; + +import java.util.Set; + +public class FakerPlugin + implements Plugin +{ + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new FakerConnectorFactory("faker")); + } + + @Override + public Set> getFunctions() + { + return ImmutableSet.>builder() + .add(FakerFunctions.class) + .build(); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplit.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplit.java new file mode 100644 index 00000000000000..debaa0226e84c8 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplit.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.trino.spi.connector.ConnectorSplit; + +public record FakerSplit() + implements ConnectorSplit +{ +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplitManager.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplitManager.java new file mode 100644 index 00000000000000..8d560ef32ea7c5 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerSplitManager.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorSplitSource; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.connector.Constraint; +import io.trino.spi.connector.DynamicFilter; +import io.trino.spi.connector.FixedSplitSource; + +import java.util.List; + +public class FakerSplitManager + implements ConnectorSplitManager +{ + @Override + public ConnectorSplitSource getSplits( + ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorTableHandle table, + DynamicFilter dynamicFilter, + Constraint constraint) + { + return new FixedSplitSource(List.of(new FakerSplit())); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java new file mode 100644 index 00000000000000..9ed8bd7bf09142 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTableHandle.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.predicate.TupleDomain; + +import static java.util.Objects.requireNonNull; + +public record FakerTableHandle(Long id, SchemaTableName schemaTableName, TupleDomain constraint, long limit) + implements ConnectorTableHandle +{ + public FakerTableHandle + { + requireNonNull(id, "id is null"); + requireNonNull(schemaTableName, "schemaTableName is null"); + requireNonNull(constraint, "constraint is null"); + } + + public FakerTableHandle withConstraint(TupleDomain constraint) + { + return new FakerTableHandle(id, schemaTableName, constraint, limit); + } + + public FakerTableHandle withLimit(long limit) + { + return new FakerTableHandle(id, schemaTableName, constraint, limit); + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTransactionHandle.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTransactionHandle.java new file mode 100644 index 00000000000000..4ef4b9c9e83248 --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerTransactionHandle.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import io.trino.spi.connector.ConnectorTransactionHandle; + +public enum FakerTransactionHandle + implements ConnectorTransactionHandle +{ + INSTANCE +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/SchemaInfo.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/SchemaInfo.java new file mode 100644 index 00000000000000..001bc3b2edb6ee --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/SchemaInfo.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +public class SchemaInfo +{ + private final String name; + private final Map properties; + + public static final String NULL_PROBABILITY_PROPERTY = "null_probability"; + public static final String DEFAULT_LIMIT_PROPERTY = "default_limit"; + + public SchemaInfo(String name) + { + this(name, Map.of()); + } + + public SchemaInfo(String name, Map properties) + { + this.name = requireNonNull(name, "name is null"); + this.properties = ImmutableMap.copyOf(requireNonNull(properties, "properties is null")); + } + + public String getName() + { + return name; + } + + public Map getProperties() + { + return properties; + } +} diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/TableInfo.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/TableInfo.java new file mode 100644 index 00000000000000..341a1426a4cf5e --- /dev/null +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/TableInfo.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ConnectorTableMetadata; +import io.trino.spi.connector.SchemaTableName; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class TableInfo +{ + private final long id; + private final String schemaName; + private final String tableName; + private List columns; + private Map properties; + private Optional comment; + + public static final String NULL_PROBABILITY_PROPERTY = "null_probability"; + public static final String DEFAULT_LIMIT_PROPERTY = "default_limit"; + + public TableInfo(long id, String schemaName, String tableName, List columns, Map properties, Optional comment) + { + this.id = id; + this.schemaName = requireNonNull(schemaName, "schemaName is null"); + this.tableName = requireNonNull(tableName, "tableName is null"); + this.columns = ImmutableList.copyOf(columns); + this.properties = ImmutableMap.copyOf(requireNonNull(properties, "properties is null")); + this.comment = requireNonNull(comment, "comment is null"); + } + + public long getId() + { + return id; + } + + public String getSchemaName() + { + return schemaName; + } + + public String getTableName() + { + return tableName; + } + + public SchemaTableName getSchemaTableName() + { + return new SchemaTableName(schemaName, tableName); + } + + public ConnectorTableMetadata getMetadata() + { + return new ConnectorTableMetadata( + new SchemaTableName(schemaName, tableName), + columns.stream() + .map(ColumnInfo::getMetadata) + .collect(toImmutableList()), + properties, + comment); + } + + public List getColumns() + { + return columns; + } + + public ColumnInfo getColumn(ColumnHandle handle) + { + return columns.stream() + .filter(column -> column.getHandle().equals(handle)) + .findFirst() + .get(); + } + + public Map getProperties() + { + return properties; + } + + public Optional getComment() + { + return comment; + } + + public TableInfo withColumns(List columns) + { + return new TableInfo(id, schemaName, tableName, columns, properties, comment); + } + + public TableInfo withProperties(Map properties) + { + return new TableInfo(id, schemaName, tableName, columns, properties, comment); + } + + public TableInfo withComment(Optional comment) + { + return new TableInfo(id, schemaName, tableName, columns, properties, comment); + } +} diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java new file mode 100644 index 00000000000000..60b686255e8489 --- /dev/null +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/FakerQueryRunner.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.faker; + +import io.airlift.log.Level; +import io.airlift.log.Logger; +import io.airlift.log.Logging; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; + +import java.util.Map; + +import static io.airlift.testing.Closeables.closeAllSuppress; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.util.Objects.requireNonNullElse; + +public class FakerQueryRunner +{ + private static final String CATALOG = "faker"; + + private FakerQueryRunner() {} + + public static Builder builder() + { + return new Builder(); + } + + public static class Builder + extends DistributedQueryRunner.Builder + { + protected Builder() + { + super(testSessionBuilder() + .setCatalog(CATALOG) + .setSchema("default") + .build()); + } + + @Override + public DistributedQueryRunner build() + throws Exception + { + setWorkerCount(0); + DistributedQueryRunner queryRunner = super.build(); + + try { + queryRunner.installPlugin(new FakerPlugin()); + queryRunner.createCatalog(CATALOG, "faker"); + + return queryRunner; + } + catch (Exception e) { + closeAllSuppress(e, queryRunner); + throw e; + } + } + } + + public static void main(String[] args) + throws Exception + { + Logging logger = Logging.initialize(); + logger.setLevel("io.trino", Level.INFO); + + QueryRunner queryRunner = builder() + .setExtraProperties(Map.of( + "http-server.http.port", requireNonNullElse(System.getenv("TRINO_PORT"), "8080"))) + .build(); + + Logger log = Logger.get(FakerQueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } +} diff --git a/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java new file mode 100644 index 00000000000000..a7519162be2def --- /dev/null +++ b/plugin/trino-faker/src/test/java/io/trino/plugin/faker/TestFakerQueries.java @@ -0,0 +1,607 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.plugin.faker; + +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.Test; + +public class TestFakerQueries + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return FakerQueryRunner.builder().build(); + } + + @Test + public void showTables() + { + assertQuery("SHOW SCHEMAS FROM faker", "VALUES 'default', 'information_schema'"); + assertUpdate("CREATE TABLE faker.default.test (id INTEGER, name VARCHAR)"); + assertTableColumnNames("faker.default.test", "id", "name"); + } + + @Test + public void selectFromTable() + { + @Language("SQL") + String tableQuery = """ + CREATE TABLE faker.default.all_types ( + rnd_bigint bigint NOT NULL, + rnd_integer integer NOT NULL, + rnd_smallint smallint NOT NULL, + rnd_tinyint tinyint NOT NULL, + rnd_boolean boolean NOT NULL, + rnd_date date NOT NULL, + rnd_decimal1 decimal NOT NULL, + rnd_decimal2 decimal(18,5) NOT NULL, + rnd_decimal3 decimal(38,0) NOT NULL, + rnd_decimal4 decimal(38,38) NOT NULL, + rnd_decimal5 decimal(5,2) NOT NULL, + rnd_real real NOT NULL, + rnd_double double NOT NULL, + rnd_interval_day_time interval day to second NOT NULL, + rnd_interval_year interval year to month NOT NULL, + rnd_timestamp timestamp NOT NULL, + rnd_timestamp0 timestamp(0) NOT NULL, + rnd_timestamp6 timestamp(6) NOT NULL, + rnd_timestamp9 timestamp(9) NOT NULL, + rnd_timestamptz timestamp with time zone NOT NULL, + rnd_timestamptz0 timestamp(0) with time zone NOT NULL, + rnd_timestamptz6 timestamp(6) with time zone NOT NULL, + rnd_timestamptz9 timestamp(9) with time zone NOT NULL, + rnd_time time NOT NULL, + rnd_time0 time(0) NOT NULL, + rnd_time6 time(6) NOT NULL, + rnd_time9 time(9) NOT NULL, + rnd_timetz time with time zone NOT NULL, + rnd_timetz0 time(0) with time zone NOT NULL, + rnd_timetz6 time(6) with time zone NOT NULL, + rnd_timetz9 time(9) with time zone NOT NULL, + rnd_timetz12 time(12) with time zone NOT NULL, + rnd_varbinary varbinary NOT NULL, + rnd_varchar varchar NOT NULL, + rnd_nvarchar varchar(1000) NOT NULL, + rnd_char char NOT NULL, + rnd_nchar char(1000) NOT NULL, + rnd_ipaddress ipaddress NOT NULL, + rnd_uuid uuid NOT NULL)"""; + assertUpdate(tableQuery); + + @Language("SQL") + String testQuery = """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(rnd_smallint), + count(rnd_tinyint), + count(distinct rnd_boolean), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(rnd_decimal5), + count(rnd_real), + count(distinct rnd_double), + count(distinct rnd_interval_day_time), + count(distinct rnd_interval_year), + count(distinct rnd_timestamp), + count(distinct rnd_timestamp0), + count(distinct rnd_timestamp6), + count(distinct rnd_timestamp9), + count(distinct rnd_timestamptz), + count(distinct rnd_timestamptz0), + count(distinct rnd_timestamptz6), + count(distinct rnd_timestamptz9), + count(rnd_time), + count(rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(rnd_timetz), + count(rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9), + count(distinct rnd_varbinary), + count(rnd_varchar), + count(rnd_nvarchar), + count(distinct rnd_char), + count(distinct rnd_nchar), + count(distinct rnd_ipaddress), + count(distinct rnd_uuid) + FROM all_types"""; + assertQuery(testQuery, + """ + VALUES ( + 1000, + 1000, + 1000, + 1000, + -- boolean, date + 2, + 1000, + -- decimal + 1000, + 1000, + 1000, + 1000, + 1000, + -- real, double + 1000, + 1000, + -- intervals + 1000, + 1000, + -- timestamps + 1000, + 1000, + 1000, + 1000, + -- timestamps with time zone + 1000, + 1000, + 1000, + 1000, + -- time + 1000, + 1000, + 1000, + 1000, + -- time with time zone + 1000, + 1000, + 1000, + 1000, + -- varbinary, varchar, nvarchar, char, nchar + 1000, + 1000, + 1000, + 19, + 1000, + -- ip address, uuid + 1000, + 1000)"""); + assertUpdate("DROP TABLE faker.default.all_types"); + } + + @Test + public void selectLimit() + { + @Language("SQL") + String tableQuery = "CREATE TABLE faker.default.single_column (rnd_bigint bigint NOT NULL)"; + assertUpdate(tableQuery); + + @Language("SQL") + String testQuery = "SELECT count(rnd_bigint) FROM (SELECT rnd_bigint FROM single_column LIMIT 5) a"; + assertQuery(testQuery, "VALUES (5)"); + + testQuery = "SELECT count(distinct rnd_bigint) FROM single_column LIMIT 5"; + assertQuery(testQuery, "VALUES (1000)"); + assertUpdate("DROP TABLE faker.default.single_column"); + } + + @Test + public void selectDefaultTableLimit() + { + @Language("SQL") + String tableQuery = "CREATE TABLE faker.default.default_table_limit (rnd_bigint bigint NOT NULL) WITH (default_limit = 100)"; + assertUpdate(tableQuery); + + @Language("SQL") + String testQuery = "SELECT count(distinct rnd_bigint) FROM default_table_limit"; + assertQuery(testQuery, "VALUES (100)"); + + assertUpdate("DROP TABLE faker.default.default_table_limit"); + } + + @Test + public void selectOnlyNulls() + { + @Language("SQL") + String tableQuery = "CREATE TABLE faker.default.only_nulls (rnd_bigint bigint) WITH (null_probability = 1.0)"; + assertUpdate(tableQuery); + tableQuery = "CREATE TABLE faker.default.only_nulls_column (rnd_bigint bigint WITH (null_probability = 1.0))"; + assertUpdate(tableQuery); + + @Language("SQL") + String testQuery = "SELECT count(distinct rnd_bigint) FROM only_nulls"; + assertQuery(testQuery, "VALUES (0)"); + testQuery = "SELECT count(distinct rnd_bigint) FROM only_nulls_column"; + assertQuery(testQuery, "VALUES (0)"); + + assertUpdate("DROP TABLE faker.default.only_nulls"); + assertUpdate("DROP TABLE faker.default.only_nulls_column"); + } + + @Test + public void selectGenerator() + { + @Language("SQL") + String tableQuery = "CREATE TABLE faker.default.generators (" + + "name VARCHAR NOT NULL WITH (generator = '#{Name.first_name} #{Name.last_name}'), " + + "age_years INTEGER NOT NULL" + + ")"; + assertUpdate(tableQuery); + + @Language("SQL") + String testQuery = "SELECT count(name) FILTER (WHERE LENGTH(name) - LENGTH(REPLACE(name, ' ', '')) = 1) FROM generators"; + assertQuery(testQuery, "VALUES (1000)"); + + assertUpdate("DROP TABLE faker.default.generators"); + } + + @Test + public void selectFunctions() + { + @Language("SQL") + String testQuery = "SELECT random_expression('#{options.option ''a'', ''b''}') IN ('a', 'b')"; + assertQuery(testQuery, "VALUES (true)"); + } + + @Test + public void selectRange() + { + @Language("SQL") + String tableQuery = """ + CREATE TABLE faker.default.all_types_range ( + rnd_bigint bigint NOT NULL, + rnd_integer integer NOT NULL, + rnd_smallint smallint NOT NULL, + rnd_tinyint tinyint NOT NULL, + rnd_boolean boolean NOT NULL, + rnd_date date NOT NULL, + rnd_decimal1 decimal NOT NULL, + rnd_decimal2 decimal(18,5) NOT NULL, + rnd_decimal3 decimal(38,0) NOT NULL, + rnd_decimal4 decimal(38,38) NOT NULL, + rnd_decimal5 decimal(5,2) NOT NULL, + rnd_real real NOT NULL, + rnd_double double NOT NULL, + rnd_interval_day_time interval day to second NOT NULL, + rnd_interval_year interval year to month NOT NULL, + rnd_timestamp timestamp NOT NULL, + rnd_timestamp0 timestamp(0) NOT NULL, + rnd_timestamp6 timestamp(6) NOT NULL, + rnd_timestamp9 timestamp(9) NOT NULL, + rnd_timestamptz timestamp with time zone NOT NULL, + rnd_timestamptz0 timestamp(0) with time zone NOT NULL, + rnd_timestamptz6 timestamp(6) with time zone NOT NULL, + rnd_timestamptz9 timestamp(9) with time zone NOT NULL, + rnd_time time NOT NULL, + rnd_time0 time(0) NOT NULL, + rnd_time6 time(6) NOT NULL, + rnd_time9 time(9) NOT NULL, + rnd_timetz time with time zone NOT NULL, + rnd_timetz0 time(0) with time zone NOT NULL, + rnd_timetz6 time(6) with time zone NOT NULL, + rnd_timetz9 time(9) with time zone NOT NULL, + rnd_timetz12 time(12) with time zone NOT NULL, + rnd_varbinary varbinary NOT NULL, + rnd_varchar varchar NOT NULL, + rnd_nvarchar varchar(1000) NOT NULL, + rnd_char char NOT NULL, + rnd_nchar char(1000) NOT NULL, + rnd_ipaddress ipaddress NOT NULL, + rnd_uuid uuid NOT NULL)"""; + assertUpdate(tableQuery); + + @Language("SQL") + String testQuery; + + // inclusive ranges (BETWEEN) that produce only 2 values + // obtained using `Math.nextUp((float) 0.0)` + testQuery = """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(distinct rnd_smallint), + count(distinct rnd_tinyint), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(distinct rnd_decimal5), + count(distinct rnd_real), + count(distinct rnd_double), + count(distinct rnd_interval_day_time), + count(distinct rnd_interval_year), + count(distinct rnd_timestamp), + count(distinct rnd_timestamp0), + count(distinct rnd_timestamp6), + count(distinct rnd_timestamp9), + count(distinct rnd_timestamptz), + count(distinct rnd_timestamptz0), + count(distinct rnd_timestamptz6), + count(distinct rnd_timestamptz9), + count(distinct rnd_time), + count(distinct rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(distinct rnd_timetz), + count(distinct rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9) + FROM all_types_range + WHERE 1=1 + AND rnd_bigint BETWEEN 0 AND 1 + AND rnd_integer BETWEEN 0 AND 1 + AND rnd_smallint BETWEEN 0 AND 1 + AND rnd_tinyint BETWEEN 0 AND 1 + AND rnd_date BETWEEN DATE '2022-03-01' AND DATE '2022-03-02' + AND rnd_decimal1 BETWEEN 0 AND 1 + AND rnd_decimal2 BETWEEN 0.00000 AND 0.00001 + AND rnd_decimal3 BETWEEN 0 AND 1 + AND rnd_decimal4 BETWEEN DECIMAL '0.00000000000000000000000000000000000000' AND DECIMAL '0.00000000000000000000000000000000000001' + AND rnd_decimal5 BETWEEN 0.00 AND 0.01 + AND rnd_real BETWEEN REAL '0.0' AND REAL '1.4E-45' + AND rnd_double BETWEEN DOUBLE '0.0' AND DOUBLE '4.9E-324' + AND rnd_interval_day_time BETWEEN INTERVAL '0.000' SECOND AND INTERVAL '0.001' SECOND + AND rnd_interval_year BETWEEN INTERVAL '0' MONTH AND INTERVAL '1' MONTH + AND rnd_timestamp BETWEEN TIMESTAMP '2022-03-21 00:00:00.000' AND TIMESTAMP '2022-03-21 00:00:00.001' + AND rnd_timestamp0 BETWEEN TIMESTAMP '2022-03-21 00:00:00' AND TIMESTAMP '2022-03-21 00:00:01' + AND rnd_timestamp6 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000' AND TIMESTAMP '2022-03-21 00:00:00.000001' + AND rnd_timestamp9 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000000' AND TIMESTAMP '2022-03-21 00:00:00.000000001' + AND rnd_timestamptz BETWEEN TIMESTAMP '2022-03-21 00:00:00.000 +01:00' AND TIMESTAMP '2022-03-21 00:00:00.001 +01:00' + AND rnd_timestamptz0 BETWEEN TIMESTAMP '2022-03-21 00:00:00 +01:00' AND TIMESTAMP '2022-03-21 00:00:01 +01:00' + AND rnd_timestamptz6 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000 +01:00' AND TIMESTAMP '2022-03-21 00:00:00.000001 +01:00' + AND rnd_timestamptz9 BETWEEN TIMESTAMP '2022-03-21 00:00:00.000000000 +01:00' AND TIMESTAMP '2022-03-21 00:00:00.000000001 +01:00' + AND rnd_time BETWEEN TIME '01:02:03.456' AND TIME '01:02:03.457' + AND rnd_time0 BETWEEN TIME '01:02:03' AND TIME '01:02:04' + AND rnd_time6 BETWEEN TIME '01:02:03.000456' AND TIME '01:02:03.000457' + AND rnd_time9 BETWEEN TIME '01:02:03.000000456' AND TIME '01:02:03.000000457' + AND rnd_timetz BETWEEN TIME '01:02:03.456 +01:00' AND TIME '01:02:03.457 +01:00' + AND rnd_timetz0 BETWEEN TIME '01:02:03 +01:00' AND TIME '01:02:04 +01:00' + AND rnd_timetz6 BETWEEN TIME '01:02:03.000456 +01:00' AND TIME '01:02:03.000457 +01:00' + AND rnd_timetz9 BETWEEN TIME '01:02:03.000000456 +01:00' AND TIME '01:02:03.000000457 +01:00'\s"""; + assertQuery(testQuery, """ + VALUES (2, + 2, + 2, + 2, + -- date + 2, + -- decimal + 2, + 2, + 2, + 2, + 2, + -- real, double + 2, + 2, + -- intervals + 2, + 2, + -- timestamps + 2, + 2, + 2, + 2, + -- timestamps with time zone + 2, + 2, + 2, + 2, + -- time + 2, + 2, + 2, + 2, + -- time with time zone + 2, + 2, + 2, + 2)"""); + + // exclusive ranges that produce only 1 value + // obtained using `Math.nextUp((float) 0.0)` + testQuery = """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(distinct rnd_smallint), + count(distinct rnd_tinyint), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(distinct rnd_decimal5), + count(distinct rnd_real), + count(distinct rnd_double), + count(distinct rnd_interval_day_time), + count(distinct rnd_interval_year), + count(distinct rnd_timestamp), + count(distinct rnd_timestamp0), + count(distinct rnd_timestamp6), + count(distinct rnd_timestamp9), + count(distinct rnd_timestamptz), + count(distinct rnd_timestamptz0), + count(distinct rnd_timestamptz6), + count(distinct rnd_timestamptz9), + count(distinct rnd_time), + count(distinct rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(distinct rnd_timetz), + count(distinct rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9) + FROM all_types_range + WHERE 1=1 + AND rnd_bigint > 0 AND rnd_bigint < 2 + AND rnd_integer > 0 AND rnd_integer < 2 + AND rnd_smallint > 0 AND rnd_smallint < 2 + AND rnd_tinyint > 0 AND rnd_tinyint < 2 + AND rnd_date > DATE '2022-03-01' AND rnd_date < DATE '2022-03-03' + AND rnd_decimal1 > 0 AND rnd_decimal1 < 2 + AND rnd_decimal2 > 0.00000 AND rnd_decimal2 < 0.00002 + AND rnd_decimal3 > 0 AND rnd_decimal3 < 2 + AND rnd_decimal4 > DECIMAL '0.00000000000000000000000000000000000000' AND rnd_decimal4 < DECIMAL '0.00000000000000000000000000000000000002' + AND rnd_decimal5 > 0.00 AND rnd_decimal5 < 0.02 + AND rnd_real > REAL '0.0' AND rnd_real < REAL '2.8E-45' + AND rnd_double > DOUBLE '0.0' AND rnd_double < DOUBLE '1.0E-323' + AND rnd_interval_day_time > INTERVAL '0.000' SECOND AND rnd_interval_day_time < INTERVAL '0.002' SECOND + AND rnd_interval_year > INTERVAL '0' MONTH AND rnd_interval_year < INTERVAL '2' MONTH + AND rnd_timestamp > TIMESTAMP '2022-03-21 00:00:00.000' AND rnd_timestamp < TIMESTAMP '2022-03-21 00:00:00.002' + AND rnd_timestamp0 > TIMESTAMP '2022-03-21 00:00:00' AND rnd_timestamp0 < TIMESTAMP '2022-03-21 00:00:02' + AND rnd_timestamp6 > TIMESTAMP '2022-03-21 00:00:00.000000' AND rnd_timestamp6 < TIMESTAMP '2022-03-21 00:00:00.000002' + AND rnd_timestamp9 > TIMESTAMP '2022-03-21 00:00:00.000000000' AND rnd_timestamp9 < TIMESTAMP '2022-03-21 00:00:00.000000002' + AND rnd_timestamptz > TIMESTAMP '2022-03-21 00:00:00.000 +01:00' AND rnd_timestamptz < TIMESTAMP '2022-03-21 00:00:00.002 +01:00' + AND rnd_timestamptz0 > TIMESTAMP '2022-03-21 00:00:00 +01:00' AND rnd_timestamptz0 < TIMESTAMP '2022-03-21 00:00:02 +01:00' + AND rnd_timestamptz6 > TIMESTAMP '2022-03-21 00:00:00.000000 +01:00' AND rnd_timestamptz6 < TIMESTAMP '2022-03-21 00:00:00.000002 +01:00' + AND rnd_timestamptz9 > TIMESTAMP '2022-03-21 00:00:00.000000000 +01:00' AND rnd_timestamptz9 < TIMESTAMP '2022-03-21 00:00:00.000000002 +01:00' + AND rnd_time > TIME '01:02:03.456' AND rnd_time < TIME '01:02:03.458' + AND rnd_time0 > TIME '01:02:03' AND rnd_time0 < TIME '01:02:05' + AND rnd_time6 > TIME '01:02:03.000456' AND rnd_time6 < TIME '01:02:03.000458' + AND rnd_time9 > TIME '01:02:03.000000456' AND rnd_time9 < TIME '01:02:03.000000458' + AND rnd_timetz > TIME '01:02:03.456 +01:00' AND rnd_timetz < TIME '01:02:03.458 +01:00' + AND rnd_timetz0 > TIME '01:02:03 +01:00' AND rnd_timetz0 < TIME '01:02:05 +01:00' + AND rnd_timetz6 > TIME '01:02:03.000456 +01:00' AND rnd_timetz6 < TIME '01:02:03.000458 +01:00' + AND rnd_timetz9 > TIME '01:02:03.000000456 +01:00' AND rnd_timetz9 < TIME '01:02:03.000000458 +01:00'\s"""; + assertQuery(testQuery, """ + VALUES (1, + 1, + 1, + 1, + -- date + 1, + -- decimal + 1, + 1, + 1, + 1, + 1, + -- real, double + 1, + 1, + -- intervals + 1, + 1, + -- timestamps + 1, + 1, + 1, + 1, + -- timestamps with time zone + 1, + 1, + 1, + 1, + -- time + 1, + 1, + 1, + 1, + -- time with time zone + 1, + 1, + 1, + 1)"""); + + // inclusive range to get the min low bound + testQuery = """ + SELECT + count(distinct rnd_bigint), + count(distinct rnd_integer), + count(distinct rnd_smallint), + count(distinct rnd_tinyint), + count(distinct rnd_date), + count(distinct rnd_decimal1), + count(distinct rnd_decimal2), + count(distinct rnd_decimal3), + count(distinct rnd_decimal4), + count(distinct rnd_decimal5), + count(distinct rnd_real), + count(distinct rnd_double), + -- interval literals can't represent smallest possible values allowed by the engine + --count(distinct rnd_interval_day_time), + --count(distinct rnd_interval_year), + -- can't count timestamps because their extreme values cannot be expressed as literals + count(distinct rnd_time), + count(distinct rnd_time0), + count(distinct rnd_time6), + count(distinct rnd_time9), + count(distinct rnd_timetz), + count(distinct rnd_timetz0), + count(distinct rnd_timetz6), + count(distinct rnd_timetz9) + FROM all_types_range + WHERE 1=1 + AND rnd_bigint <= -9223372036854775808 + AND rnd_integer <= -2147483648 + AND rnd_smallint <= -32768 + AND rnd_tinyint <= -128 + -- TODO it actually returns -5877641-06-23 - there's definitely some overflow happening in the engine + AND rnd_date <= DATE '-5877641-06-23' + AND rnd_decimal1 <= DECIMAL '-99999999999999999999999999999999999999' + AND rnd_decimal2 <= DECIMAL '-9999999999999.99999' + AND rnd_decimal3 <= DECIMAL '-99999999999999999999999999999999999999' + AND rnd_decimal4 <= DECIMAL '-0.99999999999999999999999999999999999999' + -- TODO it actually retdurns '-999.98' + AND rnd_decimal5 <= DECIMAL '-999.99' + AND rnd_real <= REAL '1.4E-45' + AND rnd_double <= DOUBLE '4.9E-324' + -- interval literals can't represent smallest possible values allowed by the engine + --AND rnd_interval_day_time <= INTERVAL '-2147483647' SECOND + --AND rnd_interval_year <= INTERVAL '-2147483647' MONTH + AND rnd_time <= TIME '00:00:00.000' + AND rnd_time0 <= TIME '00:00:00' + AND rnd_time6 <= TIME '00:00:00.000000' + AND rnd_time9 <= TIME '00:00:00.000000000' + AND rnd_timetz <= TIME '00:00:00.000 +01:00' + AND rnd_timetz0 <= TIME '00:00:00 +01:00' + AND rnd_timetz6 <= TIME '00:00:00.000000 +01:00' + AND rnd_timetz9 <= TIME '00:00:00.000000000 +01:00'"""; + assertQuery(testQuery, """ + VALUES (1, + 1, + 1, + 1, + -- date + 1, + -- decimal + 1, + 1, + 1, + 1, + 1, + -- real, double + 1, + 1, + -- intervals + --1, + --1, + -- time + 1, + 1, + 1, + 1, + -- time with time zone + 1, + 1, + 1, + 1)"""); + + // exclusive range to get the max high bound + + assertUpdate("DROP TABLE faker.default.all_types_range"); + } +} diff --git a/pom.xml b/pom.xml index 081fe2126b5e6d..d7cc99980b0bc1 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ plugin/trino-exasol plugin/trino-exchange-filesystem plugin/trino-exchange-hdfs + plugin/trino-faker plugin/trino-geospatial plugin/trino-google-sheets plugin/trino-hive @@ -1077,6 +1078,12 @@ test-jar + + io.trino + trino-faker + ${project.version} + + io.trino trino-faulttolerant-tests diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java index 0950b188a0c6f0..984c9842a8a78d 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java @@ -54,6 +54,7 @@ public void extendEnvironment(Environment.Builder builder) "druid", "delta_lake", "elasticsearch", + "faker", "gsheets", "hive", "hudi", diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/faker.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/faker.properties new file mode 100644 index 00000000000000..b39099899edefc --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/faker.properties @@ -0,0 +1 @@ +connector.name=faker diff --git a/testing/trino-server-dev/etc/catalog/faker.properties b/testing/trino-server-dev/etc/catalog/faker.properties new file mode 100644 index 00000000000000..b39099899edefc --- /dev/null +++ b/testing/trino-server-dev/etc/catalog/faker.properties @@ -0,0 +1 @@ +connector.name=faker diff --git a/testing/trino-server-dev/etc/config.properties b/testing/trino-server-dev/etc/config.properties index d9ddf55aa5f846..35fd47cf7a54c8 100644 --- a/testing/trino-server-dev/etc/config.properties +++ b/testing/trino-server-dev/etc/config.properties @@ -37,6 +37,7 @@ plugin.bundles=\ ../../plugin/trino-hive/pom.xml,\ ../../plugin/trino-hudi/pom.xml,\ ../../plugin/trino-example-http/pom.xml,\ + ../../plugin/trino-faker/pom.xml,\ ../../plugin/trino-kafka/pom.xml, \ ../../plugin/trino-tpch/pom.xml, \ ../../plugin/trino-mysql/pom.xml,\