From 79f424700ff14f709c54fe2eda4893aba4029aa9 Mon Sep 17 00:00:00 2001 From: Manfred Moser Date: Tue, 15 Oct 2024 11:15:48 -0700 Subject: [PATCH] Remove the Accumulo connector --- .github/workflows/ci.yml | 2 - core/trino-server/src/main/provisio/trino.xml | 6 - docs/release-template.md | 2 - docs/src/main/sphinx/connector.md | 1 - docs/src/main/sphinx/connector/accumulo.md | 775 -------------- docs/src/main/sphinx/release/release-0.153.md | 2 +- docs/src/main/sphinx/sql/set-session.md | 2 +- docs/src/main/sphinx/static/img/accumulo.png | Bin 12279 -> 0 bytes plugin/trino-accumulo-iterators/pom.xml | 59 -- .../iterators/MaxByteArrayCombiner.java | 47 - .../iterators/MinByteArrayCombiner.java | 47 - .../test/java/io/trino/server/TestDummy.java | 22 - plugin/trino-accumulo/pom.xml | 355 ------- .../plugin/accumulo/AccumuloConnector.java | 142 --- .../accumulo/AccumuloConnectorFactory.java | 60 -- .../plugin/accumulo/AccumuloErrorCode.java | 56 - .../plugin/accumulo/AccumuloMetadata.java | 470 --------- .../accumulo/AccumuloMetadataFactory.java | 34 - .../accumulo/AccumuloMetadataManager.java | 986 ------------------ .../trino/plugin/accumulo/AccumuloModule.java | 101 -- .../trino/plugin/accumulo/AccumuloPlugin.java | 28 - .../plugin/accumulo/AccumuloSplitManager.java | 130 --- .../plugin/accumulo/AccumuloTableManager.java | 172 --- .../accumulo/AccumuloTransactionHandle.java | 34 - .../java/io/trino/plugin/accumulo/Types.java | 75 -- .../plugin/accumulo/conf/AccumuloConfig.java | 144 --- .../conf/AccumuloSessionProperties.java | 160 --- .../conf/AccumuloTableProperties.java | 239 ----- .../index/ColumnCardinalityCache.java | 407 -------- .../plugin/accumulo/index/IndexLookup.java | 396 ------- .../trino/plugin/accumulo/index/Indexer.java | 566 ---------- .../plugin/accumulo/io/AccumuloPageSink.java | 299 ------ .../accumulo/io/AccumuloPageSinkProvider.java | 66 -- .../accumulo/io/AccumuloRecordCursor.java | 293 ------ .../plugin/accumulo/io/AccumuloRecordSet.java | 141 --- .../io/AccumuloRecordSetProvider.java | 68 -- .../accumulo/metadata/AccumuloTable.java | 215 ---- .../accumulo/metadata/AccumuloView.java | 36 - .../metadata/ZooKeeperMetadataManager.java | 354 ------- .../model/AccumuloColumnConstraint.java | 36 - .../accumulo/model/AccumuloColumnHandle.java | 62 -- .../plugin/accumulo/model/AccumuloSplit.java | 112 -- .../accumulo/model/AccumuloTableHandle.java | 179 ---- .../io/trino/plugin/accumulo/model/Field.java | 248 ----- .../io/trino/plugin/accumulo/model/Row.java | 66 -- .../accumulo/model/SerializedRange.java | 77 -- .../accumulo/model/TabletSplitMetadata.java | 31 - .../serializers/AccumuloRowSerializer.java | 620 ----------- .../serializers/BooleanLexicoder.java | 35 - .../serializers/LexicoderRowSerializer.java | 415 -------- .../accumulo/serializers/MapLexicoder.java | 73 -- .../serializers/StringRowSerializer.java | 402 ------- .../plugin/accumulo/AccumuloQueryRunner.java | 185 ---- .../accumulo/TestAccumuloConnectorTest.java | 344 ------ .../accumulo/TestAccumuloMetadataManager.java | 100 -- .../accumulo/TestingAccumuloServer.java | 95 -- .../accumulo/conf/TestAccumuloConfig.java | 67 -- .../plugin/accumulo/index/TestIndexer.java | 352 ------- .../accumulo/model/TestAccumuloSplit.java | 60 -- .../plugin/accumulo/model/TestField.java | 221 ---- .../trino/plugin/accumulo/model/TestRow.java | 76 -- .../accumulo/model/TestSerializedRange.java | 34 - .../AbstractTestAccumuloRowSerializer.java | 312 ------ .../TestLexicoderRowSerializer.java | 23 - .../serializers/TestStringRowSerializer.java | 39 - pom.xml | 4 - .../EnvMultinodeAllConnectors.java | 2 - .../multinode-all/accumulo.properties | 5 - 68 files changed, 2 insertions(+), 11265 deletions(-) delete mode 100644 docs/src/main/sphinx/connector/accumulo.md delete mode 100644 docs/src/main/sphinx/static/img/accumulo.png delete mode 100644 plugin/trino-accumulo-iterators/pom.xml delete mode 100644 plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MaxByteArrayCombiner.java delete mode 100644 plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MinByteArrayCombiner.java delete mode 100644 plugin/trino-accumulo-iterators/src/test/java/io/trino/server/TestDummy.java delete mode 100644 plugin/trino-accumulo/pom.xml delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnector.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnectorFactory.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloErrorCode.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadata.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataFactory.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataManager.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloModule.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloPlugin.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloSplitManager.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTableManager.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTransactionHandle.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/Types.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloConfig.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloSessionProperties.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloTableProperties.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/ColumnCardinalityCache.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/IndexLookup.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/Indexer.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSink.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSinkProvider.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordCursor.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSet.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSetProvider.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloTable.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloView.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/ZooKeeperMetadataManager.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnConstraint.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnHandle.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloSplit.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloTableHandle.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Field.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Row.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/SerializedRange.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/TabletSplitMetadata.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/AccumuloRowSerializer.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/BooleanLexicoder.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/LexicoderRowSerializer.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/MapLexicoder.java delete mode 100644 plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/StringRowSerializer.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/AccumuloQueryRunner.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloConnectorTest.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloMetadataManager.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestingAccumuloServer.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/conf/TestAccumuloConfig.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/index/TestIndexer.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestAccumuloSplit.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestField.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestRow.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestSerializedRange.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/AbstractTestAccumuloRowSerializer.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestLexicoderRowSerializer.java delete mode 100644 plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestStringRowSerializer.java delete mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/accumulo.properties diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f92c551677d30..f59aaa7490c44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,7 +333,6 @@ jobs: id: tests run: | $MAVEN test ${MAVEN_TEST} -pl ' - !:trino-accumulo, !:trino-base-jdbc, !:trino-bigquery, !:trino-cassandra, @@ -443,7 +442,6 @@ jobs: - { modules: lib/trino-filesystem-gcs, profile: cloud-tests } - { modules: lib/trino-filesystem-s3, profile: cloud-tests } - { modules: lib/trino-hdfs, profile: cloud-tests } - - { modules: plugin/trino-accumulo } - { modules: plugin/trino-bigquery } - { modules: plugin/trino-bigquery, profile: cloud-tests-2 } - { modules: plugin/trino-cassandra } diff --git a/core/trino-server/src/main/provisio/trino.xml b/core/trino-server/src/main/provisio/trino.xml index 5fd5c6e2415db..c8e483932712b 100644 --- a/core/trino-server/src/main/provisio/trino.xml +++ b/core/trino-server/src/main/provisio/trino.xml @@ -33,12 +33,6 @@ - - - - - - diff --git a/docs/release-template.md b/docs/release-template.md index 50df150ce128f..a091f1a4c4fc6 100644 --- a/docs/release-template.md +++ b/docs/release-template.md @@ -14,8 +14,6 @@ ## CLI -## Accumulo connector - ## BigQuery connector ## Blackhole connector diff --git a/docs/src/main/sphinx/connector.md b/docs/src/main/sphinx/connector.md index 242bcf07ad863..99af014dd71b6 100644 --- a/docs/src/main/sphinx/connector.md +++ b/docs/src/main/sphinx/connector.md @@ -6,7 +6,6 @@ from different data sources. ```{toctree} :maxdepth: 1 -Accumulo BigQuery Black Hole Cassandra diff --git a/docs/src/main/sphinx/connector/accumulo.md b/docs/src/main/sphinx/connector/accumulo.md deleted file mode 100644 index 4da0b2fb7e5b3..0000000000000 --- a/docs/src/main/sphinx/connector/accumulo.md +++ /dev/null @@ -1,775 +0,0 @@ -# Accumulo connector - -```{raw} html - -``` - -The Accumulo connector supports reading and writing data from -[Apache Accumulo](https://accumulo.apache.org/). -Please read this page thoroughly to understand the capabilities and features of the connector. - -## Installing the iterator dependency - -The Accumulo connector uses custom Accumulo iterators in -order to push various information in SQL predicate clauses to Accumulo for -server-side filtering, known as *predicate pushdown*. In order -for the server-side iterators to work, you need to add the `trino-accumulo-iterators` -JAR file to Accumulo's `lib/ext` directory on each TabletServer node. - -```bash -# For each TabletServer node: -scp $TRINO_HOME/plugins/accumulo/trino-accumulo-iterators-*.jar [tabletserver_address]:$ACCUMULO_HOME/lib/ext - -# TabletServer should pick up new JAR files in ext directory, but may require restart -``` - -## Requirements - -To connect to Accumulo, you need: - -- Accumulo versions 1.x starting with 1.7.4. Versions 2.x are not supported. -- Network access from the Trino coordinator and workers to the Accumulo - Zookeeper server. Port 2181 is the default port. - -## Connector configuration - -Create `etc/catalog/example.properties` to mount the `accumulo` connector as -the `example` catalog, with the following connector properties as appropriate -for your setup: - -```text -connector.name=accumulo -accumulo.instance=xxx -accumulo.zookeepers=xxx -accumulo.username=username -accumulo.password=password -``` - -Replace the `accumulo.xxx` properties as required. - -## Configuration variables - -| Property name | Default value | Required | Description | -| -------------------------------------------- | ----------------- | -------- | -------------------------------------------------------------------------------- | -| `accumulo.instance` | (none) | Yes | Name of the Accumulo instance | -| `accumulo.zookeepers` | (none) | Yes | ZooKeeper connect string | -| `accumulo.username` | (none) | Yes | Accumulo user for Trino | -| `accumulo.password` | (none) | Yes | Accumulo password for user | -| `accumulo.zookeeper.metadata.root` | `/trino-accumulo` | No | Root znode for storing metadata. Only relevant if using default Metadata Manager | -| `accumulo.cardinality.cache.size` | `100000` | No | Sets the size of the index cardinality cache | -| `accumulo.cardinality.cache.expire.duration` | `5m` | No | Sets the expiration duration of the cardinality cache. | - -## Usage - -Simply begin using SQL to create a new table in Accumulo to begin -working with data. By default, the first column of the table definition -is set to the Accumulo row ID. This should be the primary key of your -table, and keep in mind that any `INSERT` statements containing the same -row ID is effectively an UPDATE as far as Accumulo is concerned, as any -previous data in the cell is overwritten. The row ID can be -any valid Trino datatype. If the first column is not your primary key, you -can set the row ID column using the `row_id` table property within the `WITH` -clause of your table definition. - -Simply issue a `CREATE TABLE` statement to create a new Trino/Accumulo table: - -``` -CREATE TABLE example_schema.scientists ( - recordkey VARCHAR, - name VARCHAR, - age BIGINT, - birthday DATE -); -``` - -```sql -DESCRIBE example_schema.scientists; -``` - -```text - Column | Type | Extra | Comment ------------+---------+-------+--------------------------------------------------- - recordkey | varchar | | Accumulo row ID - name | varchar | | Accumulo column name:name. Indexed: false - age | bigint | | Accumulo column age:age. Indexed: false - birthday | date | | Accumulo column birthday:birthday. Indexed: false -``` - -This command creates a new Accumulo table with the `recordkey` column -as the Accumulo row ID. The name, age, and birthday columns are mapped to -auto-generated column family and qualifier values (which, in practice, -are both identical to the Trino column name). - -When creating a table using SQL, you can optionally specify a -`column_mapping` table property. The value of this property is a -comma-delimited list of triples, Trino column **:** Accumulo column -family **:** accumulo column qualifier, with one triple for every -non-row ID column. This sets the mapping of the Trino column name to -the corresponding Accumulo column family and column qualifier. - -If you don't specify the `column_mapping` table property, then the -connector auto-generates column names (respecting any configured locality groups). -Auto-generation of column names is only available for internal tables, so if your -table is external you must specify the column_mapping property. - -For a full list of table properties, see [Table Properties](accumulo-table-properties). - -For example: - -```sql -CREATE TABLE example_schema.scientists ( - recordkey VARCHAR, - name VARCHAR, - age BIGINT, - birthday DATE -) -WITH ( - column_mapping = 'name:metadata:name,age:metadata:age,birthday:metadata:date' -); -``` - -```sql -DESCRIBE example_schema.scientists; -``` - -```text - Column | Type | Extra | Comment ------------+---------+-------+----------------------------------------------- - recordkey | varchar | | Accumulo row ID - name | varchar | | Accumulo column metadata:name. Indexed: false - age | bigint | | Accumulo column metadata:age. Indexed: false - birthday | date | | Accumulo column metadata:date. Indexed: false -``` - -You can then issue `INSERT` statements to put data into Accumulo. - -:::{note} -While issuing `INSERT` statements is convenient, -this method of loading data into Accumulo is low-throughput. You want -to use the Accumulo APIs to write `Mutations` directly to the tables. -See the section on [Loading Data](accumulo-loading-data) for more details. -::: - -```sql -INSERT INTO example_schema.scientists VALUES -('row1', 'Grace Hopper', 109, DATE '1906-12-09' ), -('row2', 'Alan Turing', 103, DATE '1912-06-23' ); -``` - -```sql -SELECT * FROM example_schema.scientists; -``` - -```text - recordkey | name | age | birthday ------------+--------------+-----+------------ - row1 | Grace Hopper | 109 | 1906-12-09 - row2 | Alan Turing | 103 | 1912-06-23 -(2 rows) -``` - -As you'd expect, rows inserted into Accumulo via the shell or -programmatically will also show up when queried. (The Accumulo shell -thinks "-5321" is an option and not a number... so we'll just make TBL a -little younger.) - -```bash -$ accumulo shell -u root -p secret -root@default> table example_schema.scientists -root@default example_schema.scientists> insert row3 metadata name "Tim Berners-Lee" -root@default example_schema.scientists> insert row3 metadata age 60 -root@default example_schema.scientists> insert row3 metadata date 5321 -``` - -```sql -SELECT * FROM example_schema.scientists; -``` - -```text - recordkey | name | age | birthday ------------+-----------------+-----+------------ - row1 | Grace Hopper | 109 | 1906-12-09 - row2 | Alan Turing | 103 | 1912-06-23 - row3 | Tim Berners-Lee | 60 | 1984-07-27 -(3 rows) -``` - -You can also drop tables using `DROP TABLE`. This command drops both -metadata and the tables. See the below section on [External -Tables](accumulo-external-tables) for more details on internal and external -tables. - -```sql -DROP TABLE example_schema.scientists; -``` - -## Indexing columns - -Internally, the connector creates an Accumulo `Range` and packs it in -a split. This split gets passed to a Trino Worker to read the data from -the `Range` via a `BatchScanner`. When issuing a query that results -in a full table scan, each Trino Worker gets a single `Range` that -maps to a single tablet of the table. When issuing a query with a -predicate (i.e. `WHERE x = 10` clause), Trino passes the values -within the predicate (`10`) to the connector so it can use this -information to scan less data. When the Accumulo row ID is used as part -of the predicate clause, this narrows down the `Range` lookup to quickly -retrieve a subset of data from Accumulo. - -But what about the other columns? If you're frequently querying on -non-row ID columns, you should consider using the **indexing** -feature built into the Accumulo connector. This feature can drastically -reduce query runtime when selecting a handful of values from the table, -and the heavy lifting is done for you when loading data via Trino -`INSERT` statements. Keep in mind writing data to Accumulo via -`INSERT` does not have high throughput. - -To enable indexing, add the `index_columns` table property and specify -a comma-delimited list of Trino column names you wish to index (we use the -`string` serializer here to help with this example -- you -should be using the default `lexicoder` serializer). - -```sql -CREATE TABLE example_schema.scientists ( - recordkey VARCHAR, - name VARCHAR, - age BIGINT, - birthday DATE -) -WITH ( - serializer = 'string', - index_columns='name,age,birthday' -); -``` - -After creating the table, we see there are an additional two Accumulo -tables to store the index and metrics. - -```text -root@default> tables -accumulo.metadata -accumulo.root -example_schema.scientists -example_schema.scientists_idx -example_schema.scientists_idx_metrics -trace -``` - -After inserting data, we can look at the index table and see there are -indexed values for the name, age, and birthday columns. The connector -queries this index table - -```sql -INSERT INTO example_schema.scientists VALUES -('row1', 'Grace Hopper', 109, DATE '1906-12-09'), -('row2', 'Alan Turing', 103, DATE '1912-06-23'); -``` - -```text -root@default> scan -t example_schema.scientists_idx --21011 metadata_date:row2 [] --23034 metadata_date:row1 [] -103 metadata_age:row2 [] -109 metadata_age:row1 [] -Alan Turing metadata_name:row2 [] -Grace Hopper metadata_name:row1 [] -``` - -When issuing a query with a `WHERE` clause against indexed columns, -the connector searches the index table for all row IDs that contain the -value within the predicate. These row IDs are bundled into a Trino -split as single-value `Range` objects, the number of row IDs per split -is controlled by the value of `accumulo.index_rows_per_split`, and -passed to a Trino worker to be configured in the `BatchScanner` which -scans the data table. - -```sql -SELECT * FROM example_schema.scientists WHERE age = 109; -``` - -```text - recordkey | name | age | birthday ------------+--------------+-----+------------ - row1 | Grace Hopper | 109 | 1906-12-09 -(1 row) -``` - -(accumulo-loading-data)= -## Loading data - -The Accumulo connector supports loading data via INSERT statements, however -this method tends to be low-throughput and should not be relied on when -throughput is a concern. - -(accumulo-external-tables)= -## External tables - -By default, the tables created using SQL statements via Trino are -*internal* tables, that is both the Trino table metadata and the -Accumulo tables are managed by Trino. When you create an internal -table, the Accumulo table is created as well. You receive an error -if the Accumulo table already exists. When an internal table is dropped -via Trino, the Accumulo table, and any index tables, are dropped as -well. - -To change this behavior, set the `external` property to `true` when -issuing the `CREATE` statement. This makes the table an *external* -table, and a `DROP TABLE` command **only** deletes the metadata -associated with the table. If the Accumulo tables do not already exist, -they are created by the connector. - -Creating an external table *will* set any configured locality groups as well -as the iterators on the index and metrics tables, if the table is indexed. -In short, the only difference between an external table and an internal table, -is that the connector deletes the Accumulo tables when a `DROP TABLE` command -is issued. - -External tables can be a bit more difficult to work with, as the data is stored -in an expected format. If the data is not stored correctly, then you're -gonna have a bad time. Users must provide a `column_mapping` property -when creating the table. This creates the mapping of Trino column name -to the column family/qualifier for the cell of the table. The value of the -cell is stored in the `Value` of the Accumulo key/value pair. By default, -this value is expected to be serialized using Accumulo's *lexicoder* API. -If you are storing values as strings, you can specify a different serializer -using the `serializer` property of the table. See the section on -[Table Properties](accumulo-table-properties) for more information. - -Next, we create the Trino external table. - -```sql -CREATE TABLE external_table ( - a VARCHAR, - b BIGINT, - c DATE -) -WITH ( - column_mapping = 'a:md:a,b:md:b,c:md:c', - external = true, - index_columns = 'b,c', - locality_groups = 'foo:b,c' -); -``` - -After creating the table, usage of the table continues as usual: - -```sql -INSERT INTO external_table VALUES -('1', 1, DATE '2015-03-06'), -('2', 2, DATE '2015-03-07'); -``` - -```sql -SELECT * FROM external_table; -``` - -```text - a | b | c ----+---+------------ - 1 | 1 | 2015-03-06 - 2 | 2 | 2015-03-06 -(2 rows) -``` - -```sql -DROP TABLE external_table; -``` - -After dropping the table, the table still exists in Accumulo because it is *external*. - -```text -root@default> tables -accumulo.metadata -accumulo.root -external_table -external_table_idx -external_table_idx_metrics -trace -``` - -If we wanted to add a new column to the table, we can create the table again and specify a new column. -Any existing rows in the table have a value of NULL. This command re-configures the Accumulo -tables, setting the locality groups and iterator configuration. - -```sql -CREATE TABLE external_table ( - a VARCHAR, - b BIGINT, - c DATE, - d INTEGER -) -WITH ( - column_mapping = 'a:md:a,b:md:b,c:md:c,d:md:d', - external = true, - index_columns = 'b,c,d', - locality_groups = 'foo:b,c,d' -); - -SELECT * FROM external_table; -``` - -```sql - a | b | c | d ----+---+------------+------ - 1 | 1 | 2015-03-06 | NULL - 2 | 2 | 2015-03-07 | NULL -(2 rows) -``` - -(accumulo-table-properties)= -## Table properties - -Table property usage example: - -```sql -CREATE TABLE example_schema.scientists ( - recordkey VARCHAR, - name VARCHAR, - age BIGINT, - birthday DATE -) -WITH ( - column_mapping = 'name:metadata:name,age:metadata:age,birthday:metadata:date', - index_columns = 'name,age' -); -``` - -| Property name | Default value | Description | -| ----------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `column_mapping` | (generated) | Comma-delimited list of column metadata: `col_name:col_family:col_qualifier,[...]`. Required for external tables. Not setting this property results in auto-generated column names. | -| `index_columns` | (none) | A comma-delimited list of Trino columns that are indexed in this table's corresponding index table | -| `external` | `false` | If true, Trino will only do metadata operations for the table. Otherwise, Trino will create and drop Accumulo tables where appropriate. | -| `locality_groups` | (none) | List of locality groups to set on the Accumulo table. Only valid on internal tables. String format is locality group name, colon, comma delimited list of column families in the group. Groups are delimited by pipes. Example: `group1:famA,famB,famC\|group2:famD,famE,famF\|etc...` | -| `row_id` | (first column) | Trino column name that maps to the Accumulo row ID. | -| `serializer` | `default` | Serializer for Accumulo data encodings. Can either be `default`, `string`, `lexicoder` or a Java class name. Default is `default`, i.e. the value from `AccumuloRowSerializer.getDefault()`, i.e. `lexicoder`. | -| `scan_auths` | (user auths) | Scan-time authorizations set on the batch scanner. | - -## Session properties - -You can change the default value of a session property by using {doc}`/sql/set-session`. -Note that session properties are prefixed with the catalog name: - -``` -SET SESSION example.column_filter_optimizations_enabled = false; -``` - -| Property name | Default value | Description | -| ------------------------------------------ | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `optimize_locality_enabled` | `true` | Set to true to enable data locality for non-indexed scans | -| `optimize_split_ranges_enabled` | `true` | Set to true to split non-indexed queries by tablet splits. Should generally be true. | -| `optimize_index_enabled` | `true` | Set to true to enable usage of the secondary index on query | -| `index_rows_per_split` | `10000` | The number of Accumulo row IDs that are packed into a single Trino split | -| `index_threshold` | `0.2` | The ratio between number of rows to be scanned based on the index over the total number of rows. If the ratio is below this threshold, the index will be used. | -| `index_lowest_cardinality_threshold` | `0.01` | The threshold where the column with the lowest cardinality will be used instead of computing an intersection of ranges in the index. Secondary index must be enabled | -| `index_metrics_enabled` | `true` | Set to true to enable usage of the metrics table to optimize usage of the index | -| `scan_username` | (config) | User to impersonate when scanning the tables. This property trumps the `scan_auths` table property | -| `index_short_circuit_cardinality_fetch` | `true` | Short circuit the retrieval of index metrics once any column is less than the lowest cardinality threshold | -| `index_cardinality_cache_polling_duration` | `10ms` | Sets the cardinality cache polling duration for short circuit retrieval of index metrics | - -## Adding columns - -Adding a new column to an existing table cannot be done today via -`ALTER TABLE [table] ADD COLUMN [name] [type]` because of the additional -metadata required for the columns to work; the column family, qualifier, -and if the column is indexed. - -## Serializers - -The Trino connector for Accumulo has a pluggable serializer framework -for handling I/O between Trino and Accumulo. This enables end-users the -ability to programmatically serialized and deserialize their special data -formats within Accumulo, while abstracting away the complexity of the -connector itself. - -There are two types of serializers currently available; a `string` -serializer that treats values as Java `String`, and a `lexicoder` -serializer that leverages Accumulo's Lexicoder API to store values. The -default serializer is the `lexicoder` serializer, as this serializer -does not require expensive conversion operations back and forth between -`String` objects and the Trino types -- the cell's value is encoded as a -byte array. - -Additionally, the `lexicoder` serializer does proper lexigraphical ordering of -numerical types like `BIGINT` or `TIMESTAMP`. This is essential for the connector -to properly leverage the secondary index when querying for data. - -You can change the default the serializer by specifying the -`serializer` table property, using either `default` (which is -`lexicoder`), `string` or `lexicoder` for the built-in types, or -you could provide your own implementation by extending -`AccumuloRowSerializer`, adding it to the Trino `CLASSPATH`, and -specifying the fully-qualified Java class name in the connector configuration. - -```sql -CREATE TABLE example_schema.scientists ( - recordkey VARCHAR, - name VARCHAR, - age BIGINT, - birthday DATE -) -WITH ( - column_mapping = 'name:metadata:name,age:metadata:age,birthday:metadata:date', - serializer = 'default' -); -``` - -```sql -INSERT INTO example_schema.scientists VALUES -('row1', 'Grace Hopper', 109, DATE '1906-12-09' ), -('row2', 'Alan Turing', 103, DATE '1912-06-23' ); -``` - -```text -root@default> scan -t example_schema.scientists -row1 metadata:age [] \x08\x80\x00\x00\x00\x00\x00\x00m -row1 metadata:date [] \x08\x7F\xFF\xFF\xFF\xFF\xFF\xA6\x06 -row1 metadata:name [] Grace Hopper -row2 metadata:age [] \x08\x80\x00\x00\x00\x00\x00\x00g -row2 metadata:date [] \x08\x7F\xFF\xFF\xFF\xFF\xFF\xAD\xED -row2 metadata:name [] Alan Turing -``` - -```sql -CREATE TABLE example_schema.stringy_scientists ( - recordkey VARCHAR, - name VARCHAR, - age BIGINT, - birthday DATE -) -WITH ( - column_mapping = 'name:metadata:name,age:metadata:age,birthday:metadata:date', - serializer = 'string' -); -``` - -```sql -INSERT INTO example_schema.stringy_scientists VALUES -('row1', 'Grace Hopper', 109, DATE '1906-12-09' ), -('row2', 'Alan Turing', 103, DATE '1912-06-23' ); -``` - -```text -root@default> scan -t example_schema.stringy_scientists -row1 metadata:age [] 109 -row1 metadata:date [] -23034 -row1 metadata:name [] Grace Hopper -row2 metadata:age [] 103 -row2 metadata:date [] -21011 -row2 metadata:name [] Alan Turing -``` - -```sql -CREATE TABLE example_schema.custom_scientists ( - recordkey VARCHAR, - name VARCHAR, - age BIGINT, - birthday DATE -) -WITH ( - column_mapping = 'name:metadata:name,age:metadata:age,birthday:metadata:date', - serializer = 'my.serializer.package.MySerializer' -); -``` - -## Metadata management - -Metadata for the Trino/Accumulo tables is stored in ZooKeeper. You can, -and should, issue SQL statements in Trino to create and drop tables. -This is the easiest method of creating the metadata required to make the -connector work. It is best to not mess with the metadata, but here are -the details of how it is stored. - -A root node in ZooKeeper holds all the mappings, and the format is as -follows: - -```text -/metadata-root/schema/table -``` - -Where `metadata-root` is the value of `zookeeper.metadata.root` in -the config file (default is `/trino-accumulo`), `schema` is the -Trino schema (which is identical to the Accumulo namespace name), and -`table` is the Trino table name (again, identical to Accumulo name). -The data of the `table` ZooKeeper node is a serialized -`AccumuloTable` Java object (which resides in the connector code). -This table contains the schema (namespace) name, table name, column -definitions, the serializer to use for the table, and any additional -table properties. - -If you have a need to programmatically manipulate the ZooKeeper metadata -for Accumulo, take a look at -`io.trino.plugin.accumulo.metadata.ZooKeeperMetadataManager` for some -Java code to simplify the process. - -## Converting table from internal to external - -If your table is *internal*, you can convert it to an external table by deleting -the corresponding znode in ZooKeeper, effectively making the table no longer exist as -far as Trino is concerned. Then, create the table again using the same DDL, but adding the -`external = true` table property. - -For example: - -1\. We're starting with an internal table `foo.bar` that was created with the below DDL. -If you have not previously defined a table property for `column_mapping` (like this example), -be sure to describe the table **before** deleting the metadata. We need the column mappings -when creating the external table. - -```sql -CREATE TABLE foo.bar (a VARCHAR, b BIGINT, c DATE) -WITH ( - index_columns = 'b,c' -); -``` - -```sql -DESCRIBE foo.bar; -``` - -```text - Column | Type | Extra | Comment ---------+---------+-------+------------------------------------- - a | varchar | | Accumulo row ID - b | bigint | | Accumulo column b:b. Indexed: true - c | date | | Accumulo column c:c. Indexed: true -``` - -2\. Using the ZooKeeper CLI, delete the corresponding znode. Note this uses the default ZooKeeper -metadata root of `/trino-accumulo` - -```text -$ zkCli.sh -[zk: localhost:2181(CONNECTED) 1] delete /trino-accumulo/foo/bar -``` - -3\. Re-create the table using the same DDL as before, but adding the `external=true` property. -Note that if you had not previously defined the column_mapping, you need to add the property -to the new DDL (external tables require this property to be set). The column mappings are in -the output of the `DESCRIBE` statement. - -```sql -CREATE TABLE foo.bar ( - a VARCHAR, - b BIGINT, - c DATE -) -WITH ( - column_mapping = 'a:a:a,b:b:b,c:c:c', - index_columns = 'b,c', - external = true -); -``` - -(accumulo-type-mapping)= -## Type mapping - -Because Trino and Accumulo each support types that the other does not, this -connector modifies some types when reading or writing data. Data types may not -map the same way in both directions between Trino and the data source. Refer to -the following sections for type mapping in each direction. - -### Accumulo type to Trino type mapping - -The connector maps Accumulo types to the corresponding Trino types following -this table: - -:::{list-table} Accumulo type to Trino type mapping -:widths: 50, 50 -:header-rows: 1 - -* - Accumulo type - - Trino type -* - `BOOLEAN` - - `BOOLEAN` -* - `TINYINT` - - `TINYINT` -* - `SMALLINT` - - `SMALLINT` -* - `INTEGER` - - `INTEGER` -* - `BIGINT` - - `BIGINT` -* - `REAL` - - `REAL` -* - `DOUBLE` - - `DOUBLE` -* - `VARCHAR(n)` - - `VARCHAR(n)` -* - `VARBINARY` - - `VARBINARY` -* - `DATE` - - `DATE` -* - `TIME(n)` - - `TIME(n)` -* - `TIMESTAMP(n)` - - `TIMESTAMP(n)` -::: - -No other types are supported. - -### Trino type to Accumulo type mapping - -The connector maps Trino types to the corresponding Trino type to Accumulo type -mapping types following this table: - -:::{list-table} Trino type to Accumulo type mapping -:widths: 25, 25, 50 -:header-rows: 1 - -* - Trino type - - Accumulo type - - Notes -* - `BOOLEAN` - - `BOOLEAN` - - -* - `TINYINT` - - `TINYINT` - - Trino only supports writing values belonging to `[0, 127]` -* - `SMALLINT` - - `SMALLINT` - - -* - `INTEGER` - - `INTEGER` - - -* - `BIGINT` - - `BIGINT` - - -* - `REAL` - - `REAL` - - -* - `DOUBLE` - - `DOUBLE` - - -* - `VARCHAR(n)` - - `VARCHAR(n)` - - -* - `VARBINARY` - - `VARBINARY` - - -* - `DATE` - - `DATE` - - -* - `TIME(n)` - - `TIME(n)` - - -* - `TIMESTAMP(n)` - - `TIMESTAMP(n)` - - -::: - -No other types are supported - -(accumulo-sql-support)= -## SQL support - -The connector provides read and write access to data and metadata in -the Accumulo database. In addition to the {ref}`globally available -` and {ref}`read operation ` -statements, the connector supports the following features: - -- {doc}`/sql/insert` -- {doc}`/sql/create-table` -- {doc}`/sql/create-table-as` -- {doc}`/sql/drop-table` -- {doc}`/sql/create-schema` -- {doc}`/sql/drop-schema` diff --git a/docs/src/main/sphinx/release/release-0.153.md b/docs/src/main/sphinx/release/release-0.153.md index c63a6619f8775..0cd31018d4030 100644 --- a/docs/src/main/sphinx/release/release-0.153.md +++ b/docs/src/main/sphinx/release/release-0.153.md @@ -31,7 +31,7 @@ - Use nulls-last ordering for {func}`array_sort`. - Validate that `TRY` is used with exactly one argument. - Allow running Presto with early-access Java versions. -- Add {doc}`/connector/accumulo`. +- Add Accumulo connector. ## Functions and language features diff --git a/docs/src/main/sphinx/sql/set-session.md b/docs/src/main/sphinx/sql/set-session.md index e03da8f0c8bca..67b7550f0b9cf 100644 --- a/docs/src/main/sphinx/sql/set-session.md +++ b/docs/src/main/sphinx/sql/set-session.md @@ -45,7 +45,7 @@ SET SESSION optimize_hash_generation = true; ``` The following example sets the `optimize_locality_enabled` catalog session -property for an {doc}`Accumulo catalog ` named `acc01`: +property for an Accumulo catalog named `acc01`: ``` SET SESSION acc01.optimize_locality_enabled = false; diff --git a/docs/src/main/sphinx/static/img/accumulo.png b/docs/src/main/sphinx/static/img/accumulo.png deleted file mode 100644 index 2e5430bf2a25a418cd48d6221716040bddd6b202..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12279 zcmb_ihdY(;|9|YAnJpuG9*#{&Dr8gk=Ae_klaVd5_l!D34%w0IgXD-(WF8|VdvAWX z-{0^(*L9xbdR*6Y-_L!&@AvyP-f?<54=G5QNg)WL&`?)10KWs^>p)BhuEJ_`Qs9Tg zL*3LHf^PI&e{uTV-#dVtOg{Hbd<@+YKK`~|j*!2!B&qO zj#@&E9{A4tP+PG93D2dpZ@zWE>ZVBb8K_wvzSug?jKHs-=Gp6_#mxAFAgR+r+h7y zirDWx5_r76zW!qWcxQ3o=_;&lx+C@MuHSsqn?)j$DJ?n zj-E|54pnB}fU{J$9ZpJNrcEwqDcR(IHhQmod!*X9+S=aUF4K2ZU0uC3Qf2x1Nu+by z@7BP*B|e>e^`tAe{zSI%w)5S=1KUXg=rP(N2l#NSCNhwXPf9g7=1M@yH889eMuW?m)EW(B; zRV{`^q3$|LU1I9eqra+0H%i2I)#es94`2t z9nlCGq48e`BBGeEzsx?shM1U`Qc7LE;cbgUD>(0oJTwF|%rw**Y@JW$NEov3XV*O6 z>17}Ze3fUHCFNXLq+M5zr*YH9l=QW5*qoBfu7!MP33mt%WBViBqVS$4<7NHbk5SJ zS>pOKcS$+Ptu>6IPEVgRGcmbvl@LM^5CZR$0kLNdm7ROGwDwEQS3a7=oN7u%MIK3G zX8foGDdbqJ8dnN#*yUvicbW!)O0`wnBrn$xX8$Pk>S%1nA8UzQDxUVcQRIcl&Ru9Q zSQ{aqIQpjH2I_*3n#Kt6KWgUfq`?K9N+C?P3_= zU|Kj`w%lDcIAno0=Lc;Oj8rz+MLRD-oJ~p=OtyxstrVh4 z<;*akwaUaY-4>>GK@0t`Lx@XC(nZt3NES&rRU6`Ra$*H@VVoJt0W>Z>l93KHwpqj{ zm*<`h=TT>>0gB@0#MCim>#7Z}V+9Yt!@37H9qez}4CPAoAjc|CP-D-pq{c1kcD#$u zrp-%TN@?T%yNMAtDE}$J8F}+usXo~xlP5=xfCFca1}bt|o_d!)PNi@1JcBT0P9(i2 z`!>F=zR_8N1}ADi0sZx`BfBrY`u(kYI_WU-d&LXqgkF|1_1jNMB9c{82OP3vwYVyH zN_e5y(7)g{E;`<|h%zTt0WHG0C#T`Dkz>#6w^WJ${r&p|H$W(F=oXI0dX7_J^RGtN zw;(wrUW)$Lg9Am|5_6Ffj^d9H&RIZcRboDJv<5~5R!5F>I>R;dgu!#Peyx#(Hp2Mc z-mKM@Ji({BfvFTir)B2|3$f~wa`MvB%nQZA>V>WO!*{Z>39Yp`iRlo*q>vf1KvUc2 zSOE3Tb!ustx}c!o29%dmFj42!5wJbibrPp$7r)l>R#V=`=|xcEi)0d&=vX@VO3%m- zGu^)5#X}-c2K}aru*)_`YiZL!l_hgsfAFxgP9^Mt!q`E+b@eLgWt0h-Xralq;IHNdpcOV z?wO^{#f+mzes2(#py?-slWf}%)3ME}8;|oTA|fKgEr(R}rR3LiVtCM`BL1@}!mxo& zlHOztCh}_nTW(zJuYg;-ZSuvZSwo0{rYk0U0#zIjs;VR%o<|PymlOi1G<9y?%15#9 z|2C09AD}lDyY28g*@r*K0 z)c#QE_xg4@aB%sj2AL33Kdr1EYc!~!LC-&vRgTCr$nz^&C_uo>AZ|a`*CBB80M-q=}2c< z)_!re{&JR3OQd$yTJ&Rn0o2dP5ZS~u0ZQr$+;{1kNiuv5W8zbdEHN=LAN;aSG97<6 z-m#UjK(c`$BB+)#7iF2oeHR(|JT!&oWpz*#6R%8~+}K>uv{9;EjP9oVsHa+^` zIhsz&4DEmULi}S@k~9jJAQY7@Fx6PC$(u&ojsIiS$G*m-)oZcub-#U0_S*);xNn0? zt{L%NRbs&mO>UG?)!M(IJQ*3rXirbYTkJi*Rf+Wt4dGjJT@l6|yI+N-qNqrr@DM_u z2ShzEHC^4|6AB87!9FpYC@S&6T2U-0n!64Xa>>Ih5BvFVCG|AA?lwMc&CxFp$E?g8 z@uUrS_rJbL6Bp^o*heJoHho-T%P2S1m|^fLB1xIBOBU1+PE@O4iu!;3x~;1|&mZ{v zAAK$g+H4l(R3m|&KGo?cG$`kOm(Gpi9`{`;)y`uErI*0qtGGQ6O7h3*{(Vre1CGG` zL(@zY4oA`O?ORE8%7hM6jp`@^BVqa>Jfa&G^4`ms=0&Fq6XDEE1FOcsz2&_o*D2|~ ztEz@&6X8CTL0q{1s;rK1-cY-8j@H;IGI{jk5h|^JX3^MvqSmo`t)g02@a%)U5 z=>`>`whM2}d|nA^+z@v3NKmoyxRGGXOaC8tFzs7tPOZk|Pi}55)85K($5?G8(^7#G zBKY6W@-`yMTjwIwzGR%ckQcMOt`;dSH-{FdT6iqtQ`K2vW@aV|?(|J7v8giq8Z5D| zZfvln8pw0Vsu|gR?tmJ#^SaPLgzaj8$&Y$}J-IG-1n|m4;l_V0cOj`%{9w>SMfgo2f zIcdyWD^7_0g9EvTiL9&A+I!s8OF0ufTv>5a;v|S0`3&8S8)I@{@m<{9bP^e8#(ls4 z`Xx0oK7JJKl1mgt^=<<{W9N9k13iWrM%dLl4g)l=Xgmmt1fL{fa9JRtTqEvu@#&JK9AchHX1{uWfv;Mdi-!)z_tjT$*;3{n0r4xW^=6SEO^?I#U@-ZiWn< zp3pE}0J|b*IsFFIcv4T_#S%$OyK`0fW5?++Nm9%$ylpN;C6TVhhoWcCpb>&OJSc?n zmTpJ13o-GW_~nx#buvW_R@{#)vsx)H%L;a(2hcmng2Ek7f3804oD#aCP+>3i3v7XI z;2m>@3E^Pw7~%y}q~gNZXE}y!mR*_&Z=YMg4jaED>68zkPZB)(ak9olsAsb*Grvmgk-KCYPrCU z80y~1n67iuZAGE(Q&3VSiJn8?;17PNZht-&#b9*|7g##EnO%V6jNnB#d|C3+6v)_P zK>bmciSsO^ZbOU}mQgz<6$NiQQjT9)l zg`ML_r_(dhFDrWI6DEX22WPz zyet_|1|$OxUgSdqRUK^nC?CnA{&t7 zb|GE`KXH_?p8Q_wD?bKw_| z74CL5bf~oRAMHx$2GTmH_QQt{6R-plElUmWr)bYrRMx_H$EB3kC_eRFfvD^C_;hhN z1)?9fPxdXZEn?bdRJ$X1yNhTAau%eI|Fl|~x4m9BywJzuu6Xd}c2a`6fcJLkp5Zp; zr!B+>?K`9GLHMmynba)MPc<$kDV_2^sj?v9B31OiesGO#4b|m zovDwn@9V<@EshcnT|7s0Usixf1fDw?J#)!{ieAu*GWb4&f~q35HZ*iu&Jafclh zCunh`sko{d=+*x8E$-Mqkj@MKTk)ZahK-G_ z-li*D<@Q1^gY|#cJZ)_Z-2>=q_>(we>tNr9PEISg4-O83=EMo{sEz<#bfRQeI*%!J zp=S`aI38?lY-|Hyhbruc*=IB4eKogYJ=p2lnK=+Q{LLRe><o{zvo*A8)nbA^fW3ce((gkPu`Qku8;Y}o$opSrvWCOsM%YEc$YCYz=DJ_CF z8sfD^>2TnFzP|H$*mGXi{=44Zd#9jmUPz=3?B|xQ!r%mfH@iw{scWtV#6Vd4a^((S zuGl)G)YLQuSnoXdiCL6IqS7z6Be^MuH-F*WBjHrDw#HUkh&FwNC5W>GV-iBo`$PX* z9?G*d9V_Q~lzD#7(sCxnN4~%wF@%}KW~q)mfI$8oupVQVj|Uf>ReaHu{a1 z+>gI}Xm7u$F<7hF9>=_In}Al5luVxmy3(u5%S&_i=YKg{)@uhnW_-tAQ^T--vfu*II@RM^Dcs{q2|&(wM=3vh_H2De+m^{hK$Z9_qAmx_Q$!I^H9pE) ziUCW(VBe;OLofHXPPZOFvRhwLZWp~rqc7@EkFwf{J52$AhiraTJihzjwrIz>7eoc3 z0v~GEiUb&VYI*B-jr|>XXjl$1dm`X;wG2*1FB~A6Hn5#xY;`iqSM;LRts;5%T4#tC zYd`KY63iHMkca>~z8~0Lop-sfd*x$mZa!t6qZO@uxsw#?1`-H@$4tqa43Q|l_c`YT zo6}5(-{gWm+39Cr@sS^bBQSs40q>*N{Z*=IgP3pGEZaSKarO|TiBw0^R=eI;MeKK; zOM3E}u}D92*5RgU1CK5WSRffQgTwdYXxm-@QN~*^!>j7k!YlVm>yGw+$IaH~3kNpB2x?^a4b?AO2?K!}c^+E=$b1KgxtH6agKIV&yGZs>_OJ5a%c1J zUuR}kR>eVIOYm}As}?nO_j>~bO?w^@m+PeUIr=G|adA|I7Z8F0|KK|#Sr@!9z{ zmk8>GmB7KCg{TY!0e&eNbogvVs~)Hsy<0?tgyM>dZ;{s7z9H68Jg0H$hWke%6P|c2#deX0JRg%5kZ#( zoz>8J%vB&wXDDxpSo?N2+2{H(1Xs$JruurH{PJq+lds04b%-H(jz|dJv_9D|xPP_) z$v#kP?084qOH}+e>>~Ll%k0m!WQIGrcb@P6y>l(OK4!o8`>-gIkHca)m1QWJ>{8jj zFCjI()?rH8Nh0U_$3M%<%cM0O`q6$cx(rZ?9b`{k{AW7q7gc-R@=wx_srcpIroOX9rgC3wE$#tqHENg`JcdcO;L#n`( z8{-4#pset9-SCm{U5ooG%-1!=iUa65o~vSafJvYSBQ z3uZmtlw9zQBhhBPi>l@-;qXX{e3af8rQ?*}$Bj;mVoAg1PqP@e(Ru_8q7Zfve7RTD zVFT0*WkBz&G$*<+6QB@!B583{h$wU!;(Jk;1T9`e{T9n{3Y5!5*I%M-)>g)vNbIc3 zMtx>O&o46~WQ@OqKjuhKn{VjXXQ$kff5v1l4ZezIXue%5_Tr`dPQQVLPv#~jCVpr7 z>6x!5$Frzk#a%Y}u)>FuTEE7GwTM`nF0(Amg;)cSWLWarc(n{bp376L{Vtkiqr+&k zhJW9t+&eDRKzelM(=Ak#o19x-Mug10B>Hv+AnhrT<)5V+B!(_^xtG}|!r)DKjELpP z&<$6TxUEX!jBPsS^IGkGO;nm{zu7>ocE-n(zsT__RPDz@aRKU=+Z>U{1YcwRtm1Nj zlERpg1Iz5px%GI1a@9wi#>JFA!*T_fA~(OjRj%muaY_|G85~wRd-f{FIJ-2?aWL~@ z;S75fvU1dMbs^v~{d^6LYh^1%M;i5=L0M7pnIOu}Kocy*DBGV92rs1D5aVkJ&l+q) zGue1}z!15-6=>eSEmiw_VlY`*+5a$1%;wtBkWqmaosE%t=|r9iW32jVDFcp;fa6zp z|B~r>iJRxU1{|eEIp&Z5c*%`okMlNj8KC6Br3C{`JA`*Yz$d||l61}ER|BlTHGmre z8%Touwxq81hYEr~&!e}LdwX$Y>%HoZ6+1J)X&cYfM15LoX;EaCOHr4NQ!8;Wr6-M2 zeSdd(G=rJ}Qtz{?NgZSEwG*EKhcuTH&q0Fe=g*(NGac^|Q%`;N-(oVYFDv7-WsFwm zrpebt@kaijN|DsA>%zvszA731IE1fdt8k?aEfKW%)!iYsy!DPQEFx^c3^}%?BJ!rjV$ZQP@+_G`y zt1ilr?Z4=z+0!TX_V(0_<}a&_1P6d`)dD<_Sv(For*HW(mvfYta~x0?W*>mOQYUaQ zf`Xnu7Y$yo>2D7PPXRo}d)da)vfx8HmjDlS%DizrohmU8%7HU-8gS8(S2u-33AX|C z^POAm^`1&!ot z@m9AkCMqmoPsf@({*Dx93@cxqpANrRhJOYe?;*oWH6Y!k53k(U)zv-U{JeEZ10NqQ zP+-Wi^fM);#gljq0&=FL7%RllGS{rc2FW2dBN+NKdsp*W2|ADaWZK31*@KV+sXVSw zqJ+1fVA>A`ZW4~n0f!#|2m_+-TW<7E9%a;&?zPMYXzDz0@?bES$G}pSb;$5*nb~7@ zz^@yXpO8aUYqeu$hlhu}W7XDIF@RLY3lIMApAB4bghYw-*yDT6ln%;u8}1y>MVv3^ zxzC=oAGK`_d4H2z(X+RIOPkPa%lOv5I-M&t`X(T0a)E;)O{_8Q2zGg&_x8(}oc~|+ zi6(b!mhxdu5`~Pk^bLeWj(BNxda7=L{LQnG7$wQjm2U+__K2Uqez~6g;Zx2z+f%;U zvuO?NiKY(G)#4oRM~jM%TP3s#mcVsPuG$F_7MlK=BM- z5-W(^e+yXGTM#r@;D)w31OB0WzTqC4ce_Nh5P!bKZQ3)x1(47dK#5|&E-K@RP2aWj z9+Fy_28#S)(fL5pdkfK{S+Gzio&ba26^BB3(;#l@IGerrST!dnr!D)WJJR=vzY~Yi zi(15jgjR5k$-TE`+pOgPMrJ4D#BJhEX$oVa&=oLmZRzRh8CKfv#LJ|zi%a*~p^%wQ ze!iUJ^bwT0ePG>oU}tBQwA9>e7KnV#pVy6j`Jx^yn9(iLG#w7e&<-t0BvnaO)n+j6 z0%<_Eet44x@+@?3n1g-~8V zYR3KO0OE0_36QMr@$*a79BBV~ZC-B$Zw1vgxy|H$H*@NPKVXWm*;(j4{%73r6=3>S z8Y{bmT^DAVX^#jP`R%iUtrvWOf4-#f9^!G8g?E@dJ3O+40-9GqFp2dpdMfUIx&1_` z$i3W1&QsBKuof2Db*{|oDW6e~MpIBKpTodWSGryn60~o4Y@d;xaGCevgwx_9>r)yf zAowJb*i-`DYr5GRNDwe!B61*MHh)NFx>uRZq0a6oGJRv?n6a@Z0Crgo+GxMgovYW& zIPdt{(I1v&YbJ!kSbzk%-1nO0IwB$+O-YiCt0XMb`kFc}TqUKX=8LXQiWX4@0^ic` z-{V@I5?Q`)~&wQz~aPhXl_G$edaU6>({Td z#qIlys5>(hpL$p_faBE(oaZP9uSyF-`(NYZF~Mhtk1==E8A=1q z`)bv=pIEGv=ORIu%7o5Qb-4iCA4Kk* zc^41Zi9U{AS+cYmP4{m9>(r5BT%uH_tw24dnE6;=`V_11{b2~pL`K&h_S;3r+GP}K zY1BSn=a&_+yJ3J4mujOBk(g94pQ7XJ@~8sKElQ%be{es$*5ri7{wo$bV=|hJ2xJ+w z-Qg{nh)@Qr5{HR~uz5%440dsPe0H;{wXt`jE^n)x8hOU~pK}6ZP(%4M zEV6XqQ46(dtPj8!X#IH(GDP3@_D8scgakiIxKrpBRiq&?mY%}PFu_Fl$<&A*J6~=9 zecV_sz_hFUsOZ@Dk)u(>U$88sr-B&Kg-=o((zF$ z@(=#3r2Y_5^xz{UPs12jflfR?`mZj$S&1$#E;I!k(~J-6)zoe%BG}kMxg;#dwWXz{ z`)K3T=nDfoNUZ9&wIJV<7fOME-}9g_cwbVDiA4Vjb*-2xcXLgEndFp~&k5{PU*(V> zbbzHPH}Onb3}v((!0(YnL>z#N5f&M&eGO{H1%tqn7;FDams}6lekxUyKLrYRLo~VH z8Fza&QZ0N1G}aX$g24Dz*a;R2uoO5rtM0fw4F$4#C~t~-K7Td#ALTQ>XWBR_H71ph zc++I80_LN(w97{AXH7u9cAcnQazXtZ&2(Ifq?#OF^ovt(t{PM&KC~P~0jpC2ux7DB z093X_$f_YIUpA1dDQJ8%v{o3|=8_w@ao?G+J#;rp3kDZLB>*b;9c0ff*|o|#M4~;1 zlE<9NQH_27EiG=E4K<^mW*^=ZnIQ)*Q>fLr&++an?^pS^tz9cP_D^zw3F^Nw*&vfQ zbP4pazfoC&+KwR$D$c0%ADETB1Aw>uuO}W4=ERuD3X~Tskb|x9j$hj{@<7~?jA##a zHSS1aTzp4R7z%(Xnf2|?(RB1qxEH7+>*nrl2ehM-{*2M{o!)4n4ViCSMff(diT0C? z7n@{2#5M%-72{&DzNRez7%^w3ry<$u+=Sc*yw>tawD}!R zj@D)vgT4}EGY*i%2EctSq*2`+%+a3tIb^EDc@qj+DLE*B^T#HCyi4=zDg@m^p?kYa zg9OwKcmYgS4##ppz-GTL{Ierw9d?WlX5GoU;?HeD6<)^1o_G7^Yw`A=;4nB)GLZ4t zzV*Ql79e@MK9EcOS#&+$nVf9G;G@8JRvMcK^oZ>*@d5A2U=5b&L-W^}Ki9$@1&j3g zL}#Z;dw%j-%jUJV>*M2tu~pVS(kkkSp&WOgX*uwmYMk*h16=^Kz{>A{)tQQ*8NfQ% zUYs3miIu%=Se~7o?S3TIw*L!s%E4WKf1ED_cT|(-8Evrb!4Izs(1&{?64rrB87q4y zt%7CtQ_?_iSW*VV$<&#d0b?}h+WHNyQb4>9u(45b2aljWtPko9>hI{e9_k+$>WT+$ z-GlMY4kUnwo(#!g&tpJKfVHuR*mW1(`H*|a-qOpLFOwwpv<}}gW>5SEa~+p)oveRj zW`@)K#BIf}Oo!sX#hvyYEK{^TjC}DJSSw{(se)JjK;D6Cd%sZI`AB6K1b9?th1RKQ z9pW`G$%9p6#p`3$b4n-dbC-z=zF8OTg4DC1E)e4~onog-+%Gv~zpY*QR7o|KHrkez z4Qs_2DM9|W+D!gZHh*8{PhY3xkkVi+;k^<4Oh-1#e_UA|l!LX2j|4v466*K#Ds{zp{AUk2pSrde|}d$inIPs-ViC*51_!^psmqR+)*<{AbLEzm-=A3ibcs6L*HQE z-;0Qda7V)K+V@B2FcrQwmqG_9Zl)UN)){#B@Lql5=8Z0-(&ywBL3Ge}ep&;LdgDgL z?b5Y>|9BNjaks~C9w%kkG76(AV^&1k7_G#+&3a(DW!}>T!<6=?B@Ofw&;B}$Y^$bf z@#-?PQRb`2{7a3Fknjd@|F~)phrW8IhyM!+Eji_rx0)WqGL^x@pHA&SwmX z5OXdYJ!dd{`68i1G%^-7x;iq8UZo4S5m>?>T4GnAI;`x{RJEhhw+x(BliGG2-Z@I9 z>c$}I9yhK2EH}J7`EKuC>^WQZbS|mi=i=O+1NBFU6- z%8>VyxU#!?Mp6wl(Y`dZm*!%v@*Oh0AA7M_=bVR~En5P(LRoIYMCBQZaZl<+Lhnsy+H3}<^NhUDlY z!#KTkdF79hM~K8Ys(z0%-a`j1hHslkZQ-(P`Q;ZKi5XPkLOML7 z_jpu`&sN~mku?+w@zC2d)y~zYy!c|jVd#6T#_dmk3F3=h>|jsI$!+uKt<=4mSu=c{ z=`!dwon8bi!B0t{-6c7Gr*4PInT|`Hf}y*H&O@)ZrELDL2GeNu1<*ZE(CO_|Q!CRc zsK|Z7o0j9V-eabIdAAm~%&?K5HQ<99qZOq1lI+#nq@V}yr3r}JV$0F!0!^}Zw#Q}j zy}wzXXli@p749r2m(Dscva-I@c;**bFvDxE(%v)oaplw5w<4^*cPT^mQ%1sUxi3XH zikuo$)Q^eZWz!gZ9e9Pe>*FBs7`4!!q2iJ!+>9H#u-cccXNjk_ehn$(um~ruH7Ye+FJv&6jhaek##!L@?NoQob=1 z@Fgv~w>L}-EVe3Ss}N1JEnRm&Xu82AH zE-^KNV`y_C&d80H&B&nhwT`nEb3o4nH%E%^DfC%e*E{QflN>Vc7M{(^&28jM8Hk=LLI$+?=goTjkrnmEz&n1_14}G^RxFJAqs2Qq16a3(NC5yTs!ByyVru(FC%yux zso%dp32Eh}4*W?m5Yskk$ - - 4.0.0 - - - io.trino - trino-root - 463-SNAPSHOT - ../../pom.xml - - - trino-accumulo-iterators - jar - Trino - Accumulo iterators - - - 11 - - - - - com.google.guava - guava - - - - org.apache.accumulo - accumulo-core - ${dep.accumulo.version} - provided - - - commons-logging - commons-logging - - - org.apache.hadoop - hadoop-client-runtime - - - org.apache.logging.log4j - log4j-1.2-api - - - - - - io.airlift - junit-extensions - test - - - - org.junit.jupiter - junit-jupiter-api - test - - - diff --git a/plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MaxByteArrayCombiner.java b/plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MaxByteArrayCombiner.java deleted file mode 100644 index 9f34bf0455676..0000000000000 --- a/plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MaxByteArrayCombiner.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.iterators; - -import com.google.common.primitives.UnsignedBytes; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.Combiner; - -import java.util.Comparator; -import java.util.Iterator; - -/** - * A Combiner that does a lexicographic compare against values, returning the 'largest' value - */ -public class MaxByteArrayCombiner - extends Combiner -{ - private final Comparator comparator = UnsignedBytes.lexicographicalComparator(); - - @Override - public Value reduce(Key key, Iterator iter) - { - Value max = null; - while (iter.hasNext()) { - Value test = iter.next(); - if (max == null) { - max = new Value(test.get()); - } - else if (comparator.compare(test.get(), max.get()) > 0) { - max.set(test.get()); - } - } - return max; - } -} diff --git a/plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MinByteArrayCombiner.java b/plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MinByteArrayCombiner.java deleted file mode 100644 index 8d47fb79319ef..0000000000000 --- a/plugin/trino-accumulo-iterators/src/main/java/io/trino/plugin/accumulo/iterators/MinByteArrayCombiner.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.iterators; - -import com.google.common.primitives.UnsignedBytes; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.Combiner; - -import java.util.Comparator; -import java.util.Iterator; - -/** - * A Combiner that does a lexicographic compare against values, returning the 'smallest' value - */ -public class MinByteArrayCombiner - extends Combiner -{ - private final Comparator comparator = UnsignedBytes.lexicographicalComparator(); - - @Override - public Value reduce(Key key, Iterator iter) - { - Value min = null; - while (iter.hasNext()) { - Value test = iter.next(); - if (min == null) { - min = new Value(test.get()); - } - else if (comparator.compare(test.get(), min.get()) < 0) { - min.set(test.get()); - } - } - return min; - } -} diff --git a/plugin/trino-accumulo-iterators/src/test/java/io/trino/server/TestDummy.java b/plugin/trino-accumulo-iterators/src/test/java/io/trino/server/TestDummy.java deleted file mode 100644 index b560df431cb69..0000000000000 --- a/plugin/trino-accumulo-iterators/src/test/java/io/trino/server/TestDummy.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.server; - -import org.junit.jupiter.api.Test; - -public class TestDummy -{ - @Test - public void buildRequiresTestToExist() {} -} diff --git a/plugin/trino-accumulo/pom.xml b/plugin/trino-accumulo/pom.xml deleted file mode 100644 index f71caf237b2ad..0000000000000 --- a/plugin/trino-accumulo/pom.xml +++ /dev/null @@ -1,355 +0,0 @@ - - - 4.0.0 - - - io.trino - trino-root - 463-SNAPSHOT - ../../pom.xml - - - trino-accumulo - trino-plugin - Trino - Accumulo connector - - - 5.7.1 - - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.google.guava - guava - - - - com.google.inject - guice - - - - io.airlift - bootstrap - - - - io.airlift - concurrent - - - - io.airlift - configuration - - - - io.airlift - json - - - - io.airlift - log - - - - io.airlift - units - - - - io.trino - trino-accumulo-iterators - ${project.version} - - - - io.trino - trino-cache - - - - io.trino - trino-plugin-toolkit - - - - io.trino.hadoop - hadoop-apache - - - - jakarta.annotation - jakarta.annotation-api - - - - jakarta.validation - jakarta.validation-api - - - - org.apache.accumulo - accumulo-core - ${dep.accumulo.version} - - - com.beust - jcommander - - - commons-logging - commons-logging - - - jline - jline - - - log4j - log4j - - - org.apache.hadoop - hadoop-client - - - org.apache.hadoop - hadoop-client-api - - - org.apache.hadoop - hadoop-client-runtime - - - org.apache.htrace - htrace-core - - - - - - org.apache.curator - curator-client - ${dep.curator.version} - - - - org.apache.curator - curator-framework - ${dep.curator.version} - - - - org.apache.zookeeper - zookeeper - - - - com.fasterxml.jackson.core - jackson-annotations - provided - - - - io.airlift - slice - provided - - - - io.opentelemetry - opentelemetry-api - provided - - - - io.opentelemetry - opentelemetry-context - provided - - - - io.trino - trino-spi - provided - - - - org.openjdk.jol - jol-core - provided - - - - io.airlift - log-manager - runtime - - - org.slf4j - log4j-over-slf4j - - - - - - io.dropwizard.metrics - metrics-core - runtime - - - - com.github.docker-java - docker-java-api - test - - - - io.airlift - junit-extensions - test - - - - io.trino - trino-main - test - - - commons-codec - commons-codec - - - - - - io.trino - trino-testing - test - - - commons-codec - commons-codec - - - - - - io.trino - trino-testing-containers - test - - - - io.trino - trino-testing-services - test - - - - io.trino - trino-tpch - test - - - - io.trino.tpch - tpch - test - - - - org.apache.accumulo - accumulo-minicluster - ${dep.accumulo.version} - test - - - org.apache.hadoop - hadoop-client-api - - - org.apache.hadoop - hadoop-client-runtime - - - org.apache.logging.log4j - log4j-core - - - org.glassfish.hk2.external - aopalliance-repackaged - - - - - - org.assertj - assertj-core - test - - - - org.jetbrains - annotations - test - - - - org.junit.jupiter - junit-jupiter-api - test - - - - org.junit.jupiter - junit-jupiter-engine - test - - - - org.testcontainers - testcontainers - test - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-iterators - - copy - - - process-test-classes - - false - - - io.trino - trino-accumulo-iterators - ${project.version} - jar - - - true - ${project.build.testOutputDirectory} - - - - - - - diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnector.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnector.java deleted file mode 100644 index 6caf14fa623fa..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnector.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.inject.Inject; -import io.airlift.bootstrap.LifeCycleManager; -import io.trino.plugin.accumulo.conf.AccumuloSessionProperties; -import io.trino.plugin.accumulo.conf.AccumuloTableProperties; -import io.trino.plugin.accumulo.io.AccumuloPageSinkProvider; -import io.trino.plugin.accumulo.io.AccumuloRecordSetProvider; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorPageSinkProvider; -import io.trino.spi.connector.ConnectorRecordSetProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.session.PropertyMetadata; -import io.trino.spi.transaction.IsolationLevel; - -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.spi.transaction.IsolationLevel.READ_UNCOMMITTED; -import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports; -import static java.util.Objects.requireNonNull; - -/** - * Trino Connector for Accumulo. - * Defines several high-level classes for properties, metadata, retrieving splits, providing I/O operations, etc. - */ -public class AccumuloConnector - implements Connector -{ - private final LifeCycleManager lifeCycleManager; - private final AccumuloMetadataFactory metadataFactory; - private final AccumuloSplitManager splitManager; - private final AccumuloRecordSetProvider recordSetProvider; - private final AccumuloPageSinkProvider pageSinkProvider; - private final AccumuloSessionProperties sessionProperties; - private final AccumuloTableProperties tableProperties; - private final ConcurrentMap transactions = new ConcurrentHashMap<>(); - - @Inject - public AccumuloConnector( - LifeCycleManager lifeCycleManager, - AccumuloMetadataFactory metadataFactory, - AccumuloSplitManager splitManager, - AccumuloRecordSetProvider recordSetProvider, - AccumuloPageSinkProvider pageSinkProvider, - AccumuloSessionProperties sessionProperties, - AccumuloTableProperties tableProperties) - { - this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); - this.metadataFactory = requireNonNull(metadataFactory, "metadataFactory is null"); - this.splitManager = requireNonNull(splitManager, "splitManager is null"); - this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); - this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); - this.sessionProperties = requireNonNull(sessionProperties, "sessionProperties is null"); - this.tableProperties = requireNonNull(tableProperties, "tableProperties is null"); - } - - @Override - public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transactionHandle) - { - ConnectorMetadata metadata = transactions.get(transactionHandle); - checkArgument(metadata != null, "no such transaction: %s", transactionHandle); - return metadata; - } - - @Override - public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit) - { - checkConnectorSupports(READ_UNCOMMITTED, isolationLevel); - ConnectorTransactionHandle transaction = new AccumuloTransactionHandle(); - transactions.put(transaction, metadataFactory.create()); - return transaction; - } - - @Override - public void commit(ConnectorTransactionHandle transactionHandle) - { - checkArgument(transactions.remove(transactionHandle) != null, "no such transaction: %s", transactionHandle); - } - - @Override - public void rollback(ConnectorTransactionHandle transactionHandle) - { - AccumuloMetadata metadata = transactions.remove(transactionHandle); - checkArgument(metadata != null, "no such transaction: %s", transactionHandle); - metadata.rollback(); - } - - @Override - public ConnectorSplitManager getSplitManager() - { - return splitManager; - } - - @Override - public ConnectorRecordSetProvider getRecordSetProvider() - { - return recordSetProvider; - } - - @Override - public ConnectorPageSinkProvider getPageSinkProvider() - { - return pageSinkProvider; - } - - @Override - public List> getTableProperties() - { - return tableProperties.getTableProperties(); - } - - @Override - public List> getSessionProperties() - { - return sessionProperties.getSessionProperties(); - } - - @Override - public final void shutdown() - { - lifeCycleManager.stop(); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnectorFactory.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnectorFactory.java deleted file mode 100644 index 0f33a73f145d9..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloConnectorFactory.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; -import io.trino.plugin.base.TypeDeserializerModule; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.connector.ConnectorFactory; - -import java.util.Map; - -import static io.trino.plugin.base.Versions.checkStrictSpiVersionMatch; -import static java.util.Objects.requireNonNull; - -public class AccumuloConnectorFactory - implements ConnectorFactory -{ - public static final String CONNECTOR_NAME = "accumulo"; - - @Override - public String getName() - { - return CONNECTOR_NAME; - } - - @Override - public Connector create(String catalogName, Map config, ConnectorContext context) - { - requireNonNull(catalogName, "catalogName is null"); - requireNonNull(config, "config is null"); - requireNonNull(context, "context is null"); - checkStrictSpiVersionMatch(context, this); - - Bootstrap app = new Bootstrap( - new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), - new AccumuloModule()); - - Injector injector = app - .doNotInitializeLogging() - .setRequiredConfigurationProperties(config) - .initialize(); - - return injector.getInstance(AccumuloConnector.class); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloErrorCode.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloErrorCode.java deleted file mode 100644 index 25c6f8d0a80d9..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloErrorCode.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import io.trino.spi.ErrorCode; -import io.trino.spi.ErrorCodeSupplier; -import io.trino.spi.ErrorType; - -import static io.trino.spi.ErrorType.EXTERNAL; - -public enum AccumuloErrorCode - implements ErrorCodeSupplier -{ - // Thrown when an Accumulo error is caught that we were not expecting, - // such as when a create table operation fails (even though we know it will succeed due to our validation steps) - UNEXPECTED_ACCUMULO_ERROR(1, EXTERNAL), - - // Thrown when a ZooKeeper error is caught due to a failed operation - ZOOKEEPER_ERROR(2, EXTERNAL), - - // Thrown when a serialization error occurs when reading/writing data from/to Accumulo - IO_ERROR(3, EXTERNAL), - - // Thrown when a table that is expected to exist does not exist - ACCUMULO_TABLE_DNE(4, EXTERNAL), - - // Thrown when a table that is *not* expected to exist, does exist - ACCUMULO_TABLE_EXISTS(5, EXTERNAL), - - // Thrown when an attempt to start/stop MiniAccumuloCluster fails (testing only) - MINI_ACCUMULO(6, EXTERNAL); - - private final ErrorCode errorCode; - - AccumuloErrorCode(int code, ErrorType type) - { - errorCode = new ErrorCode(code + 0x0103_0000, name(), type); - } - - @Override - public ErrorCode toErrorCode() - { - return errorCode; - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadata.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadata.java deleted file mode 100644 index adb3d34bb536d..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadata.java +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; -import io.airlift.slice.Slice; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.model.AccumuloTableHandle; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorOutputMetadata; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableLayout; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTableVersion; -import io.trino.spi.connector.ConnectorViewDefinition; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.ConstraintApplicationResult; -import io.trino.spi.connector.RetryMode; -import io.trino.spi.connector.SaveMode; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.spi.connector.TableNotFoundException; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.security.TrinoPrincipal; -import io.trino.spi.statistics.ComputedStatistics; - -import java.util.Collection; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static io.trino.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_EXISTS; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.connector.RetryMode.NO_RETRIES; -import static io.trino.spi.connector.SaveMode.REPLACE; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -/** - * Trino metadata provider for Accumulo. - * Responsible for creating/dropping/listing tables, schemas, columns, all sorts of goodness. Heavily leverages {@link AccumuloMetadataManager}. - */ -public class AccumuloMetadata - implements ConnectorMetadata -{ - private static final JsonCodec VIEW_CODEC = - new JsonCodecFactory(new ObjectMapperProvider()).jsonCodec(ConnectorViewDefinition.class); - - private final AccumuloMetadataManager metadataManager; - private final AtomicReference rollbackAction = new AtomicReference<>(); - - @Inject - public AccumuloMetadata(AccumuloMetadataManager metadataManager) - { - this.metadataManager = requireNonNull(metadataManager, "metadataManager is null"); - } - - @Override - public void createSchema(ConnectorSession session, String schemaName, Map properties, TrinoPrincipal owner) - { - checkArgument(properties.isEmpty(), "Can't have properties for schema creation"); - metadataManager.createSchema(schemaName); - } - - @Override - public void dropSchema(ConnectorSession session, String schemaName, boolean cascade) - { - if (cascade) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support dropping schemas with CASCADE option"); - } - metadataManager.dropSchema(schemaName); - } - - @Override - public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout, RetryMode retryMode, boolean replace) - { - if (retryMode != NO_RETRIES) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support query retries"); - } - if (replace) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support replacing tables"); - } - if (tableMetadata.getComment().isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support creating tables with table comment"); - } - - checkNoRollback(); - - SchemaTableName tableName = tableMetadata.getTable(); - AccumuloTable table = metadataManager.createTable(tableMetadata); - - AccumuloTableHandle handle = new AccumuloTableHandle( - tableName.getSchemaName(), - tableName.getTableName(), - table.getRowId(), - table.isExternal(), - table.getSerializerClassName(), - table.getScanAuthorizations()); - - setRollback(() -> rollbackCreateTable(table)); - - return handle; - } - - @Override - public Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) - { - clearRollback(); - return Optional.empty(); - } - - private void rollbackCreateTable(AccumuloTable table) - { - metadataManager.dropTable(table); - } - - @Override - public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, SaveMode saveMode) - { - if (saveMode == REPLACE) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support replacing tables"); - } - if (tableMetadata.getComment().isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support creating tables with table comment"); - } - metadataManager.createTable(tableMetadata); - } - - @Override - public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) - { - AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle; - AccumuloTable table = metadataManager.getTable(handle.toSchemaTableName()); - if (table != null) { - metadataManager.dropTable(table); - } - } - - @Override - public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, - SchemaTableName newTableName) - { - if (metadataManager.getTable(newTableName) != null) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, "Table " + newTableName + " already exists"); - } - - AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle; - metadataManager.renameTable(handle.toSchemaTableName(), newTableName); - } - - @Override - public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, Map viewProperties, boolean replace) - { - checkArgument(viewProperties.isEmpty(), "This connector does not support creating views with properties"); - String viewData = VIEW_CODEC.toJson(definition); - if (replace) { - metadataManager.createOrReplaceView(viewName, viewData); - } - else { - metadataManager.createView(viewName, viewData); - } - } - - @Override - public void dropView(ConnectorSession session, SchemaTableName viewName) - { - metadataManager.dropView(viewName); - } - - @Override - public Optional getView(ConnectorSession session, SchemaTableName viewName) - { - return Optional.ofNullable(metadataManager.getView(viewName)) - .map(view -> VIEW_CODEC.fromJson(view.data())); - } - - @Override - public List listViews(ConnectorSession session, Optional schemaName) - { - return listViews(schemaName); - } - - /** - * Gets all views in the given schema, or all schemas if null. - * - * @param filterSchema Schema to filter the views, or absent to list all schemas - * @return List of views - */ - private List listViews(Optional filterSchema) - { - ImmutableList.Builder builder = ImmutableList.builder(); - if (filterSchema.isPresent()) { - for (String view : metadataManager.getViewNames(filterSchema.get())) { - builder.add(new SchemaTableName(filterSchema.get(), view)); - } - } - else { - for (String schemaName : metadataManager.getSchemaNames()) { - for (String view : metadataManager.getViewNames(schemaName)) { - builder.add(new SchemaTableName(schemaName, view)); - } - } - } - - return builder.build(); - } - - @Override - public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle, List columns, RetryMode retryMode) - { - if (retryMode != NO_RETRIES) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support query retries"); - } - - checkNoRollback(); - AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle; - setRollback(() -> rollbackInsert(handle)); - return handle; - } - - @Override - public Optional finishInsert( - ConnectorSession session, - ConnectorInsertTableHandle insertHandle, - List sourceTableHandles, - Collection fragments, - Collection computedStatistics) - { - clearRollback(); - return Optional.empty(); - } - - private static void rollbackInsert(ConnectorInsertTableHandle insertHandle) - { - // Rollbacks for inserts are off the table when it comes to data in Accumulo. - // When a batch of Mutations fails to be inserted, the general strategy - // is to run the insert operation again until it is successful - // Any mutations that were successfully written will be overwritten - // with the same values, so that isn't a problem. - AccumuloTableHandle handle = (AccumuloTableHandle) insertHandle; - throw new TrinoException(NOT_SUPPORTED, format("Unable to rollback insert for table %s.%s. Some rows may have been written. Please run your insert again.", handle.getSchema(), handle.getTable())); - } - - @Override - public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName, Optional startVersion, Optional endVersion) - { - if (startVersion.isPresent() || endVersion.isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables"); - } - - if (!listSchemaNames(session).contains(tableName.getSchemaName().toLowerCase(Locale.ENGLISH))) { - return null; - } - - // Need to validate that SchemaTableName is a table - if (!this.listViews(session, Optional.of(tableName.getSchemaName())).contains(tableName)) { - AccumuloTable table = metadataManager.getTable(tableName); - if (table == null) { - return null; - } - - return new AccumuloTableHandle( - table.getSchema(), - table.getTable(), - table.getRowId(), - table.isExternal(), - table.getSerializerClassName(), - table.getScanAuthorizations()); - } - - return null; - } - - @Override - public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) - { - AccumuloTableHandle handle = (AccumuloTableHandle) table; - SchemaTableName tableName = new SchemaTableName(handle.getSchema(), handle.getTable()); - ConnectorTableMetadata metadata = getTableMetadata(tableName); - if (metadata == null) { - throw new TableNotFoundException(tableName); - } - return metadata; - } - - @Override - public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) - { - AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle; - - AccumuloTable table = metadataManager.getTable(handle.toSchemaTableName()); - if (table == null) { - throw new TableNotFoundException(handle.toSchemaTableName()); - } - - ImmutableMap.Builder columnHandles = ImmutableMap.builder(); - for (AccumuloColumnHandle column : table.getColumns()) { - columnHandles.put(column.name(), column); - } - return columnHandles.buildOrThrow(); - } - - @Override - public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) - { - return ((AccumuloColumnHandle) columnHandle).columnMetadata(); - } - - @Override - public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) - { - AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle; - AccumuloColumnHandle columnHandle = (AccumuloColumnHandle) source; - AccumuloTable table = metadataManager.getTable(handle.toSchemaTableName()); - if (table == null) { - throw new TableNotFoundException(new SchemaTableName(handle.getSchema(), handle.getTable())); - } - - metadataManager.renameColumn(table, columnHandle.name(), target); - } - - @Override - public List listSchemaNames(ConnectorSession session) - { - return ImmutableList.copyOf(metadataManager.getSchemaNames()); - } - - @Override - public List listTables(ConnectorSession session, Optional filterSchema) - { - Set schemaNames = filterSchema.>map(ImmutableSet::of) - .orElseGet(metadataManager::getSchemaNames); - - ImmutableSet.Builder builder = ImmutableSet.builder(); - for (String schemaName : schemaNames) { - for (String tableName : metadataManager.getTableNames(schemaName)) { - builder.add(new SchemaTableName(schemaName, tableName)); - } - } - builder.addAll(listViews(session, filterSchema)); - // Deduplicate with set because state may change concurrently - return builder.build().asList(); - } - - @Override - public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) - { - requireNonNull(prefix, "prefix is null"); - ImmutableMap.Builder> columns = ImmutableMap.builder(); - for (SchemaTableName tableName : listTables(session, prefix)) { - ConnectorTableMetadata tableMetadata = getTableMetadata(tableName); - // table can disappear during listing operation - if (tableMetadata != null) { - columns.put(tableName, tableMetadata.getColumns()); - } - } - return columns.buildOrThrow(); - } - - @Override - public Optional> applyFilter(ConnectorSession session, ConnectorTableHandle table, Constraint constraint) - { - AccumuloTableHandle handle = (AccumuloTableHandle) table; - - TupleDomain oldDomain = handle.getConstraint(); - TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); - if (oldDomain.equals(newDomain)) { - return Optional.empty(); - } - - handle = new AccumuloTableHandle( - handle.getSchema(), - handle.getTable(), - handle.getRowId(), - newDomain, - handle.isExternal(), - handle.getSerializerClassName(), - handle.getScanAuthorizations()); - - return Optional.of(new ConstraintApplicationResult<>(handle, constraint.getSummary(), constraint.getExpression(), false)); - } - - private void checkNoRollback() - { - checkState(rollbackAction.get() == null, "Cannot begin a new write while in an existing one"); - } - - private void setRollback(Runnable action) - { - checkState(rollbackAction.compareAndSet(null, action), "Should not have to override existing rollback action"); - } - - private void clearRollback() - { - rollbackAction.set(null); - } - - public void rollback() - { - Runnable rollbackAction = this.rollbackAction.getAndSet(null); - if (rollbackAction != null) { - rollbackAction.run(); - } - } - - private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) - { - if (!metadataManager.getSchemaNames().contains(tableName.getSchemaName())) { - return null; - } - - // Need to validate that SchemaTableName is a table - if (!this.listViews(Optional.ofNullable(tableName.getSchemaName())).contains(tableName)) { - AccumuloTable table = metadataManager.getTable(tableName); - if (table == null) { - return null; - } - - return new ConnectorTableMetadata(tableName, table.getColumnsMetadata()); - } - - return null; - } - - private List listTables(ConnectorSession session, SchemaTablePrefix prefix) - { - // List all tables if schema or table is null - if (prefix.getTable().isEmpty()) { - return listTables(session, prefix.getSchema()); - } - - // Make sure requested table exists, returning the single table of it does - SchemaTableName table = prefix.toSchemaTableName(); - if (getTableHandle(session, table, Optional.empty(), Optional.empty()) != null) { - return ImmutableList.of(table); - } - - // Else, return empty list - return ImmutableList.of(); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataFactory.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataFactory.java deleted file mode 100644 index 765997ac4cf25..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.inject.Inject; - -import static java.util.Objects.requireNonNull; - -public class AccumuloMetadataFactory -{ - private final AccumuloMetadataManager metadataManager; - - @Inject - public AccumuloMetadataFactory(AccumuloMetadataManager metadataManager) - { - this.metadataManager = requireNonNull(metadataManager, "metadataManager is null"); - } - - public AccumuloMetadata create() - { - return new AccumuloMetadata(metadataManager); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataManager.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataManager.java deleted file mode 100644 index 7cef53573b807..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloMetadataManager.java +++ /dev/null @@ -1,986 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.accumulo.conf.AccumuloSessionProperties; -import io.trino.plugin.accumulo.conf.AccumuloTableProperties; -import io.trino.plugin.accumulo.index.IndexLookup; -import io.trino.plugin.accumulo.index.Indexer; -import io.trino.plugin.accumulo.io.AccumuloPageSink; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.metadata.AccumuloView; -import io.trino.plugin.accumulo.metadata.ZooKeeperMetadataManager; -import io.trino.plugin.accumulo.model.AccumuloColumnConstraint; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.model.TabletSplitMetadata; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.SchemaNotFoundException; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.TableNotFoundException; -import io.trino.spi.predicate.Domain; -import io.trino.spi.type.TimestampType; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.PartialKey; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.hadoop.io.Text; - -import java.security.InvalidParameterException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static io.trino.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE; -import static io.trino.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_EXISTS; -import static io.trino.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static io.trino.plugin.accumulo.metadata.ZooKeeperMetadataManager.DEFAULT_SCHEMA; -import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; -import static io.trino.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; -import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; -import static io.trino.spi.StandardErrorCode.NOT_FOUND; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -/** - * This class is the main access point for the Trino connector to interact with Accumulo. - * It is responsible for creating tables, dropping tables, retrieving table metadata, and getting the ConnectorSplits from a table. - */ -public class AccumuloMetadataManager -{ - private static final Logger LOG = Logger.get(AccumuloMetadataManager.class); - private static final Splitter COMMA_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults(); - - private final ZooKeeperMetadataManager metaManager; - private final Authorizations auths; - private final AccumuloTableManager tableManager; - private final AccumuloClient client; - private final IndexLookup indexLookup; - private final String username; - - @Inject - public AccumuloMetadataManager( - AccumuloClient client, - AccumuloConfig config, - ZooKeeperMetadataManager metaManager, - AccumuloTableManager tableManager, - IndexLookup indexLookup) - throws AccumuloException, AccumuloSecurityException - { - this.client = requireNonNull(client, "client is null"); - this.username = config.getUsername(); - this.metaManager = requireNonNull(metaManager, "metaManager is null"); - this.tableManager = requireNonNull(tableManager, "tableManager is null"); - this.indexLookup = requireNonNull(indexLookup, "indexLookup is null"); - - this.auths = client.securityOperations().getUserAuthorizations(username); - - // The default namespace is created in ZooKeeperMetadataManager's constructor - if (!tableManager.namespaceExists(DEFAULT_SCHEMA)) { - try { - tableManager.createNamespace(DEFAULT_SCHEMA); - } - catch (TrinoException e) { - if (!e.getErrorCode().equals(ALREADY_EXISTS.toErrorCode())) { - throw e; - } - } - } - } - - public void createSchema(String schemaName) - { - metaManager.createSchema(schemaName); - tableManager.createNamespace(schemaName); - } - - public void dropSchema(String schemaName) - { - metaManager.dropSchema(schemaName); - tableManager.dropNamespace(schemaName); - } - - public AccumuloTable createTable(ConnectorTableMetadata meta) - { - // Validate the DDL is something we can handle - validateCreateTable(meta); - - Map tableProperties = meta.getProperties(); - String rowIdColumn = getRowIdColumn(meta); - - // Get the list of column handles - List columns = getColumnHandles(meta, rowIdColumn); - - // Create the AccumuloTable object - AccumuloTable table = new AccumuloTable( - meta.getTable().getSchemaName(), - meta.getTable().getTableName(), - columns, - rowIdColumn, - AccumuloTableProperties.isExternal(tableProperties), - AccumuloTableProperties.getSerializerClass(tableProperties), - AccumuloTableProperties.getScanAuthorizations(tableProperties)); - - // Make sure the namespace exists - if (!tableManager.namespaceExists(table.getSchema())) { - throw new SchemaNotFoundException(table.getSchema()); - } - - // First, create the metadata - metaManager.createTableMetadata(table); - - // Create the Accumulo table if it does not exist (for 'external' table) - if (!tableManager.exists(table.getFullTableName())) { - tableManager.createAccumuloTable(table.getFullTableName()); - } - - // Set any locality groups on the data table - setLocalityGroups(tableProperties, table); - - // Create index tables, if appropriate - createIndexTables(table); - - return table; - } - - /** - * Validates the given metadata for a series of conditions to ensure the table is well-formed. - * - * @param meta Table metadata - */ - private void validateCreateTable(ConnectorTableMetadata meta) - { - validateColumns(meta); - validateLocalityGroups(meta); - if (!AccumuloTableProperties.isExternal(meta.getProperties())) { - validateInternalTable(meta); - } - } - - private static void validateColumns(ConnectorTableMetadata meta) - { - // Check all the column types, and throw an exception if the types of a map are complex - // While it is a rare case, this is not supported by the Accumulo connector - ImmutableSet.Builder columnNameBuilder = ImmutableSet.builder(); - for (ColumnMetadata column : meta.getColumns()) { - if (Types.isMapType(column.getType())) { - if (Types.isMapType(Types.getKeyType(column.getType())) - || Types.isMapType(Types.getValueType(column.getType())) - || Types.isArrayType(Types.getKeyType(column.getType())) - || Types.isArrayType(Types.getValueType(column.getType()))) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Key/value types of a MAP column must be plain types"); - } - } - - if (column.getType() instanceof TimestampType && ((TimestampType) column.getType()).getPrecision() != 3) { - throw new TrinoException(NOT_SUPPORTED, format("%s type not supported", column.getType())); - } - - columnNameBuilder.add(column.getName().toLowerCase(Locale.ENGLISH)); - } - - // Validate the columns are distinct - if (columnNameBuilder.build().size() != meta.getColumns().size()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Duplicate column names are not supported"); - } - - Optional>> columnMapping = AccumuloTableProperties.getColumnMapping(meta.getProperties()); - if (columnMapping.isPresent()) { - // Validate there are no duplicates in the column mapping - long distinctMappings = columnMapping.get().values().stream().distinct().count(); - if (distinctMappings != columnMapping.get().size()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Duplicate column family/qualifier pair detected in column mapping, check the value of " + AccumuloTableProperties.COLUMN_MAPPING); - } - - // Validate no column is mapped to the reserved entry - String reservedRowIdColumn = AccumuloPageSink.ROW_ID_COLUMN.toString(); - if (columnMapping.get().values().stream() - .filter(pair -> pair.getKey().equals(reservedRowIdColumn) && pair.getValue().equals(reservedRowIdColumn)) - .count() > 0) { - throw new TrinoException(INVALID_TABLE_PROPERTY, format("Column familiy/qualifier mapping of %s:%s is reserved", reservedRowIdColumn, reservedRowIdColumn)); - } - } - else if (AccumuloTableProperties.isExternal(meta.getProperties())) { - // Column mapping is not defined (i.e. use column generation) and table is external - // But column generation is for internal tables only - throw new TrinoException(INVALID_TABLE_PROPERTY, "Column generation for external tables is not supported, must specify " + AccumuloTableProperties.COLUMN_MAPPING); - } - } - - private static void validateLocalityGroups(ConnectorTableMetadata meta) - { - // Validate any configured locality groups - Optional>> groups = AccumuloTableProperties.getLocalityGroups(meta.getProperties()); - if (groups.isEmpty()) { - return; - } - - String rowIdColumn = getRowIdColumn(meta); - - // For each locality group - for (Map.Entry> g : groups.get().entrySet()) { - if (g.getValue().contains(rowIdColumn)) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Row ID column cannot be in a locality group"); - } - - // Validate the specified column names exist in the table definition, - // incrementing a counter for each matching column - int matchingColumns = 0; - for (ColumnMetadata column : meta.getColumns()) { - if (g.getValue().contains(column.getName().toLowerCase(Locale.ENGLISH))) { - ++matchingColumns; - - // Break out early if all columns are found - if (matchingColumns == g.getValue().size()) { - break; - } - } - } - - // If the number of matched columns does not equal the defined size, - // then a column was specified that does not exist - // (or there is a duplicate column in the table DDL, which is also an issue but has been checked before in validateColumns). - if (matchingColumns != g.getValue().size()) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Unknown Trino column defined for locality group " + g.getKey()); - } - } - } - - private void validateInternalTable(ConnectorTableMetadata meta) - { - String table = AccumuloTable.getFullTableName(meta.getTable()); - String indexTable = Indexer.getIndexTableName(meta.getTable()); - String metricsTable = Indexer.getMetricsTableName(meta.getTable()); - - if (tableManager.exists(table)) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, "Cannot create internal table when an Accumulo table already exists"); - } - - if (AccumuloTableProperties.getIndexColumns(meta.getProperties()).isPresent()) { - if (tableManager.exists(indexTable) || tableManager.exists(metricsTable)) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, "Internal table is indexed, but the index table and/or index metrics table(s) already exist"); - } - } - } - - /** - * Gets the row ID based on a table properties or the first column name. - * - * @param meta ConnectorTableMetadata - * @return Lowercase Trino column name mapped to the Accumulo row ID - */ - private static String getRowIdColumn(ConnectorTableMetadata meta) - { - Optional rowIdColumn = AccumuloTableProperties.getRowId(meta.getProperties()); - return rowIdColumn.orElseGet(() -> meta.getColumns().get(0).getName()).toLowerCase(Locale.ENGLISH); - } - - private static List getColumnHandles(ConnectorTableMetadata meta, String rowIdColumn) - { - // Get the column mappings from the table property or auto-generate columns if not defined - Map> mapping = AccumuloTableProperties.getColumnMapping(meta.getProperties()) - .orElseGet(() -> autoGenerateMapping(meta.getColumns(), AccumuloTableProperties.getLocalityGroups(meta.getProperties()))); - - // The list of indexed columns - Optional> indexedColumns = AccumuloTableProperties.getIndexColumns(meta.getProperties()); - - // And now we parse the configured columns and create handles for the metadata manager - ImmutableList.Builder cBuilder = ImmutableList.builder(); - for (int ordinal = 0; ordinal < meta.getColumns().size(); ++ordinal) { - ColumnMetadata cm = meta.getColumns().get(ordinal); - - // Special case if this column is the row ID - if (cm.getName().equalsIgnoreCase(rowIdColumn)) { - cBuilder.add( - new AccumuloColumnHandle( - rowIdColumn, - Optional.empty(), - Optional.empty(), - cm.getType(), - ordinal, - "Accumulo row ID", - Optional.ofNullable(cm.getComment()), - false)); - } - else { - if (!mapping.containsKey(cm.getName())) { - throw new InvalidParameterException(format("Misconfigured mapping for Trino column %s", cm.getName())); - } - - // Get the mapping for this column - Entry famqual = mapping.get(cm.getName()); - boolean indexed = indexedColumns.isPresent() && indexedColumns.get().contains(cm.getName().toLowerCase(Locale.ENGLISH)); - String extraInfo = format("Accumulo column %s:%s. Indexed: %b", famqual.getKey(), famqual.getValue(), indexed); - - // Create a new AccumuloColumnHandle object - cBuilder.add( - new AccumuloColumnHandle( - cm.getName(), - Optional.of(famqual.getKey()), - Optional.of(famqual.getValue()), - cm.getType(), - ordinal, - extraInfo, - Optional.ofNullable(cm.getComment()), - indexed)); - } - } - - return cBuilder.build(); - } - - private void setLocalityGroups(Map tableProperties, AccumuloTable table) - { - Optional>> groups = AccumuloTableProperties.getLocalityGroups(tableProperties); - if (groups.isEmpty()) { - LOG.debug("No locality groups to set"); - return; - } - - ImmutableMap.Builder> localityGroupsBuilder = ImmutableMap.builder(); - for (Map.Entry> g : groups.get().entrySet()) { - ImmutableSet.Builder familyBuilder = ImmutableSet.builder(); - // For each configured column for this locality group - for (String col : g.getValue()) { - // Locate the column family mapping via the Handle - // We already validated this earlier, so it'll exist - AccumuloColumnHandle handle = table.getColumns() - .stream() - .filter(x -> x.name().equals(col)) - .collect(Collectors.toList()) - .get(0); - familyBuilder.add(new Text(handle.family().get())); - } - - localityGroupsBuilder.put(g.getKey(), familyBuilder.build()); - } - - Map> localityGroups = localityGroupsBuilder.buildOrThrow(); - LOG.debug("Setting locality groups: %s", localityGroups); - tableManager.setLocalityGroups(table.getFullTableName(), localityGroups); - } - - /** - * Creates the index tables from the given Accumulo table. No op if - * {@link AccumuloTable#isIndexed()} is false. - * - * @param table Table to create index tables - */ - private void createIndexTables(AccumuloTable table) - { - // Early-out if table is not indexed - if (!table.isIndexed()) { - return; - } - - // Create index table if it does not exist (for 'external' table) - if (!tableManager.exists(table.getIndexTableName())) { - tableManager.createAccumuloTable(table.getIndexTableName()); - } - - // Create index metrics table if it does not exist - if (!tableManager.exists(table.getMetricsTableName())) { - tableManager.createAccumuloTable(table.getMetricsTableName()); - } - - // Set locality groups on index and metrics table - Map> indexGroups = Indexer.getLocalityGroups(table); - tableManager.setLocalityGroups(table.getIndexTableName(), indexGroups); - tableManager.setLocalityGroups(table.getMetricsTableName(), indexGroups); - - // Attach iterators to metrics table - for (IteratorSetting setting : Indexer.getMetricIterators(table)) { - tableManager.setIterator(table.getMetricsTableName(), setting); - } - } - - /** - * Auto-generates the mapping of Trino column name to Accumulo family/qualifier, respecting the locality groups (if any). - * - * @param columns Trino columns for the table - * @param groups Mapping of locality groups to a set of Trino columns, or null if none - * @return Column mappings - */ - private static Map> autoGenerateMapping(List columns, Optional>> groups) - { - Map> mapping = new HashMap<>(); - for (ColumnMetadata column : columns) { - Optional family = getColumnLocalityGroup(column.getName(), groups); - mapping.put(column.getName(), Map.entry(family.orElse(column.getName()), column.getName())); - } - return mapping; - } - - /** - * Searches through the given locality groups to find if this column has a locality group. - * - * @param columnName Column name to get the locality group of - * @param groups Optional locality group configuration - * @return Optional string containing the name of the locality group, if present - */ - private static Optional getColumnLocalityGroup(String columnName, Optional>> groups) - { - if (groups.isPresent()) { - for (Map.Entry> group : groups.get().entrySet()) { - if (group.getValue().contains(columnName.toLowerCase(Locale.ENGLISH))) { - return Optional.of(group.getKey()); - } - } - } - - return Optional.empty(); - } - - public void dropTable(AccumuloTable table) - { - SchemaTableName tableName = new SchemaTableName(table.getSchema(), table.getTable()); - - // Remove the table metadata from Trino - if (metaManager.getTable(tableName) != null) { - metaManager.deleteTableMetadata(tableName); - } - - if (!table.isExternal()) { - // delete the table and index tables - String fullTableName = table.getFullTableName(); - if (tableManager.exists(fullTableName)) { - tableManager.deleteAccumuloTable(fullTableName); - } - - if (table.isIndexed()) { - String indexTableName = Indexer.getIndexTableName(tableName); - if (tableManager.exists(indexTableName)) { - tableManager.deleteAccumuloTable(indexTableName); - } - - String metricsTableName = Indexer.getMetricsTableName(tableName); - if (tableManager.exists(metricsTableName)) { - tableManager.deleteAccumuloTable(metricsTableName); - } - } - } - } - - public void renameTable(SchemaTableName oldName, SchemaTableName newName) - { - if (!oldName.getSchemaName().equals(newName.getSchemaName())) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support renaming tables across schemas"); - } - - AccumuloTable oldTable = getTable(oldName); - if (oldTable == null) { - throw new TableNotFoundException(oldName); - } - - AccumuloTable newTable = new AccumuloTable( - oldTable.getSchema(), - newName.getTableName(), - oldTable.getColumns(), - oldTable.getRowId(), - oldTable.isExternal(), - oldTable.getSerializerClassName(), - oldTable.getScanAuthorizations()); - - // Validate table existence - if (!tableManager.exists(oldTable.getFullTableName())) { - throw new TrinoException(ACCUMULO_TABLE_DNE, format("Table '%s' does not exist", oldTable.getFullTableName())); - } - - if (tableManager.exists(newTable.getFullTableName())) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, format("Table '%s' already exists", newTable.getFullTableName())); - } - - // Rename index tables (which will also validate table existence) - renameIndexTables(oldTable, newTable); - - // Rename the Accumulo table - tableManager.renameAccumuloTable(oldTable.getFullTableName(), newTable.getFullTableName()); - - // We'll then create the metadata - metaManager.deleteTableMetadata(oldTable.getSchemaTableName()); - metaManager.createTableMetadata(newTable); - } - - /** - * Renames the index tables (if applicable) for the old table to the new table. - * - * @param oldTable Old AccumuloTable - * @param newTable New AccumuloTable - */ - private void renameIndexTables(AccumuloTable oldTable, AccumuloTable newTable) - { - if (!oldTable.isIndexed()) { - return; - } - - if (!tableManager.exists(oldTable.getIndexTableName())) { - throw new TrinoException(ACCUMULO_TABLE_DNE, format("Table '%s' does not exist", oldTable.getIndexTableName())); - } - - if (tableManager.exists(newTable.getIndexTableName())) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, format("Table '%s' already exists", newTable.getIndexTableName())); - } - - if (!tableManager.exists(oldTable.getMetricsTableName())) { - throw new TrinoException(ACCUMULO_TABLE_DNE, format("Table '%s' does not exist", oldTable.getMetricsTableName())); - } - - if (tableManager.exists(newTable.getMetricsTableName())) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, format("Table '%s' already exists", newTable.getMetricsTableName())); - } - - tableManager.renameAccumuloTable(oldTable.getIndexTableName(), newTable.getIndexTableName()); - tableManager.renameAccumuloTable(oldTable.getMetricsTableName(), newTable.getMetricsTableName()); - } - - public void createView(SchemaTableName viewName, String viewData) - { - if (!tableManager.namespaceExists(viewName.getSchemaName())) { - throw new SchemaNotFoundException(viewName.getSchemaName()); - } - - if (getSchemaNames().contains(viewName.getSchemaName())) { - if (getViewNames(viewName.getSchemaName()).contains(viewName.getTableName())) { - throw new TrinoException(ALREADY_EXISTS, "View already exists"); - } - - if (getTableNames(viewName.getSchemaName()).contains(viewName.getTableName())) { - throw new TrinoException(ALREADY_EXISTS, "View already exists as data table"); - } - } - - metaManager.createViewMetadata(new AccumuloView(viewName.getSchemaName(), viewName.getTableName(), viewData)); - } - - public void createOrReplaceView(SchemaTableName viewName, String viewData) - { - if (!tableManager.namespaceExists(viewName.getSchemaName())) { - throw new SchemaNotFoundException(viewName.getSchemaName()); - } - - if (getView(viewName) != null) { - metaManager.deleteViewMetadata(viewName); - } - - metaManager.createViewMetadata(new AccumuloView(viewName.getSchemaName(), viewName.getTableName(), viewData)); - } - - public void dropView(SchemaTableName viewName) - { - metaManager.deleteViewMetadata(viewName); - } - - public void renameColumn(AccumuloTable table, String source, String target) - { - if (table.getColumns().stream().noneMatch(columnHandle -> columnHandle.name().equalsIgnoreCase(source))) { - throw new TrinoException(NOT_FOUND, format("Failed to find source column %s to rename to %s", source, target)); - } - - // Copy existing column list, replacing the old column name with the new - ImmutableList.Builder newColumnList = ImmutableList.builder(); - for (AccumuloColumnHandle columnHandle : table.getColumns()) { - if (columnHandle.name().equalsIgnoreCase(source)) { - newColumnList.add(new AccumuloColumnHandle( - target, - columnHandle.family(), - columnHandle.qualifier(), - columnHandle.type(), - columnHandle.ordinal(), - columnHandle.extraInfo(), - columnHandle.comment(), - columnHandle.indexed())); - } - else { - newColumnList.add(columnHandle); - } - } - - // Create new table metadata - AccumuloTable newTable = new AccumuloTable( - table.getSchema(), - table.getTable(), - newColumnList.build(), - table.getRowId().equalsIgnoreCase(source) ? target : table.getRowId(), - table.isExternal(), - table.getSerializerClassName(), - table.getScanAuthorizations()); - - // Replace the table metadata - metaManager.deleteTableMetadata(new SchemaTableName(table.getSchema(), table.getTable())); - metaManager.createTableMetadata(newTable); - } - - public Set getSchemaNames() - { - return metaManager.getSchemaNames(); - } - - public Set getTableNames(String schema) - { - requireNonNull(schema, "schema is null"); - return metaManager.getTableNames(schema); - } - - public AccumuloTable getTable(SchemaTableName table) - { - requireNonNull(table, "table is null"); - return metaManager.getTable(table); - } - - public Set getViewNames(String schema) - { - requireNonNull(schema, "schema is null"); - return metaManager.getViewNames(schema); - } - - public AccumuloView getView(SchemaTableName viewName) - { - requireNonNull(viewName, "viewName is null"); - return metaManager.getView(viewName); - } - - /** - * Fetches the TabletSplitMetadata for a query against an Accumulo table. - *

- * Does a whole bunch of fun stuff! Splitting on row ID ranges, applying secondary indexes, column pruning, - * all sorts of sweet optimizations. What you have here is an important method. - * - * @param session Current session - * @param schema Schema name - * @param table Table Name - * @param rowIdDomain Domain for the row ID - * @param constraints Column constraints for the query - * @param serializer Instance of a row serializer - * @return List of TabletSplitMetadata objects for Trino - */ - public List getTabletSplits( - ConnectorSession session, - String schema, - String table, - Optional rowIdDomain, - List constraints, - AccumuloRowSerializer serializer) - { - try { - String tableName = AccumuloTable.getFullTableName(schema, table); - LOG.debug("Getting tablet splits for table %s", tableName); - - // Get the initial Range based on the row ID domain - Collection rowIdRanges = getRangesFromDomain(rowIdDomain, serializer); - List tabletSplits = new ArrayList<>(); - - // Use the secondary index, if enabled - if (AccumuloSessionProperties.isOptimizeIndexEnabled(session)) { - // Get the scan authorizations to query the index - Authorizations auths = getScanAuthorizations(session, schema, table); - - // Check the secondary index based on the column constraints - // If this returns true, return the tablet splits to Trino - if (indexLookup.applyIndex(schema, table, session, constraints, rowIdRanges, tabletSplits, serializer, auths)) { - return tabletSplits; - } - } - - // If we can't (or shouldn't) use the secondary index, we will just use the Range from the row ID domain - - // Split the ranges on tablet boundaries, if enabled - Collection splitRanges; - if (AccumuloSessionProperties.isOptimizeSplitRangesEnabled(session)) { - splitRanges = splitByTabletBoundaries(tableName, rowIdRanges); - } - else { - // if not enabled, just use the same collection - splitRanges = rowIdRanges; - } - - // Create TabletSplitMetadata objects for each range - boolean fetchTabletLocations = AccumuloSessionProperties.isOptimizeLocalityEnabled(session); - - LOG.debug("Fetching tablet locations: %s", fetchTabletLocations); - - for (Range range : splitRanges) { - // If locality is enabled, then fetch tablet location - if (fetchTabletLocations) { - tabletSplits.add(new TabletSplitMetadata(getTabletLocation(tableName, range.getStartKey()), ImmutableList.of(range))); - } - else { - // else, just use the default location - tabletSplits.add(new TabletSplitMetadata(Optional.empty(), ImmutableList.of(range))); - } - } - - // Log some fun stuff and return the tablet splits - LOG.debug("Number of splits for table %s is %d with %d ranges", tableName, tabletSplits.size(), splitRanges.size()); - return tabletSplits; - } - catch (Exception e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get splits from Accumulo", e); - } - } - - /** - * Gets the scan authorizations to use for scanning tables. - *

- * In order of priority: session username authorizations, then table property, then the default connector auths. - * - * @param session Current session - * @param schema Schema name - * @param table Table Name - * @return Scan authorizations - * @throws AccumuloException If a generic Accumulo error occurs - * @throws AccumuloSecurityException If a security exception occurs - */ - private Authorizations getScanAuthorizations(ConnectorSession session, String schema, - String table) - throws AccumuloException, AccumuloSecurityException - { - String sessionScanUser = AccumuloSessionProperties.getScanUsername(session); - if (sessionScanUser != null) { - Authorizations scanAuths = client.securityOperations().getUserAuthorizations(sessionScanUser); - LOG.debug("Using session scan auths for user %s: %s", sessionScanUser, scanAuths); - return scanAuths; - } - - AccumuloTable accumuloTable = this.getTable(new SchemaTableName(schema, table)); - if (accumuloTable == null) { - throw new TableNotFoundException(new SchemaTableName(schema, table)); - } - - Optional strAuths = accumuloTable.getScanAuthorizations(); - if (strAuths.isPresent()) { - Authorizations scanAuths = new Authorizations(Iterables.toArray(COMMA_SPLITTER.split(strAuths.get()), String.class)); - LOG.debug("scan_auths table property set, using: %s", scanAuths); - return scanAuths; - } - - LOG.debug("scan_auths table property not set, using connector auths: %s", this.auths); - return this.auths; - } - - private Collection splitByTabletBoundaries(String tableName, Collection ranges) - throws org.apache.accumulo.core.client.TableNotFoundException, AccumuloException, AccumuloSecurityException - { - ImmutableSet.Builder rangeBuilder = ImmutableSet.builder(); - for (Range range : ranges) { - // if start and end key are equivalent, no need to split the range - if (range.getStartKey() != null && range.getEndKey() != null && range.getStartKey().equals(range.getEndKey())) { - rangeBuilder.add(range); - } - else { - // Call out to Accumulo to split the range on tablets - rangeBuilder.addAll(client.tableOperations().splitRangeByTablets(tableName, range, Integer.MAX_VALUE)); - } - } - return rangeBuilder.build(); - } - - /** - * Gets the TabletServer hostname for where the given key is located in the given table - * - * @param table Fully-qualified table name - * @param key Key to locate - * @return The tablet location, or DUMMY_LOCATION if an error occurs - */ - private Optional getTabletLocation(String table, Key key) - { - try { - // Get the Accumulo table ID so we can scan some fun stuff - String tableId = client.tableOperations().tableIdMap().get(table); - - // Create our scanner against the metadata table, fetching 'loc' family - Scanner scanner = client.createScanner("accumulo.metadata", auths); - scanner.fetchColumnFamily(new Text("loc")); - - // Set the scan range to just this table, from the table ID to the default tablet - // row, which is the last listed tablet - Key defaultTabletRow = new Key(tableId + '<'); - Key start = new Key(tableId); - Key end = defaultTabletRow.followingKey(PartialKey.ROW); - scanner.setRange(new Range(start, end)); - - Optional location = Optional.empty(); - if (key == null) { - // if the key is null, then it is -inf, so get first tablet location - Iterator> iter = scanner.iterator(); - if (iter.hasNext()) { - location = Optional.of(iter.next().getValue().toString()); - } - } - else { - // Else, we will need to scan through the tablet location data and find the location - - // Create some text objects to do comparison for what we are looking for - Text splitCompareKey = new Text(); - key.getRow(splitCompareKey); - Text scannedCompareKey = new Text(); - - // Scan the table! - for (Entry entry : scanner) { - // Get the bytes of the key - byte[] keyBytes = entry.getKey().getRow().copyBytes(); - - // If the last byte is <, then we have hit the default tablet, so use this location - if (keyBytes[keyBytes.length - 1] == '<') { - location = Optional.of(entry.getValue().toString()); - break; - } - // Chop off some magic nonsense - scannedCompareKey.set(keyBytes, 3, keyBytes.length - 3); - - // Compare the keys, moving along the tablets until the location is found - if (scannedCompareKey.getLength() > 0) { - int compareTo = splitCompareKey.compareTo(scannedCompareKey); - if (compareTo <= 0) { - location = Optional.of(entry.getValue().toString()); - } - else { - // all future tablets will be greater than this key - break; - } - } - } - scanner.close(); - } - - // If we were unable to find the location for some reason, return the default tablet - // location - return location.isPresent() ? location : getDefaultTabletLocation(table); - } - catch (Exception e) { - // Swallow this exception so the query does not fail due to being unable - // to locate the tablet server for the provided Key. - // This is purely an optimization, but we will want to log the error. - LOG.error(e, "Failed to get tablet location, returning dummy location"); - return Optional.empty(); - } - } - - private Optional getDefaultTabletLocation(String fulltable) - { - try { - String tableId = client.tableOperations().tableIdMap().get(fulltable); - - // Create a scanner over the metadata table, fetching the 'loc' column of the default tablet row - Scanner scan = client.createScanner("accumulo.metadata", client.securityOperations().getUserAuthorizations(username)); - scan.fetchColumnFamily(new Text("loc")); - scan.setRange(new Range(tableId + '<')); - - // scan the entry - Optional location = Optional.empty(); - for (Entry entry : scan) { - if (location.isPresent()) { - throw new TrinoException(FUNCTION_IMPLEMENTATION_ERROR, "Scan for default tablet returned more than one entry"); - } - - location = Optional.of(entry.getValue().toString()); - } - - scan.close(); - return location; - } - catch (Exception e) { - // Swallow this exception so the query does not fail due to being unable to locate the tablet server for the default tablet. - // This is purely an optimization, but we will want to log the error. - LOG.error(e, "Failed to get tablet location, returning dummy location"); - return Optional.empty(); - } - } - - /** - * Gets a collection of Accumulo Range objects from the given Trino domain. - * This maps the column constraints of the given Domain to an Accumulo Range scan. - * - * @param domain Domain, can be null (returns (-inf, +inf) Range) - * @param serializer Instance of an {@link AccumuloRowSerializer} - * @return A collection of Accumulo Range objects - * @throws TableNotFoundException If the Accumulo table is not found - */ - public static Collection getRangesFromDomain(Optional domain, AccumuloRowSerializer serializer) - throws TableNotFoundException - { - // if we have no predicate pushdown, use the full range - if (domain.isEmpty()) { - return ImmutableSet.of(new Range()); - } - - ImmutableSet.Builder rangeBuilder = ImmutableSet.builder(); - for (io.trino.spi.predicate.Range range : domain.get().getValues().getRanges().getOrderedRanges()) { - rangeBuilder.add(getRangeFromTrinoRange(range, serializer)); - } - - return rangeBuilder.build(); - } - - private static Range getRangeFromTrinoRange(io.trino.spi.predicate.Range trinoRange, AccumuloRowSerializer serializer) - throws TableNotFoundException - { - Range accumuloRange; - if (trinoRange.isAll()) { - accumuloRange = new Range(); - } - else if (trinoRange.isSingleValue()) { - Text split = new Text(serializer.encode(trinoRange.getType(), trinoRange.getSingleValue())); - accumuloRange = new Range(split); - } - else { - if (trinoRange.isLowUnbounded()) { - // If low is unbounded, then create a range from (-inf, value), checking inclusivity - Text split = new Text(serializer.encode(trinoRange.getType(), trinoRange.getHighBoundedValue())); - accumuloRange = new Range(null, false, split, trinoRange.isLowInclusive()); - } - else if (trinoRange.isHighUnbounded()) { - // If high is unbounded, then create a range from (value, +inf), checking inclusivity - Text split = new Text(serializer.encode(trinoRange.getType(), trinoRange.getLowBoundedValue())); - accumuloRange = new Range(split, trinoRange.isHighInclusive(), null, false); - } - else { - // If high is unbounded, then create a range from low to high, checking inclusivity - boolean startKeyInclusive = trinoRange.isLowInclusive(); - Text startSplit = new Text(serializer.encode(trinoRange.getType(), trinoRange.getLowBoundedValue())); - - boolean endKeyInclusive = trinoRange.isHighInclusive(); - Text endSplit = new Text(serializer.encode(trinoRange.getType(), trinoRange.getHighBoundedValue())); - accumuloRange = new Range(startSplit, startKeyInclusive, endSplit, endKeyInclusive); - } - } - - return accumuloRange; - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloModule.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloModule.java deleted file mode 100644 index cb3e56a184dea..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloModule.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.inject.Binder; -import com.google.inject.Inject; -import com.google.inject.Module; -import com.google.inject.Provider; -import com.google.inject.Scopes; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.accumulo.conf.AccumuloSessionProperties; -import io.trino.plugin.accumulo.conf.AccumuloTableProperties; -import io.trino.plugin.accumulo.index.ColumnCardinalityCache; -import io.trino.plugin.accumulo.index.IndexLookup; -import io.trino.plugin.accumulo.io.AccumuloPageSinkProvider; -import io.trino.plugin.accumulo.io.AccumuloRecordSetProvider; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.metadata.ZooKeeperMetadataManager; -import org.apache.accumulo.core.client.Accumulo; -import org.apache.accumulo.core.client.AccumuloClient; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; -import static io.trino.plugin.base.ClosingBinder.closingBinder; - -/** - * Trino module to do all kinds of run Guice injection stuff! - *

- * WARNING: Contains black magick - */ -public class AccumuloModule - implements Module -{ - @Override - public void configure(Binder binder) - { - binder.bind(AccumuloConnector.class).in(Scopes.SINGLETON); - binder.bind(AccumuloMetadata.class).in(Scopes.SINGLETON); - binder.bind(AccumuloMetadataFactory.class).in(Scopes.SINGLETON); - binder.bind(AccumuloMetadataManager.class).in(Scopes.SINGLETON); - binder.bind(AccumuloSplitManager.class).in(Scopes.SINGLETON); - binder.bind(AccumuloRecordSetProvider.class).in(Scopes.SINGLETON); - binder.bind(AccumuloPageSinkProvider.class).in(Scopes.SINGLETON); - binder.bind(AccumuloSessionProperties.class).in(Scopes.SINGLETON); - binder.bind(AccumuloTableProperties.class).in(Scopes.SINGLETON); - binder.bind(ZooKeeperMetadataManager.class).in(Scopes.SINGLETON); - binder.bind(AccumuloTableManager.class).in(Scopes.SINGLETON); - binder.bind(IndexLookup.class).in(Scopes.SINGLETON); - binder.bind(ColumnCardinalityCache.class).in(Scopes.SINGLETON); - binder.bind(AccumuloClient.class).toProvider(ClientProvider.class).in(Scopes.SINGLETON); - closingBinder(binder).registerCloseable(AccumuloClient.class); - - configBinder(binder).bindConfig(AccumuloConfig.class); - - jsonCodecBinder(binder).bindMapJsonCodec(String.class, JsonCodec.listJsonCodec(AccumuloTable.class)); - } - - private static class ClientProvider - implements Provider - { - private static final Logger LOG = Logger.get(ClientProvider.class); - - private final String instance; - private final String zooKeepers; - private final String username; - private final String password; - - @Inject - public ClientProvider(AccumuloConfig config) - { - this.instance = config.getInstance(); - this.zooKeepers = config.getZooKeepers(); - this.username = config.getUsername(); - this.password = config.getPassword(); - } - - @Override - public AccumuloClient get() - { - AccumuloClient client = Accumulo.newClient() - .to(instance, zooKeepers) - .as(username, password) - .build(); - LOG.info("Connection to instance %s at %s established, user %s", instance, zooKeepers, username); - return client; - } - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloPlugin.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloPlugin.java deleted file mode 100644 index 886aee388cdb0..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloPlugin.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; - -public class AccumuloPlugin - implements Plugin -{ - @Override - public Iterable getConnectorFactories() - { - return ImmutableList.of(new AccumuloConnectorFactory()); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloSplitManager.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloSplitManager.java deleted file mode 100644 index be9436e402cc5..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloSplitManager.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.trino.plugin.accumulo.model.AccumuloColumnConstraint; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.model.AccumuloSplit; -import io.trino.plugin.accumulo.model.AccumuloTableHandle; -import io.trino.plugin.accumulo.model.SerializedRange; -import io.trino.plugin.accumulo.model.TabletSplitMetadata; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.connector.FixedSplitSource; -import io.trino.spi.predicate.Domain; -import io.trino.spi.predicate.TupleDomain; - -import java.util.List; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.stream.Collectors; - -import static java.util.Objects.requireNonNull; - -public class AccumuloSplitManager - implements ConnectorSplitManager -{ - private final AccumuloMetadataManager metadataManager; - - @Inject - public AccumuloSplitManager(AccumuloMetadataManager metadataManager) - { - this.metadataManager = requireNonNull(metadataManager, "metadataManager is null"); - } - - @Override - public ConnectorSplitSource getSplits( - ConnectorTransactionHandle transactionHandle, - ConnectorSession session, - ConnectorTableHandle tableHandle, - DynamicFilter dynamicFilter, - Constraint constraint) - { - AccumuloTableHandle handle = (AccumuloTableHandle) tableHandle; - - String schemaName = handle.getSchema(); - String tableName = handle.getTable(); - String rowIdName = handle.getRowId(); - - // Get non-row ID column constraints - List constraints = getColumnConstraints(rowIdName, handle.getConstraint()); - - // Get the row domain column range - Optional rDom = getRangeDomain(rowIdName, handle.getConstraint()); - - // Call out to our client to retrieve all tablet split metadata using the row ID domain and the secondary index - List tabletSplits = metadataManager.getTabletSplits(session, schemaName, tableName, rDom, constraints, handle.getSerializerInstance()); - - // Pack the tablet split metadata into a connector split - ImmutableList.Builder cSplits = ImmutableList.builder(); - for (TabletSplitMetadata splitMetadata : tabletSplits) { - AccumuloSplit split = new AccumuloSplit( - splitMetadata.ranges().stream().map(SerializedRange::serialize).collect(Collectors.toList()), - splitMetadata.hostPort()); - cSplits.add(split); - } - - return new FixedSplitSource(cSplits.build()); - } - - private static Optional getRangeDomain(String rowIdName, TupleDomain constraint) - { - if (constraint.getDomains().isPresent()) { - for (Entry columnDomain : constraint.getDomains().get().entrySet()) { - AccumuloColumnHandle col = (AccumuloColumnHandle) columnDomain.getKey(); - if (col.name().equals(rowIdName)) { - return Optional.of(columnDomain.getValue()); - } - } - } - - return Optional.empty(); - } - - /** - * Gets a list of {@link AccumuloColumnConstraint} based on the given constraint ID, excluding the row ID column - * - * @param rowIdName Trino column name mapping to the Accumulo row ID - * @param constraint Set of query constraints - * @return List of all column constraints - */ - private static List getColumnConstraints(String rowIdName, TupleDomain constraint) - { - ImmutableList.Builder constraintBuilder = ImmutableList.builder(); - constraint.getDomains().orElseThrow().forEach((handle, domain) -> { - AccumuloColumnHandle columnHandle = (AccumuloColumnHandle) handle; - - if (!columnHandle.name().equals(rowIdName)) { - // Family and qualifier will exist for non-row ID columns - constraintBuilder.add(new AccumuloColumnConstraint( - columnHandle.name(), - columnHandle.family().get(), - columnHandle.qualifier().get(), - Optional.of(domain), - columnHandle.indexed())); - } - }); - - return constraintBuilder.build(); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTableManager.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTableManager.java deleted file mode 100644 index a0ecfd709bc8c..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTableManager.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.inject.Inject; -import io.airlift.log.Logger; -import io.trino.spi.TrinoException; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.NamespaceExistsException; -import org.apache.accumulo.core.client.NamespaceNotEmptyException; -import org.apache.accumulo.core.client.NamespaceNotFoundException; -import org.apache.accumulo.core.client.TableExistsException; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; -import org.apache.hadoop.io.Text; - -import java.util.EnumSet; -import java.util.Map; -import java.util.Set; - -import static io.trino.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE; -import static io.trino.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_EXISTS; -import static io.trino.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static io.trino.spi.StandardErrorCode.ALREADY_EXISTS; -import static java.util.Objects.requireNonNull; - -/** - * This class is a light wrapper for Accumulo's Connector object. - * It will perform the given operation, or throw an exception if an Accumulo- or ZooKeeper-based error occurs. - */ -public class AccumuloTableManager -{ - private static final Logger LOG = Logger.get(AccumuloTableManager.class); - private final AccumuloClient client; - - @Inject - public AccumuloTableManager(AccumuloClient client) - { - this.client = requireNonNull(client, "client is null"); - } - - public void createNamespace(String schema) - { - try { - client.namespaceOperations().create(schema); - } - catch (NamespaceExistsException e) { - throw new TrinoException(ALREADY_EXISTS, "Namespace already exists: " + schema, e); - } - catch (AccumuloException | AccumuloSecurityException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to create Accumulo namespace: " + schema, e); - } - } - - public void dropNamespace(String schema) - { - try { - client.namespaceOperations().delete(schema); - } - catch (AccumuloException | AccumuloSecurityException | NamespaceNotFoundException | NamespaceNotEmptyException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to delete Accumulo namespace: " + schema, e); - } - } - - public boolean namespaceExists(String schema) - { - try { - return client.namespaceOperations().exists(schema); - } - catch (AccumuloException | AccumuloSecurityException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to check for existence Accumulo namespace: " + schema, e); - } - } - - public boolean exists(String table) - { - return client.tableOperations().exists(table); - } - - public void createAccumuloTable(String table) - { - try { - client.tableOperations().create(table); - } - catch (AccumuloException | AccumuloSecurityException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to create Accumulo table", e); - } - catch (TableExistsException e) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, "Accumulo table already exists", e); - } - } - - public void setLocalityGroups(String tableName, Map> groups) - { - if (groups.isEmpty()) { - return; - } - - try { - client.tableOperations().setLocalityGroups(tableName, groups); - LOG.debug("Set locality groups for %s to %s", tableName, groups); - } - catch (AccumuloException | AccumuloSecurityException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to set locality groups", e); - } - catch (TableNotFoundException e) { - throw new TrinoException(ACCUMULO_TABLE_DNE, "Failed to set locality groups, table does not exist", e); - } - } - - public void setIterator(String table, IteratorSetting setting) - { - try { - // Remove any existing iterator settings of the same name, if applicable - Map> iterators = client.tableOperations().listIterators(table); - if (iterators.containsKey(setting.getName())) { - client.tableOperations().removeIterator(table, setting.getName(), iterators.get(setting.getName())); - } - - client.tableOperations().attachIterator(table, setting); - } - catch (AccumuloSecurityException | AccumuloException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to set iterator on table " + table, e); - } - catch (TableNotFoundException e) { - throw new TrinoException(ACCUMULO_TABLE_DNE, "Failed to set iterator, table does not exist", e); - } - } - - public void deleteAccumuloTable(String tableName) - { - try { - client.tableOperations().delete(tableName); - } - catch (AccumuloException | AccumuloSecurityException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to delete Accumulo table", e); - } - catch (TableNotFoundException e) { - throw new TrinoException(ACCUMULO_TABLE_DNE, "Failed to delete Accumulo table, does not exist", e); - } - } - - public void renameAccumuloTable(String oldName, String newName) - { - try { - client.tableOperations().rename(oldName, newName); - } - catch (AccumuloSecurityException | AccumuloException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to rename table", e); - } - catch (TableNotFoundException e) { - throw new TrinoException(ACCUMULO_TABLE_DNE, "Failed to rename table, old table does not exist", e); - } - catch (TableExistsException e) { - throw new TrinoException(ACCUMULO_TABLE_EXISTS, "Failed to rename table, new table already exists", e); - } - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTransactionHandle.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTransactionHandle.java deleted file mode 100644 index 5eda4d453e927..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/AccumuloTransactionHandle.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import io.trino.spi.connector.ConnectorTransactionHandle; - -import java.util.UUID; - -import static java.util.Objects.requireNonNull; - -public record AccumuloTransactionHandle(UUID uuid) - implements ConnectorTransactionHandle -{ - public AccumuloTransactionHandle() - { - this(UUID.randomUUID()); - } - - public AccumuloTransactionHandle - { - requireNonNull(uuid, "uuid is null"); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/Types.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/Types.java deleted file mode 100644 index cac7973e7ac0d..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/Types.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.Type; - -/** - * Utility class for Trino Type-related functionality. - */ -public final class Types -{ - private Types() {} - - public static boolean isArrayType(Type type) - { - return type instanceof ArrayType; - } - - public static boolean isMapType(Type type) - { - return type instanceof MapType; - } - - /** - * Gets the element type of the given array type. Does not validate that the given type is an array. - * - * @param type An array type - * @return Element type of the array - * @throws IndexOutOfBoundsException If type is not an array - * @see Types#isArrayType - */ - public static Type getElementType(Type type) - { - return type.getTypeParameters().get(0); - } - - /** - * Gets the key type of the given map type. Does not validate that the given type is a map. - * - * @param type A map type - * @return Key type of the map - * @throws IndexOutOfBoundsException If type is not a map - * @see Types#isMapType - */ - public static Type getKeyType(Type type) - { - return type.getTypeParameters().get(0); - } - - /** - * Gets the value type of the given map type. Does not validate that the given type is a map. - * - * @param type A map type - * @return Value type of the map - * @throws IndexOutOfBoundsException If type is not a map - * @see Types#isMapType - */ - public static Type getValueType(Type type) - { - return type.getTypeParameters().get(1); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloConfig.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloConfig.java deleted file mode 100644 index 0535c8792b017..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloConfig.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.conf; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; -import io.airlift.units.Duration; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import java.util.concurrent.TimeUnit; - -/** - * File-based configuration properties for the Accumulo connector - */ -public class AccumuloConfig -{ - public static final String INSTANCE = "accumulo.instance"; - public static final String ZOOKEEPERS = "accumulo.zookeepers"; - public static final String USERNAME = "accumulo.username"; - public static final String PASSWORD = "accumulo.password"; - public static final String ZOOKEEPER_METADATA_ROOT = "accumulo.zookeeper.metadata.root"; - public static final String CARDINALITY_CACHE_SIZE = "accumulo.cardinality.cache.size"; - public static final String CARDINALITY_CACHE_EXPIRE_DURATION = "accumulo.cardinality.cache.expire.duration"; - - private String instance; - private String zooKeepers; - private String username; - private String password; - private String zkMetadataRoot = "/trino-accumulo"; - private int cardinalityCacheSize = 100_000; - private Duration cardinalityCacheExpiration = new Duration(5, TimeUnit.MINUTES); - - @NotNull - public String getInstance() - { - return this.instance; - } - - @Config(INSTANCE) - @ConfigDescription("Accumulo instance name") - public AccumuloConfig setInstance(String instance) - { - this.instance = instance; - return this; - } - - @NotNull - public String getZooKeepers() - { - return this.zooKeepers; - } - - @Config(ZOOKEEPERS) - @ConfigDescription("ZooKeeper quorum connect string for Accumulo") - public AccumuloConfig setZooKeepers(String zooKeepers) - { - this.zooKeepers = zooKeepers; - return this; - } - - @NotNull - public String getUsername() - { - return this.username; - } - - @Config(USERNAME) - @ConfigDescription("Sets the user to use when interacting with Accumulo. This user will require administrative permissions") - public AccumuloConfig setUsername(String username) - { - this.username = username; - return this; - } - - @NotNull - public String getPassword() - { - return this.password; - } - - @Config(PASSWORD) - @ConfigSecuritySensitive - @ConfigDescription("Sets the password for the configured user") - public AccumuloConfig setPassword(String password) - { - this.password = password; - return this; - } - - @NotNull - public String getZkMetadataRoot() - { - return zkMetadataRoot; - } - - @Config(ZOOKEEPER_METADATA_ROOT) - @ConfigDescription("Sets the root znode for metadata storage") - public AccumuloConfig setZkMetadataRoot(String zkMetadataRoot) - { - this.zkMetadataRoot = zkMetadataRoot; - return this; - } - - @Min(1) - public int getCardinalityCacheSize() - { - return cardinalityCacheSize; - } - - @Config(CARDINALITY_CACHE_SIZE) - @ConfigDescription("Sets the cardinality cache size") - public AccumuloConfig setCardinalityCacheSize(int cardinalityCacheSize) - { - this.cardinalityCacheSize = cardinalityCacheSize; - return this; - } - - @NotNull - public Duration getCardinalityCacheExpiration() - { - return cardinalityCacheExpiration; - } - - @Config(CARDINALITY_CACHE_EXPIRE_DURATION) - @ConfigDescription("Sets the cardinality cache expiration") - public AccumuloConfig setCardinalityCacheExpiration(Duration cardinalityCacheExpiration) - { - this.cardinalityCacheExpiration = cardinalityCacheExpiration; - return this; - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloSessionProperties.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloSessionProperties.java deleted file mode 100644 index b3be64afe77e9..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloSessionProperties.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.conf; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.units.Duration; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.session.PropertyMetadata; - -import java.util.List; - -import static io.trino.plugin.base.session.PropertyMetadataUtil.durationProperty; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; -import static io.trino.spi.session.PropertyMetadata.doubleProperty; -import static io.trino.spi.session.PropertyMetadata.integerProperty; -import static io.trino.spi.session.PropertyMetadata.stringProperty; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -/** - * Class contains all session-based properties for the Accumulo connector. - * Use SHOW SESSION to view all available properties. - *

- * Can set the property using: - *

- * SET SESSION <property> = <value>; - */ -public final class AccumuloSessionProperties -{ - private static final String OPTIMIZE_LOCALITY_ENABLED = "optimize_locality_enabled"; - private static final String OPTIMIZE_SPLIT_RANGES_ENABLED = "optimize_split_ranges_enabled"; - private static final String OPTIMIZE_INDEX_ENABLED = "optimize_index_enabled"; - private static final String INDEX_ROWS_PER_SPLIT = "index_rows_per_split"; - private static final String INDEX_THRESHOLD = "index_threshold"; - private static final String INDEX_LOWEST_CARDINALITY_THRESHOLD = "index_lowest_cardinality_threshold"; - private static final String INDEX_METRICS_ENABLED = "index_metrics_enabled"; - private static final String SCAN_USERNAME = "scan_username"; - private static final String INDEX_SHORT_CIRCUIT_CARDINALITY_FETCH = "index_short_circuit_cardinality_fetch"; - private static final String INDEX_CARDINALITY_CACHE_POLLING_DURATION = "index_cardinality_cache_polling_duration"; - - private final List> sessionProperties; - - @Inject - public AccumuloSessionProperties() - { - sessionProperties = ImmutableList.of( - booleanProperty( - OPTIMIZE_LOCALITY_ENABLED, - "Set to true to enable data locality for non-indexed scans. Default true.", true, - false), - booleanProperty( - OPTIMIZE_SPLIT_RANGES_ENABLED, - "Set to true to split non-indexed queries by tablet splits. Should generally be true.", - true, false), - stringProperty( - SCAN_USERNAME, - "User to impersonate when scanning the tables. This property trumps the scan_auths table property. Default is the user in the configuration file.", null, false), - booleanProperty( - OPTIMIZE_INDEX_ENABLED, - "Set to true to enable usage of the secondary index on query. Default true.", - true, - false), - integerProperty( - INDEX_ROWS_PER_SPLIT, - "The number of Accumulo row IDs that are packed into a single Trino split. Default 10000", - 10000, - false), - doubleProperty( - INDEX_THRESHOLD, - "The ratio between number of rows to be scanned based on the index over the total number of rows. If the ratio is below this threshold, the index will be used. Default .2", - 0.2, - false), - doubleProperty( - INDEX_LOWEST_CARDINALITY_THRESHOLD, - "The threshold where the column with the lowest cardinality will be used instead of computing an intersection of ranges in the secondary index. Secondary index must be enabled. Default .01", - 0.01, - false), - booleanProperty( - INDEX_METRICS_ENABLED, - "Set to true to enable usage of the metrics table to optimize usage of the index. Default true", - true, - false), - booleanProperty( - INDEX_SHORT_CIRCUIT_CARDINALITY_FETCH, - "Short circuit the retrieval of index metrics once any column is less than the lowest cardinality threshold. Default true", - true, - false), - durationProperty( - INDEX_CARDINALITY_CACHE_POLLING_DURATION, - "Sets the cardinality cache polling duration for short circuit retrieval of index metrics. Default 10ms", - new Duration(10, MILLISECONDS), - false)); - } - - public List> getSessionProperties() - { - return sessionProperties; - } - - public static boolean isOptimizeLocalityEnabled(ConnectorSession session) - { - return session.getProperty(OPTIMIZE_LOCALITY_ENABLED, Boolean.class); - } - - public static boolean isOptimizeSplitRangesEnabled(ConnectorSession session) - { - return session.getProperty(OPTIMIZE_SPLIT_RANGES_ENABLED, Boolean.class); - } - - public static boolean isOptimizeIndexEnabled(ConnectorSession session) - { - return session.getProperty(OPTIMIZE_INDEX_ENABLED, Boolean.class); - } - - public static double getIndexThreshold(ConnectorSession session) - { - return session.getProperty(INDEX_THRESHOLD, Double.class); - } - - public static int getNumIndexRowsPerSplit(ConnectorSession session) - { - return session.getProperty(INDEX_ROWS_PER_SPLIT, Integer.class); - } - - public static double getIndexSmallCardThreshold(ConnectorSession session) - { - return session.getProperty(INDEX_LOWEST_CARDINALITY_THRESHOLD, Double.class); - } - - public static Duration getIndexCardinalityCachePollingDuration(ConnectorSession session) - { - return session.getProperty(INDEX_CARDINALITY_CACHE_POLLING_DURATION, Duration.class); - } - - public static boolean isIndexMetricsEnabled(ConnectorSession session) - { - return session.getProperty(INDEX_METRICS_ENABLED, Boolean.class); - } - - public static String getScanUsername(ConnectorSession session) - { - return session.getProperty(SCAN_USERNAME, String.class); - } - - public static boolean isIndexShortCircuitEnabled(ConnectorSession session) - { - return session.getProperty(INDEX_SHORT_CIRCUIT_CARDINALITY_FETCH, Boolean.class); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloTableProperties.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloTableProperties.java deleted file mode 100644 index 8ec8b25c528f2..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/conf/AccumuloTableProperties.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.conf; - -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.plugin.accumulo.serializers.LexicoderRowSerializer; -import io.trino.plugin.accumulo.serializers.StringRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.session.PropertyMetadata; -import io.trino.spi.type.VarcharType; - -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkState; -import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY; -import static io.trino.spi.session.PropertyMetadata.booleanProperty; -import static io.trino.spi.session.PropertyMetadata.stringProperty; -import static java.util.Objects.requireNonNull; - -/** - * Class contains all table properties for the Accumulo connector. Used when creating a table: - *

- * CREATE TABLE foo (a VARCHAR, b INT) - * WITH (column_mapping = 'b:md:b', external = true); - */ -public final class AccumuloTableProperties -{ - public static final String COLUMN_MAPPING = "column_mapping"; - public static final String INDEX_COLUMNS = "index_columns"; - public static final String EXTERNAL = "external"; - public static final String LOCALITY_GROUPS = "locality_groups"; - public static final String ROW_ID = "row_id"; - public static final String SERIALIZER = "serializer"; - public static final String SCAN_AUTHS = "scan_auths"; - private static final Splitter COLON_SPLITTER = Splitter.on(':').trimResults(); - private static final Splitter COMMA_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults(); - private static final Splitter PIPE_SPLITTER = Splitter.on('|').omitEmptyStrings().trimResults(); - - private final List> tableProperties; - - public AccumuloTableProperties() - { - PropertyMetadata s1 = stringProperty( - COLUMN_MAPPING, - "Comma-delimited list of column metadata: col_name:col_family:col_qualifier,[...]. Required for external tables. Not setting this property results in auto-generated column names.", - null, - false); - - PropertyMetadata s2 = stringProperty( - INDEX_COLUMNS, - "A comma-delimited list of Trino columns that are indexed in this table's corresponding index table. Default is no indexed columns.", - "", - false); - - PropertyMetadata s3 = booleanProperty( - EXTERNAL, - "If true, Trino will only do metadata operations for the table. Else, Trino will create and drop Accumulo tables where appropriate. Default false.", - false, - false); - - PropertyMetadata s4 = stringProperty( - LOCALITY_GROUPS, - "List of locality groups to set on the Accumulo table. Only valid on internal tables. String format is locality group name, colon, comma delimited list of Trino column names in the group. Groups are delimited by pipes. Example: group1:colA,colB,colC|group2:colD,colE,colF|etc.... Default is no locality groups.", - null, - false); - - PropertyMetadata s5 = stringProperty( - ROW_ID, - "Trino column name that maps to the Accumulo row ID. Default is the first column.", - null, - false); - - PropertyMetadata s6 = new PropertyMetadata<>( - SERIALIZER, - "Serializer for Accumulo data encodings. Can either be 'default', 'string', 'lexicoder', or a Java class name. Default is 'default', i.e. the value from AccumuloRowSerializer.getDefault(), i.e. 'lexicoder'.", - VarcharType.VARCHAR, String.class, - AccumuloRowSerializer.getDefault().getClass().getName(), - false, - x -> x.equals("default") - ? AccumuloRowSerializer.getDefault().getClass().getName() - : (x.equals("string") ? StringRowSerializer.class.getName() - : (x.equals("lexicoder") - ? LexicoderRowSerializer.class.getName() - : (String) x)), - object -> object); - - PropertyMetadata s7 = stringProperty( - SCAN_AUTHS, - "Scan-time authorizations set on the batch scanner. Default is all scan authorizations for the user", - null, - false); - - tableProperties = ImmutableList.of(s1, s2, s3, s4, s5, s6, s7); - } - - public List> getTableProperties() - { - return tableProperties; - } - - /** - * Gets the value of the column_mapping property, or Optional.empty() if not set. - *

- * Parses the value into a map of Trino column name to a pair of strings, the Accumulo column family and qualifier. - * - * @param tableProperties The map of table properties - * @return The column mapping, Trino name to (accumulo column family, qualifier) - */ - public static Optional>> getColumnMapping( - Map tableProperties) - { - requireNonNull(tableProperties); - - String strMapping = (String) tableProperties.get(COLUMN_MAPPING); - if (strMapping == null) { - return Optional.empty(); - } - - // Parse out the column mapping - // This is a comma-delimited list of "(trinoName, column:accumulo, fam:accumulo qualifier)" triplets - ImmutableMap.Builder> mapping = ImmutableMap.builder(); - for (String m : COMMA_SPLITTER.split(strMapping)) { - String[] tokens = Iterables.toArray(COLON_SPLITTER.split(m), String.class); - checkState(tokens.length == 3, "Mapping of %s contains %s tokens instead of 3", m, tokens.length); - mapping.put(tokens[0], Map.entry(tokens[1], tokens[2])); - } - - return Optional.of(mapping.buildOrThrow()); - } - - public static Optional> getIndexColumns(Map tableProperties) - { - requireNonNull(tableProperties); - - String indexColumns = (String) tableProperties.get(INDEX_COLUMNS); - if (indexColumns == null) { - return Optional.empty(); - } - - return Optional.of(Splitter.on(',').splitToList(indexColumns)); - } - - /** - * Gets the configured locality groups for the table, or Optional.empty() if not set. - *

- * All strings are lowercase. - * - * @param tableProperties The map of table properties - * @return Optional map of locality groups - */ - public static Optional>> getLocalityGroups(Map tableProperties) - { - requireNonNull(tableProperties); - - String groupStr = (String) tableProperties.get(LOCALITY_GROUPS); - if (groupStr == null) { - return Optional.empty(); - } - - ImmutableMap.Builder> groups = ImmutableMap.builder(); - - // Split all configured locality groups - for (String group : PIPE_SPLITTER.split(groupStr)) { - String[] locGroups = Iterables.toArray(COLON_SPLITTER.split(group), String.class); - - if (locGroups.length != 2) { - throw new TrinoException(INVALID_TABLE_PROPERTY, "Locality groups string is malformed. See documentation for proper format."); - } - - String grpName = locGroups[0]; - ImmutableSet.Builder colSet = ImmutableSet.builder(); - - for (String f : COMMA_SPLITTER.split(locGroups[1])) { - colSet.add(f.toLowerCase(Locale.ENGLISH)); - } - - groups.put(grpName.toLowerCase(Locale.ENGLISH), colSet.build()); - } - - return Optional.of(groups.buildOrThrow()); - } - - public static Optional getRowId(Map tableProperties) - { - requireNonNull(tableProperties); - - String rowId = (String) tableProperties.get(ROW_ID); - return Optional.ofNullable(rowId); - } - - public static Optional getScanAuthorizations(Map tableProperties) - { - requireNonNull(tableProperties); - - String scanAuths = (String) tableProperties.get(SCAN_AUTHS); - return Optional.ofNullable(scanAuths); - } - - /** - * Gets the {@link AccumuloRowSerializer} class name to use for this table - * - * @param tableProperties The map of table properties - * @return The name of the AccumuloRowSerializer class - */ - public static String getSerializerClass(Map tableProperties) - { - requireNonNull(tableProperties); - - return (String) tableProperties.get(SERIALIZER); - } - - public static boolean isExternal(Map tableProperties) - { - requireNonNull(tableProperties); - - return (Boolean) tableProperties.get(EXTERNAL); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/ColumnCardinalityCache.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/ColumnCardinalityCache.java deleted file mode 100644 index a95939ca5facc..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/ColumnCardinalityCache.java +++ /dev/null @@ -1,407 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.index; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Iterables; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.MultimapBuilder; -import com.google.inject.Inject; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.cache.NonEvictableLoadingCache; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.accumulo.model.AccumuloColumnConstraint; -import io.trino.spi.TrinoException; -import jakarta.annotation.PreDestroy; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.PartialKey; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.hadoop.io.Text; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.CompletionService; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.collect.Streams.stream; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.cache.SafeCaches.buildNonEvictableCache; -import static io.trino.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static io.trino.plugin.accumulo.index.Indexer.CARDINALITY_CQ_AS_TEXT; -import static io.trino.plugin.accumulo.index.Indexer.getIndexColumnFamily; -import static io.trino.plugin.accumulo.index.Indexer.getMetricsTableName; -import static io.trino.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; -import static java.lang.Long.parseLong; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newCachedThreadPool; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -/** - * This class is an indexing utility to cache the cardinality of a column value for every table. - * Each table has its own cache that is independent of every other, and every column also has its - * own Guava cache. Use of this utility can have a significant impact for retrieving the cardinality - * of many columns, preventing unnecessary accesses to the metrics table in Accumulo for a - * cardinality that won't change much. - */ -public class ColumnCardinalityCache -{ - private static final Logger LOG = Logger.get(ColumnCardinalityCache.class); - private final AccumuloClient client; - private final ExecutorService coreExecutor; - private final BoundedExecutor executorService; - private final NonEvictableLoadingCache cache; - - @Inject - public ColumnCardinalityCache(AccumuloClient client, AccumuloConfig config) - { - this.client = requireNonNull(client, "client is null"); - int size = config.getCardinalityCacheSize(); - Duration expireDuration = config.getCardinalityCacheExpiration(); - - // Create a bounded executor with a pool size at 4x number of processors - this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s")); - this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors()); - - LOG.debug("Created new cache size %d expiry %s", size, expireDuration); - cache = buildNonEvictableCache( - CacheBuilder.newBuilder() - .maximumSize(size) - .expireAfterWrite(expireDuration.toMillis(), MILLISECONDS), - new CardinalityCacheLoader()); - } - - @PreDestroy - public void shutdown() - { - coreExecutor.shutdownNow(); - } - - /** - * Gets the cardinality for each {@link AccumuloColumnConstraint}. - * Given constraints are expected to be indexed! Who knows what would happen if they weren't! - * - * @param schema Schema name - * @param table Table name - * @param auths Scan authorizations - * @param idxConstraintRangePairs Mapping of all ranges for a given constraint - * @param earlyReturnThreshold Smallest acceptable cardinality to return early while other tasks complete - * @param pollingDuration Duration for polling the cardinality completion service - * @return An immutable multimap of cardinality to column constraint, sorted by cardinality from smallest to largest - */ - public Multimap getCardinalities(String schema, String table, Authorizations auths, Multimap idxConstraintRangePairs, long earlyReturnThreshold, Duration pollingDuration) - { - // Submit tasks to the executor to fetch column cardinality, adding it to the Guava cache if necessary - CompletionService> executor = new ExecutorCompletionService<>(executorService); - idxConstraintRangePairs.asMap().forEach((key, value) -> executor.submit(() -> { - long cardinality = getColumnCardinality(schema, table, auths, key.family(), key.qualifier(), value); - LOG.debug("Cardinality for column %s is %s", key.name(), cardinality); - return Map.entry(cardinality, key); - })); - - // Create a multi map sorted by cardinality - ListMultimap cardinalityToConstraints = MultimapBuilder.treeKeys().arrayListValues().build(); - try { - boolean earlyReturn = false; - int numTasks = idxConstraintRangePairs.asMap().entrySet().size(); - do { - // Sleep for the polling duration to allow concurrent tasks to run for this time - Thread.sleep(pollingDuration.toMillis()); - - // Poll each task, retrieving the result if it is done - for (int i = 0; i < numTasks; ++i) { - Future> futureCardinality = executor.poll(); - if (futureCardinality != null && futureCardinality.isDone()) { - Entry columnCardinality = futureCardinality.get(); - cardinalityToConstraints.put(columnCardinality.getKey(), columnCardinality.getValue()); - } - } - - // If the smallest cardinality is present and below the threshold, set the earlyReturn flag - Optional> smallestCardinality = cardinalityToConstraints.entries().stream().findFirst(); - if (smallestCardinality.isPresent()) { - if (smallestCardinality.get().getKey() <= earlyReturnThreshold) { - LOG.info("Cardinality %s, is below threshold. Returning early while other tasks finish", smallestCardinality); - earlyReturn = true; - } - } - } - while (!earlyReturn && cardinalityToConstraints.entries().size() < numTasks); - } - catch (ExecutionException | InterruptedException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Exception when getting cardinality", e); - } - - // Create a copy of the cardinalities - return ImmutableMultimap.copyOf(cardinalityToConstraints); - } - - /** - * Gets the column cardinality for all of the given range values. May reach out to the - * metrics table in Accumulo to retrieve new cache elements. - * - * @param schema Table schema - * @param table Table name - * @param auths Scan authorizations - * @param family Accumulo column family - * @param qualifier Accumulo column qualifier - * @param colValues All range values to summarize for the cardinality - * @return The cardinality of the column - */ - public long getColumnCardinality(String schema, String table, Authorizations auths, String family, String qualifier, Collection colValues) - throws ExecutionException - { - LOG.debug("Getting cardinality for %s:%s", family, qualifier); - - // Collect all exact Accumulo Ranges, i.e. single value entries vs. a full scan - Collection exactRanges = colValues.stream() - .filter(ColumnCardinalityCache::isExact) - .map(range -> new CacheKey(schema, table, family, qualifier, range, auths)) - .collect(Collectors.toList()); - - LOG.debug("Column values contain %s exact ranges of %s", exactRanges.size(), colValues.size()); - - // Sum the cardinalities for the exact-value Ranges - // This is where the reach-out to Accumulo occurs for all Ranges that have not - // previously been fetched - long sum = cache.getAll(exactRanges).values().stream().mapToLong(Long::longValue).sum(); - - // If these collection sizes are not equal, - // then there is at least one non-exact range - if (exactRanges.size() != colValues.size()) { - // for each range in the column value - for (Range range : colValues) { - // if this range is not exact - if (!isExact(range)) { - // Then get the value for this range using the single-value cache lookup - sum += cache.get(new CacheKey(schema, table, family, qualifier, range, auths)); - } - } - } - - return sum; - } - - private static boolean isExact(Range range) - { - return !range.isInfiniteStartKey() && !range.isInfiniteStopKey() && - range.getStartKey().followingKey(PartialKey.ROW).equals(range.getEndKey()); - } - - /** - * Complex key for the CacheLoader - */ - private static class CacheKey - { - private final String schema; - private final String table; - private final String family; - private final String qualifier; - private final Range range; - private final Authorizations auths; - - public CacheKey( - String schema, - String table, - String family, - String qualifier, - Range range, - Authorizations auths) - { - this.schema = requireNonNull(schema, "schema is null"); - this.table = requireNonNull(table, "table is null"); - this.family = requireNonNull(family, "family is null"); - this.qualifier = requireNonNull(qualifier, "qualifier is null"); - this.range = requireNonNull(range, "range is null"); - this.auths = requireNonNull(auths, "auths is null"); - } - - public String getSchema() - { - return schema; - } - - public String getTable() - { - return table; - } - - public String getFamily() - { - return family; - } - - public String getQualifier() - { - return qualifier; - } - - public Range getRange() - { - return range; - } - - public Authorizations getAuths() - { - return auths; - } - - @Override - public int hashCode() - { - return Objects.hash(schema, table, family, qualifier, range); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - - CacheKey other = (CacheKey) obj; - return Objects.equals(this.schema, other.schema) - && Objects.equals(this.table, other.table) - && Objects.equals(this.family, other.family) - && Objects.equals(this.qualifier, other.qualifier) - && Objects.equals(this.range, other.range); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("schema", schema) - .add("table", table) - .add("family", family) - .add("qualifier", qualifier) - .add("range", range).toString(); - } - } - - /** - * Internal class for loading the cardinality from Accumulo - */ - private class CardinalityCacheLoader - extends CacheLoader - { - /** - * Loads the cardinality for the given Range. Uses a BatchScanner and sums the cardinality for all values that encapsulate the Range. - * - * @param key Range to get the cardinality for - * @return The cardinality of the column, which would be zero if the value does not exist - */ - @Override - public Long load(CacheKey key) - throws Exception - { - LOG.debug("Loading a non-exact range from Accumulo: %s", key); - // Get metrics table name and the column family for the scanner - String metricsTable = getMetricsTableName(key.getSchema(), key.getTable()); - Text columnFamily = new Text(getIndexColumnFamily(key.getFamily().getBytes(UTF_8), key.getQualifier().getBytes(UTF_8)).array()); - - // Create scanner for querying the range - BatchScanner scanner = client.createBatchScanner(metricsTable, key.auths, 10); - scanner.setRanges(client.tableOperations().splitRangeByTablets(metricsTable, key.range, Integer.MAX_VALUE)); - scanner.fetchColumn(columnFamily, CARDINALITY_CQ_AS_TEXT); - - try { - return stream(scanner) - .map(Entry::getValue) - .map(Value::toString) - .mapToLong(Long::parseLong) - .sum(); - } - finally { - scanner.close(); - } - } - - @Override - public Map loadAll(Iterable keys) - throws Exception - { - int size = Iterables.size(keys); - if (size == 0) { - return ImmutableMap.of(); - } - - LOG.debug("Loading %s exact ranges from Accumulo", size); - - // In order to simplify the implementation, we are making a (safe) assumption - // that the CacheKeys will all contain the same combination of schema/table/family/qualifier - // This is asserted with the below implementation error just to make sure - CacheKey anyKey = stream(keys).findAny().get(); - if (stream(keys).anyMatch(k -> !k.getSchema().equals(anyKey.getSchema()) || !k.getTable().equals(anyKey.getTable()) || !k.getFamily().equals(anyKey.getFamily()) || !k.getQualifier().equals(anyKey.getQualifier()))) { - throw new TrinoException(FUNCTION_IMPLEMENTATION_ERROR, "loadAll called with a non-homogeneous collection of cache keys"); - } - - Map rangeToKey = stream(keys).collect(Collectors.toMap(CacheKey::getRange, Function.identity())); - LOG.debug("rangeToKey size is %s", rangeToKey.size()); - - // Get metrics table name and the column family for the scanner - String metricsTable = getMetricsTableName(anyKey.getSchema(), anyKey.getTable()); - Text columnFamily = new Text(getIndexColumnFamily(anyKey.getFamily().getBytes(UTF_8), anyKey.getQualifier().getBytes(UTF_8)).array()); - - BatchScanner scanner = client.createBatchScanner(metricsTable, anyKey.getAuths(), 10); - try { - scanner.setRanges(stream(keys).map(CacheKey::getRange).collect(Collectors.toList())); - scanner.fetchColumn(columnFamily, CARDINALITY_CQ_AS_TEXT); - - // Create a new map to hold our cardinalities for each range, returning a default of - // Zero for each non-existent Key - Map rangeValues = new HashMap<>(); - stream(keys).forEach(key -> rangeValues.put(key, 0L)); - - for (Entry entry : scanner) { - rangeValues.put(rangeToKey.get(Range.exact(entry.getKey().getRow())), parseLong(entry.getValue().toString())); - } - - return rangeValues; - } - finally { - if (scanner != null) { - scanner.close(); - } - } - } - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/IndexLookup.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/IndexLookup.java deleted file mode 100644 index a579ae7022bab..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/IndexLookup.java +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.index; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Multimap; -import com.google.inject.Inject; -import io.airlift.concurrent.BoundedExecutor; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.plugin.accumulo.model.AccumuloColumnConstraint; -import io.trino.plugin.accumulo.model.TabletSplitMetadata; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ConnectorSession; -import jakarta.annotation.PreDestroy; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.hadoop.io.Text; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletionService; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static io.trino.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static io.trino.plugin.accumulo.AccumuloMetadataManager.getRangesFromDomain; -import static io.trino.plugin.accumulo.conf.AccumuloSessionProperties.getIndexCardinalityCachePollingDuration; -import static io.trino.plugin.accumulo.conf.AccumuloSessionProperties.getIndexSmallCardThreshold; -import static io.trino.plugin.accumulo.conf.AccumuloSessionProperties.getIndexThreshold; -import static io.trino.plugin.accumulo.conf.AccumuloSessionProperties.getNumIndexRowsPerSplit; -import static io.trino.plugin.accumulo.conf.AccumuloSessionProperties.isIndexMetricsEnabled; -import static io.trino.plugin.accumulo.conf.AccumuloSessionProperties.isIndexShortCircuitEnabled; -import static io.trino.plugin.accumulo.conf.AccumuloSessionProperties.isOptimizeIndexEnabled; -import static io.trino.plugin.accumulo.index.Indexer.CARDINALITY_CQ_AS_TEXT; -import static io.trino.plugin.accumulo.index.Indexer.METRICS_TABLE_ROWID_AS_TEXT; -import static io.trino.plugin.accumulo.index.Indexer.METRICS_TABLE_ROWS_CF_AS_TEXT; -import static io.trino.plugin.accumulo.index.Indexer.getIndexTableName; -import static io.trino.plugin.accumulo.index.Indexer.getMetricsTableName; -import static io.trino.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newCachedThreadPool; - -/** - * Class to assist the Trino connector, and maybe external applications, - * leverage the secondary * index built by the {@link Indexer}. - * Leverages {@link ColumnCardinalityCache} to assist in * retrieving row IDs. - * Currently pretty bound to the Trino connector APIs. - */ -public class IndexLookup -{ - private static final Logger LOG = Logger.get(IndexLookup.class); - private static final Range METRICS_TABLE_ROWID_RANGE = new Range(METRICS_TABLE_ROWID_AS_TEXT); - private final ColumnCardinalityCache cardinalityCache; - private final AccumuloClient connector; - private final ExecutorService coreExecutor; - private final BoundedExecutor executorService; - - @Inject - public IndexLookup(AccumuloClient client, ColumnCardinalityCache cardinalityCache) - { - this.connector = requireNonNull(client, "client is null"); - this.cardinalityCache = requireNonNull(cardinalityCache, "cardinalityCache is null"); - - // Create a bounded executor with a pool size at 4x number of processors - this.coreExecutor = newCachedThreadPool(daemonThreadsNamed("cardinality-lookup-%s")); - this.executorService = new BoundedExecutor(coreExecutor, 4 * Runtime.getRuntime().availableProcessors()); - } - - @PreDestroy - public void shutdown() - { - coreExecutor.shutdownNow(); - } - - /** - * Scans the index table, applying the index based on the given column constraints to return a set of tablet splits. - *

- * If this function returns true, the output parameter tabletSplits contains a list of TabletSplitMetadata objects. - * These in turn contain a collection of Ranges containing the exact row IDs determined using the index. - *

- * If this function returns false, the secondary index should not be used. In this case, - * either the accumulo session has disabled secondary indexing, - * or the number of row IDs that would be used by the secondary index is greater than the configured threshold - * (again retrieved from the session). - * - * @param schema Schema name - * @param table Table name - * @param session Current client session - * @param constraints All column constraints (this method will filter for if the column is indexed) - * @param rowIdRanges Collection of Accumulo ranges based on any predicate against a record key - * @param tabletSplits Output parameter containing the bundles of row IDs determined by the use of the index. - * @param serializer Instance of a row serializer - * @param auths Scan-time authorizations - * @return True if the tablet splits are valid and should be used, false otherwise - * @throws Exception If something bad happens. What are the odds? - */ - public boolean applyIndex( - String schema, - String table, - ConnectorSession session, - Collection constraints, - Collection rowIdRanges, - List tabletSplits, - AccumuloRowSerializer serializer, - Authorizations auths) - throws Exception - { - // Early out if index is disabled - if (!isOptimizeIndexEnabled(session)) { - LOG.debug("Secondary index is disabled"); - return false; - } - - LOG.debug("Secondary index is enabled"); - - // Collect Accumulo ranges for each indexed column constraint - Multimap constraintRanges = getIndexedConstraintRanges(constraints, serializer); - - // If there is no constraints on an index column, we again will bail out - if (constraintRanges.isEmpty()) { - LOG.debug("Query contains no constraints on indexed columns, skipping secondary index"); - return false; - } - - // If metrics are not enabled - if (!isIndexMetricsEnabled(session)) { - LOG.debug("Use of index metrics is disabled"); - // Get the ranges via the index table - List indexRanges = getIndexRanges(getIndexTableName(schema, table), constraintRanges, rowIdRanges, auths); - - if (!indexRanges.isEmpty()) { - // Bin the ranges into TabletMetadataSplits and return true to use the tablet splits - binRanges(getNumIndexRowsPerSplit(session), indexRanges, tabletSplits); - LOG.debug("Number of splits for %s.%s is %d with %d ranges", schema, table, tabletSplits.size(), indexRanges.size()); - } - else { - LOG.debug("Query would return no results, returning empty list of splits"); - } - - return true; - } - LOG.debug("Use of index metrics is enabled"); - // Get ranges using the metrics - return getRangesWithMetrics(session, schema, table, constraintRanges, rowIdRanges, tabletSplits, auths); - } - - private static Multimap getIndexedConstraintRanges(Collection constraints, AccumuloRowSerializer serializer) - { - ImmutableListMultimap.Builder builder = ImmutableListMultimap.builder(); - for (AccumuloColumnConstraint columnConstraint : constraints) { - if (columnConstraint.indexed()) { - for (Range range : getRangesFromDomain(columnConstraint.domain(), serializer)) { - builder.put(columnConstraint, range); - } - } - else { - LOG.warn("Query contains constraint on non-indexed column %s. Is it worth indexing?", columnConstraint.name()); - } - } - return builder.build(); - } - - private boolean getRangesWithMetrics( - ConnectorSession session, - String schema, - String table, - Multimap constraintRanges, - Collection rowIdRanges, - List tabletSplits, - Authorizations auths) - throws Exception - { - String metricsTable = getMetricsTableName(schema, table); - long numRows = getNumRowsInTable(metricsTable, auths); - - // Get the cardinalities from the metrics table - Multimap cardinalities; - if (isIndexShortCircuitEnabled(session)) { - cardinalities = cardinalityCache.getCardinalities( - schema, - table, - auths, - constraintRanges, - (long) (numRows * getIndexSmallCardThreshold(session)), - getIndexCardinalityCachePollingDuration(session)); - } - else { - // disable short circuit using 0 - cardinalities = cardinalityCache.getCardinalities(schema, table, auths, constraintRanges, 0, new Duration(0, TimeUnit.MILLISECONDS)); - } - - Optional> entry = cardinalities.entries().stream().findFirst(); - if (entry.isEmpty()) { - return false; - } - - Entry lowestCardinality = entry.get(); - String indexTable = getIndexTableName(schema, table); - double threshold = getIndexThreshold(session); - List indexRanges; - - // If the smallest cardinality in our list is above the lowest cardinality threshold, - // we should look at intersecting the row ID ranges to try and get under the threshold. - if (smallestCardAboveThreshold(session, numRows, lowestCardinality.getKey())) { - // If we only have one column, we can skip the intersection process and just check the index threshold - if (cardinalities.size() == 1) { - long numEntries = lowestCardinality.getKey(); - double ratio = ((double) numEntries / (double) numRows); - LOG.debug("Use of index would scan %s of %s rows, ratio %s. Threshold %2f, Using for index table? %s", numEntries, numRows, ratio, threshold, ratio < threshold); - if (ratio >= threshold) { - return false; - } - } - - // Else, get the intersection of all row IDs for all column constraints - LOG.debug("%d indexed columns, intersecting ranges", constraintRanges.size()); - indexRanges = getIndexRanges(indexTable, constraintRanges, rowIdRanges, auths); - LOG.debug("Intersection results in %d ranges from secondary index", indexRanges.size()); - } - else { - // Else, we don't need to intersect the columns and we can just use the column with the lowest cardinality, - // so get all those row IDs in a set of ranges. - LOG.debug("Not intersecting columns, using column with lowest cardinality "); - ImmutableMultimap.Builder lcBldr = ImmutableMultimap.builder(); - lcBldr.putAll(lowestCardinality.getValue(), constraintRanges.get(lowestCardinality.getValue())); - indexRanges = getIndexRanges(indexTable, lcBldr.build(), rowIdRanges, auths); - } - - if (indexRanges.isEmpty()) { - LOG.debug("Query would return no results, returning empty list of splits"); - return true; - } - - // Okay, we now check how many rows we would scan by using the index vs. the overall number - // of rows - long numEntries = indexRanges.size(); - double ratio = (double) numEntries / (double) numRows; - LOG.debug("Use of index would scan %d of %d rows, ratio %s. Threshold %2f, Using for table %s? %b", numEntries, numRows, ratio, threshold, table, ratio < threshold); - - // If the percentage of scanned rows, the ratio, less than the configured threshold - if (ratio < threshold) { - // Bin the ranges into TabletMetadataSplits and return true to use the tablet splits - binRanges(getNumIndexRowsPerSplit(session), indexRanges, tabletSplits); - LOG.debug("Number of splits for %s.%s is %d with %d ranges", schema, table, tabletSplits.size(), indexRanges.size()); - return true; - } - // We are going to do too much work to use the secondary index, so return false - return false; - } - - private static boolean smallestCardAboveThreshold(ConnectorSession session, long numRows, long smallestCardinality) - { - double ratio = ((double) smallestCardinality / (double) numRows); - double threshold = getIndexSmallCardThreshold(session); - LOG.debug("Smallest cardinality is %d, num rows is %d, ratio is %2f with threshold of %f", smallestCardinality, numRows, ratio, threshold); - return ratio > threshold; - } - - private long getNumRowsInTable(String metricsTable, Authorizations auths) - throws TableNotFoundException - { - // Create scanner against the metrics table, pulling the special column and the rows column - Scanner scanner = connector.createScanner(metricsTable, auths); - scanner.setRange(METRICS_TABLE_ROWID_RANGE); - scanner.fetchColumn(METRICS_TABLE_ROWS_CF_AS_TEXT, CARDINALITY_CQ_AS_TEXT); - - // Scan the entry and get the number of rows - long numRows = -1; - for (Entry entry : scanner) { - if (numRows > 0) { - throw new TrinoException(FUNCTION_IMPLEMENTATION_ERROR, "Should have received only one entry when scanning for number of rows in metrics table"); - } - numRows = Long.parseLong(entry.getValue().toString()); - } - scanner.close(); - - LOG.debug("Number of rows in table is %d", numRows); - return numRows; - } - - private List getIndexRanges(String indexTable, Multimap constraintRanges, Collection rowIDRanges, Authorizations auths) - { - Set finalRanges = new HashSet<>(); - // For each column/constraint pair we submit a task to scan the index ranges - List>> tasks = new ArrayList<>(); - CompletionService> executor = new ExecutorCompletionService<>(executorService); - for (Entry> constraintEntry : constraintRanges.asMap().entrySet()) { - tasks.add(executor.submit(() -> { - // Create a batch scanner against the index table, setting the ranges - BatchScanner scan = connector.createBatchScanner(indexTable, auths, 10); - scan.setRanges(constraintEntry.getValue()); - - // Fetch the column family for this specific column - scan.fetchColumnFamily(new Text(Indexer.getIndexColumnFamily(constraintEntry.getKey().family().getBytes(UTF_8), constraintEntry.getKey().qualifier().getBytes(UTF_8)).array())); - - // For each entry in the scanner - Text tmpQualifier = new Text(); - Set columnRanges = new HashSet<>(); - for (Entry entry : scan) { - entry.getKey().getColumnQualifier(tmpQualifier); - - // Add to our column ranges if it is in one of the row ID ranges - if (inRange(tmpQualifier, rowIDRanges)) { - columnRanges.add(new Range(tmpQualifier)); - } - } - - LOG.debug("Retrieved %d ranges for index column %s", columnRanges.size(), constraintEntry.getKey().name()); - scan.close(); - return columnRanges; - })); - } - tasks.forEach(future -> { - try { - // If finalRanges is null, we have not yet added any column ranges - if (finalRanges.isEmpty()) { - finalRanges.addAll(future.get()); - } - else { - // Retain only the row IDs for this column that have already been added - // This is your set intersection operation! - finalRanges.retainAll(future.get()); - } - } - catch (ExecutionException | InterruptedException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Exception when getting index ranges", e.getCause()); - } - }); - return ImmutableList.copyOf(finalRanges); - } - - private static void binRanges(int numRangesPerBin, List splitRanges, List trinoSplits) - { - checkArgument(numRangesPerBin > 0, "number of ranges per bin must positivebe greater than zero"); - int toAdd = splitRanges.size(); - int fromIndex = 0; - int toIndex = Math.min(toAdd, numRangesPerBin); - do { - // Add the sublist of range handles - // Use an empty location because we are binning multiple Ranges spread across many tablet servers - trinoSplits.add(new TabletSplitMetadata(Optional.empty(), splitRanges.subList(fromIndex, toIndex))); - toAdd -= toIndex - fromIndex; - fromIndex = toIndex; - toIndex += Math.min(toAdd, numRangesPerBin); - } - while (toAdd > 0); - } - - /** - * Gets a Boolean value indicating if the given value is in one of the Ranges in the given collection - * - * @param text Text object to check against the Range collection - * @param ranges Ranges to look into - * @return True if the text object is in one of the ranges, false otherwise - */ - private static boolean inRange(Text text, Collection ranges) - { - Key kCq = new Key(text); - return ranges.stream().anyMatch(r -> !r.beforeStartKey(kCq) && !r.afterEndKey(kCq)); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/Indexer.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/Indexer.java deleted file mode 100644 index 6dfc421dd0977..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/index/Indexer.java +++ /dev/null @@ -1,566 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.index; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; -import com.google.common.collect.Table; -import com.google.common.primitives.Bytes; -import com.google.common.primitives.UnsignedBytes; -import io.trino.annotation.NotThreadSafe; -import io.trino.plugin.accumulo.Types; -import io.trino.plugin.accumulo.iterators.MaxByteArrayCombiner; -import io.trino.plugin.accumulo.iterators.MinByteArrayCombiner; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.Type; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.MutationsRejectedException; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.client.lexicoder.AbstractLexicoder; -import org.apache.accumulo.core.data.ColumnUpdate; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.LongCombiner; -import org.apache.accumulo.core.iterators.user.SummingCombiner; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.hadoop.io.Text; - -import java.io.Closeable; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Maps.immutableEntry; -import static io.trino.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE; -import static io.trino.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static java.nio.ByteBuffer.wrap; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; - -/** - * This utility class assists the Trino connector, and external applications, - * in populating the index table and metrics table for Accumulo-backed Trino tables. - *

- * This class is totally not thread safe. - *

- * When creating a table, if it contains indexed columns, users will have to create the index table - * and the index metrics table, the names of which can be retrieved using the static functions in - * this class. Additionally, users MUST add iterators to the index metrics table (also available via - * static function), and, while not required, recommended to add the locality groups to the index - * table to improve index lookup times. - *

- * Sample usage of an indexer: - * 

- * 
- * Indexer indexer = new Indexer(connector, userAuths, table, writerConf);
- * for (Mutation m : mutationsToNormalTable) {
- *      indexer.index(m);
- * }
- *
- * // can flush indexer w/regular BatchWriter
- * indexer.flush()
- *
- * // finished adding new mutations, close the indexer
- * indexer.close();
- * 
- *
- */ -@NotThreadSafe -public class Indexer - implements Closeable -{ - public static final ByteBuffer METRICS_TABLE_ROW_ID = wrap("___METRICS_TABLE___".getBytes(UTF_8)); - public static final ByteBuffer METRICS_TABLE_ROWS_CF = wrap("___rows___".getBytes(UTF_8)); - public static final MetricsKey METRICS_TABLE_ROW_COUNT = new MetricsKey(METRICS_TABLE_ROW_ID, METRICS_TABLE_ROWS_CF); - public static final ByteBuffer METRICS_TABLE_FIRST_ROW_CQ = wrap("___first_row___".getBytes(UTF_8)); - public static final ByteBuffer METRICS_TABLE_LAST_ROW_CQ = wrap("___last_row___".getBytes(UTF_8)); - public static final byte[] CARDINALITY_CQ = "___card___".getBytes(UTF_8); - public static final Text CARDINALITY_CQ_AS_TEXT = new Text(CARDINALITY_CQ); - public static final Text METRICS_TABLE_ROWS_CF_AS_TEXT = new Text(METRICS_TABLE_ROWS_CF.array()); - public static final Text METRICS_TABLE_ROWID_AS_TEXT = new Text(METRICS_TABLE_ROW_ID.array()); - - private static final byte[] EMPTY_BYTES = new byte[0]; - private static final byte[] UNDERSCORE = {'_'}; - private static final AbstractLexicoder ENCODER = new LongCombiner.StringEncoder(); - - private final AccumuloTable table; - private final BatchWriter indexWriter; - private final BatchWriterConfig writerConfig; - private final AccumuloClient client; - private final Map metrics = new HashMap<>(); - private final Multimap indexColumns; - private final Map> indexColumnTypes; - private final AccumuloRowSerializer serializer; - private final Comparator byteArrayComparator = UnsignedBytes.lexicographicalComparator(); - - private byte[] firstRow; - private byte[] lastRow; - - public Indexer( - AccumuloClient client, - Authorizations auths, - AccumuloTable table, - BatchWriterConfig writerConfig) - throws TableNotFoundException - { - this.client = requireNonNull(client, "client is null"); - this.table = requireNonNull(table, "table is null"); - this.writerConfig = requireNonNull(writerConfig, "writerConfig is null"); - requireNonNull(auths, "auths is null"); - - this.serializer = table.getSerializerInstance(); - - // Create our batch writer - indexWriter = client.createBatchWriter(table.getIndexTableName(), writerConfig); - - ImmutableMultimap.Builder indexColumnsBuilder = ImmutableMultimap.builder(); - Table indexColumnTypesBuilder = HashBasedTable.create(); - - // Initialize metadata - table.getColumns().forEach(columnHandle -> { - if (columnHandle.indexed()) { - // Wrap the column family and qualifier for this column and add it to - // collection of indexed columns - ByteBuffer family = wrap(columnHandle.family().get().getBytes(UTF_8)); - ByteBuffer qualifier = wrap(columnHandle.qualifier().get().getBytes(UTF_8)); - indexColumnsBuilder.put(family, qualifier); - - // Create a mapping for this column's Trino type, again creating a new one for the - // family if necessary - indexColumnTypesBuilder.put(family, qualifier, columnHandle.type()); - } - }); - - indexColumns = indexColumnsBuilder.build(); - indexColumnTypes = ImmutableMap.copyOf(indexColumnTypesBuilder.rowMap()); - - // If there are no indexed columns, throw an exception - if (indexColumns.isEmpty()) { - throw new TrinoException(NOT_SUPPORTED, "No indexed columns in table metadata. Refusing to index a table with no indexed columns"); - } - - // Initialize metrics map - // This metrics map is for column cardinality - metrics.put(METRICS_TABLE_ROW_COUNT, new AtomicLong(0)); - - // Scan the metrics table for existing first row and last row - Entry minmax = getMinMaxRowIds(client, table, auths); - firstRow = minmax.getKey(); - lastRow = minmax.getValue(); - } - - /** - * Index the given mutation, adding mutations to the index and metrics table - *

- * Like typical use of a BatchWriter, this method does not flush mutations to the underlying index table. - * For higher throughput the modifications to the metrics table are tracked in memory and added to the metrics table when the indexer is flushed or closed. - * - * @param mutation Mutation to index - */ - public void index(Mutation mutation) - { - // Increment the cardinality for the number of rows in the table - metrics.get(METRICS_TABLE_ROW_COUNT).incrementAndGet(); - - // Set the first and last row values of the table based on existing row IDs - if (firstRow == null || byteArrayComparator.compare(mutation.getRow(), firstRow) < 0) { - firstRow = mutation.getRow(); - } - - if (lastRow == null || byteArrayComparator.compare(mutation.getRow(), lastRow) > 0) { - lastRow = mutation.getRow(); - } - - // For each column update in this mutation - for (ColumnUpdate columnUpdate : mutation.getUpdates()) { - // Get the column qualifiers we want to index for this column family (if any) - ByteBuffer family = wrap(columnUpdate.getColumnFamily()); - Collection indexQualifiers = indexColumns.get(family); - - // If we have column qualifiers we want to index for this column family - if (indexQualifiers != null) { - // Check if we want to index this particular qualifier - ByteBuffer qualifier = wrap(columnUpdate.getColumnQualifier()); - if (indexQualifiers.contains(qualifier)) { - // If so, create a mutation using the following mapping: - // Row ID = column value - // Column Family = columnqualifier_columnfamily - // Column Qualifier = row ID - // Value = empty - ByteBuffer indexFamily = getIndexColumnFamily(columnUpdate.getColumnFamily(), columnUpdate.getColumnQualifier()); - Type type = indexColumnTypes.get(family).get(qualifier); - ColumnVisibility visibility = new ColumnVisibility(columnUpdate.getColumnVisibility()); - - // If this is an array type, then index each individual element in the array - if (Types.isArrayType(type)) { - Type elementType = Types.getElementType(type); - List elements = serializer.decode(type, columnUpdate.getValue()); - for (Object element : elements) { - addIndexMutation(wrap(serializer.encode(elementType, element)), indexFamily, visibility, mutation.getRow()); - } - } - else { - addIndexMutation(wrap(columnUpdate.getValue()), indexFamily, visibility, mutation.getRow()); - } - } - } - } - } - - public void index(Iterable mutations) - { - for (Mutation mutation : mutations) { - index(mutation); - } - } - - private void addIndexMutation(ByteBuffer row, ByteBuffer family, ColumnVisibility visibility, byte[] qualifier) - { - // Create the mutation and add it to the batch writer - checkArgument(row.arrayOffset() == 0, "Unexpected row buffer offset: %s", row.arrayOffset()); - Mutation indexMutation = new Mutation(row.array()); - checkArgument(family.arrayOffset() == 0, "Unexpected family buffer offset: %s", family.arrayOffset()); - indexMutation.put(family.array(), qualifier, visibility, EMPTY_BYTES); - try { - indexWriter.addMutation(indexMutation); - } - catch (MutationsRejectedException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Index mutation rejected by server", e); - } - - // Increment the cardinality metrics for this value of index - // metrics is a mapping of row ID to column family - MetricsKey key = new MetricsKey(row, family, visibility); - AtomicLong count = metrics.get(key); - if (count == null) { - count = new AtomicLong(0); - metrics.put(key, count); - } - - count.incrementAndGet(); - } - - /** - * Flushes all Mutations in the index writer. And all metric mutations to the metrics table. - * Note that the metrics table is not updated until this method is explicitly called (or implicitly via close). - */ - public void flush() - { - try { - // Flush index writer - indexWriter.flush(); - - // Write out metrics mutations - BatchWriter metricsWriter = client.createBatchWriter(table.getMetricsTableName(), writerConfig); - metricsWriter.addMutations(getMetricsMutations()); - metricsWriter.close(); - - // Re-initialize the metrics - metrics.clear(); - metrics.put(METRICS_TABLE_ROW_COUNT, new AtomicLong(0)); - } - catch (MutationsRejectedException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Index mutation was rejected by server on flush", e); - } - catch (TableNotFoundException e) { - throw new TrinoException(ACCUMULO_TABLE_DNE, "Accumulo table does not exist", e); - } - } - - /** - * Flushes all remaining mutations via {@link Indexer#flush} and closes the index writer. - */ - @Override - public void close() - { - try { - flush(); - indexWriter.close(); - } - catch (MutationsRejectedException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation was rejected by server on close", e); - } - } - - private Collection getMetricsMutations() - { - ImmutableList.Builder mutationBuilder = ImmutableList.builder(); - // Mapping of column value to column to number of row IDs that contain that value - for (Entry entry : metrics.entrySet()) { - // Row ID: Column value - // Family: columnfamily_columnqualifier - // Qualifier: CARDINALITY_CQ - // Visibility: Inherited from indexed Mutation - // Value: Cardinality - checkState(entry.getKey().row.arrayOffset() == 0, "Unexpected row buffer offset: %s", entry.getKey().row.arrayOffset()); - Mutation mut = new Mutation(entry.getKey().row.array()); - checkState(entry.getKey().family.arrayOffset() == 0, "Unexpected family buffer offset: %s", entry.getKey().family.arrayOffset()); - mut.put(entry.getKey().family.array(), CARDINALITY_CQ, entry.getKey().visibility, ENCODER.encode(entry.getValue().get())); - - // Add to our list of mutations - mutationBuilder.add(mut); - } - - // If the first row and last row are both not null, - // which would really be for a brand new table that has zero rows and no indexed elements... - // Talk about your edge cases! - if (firstRow != null && lastRow != null) { - // Add a some columns to the special metrics table row ID for the first/last row. - // Note that if the values on the server side are greater/lesser, - // the configured iterator will take care of this at scan/compaction time - Mutation firstLastMutation = new Mutation(METRICS_TABLE_ROW_ID.array()); - firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_FIRST_ROW_CQ.array(), firstRow); - firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_LAST_ROW_CQ.array(), lastRow); - mutationBuilder.add(firstLastMutation); - } - - return mutationBuilder.build(); - } - - /** - * Gets a collection of iterator settings that should be added to the metric table for the given Accumulo table. Don't forget! Please! - * - * @param table Table for retrieving metrics iterators, see AccumuloClient#getTable - * @return Collection of iterator settings - */ - public static Collection getMetricIterators(AccumuloTable table) - { - String cardQualifier = new String(CARDINALITY_CQ, UTF_8); - String rowsFamily = new String(METRICS_TABLE_ROWS_CF.array(), UTF_8); - - // Build a string for all columns where the summing combiner should be applied, - // i.e. all indexed columns - StringBuilder cardBuilder = new StringBuilder(rowsFamily + ":" + cardQualifier + ","); - for (String s : getLocalityGroups(table).keySet()) { - cardBuilder.append(s).append(":").append(cardQualifier).append(','); - } - cardBuilder.deleteCharAt(cardBuilder.length() - 1); - - // Configuration rows for the Min/Max combiners - String firstRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_FIRST_ROW_CQ.array(), UTF_8); - String lastRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_LAST_ROW_CQ.array(), UTF_8); - - // Summing combiner for cardinality columns - IteratorSetting s1 = new IteratorSetting(1, SummingCombiner.class, ImmutableMap.of("columns", cardBuilder.toString(), "type", "STRING")); - - // Min/Max combiner for the first/last rows of the table - IteratorSetting s2 = new IteratorSetting(2, MinByteArrayCombiner.class, ImmutableMap.of("columns", firstRowColumn)); - IteratorSetting s3 = new IteratorSetting(3, MaxByteArrayCombiner.class, ImmutableMap.of("columns", lastRowColumn)); - - return ImmutableList.of(s1, s2, s3); - } - - /** - * Gets the column family of the index table based on the given column family and qualifier. - * - * @param columnFamily Trino column family - * @param columnQualifier Trino column qualifier - * @return ByteBuffer of the given index column family - */ - public static ByteBuffer getIndexColumnFamily(byte[] columnFamily, byte[] columnQualifier) - { - return wrap(Bytes.concat(columnFamily, UNDERSCORE, columnQualifier)); - } - - /** - * Gets a set of locality groups that should be added to the index table (not the metrics table). - * - * @param table Table for the locality groups, see AccumuloClient#getTable - * @return Mapping of locality group to column families in the locality group, 1:1 mapping in - * this case - */ - public static Map> getLocalityGroups(AccumuloTable table) - { - Map> groups = new HashMap<>(); - // For each indexed column - for (AccumuloColumnHandle columnHandle : table.getColumns().stream().filter(AccumuloColumnHandle::indexed).collect(Collectors.toList())) { - // Create a Text version of the index column family - Text indexColumnFamily = new Text(getIndexColumnFamily(columnHandle.family().get().getBytes(UTF_8), columnHandle.qualifier().get().getBytes(UTF_8)).array()); - - // Add this to the locality groups, - // it is a 1:1 mapping of locality group to column families - groups.put(indexColumnFamily.toString(), ImmutableSet.of(indexColumnFamily)); - } - return groups; - } - - /** - * Gets the fully-qualified index table name for the given table. - * - * @param schema Schema name - * @param table Table name - * @return Qualified index table name - */ - public static String getIndexTableName(String schema, String table) - { - return schema.equals("default") ? table + "_idx" : schema + '.' + table + "_idx"; - } - - /** - * Gets the fully-qualified index table name for the given table. - * - * @param tableName Schema table name - * @return Qualified index table name - */ - public static String getIndexTableName(SchemaTableName tableName) - { - return getIndexTableName(tableName.getSchemaName(), tableName.getTableName()); - } - - /** - * Gets the fully-qualified index metrics table name for the given table. - * - * @param schema Schema name - * @param table Table name - * @return Qualified index metrics table name - */ - public static String getMetricsTableName(String schema, String table) - { - return schema.equals("default") ? table + "_idx_metrics" - : schema + '.' + table + "_idx_metrics"; - } - - /** - * Gets the fully-qualified index metrics table name for the given table. - * - * @param tableName Schema table name - * @return Qualified index metrics table name - */ - public static String getMetricsTableName(SchemaTableName tableName) - { - return getMetricsTableName(tableName.getSchemaName(), tableName.getTableName()); - } - - public static Entry getMinMaxRowIds(AccumuloClient connector, AccumuloTable table, Authorizations auths) - throws TableNotFoundException - { - Scanner scanner = connector.createScanner(table.getMetricsTableName(), auths); - scanner.setRange(new Range(new Text(Indexer.METRICS_TABLE_ROW_ID.array()))); - Text family = new Text(Indexer.METRICS_TABLE_ROWS_CF.array()); - Text firstRowQualifier = new Text(Indexer.METRICS_TABLE_FIRST_ROW_CQ.array()); - Text lastRowQualifier = new Text(Indexer.METRICS_TABLE_LAST_ROW_CQ.array()); - scanner.fetchColumn(family, firstRowQualifier); - scanner.fetchColumn(family, lastRowQualifier); - - byte[] firstRow = null; - byte[] lastRow = null; - for (Entry entry : scanner) { - if (entry.getKey().compareColumnQualifier(firstRowQualifier) == 0) { - firstRow = entry.getValue().get(); - } - - if (entry.getKey().compareColumnQualifier(lastRowQualifier) == 0) { - lastRow = entry.getValue().get(); - } - } - scanner.close(); - return immutableEntry(firstRow, lastRow); - } - - /** - * Class containing the key for aggregating the local metrics counter. - */ - private static class MetricsKey - { - private static final ColumnVisibility EMPTY_VISIBILITY = new ColumnVisibility(); - - public final ByteBuffer row; - public final ByteBuffer family; - public final ColumnVisibility visibility; - - public MetricsKey(ByteBuffer row, ByteBuffer family) - { - requireNonNull(row, "row is null"); - requireNonNull(family, "family is null"); - checkArgument(row.arrayOffset() == 0, "Unexpected row buffer offset: %s", row.arrayOffset()); - checkArgument(family.arrayOffset() == 0, "Unexpected family buffer offset: %s", family.arrayOffset()); - this.row = row; - this.family = family; - this.visibility = EMPTY_VISIBILITY; - } - - public MetricsKey(ByteBuffer row, ByteBuffer family, ColumnVisibility visibility) - { - requireNonNull(row, "row is null"); - requireNonNull(family, "family is null"); - requireNonNull(visibility, "visibility is null"); - checkArgument(row.arrayOffset() == 0, "Unexpected row buffer offset: %s", row.arrayOffset()); - checkArgument(family.arrayOffset() == 0, "Unexpected family buffer offset: %s", family.arrayOffset()); - this.row = row; - this.family = family; - this.visibility = visibility.getExpression() != null ? visibility : EMPTY_VISIBILITY; - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - - MetricsKey other = (MetricsKey) obj; - return Objects.equals(this.row, other.row) - && Objects.equals(this.family, other.family) - && Objects.equals(this.visibility, other.visibility); - } - - @Override - public int hashCode() - { - return Objects.hash(row, family, visibility); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("row", new String(row.array(), UTF_8)) - .add("rowOffset", row.arrayOffset()) - .add("family", new String(family.array(), UTF_8)) - .add("familyOffset", family.arrayOffset()) - .add("visibility", visibility.toString()) - .toString(); - } - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSink.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSink.java deleted file mode 100644 index 9f0ffbfb204d1..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSink.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.io; - -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.plugin.accumulo.Types; -import io.trino.plugin.accumulo.index.Indexer; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.model.Field; -import io.trino.plugin.accumulo.model.Row; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.Page; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeUtils; -import io.trino.spi.type.VarcharType; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.MutationsRejectedException; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.hadoop.io.Text; - -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; - -import static io.airlift.concurrent.MoreFutures.getFutureValue; -import static io.trino.plugin.accumulo.AccumuloErrorCode.ACCUMULO_TABLE_DNE; -import static io.trino.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static io.trino.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; -import static io.trino.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.CompletableFuture.completedFuture; - -/** - * Output class for serializing Trino pages (blocks of rows of data) to Accumulo. - * This class converts the rows from within a page to a collection of Accumulo Mutations, - * writing and indexed the rows. Writers are flushed and closed on commit, and if a rollback occurs... - * we'll you're gonna have a bad time. - * - * @see AccumuloPageSinkProvider - */ -public class AccumuloPageSink - implements ConnectorPageSink -{ - public static final Text ROW_ID_COLUMN = new Text("___ROW___"); - private final AccumuloRowSerializer serializer; - private final BatchWriter writer; - private final Optional indexer; - private final List columns; - private final int rowIdOrdinal; - private long numRows; - - public AccumuloPageSink( - AccumuloClient client, - AccumuloTable table, - String username) - { - requireNonNull(table, "table is null"); - - this.columns = table.getColumns(); - - // Fetch the row ID ordinal, throwing an exception if not found for safety - this.rowIdOrdinal = columns.stream() - .filter(columnHandle -> columnHandle.name().equals(table.getRowId())) - .map(AccumuloColumnHandle::ordinal) - .findAny() - .orElseThrow(() -> new TrinoException(FUNCTION_IMPLEMENTATION_ERROR, "Row ID ordinal not found")); - this.serializer = table.getSerializerInstance(); - - try { - // Create a BatchWriter to the Accumulo table - BatchWriterConfig conf = new BatchWriterConfig(); - writer = client.createBatchWriter(table.getFullTableName(), conf); - - // If the table is indexed, create an instance of an Indexer, else empty - if (table.isIndexed()) { - indexer = Optional.of( - new Indexer( - client, - client.securityOperations().getUserAuthorizations(username), - table, - conf)); - } - else { - indexer = Optional.empty(); - } - } - catch (AccumuloException | AccumuloSecurityException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Accumulo error when creating BatchWriter and/or Indexer", e); - } - catch (TableNotFoundException e) { - throw new TrinoException(ACCUMULO_TABLE_DNE, "Accumulo error when creating BatchWriter and/or Indexer, table does not exist", e); - } - } - - /** - * Converts a {@link Row} to an Accumulo mutation. - * - * @param row Row object - * @param rowIdOrdinal Ordinal in the list of columns that is the row ID. This isn't checked at all, so I hope you're right. Also, it is expected that the list of column handles is sorted in ordinal order. This is a very demanding function. - * @param columns All column handles for the Row, sorted by ordinal. - * @param serializer Instance of {@link AccumuloRowSerializer} used to encode the values of the row to the Mutation - * @return Mutation - */ - public static Mutation toMutation(Row row, int rowIdOrdinal, List columns, AccumuloRowSerializer serializer) - { - // Set our value to the row ID - Text value = new Text(); - Field rowField = row.getField(rowIdOrdinal); - if (rowField.isNull()) { - throw new TrinoException(INVALID_FUNCTION_ARGUMENT, "Column mapped as the Accumulo row ID cannot be null"); - } - - setText(rowField, value, serializer); - - // Iterate through all the column handles, setting the Mutation's columns - Mutation mutation = new Mutation(value); - - // Store row ID in a special column - mutation.put(ROW_ID_COLUMN, ROW_ID_COLUMN, new Value(value.copyBytes())); - for (AccumuloColumnHandle columnHandle : columns) { - // Skip the row ID ordinal - if (columnHandle.ordinal() == rowIdOrdinal) { - continue; - } - - // If the value of the field is not null - if (!row.getField(columnHandle.ordinal()).isNull()) { - // Serialize the value to the text - setText(row.getField(columnHandle.ordinal()), value, serializer); - - // And add the bytes to the Mutation - mutation.put(columnHandle.family().get(), columnHandle.qualifier().get(), new Value(value.copyBytes())); - } - } - - return mutation; - } - - private static void setText(Field field, Text value, AccumuloRowSerializer serializer) - { - Type type = field.getType(); - if (Types.isArrayType(type)) { - serializer.setArray(value, type, field.getArray()); - } - else if (Types.isMapType(type)) { - serializer.setMap(value, type, field.getMap()); - } - else { - if (type.equals(BIGINT)) { - serializer.setLong(value, field.getLong()); - } - else if (type.equals(BOOLEAN)) { - serializer.setBoolean(value, field.getBoolean()); - } - else if (type.equals(DATE)) { - serializer.setDate(value, field.getDate()); - } - else if (type.equals(DOUBLE)) { - serializer.setDouble(value, field.getDouble()); - } - else if (type.equals(INTEGER)) { - serializer.setInt(value, field.getInt()); - } - else if (type.equals(REAL)) { - serializer.setFloat(value, field.getFloat()); - } - else if (type.equals(SMALLINT)) { - serializer.setShort(value, field.getShort()); - } - else if (type.equals(TIME_MILLIS)) { - serializer.setTime(value, field.getTime()); - } - else if (type.equals(TINYINT)) { - serializer.setByte(value, field.getByte()); - } - else if (type.equals(TIMESTAMP_MILLIS)) { - serializer.setTimestamp(value, field.getTimestamp()); - } - else if (type.equals(VARBINARY)) { - serializer.setVarbinary(value, field.getVarbinary()); - } - else if (type instanceof VarcharType) { - serializer.setVarchar(value, field.getVarchar()); - } - else { - throw new UnsupportedOperationException("Unsupported type " + type); - } - } - } - - @Override - public CompletableFuture appendPage(Page page) - { - // For each position within the page, i.e. row - for (int position = 0; position < page.getPositionCount(); ++position) { - Row row = new Row(); - // For each channel within the page, i.e. column - for (int channel = 0; channel < page.getChannelCount(); ++channel) { - // Get the type for this channel - Type type = columns.get(channel).type(); - - // Read the value from the page and append the field to the row - row.addField(TypeUtils.readNativeValue(type, page.getBlock(channel), position), type); - } - - try { - // Convert row to a Mutation, writing and indexing it - Mutation mutation = toMutation(row, rowIdOrdinal, columns, serializer); - writer.addMutation(mutation); - if (indexer.isPresent()) { - indexer.get().index(mutation); - } - ++numRows; - } - catch (MutationsRejectedException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server", e); - } - - // TODO Fix arbitrary flush every 100k rows - if (numRows % 100_000 == 0) { - flush(); - } - } - - return NOT_BLOCKED; - } - - @Override - public CompletableFuture> finish() - { - try { - // Done serializing rows, so flush and close the writer and indexer - writer.flush(); - writer.close(); - if (indexer.isPresent()) { - indexer.get().close(); - } - } - catch (MutationsRejectedException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server on flush", e); - } - - // TODO Look into any use of the metadata for writing out the rows - return completedFuture(ImmutableList.of()); - } - - @Override - public void abort() - { - getFutureValue(finish()); - } - - private void flush() - { - try { - if (indexer.isPresent()) { - indexer.get().flush(); - // MetricsWriter is non-null if Indexer is present - } - writer.flush(); - } - catch (MutationsRejectedException e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server on flush", e); - } - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSinkProvider.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSinkProvider.java deleted file mode 100644 index b40d0d7982a12..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloPageSinkProvider.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.io; - -import com.google.inject.Inject; -import io.trino.plugin.accumulo.AccumuloMetadataManager; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.accumulo.model.AccumuloTableHandle; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorPageSink; -import io.trino.spi.connector.ConnectorPageSinkId; -import io.trino.spi.connector.ConnectorPageSinkProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTransactionHandle; -import org.apache.accumulo.core.client.AccumuloClient; - -import static java.util.Objects.requireNonNull; - -/** - * Page sink provider for Accumulo connector. Creates {@link AccumuloPageSink} objects for output tables (CTAS) and inserts. - * - * @see AccumuloPageSink - */ -public class AccumuloPageSinkProvider - implements ConnectorPageSinkProvider -{ - private final AccumuloMetadataManager metadataManager; - private final AccumuloClient client; - private final String username; - - @Inject - public AccumuloPageSinkProvider( - AccumuloClient client, - AccumuloConfig config, - AccumuloMetadataManager metadataManager) - { - this.metadataManager = requireNonNull(metadataManager, "metadataManager is null"); - this.client = requireNonNull(client, "client is null"); - this.username = config.getUsername(); - } - - @Override - public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle outputTableHandle, ConnectorPageSinkId pageSinkId) - { - AccumuloTableHandle tableHandle = (AccumuloTableHandle) outputTableHandle; - return new AccumuloPageSink(client, metadataManager.getTable(tableHandle.toSchemaTableName()), username); - } - - @Override - public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle insertTableHandle, ConnectorPageSinkId pageSinkId) - { - return createPageSink(transactionHandle, session, (ConnectorOutputTableHandle) insertTableHandle, pageSinkId); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordCursor.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordCursor.java deleted file mode 100644 index 707922f858829..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordCursor.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.io; - -import io.airlift.slice.Slice; -import io.airlift.slice.Slices; -import io.trino.plugin.accumulo.Types; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarbinaryType; -import io.trino.spi.type.VarcharType; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.FirstEntryInRowIterator; -import org.apache.accumulo.core.iterators.user.WholeRowIterator; -import org.apache.hadoop.io.Text; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.plugin.accumulo.AccumuloErrorCode.IO_ERROR; -import static io.trino.plugin.accumulo.io.AccumuloPageSink.ROW_ID_COLUMN; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static io.trino.spi.type.TinyintType.TINYINT; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -/** - * Implementation of Trino RecordCursor, responsible for iterating over a Trino split, - * reading rows of data and then implementing various methods to retrieve columns within each row. - * - * @see AccumuloRecordSet - * @see AccumuloRecordSetProvider - */ -public class AccumuloRecordCursor - implements RecordCursor -{ - private static final int WHOLE_ROW_ITERATOR_PRIORITY = Integer.MAX_VALUE; - - private final List columnHandles; - private final String[] fieldToColumnName; - private final BatchScanner scanner; - private final Iterator> iterator; - private final AccumuloRowSerializer serializer; - - private long bytesRead; - - public AccumuloRecordCursor( - AccumuloRowSerializer serializer, - BatchScanner scanner, - String rowIdName, - List columnHandles) - { - this.columnHandles = requireNonNull(columnHandles, "columnHandles is null"); - this.scanner = requireNonNull(scanner, "scanner is null"); - this.serializer = requireNonNull(serializer, "serializer is null"); - this.serializer.setRowIdName(requireNonNull(rowIdName, "rowIdName is null")); - - requireNonNull(columnHandles, "columnHandles is null"); - - if (retrieveOnlyRowIds(rowIdName)) { - this.scanner.addScanIterator(new IteratorSetting(1, "firstentryiter", FirstEntryInRowIterator.class)); - - fieldToColumnName = new String[1]; - fieldToColumnName[0] = rowIdName; - - // Set a flag on the serializer saying we are only going to be retrieving the row ID - this.serializer.setRowOnly(true); - } - else { - // Else, we will be scanning some more columns here - this.serializer.setRowOnly(false); - - // Fetch the reserved row ID column - this.scanner.fetchColumn(ROW_ID_COLUMN, ROW_ID_COLUMN); - - Text family = new Text(); - Text qualifier = new Text(); - - // Create an array which maps the column ordinal to the name of the column - fieldToColumnName = new String[columnHandles.size()]; - for (int i = 0; i < columnHandles.size(); ++i) { - AccumuloColumnHandle columnHandle = columnHandles.get(i); - fieldToColumnName[i] = columnHandle.name(); - - // Make sure to skip the row ID! - if (!columnHandle.name().equals(rowIdName)) { - // Set the mapping of Trino column name to the family/qualifier - this.serializer.setMapping(columnHandle.name(), columnHandle.family().get(), columnHandle.qualifier().get()); - - // Set our scanner to fetch this family/qualifier column - // This will help us prune which data we receive from Accumulo - family.set(columnHandle.family().get()); - qualifier.set(columnHandle.qualifier().get()); - this.scanner.fetchColumn(family, qualifier); - } - } - } - - IteratorSetting setting = new IteratorSetting(WHOLE_ROW_ITERATOR_PRIORITY, WholeRowIterator.class); - scanner.addScanIterator(setting); - - iterator = this.scanner.iterator(); - } - - @Override - public long getCompletedBytes() - { - return bytesRead; - } - - @Override - public long getReadTimeNanos() - { - return 0; - } - - @Override - public Type getType(int field) - { - checkArgument(field >= 0 && field < columnHandles.size(), "Invalid field index"); - return columnHandles.get(field).type(); - } - - @Override - public boolean advanceNextPosition() - { - try { - if (iterator.hasNext()) { - serializer.reset(); - Entry row = iterator.next(); - for (Entry entry : WholeRowIterator.decodeRow(row.getKey(), row.getValue()).entrySet()) { - bytesRead += entry.getKey().getSize() + entry.getValue().getSize(); - serializer.deserialize(entry); - } - return true; - } - return false; - } - catch (IOException e) { - throw new TrinoException(IO_ERROR, "Caught IO error from serializer on read", e); - } - } - - @Override - public boolean isNull(int field) - { - checkArgument(field < columnHandles.size(), "Invalid field index"); - return serializer.isNull(fieldToColumnName[field]); - } - - @Override - public boolean getBoolean(int field) - { - checkFieldType(field, BOOLEAN); - return serializer.getBoolean(fieldToColumnName[field]); - } - - @Override - public double getDouble(int field) - { - checkFieldType(field, DOUBLE); - return serializer.getDouble(fieldToColumnName[field]); - } - - @Override - public long getLong(int field) - { - checkFieldType(field, BIGINT, DATE, INTEGER, REAL, SMALLINT, TIME_MILLIS, TIMESTAMP_MILLIS, TINYINT); - Type type = getType(field); - if (type.equals(BIGINT)) { - return serializer.getLong(fieldToColumnName[field]); - } - if (type.equals(DATE)) { - return serializer.getDate(fieldToColumnName[field]); - } - if (type.equals(INTEGER)) { - return serializer.getInt(fieldToColumnName[field]); - } - if (type.equals(REAL)) { - return Float.floatToIntBits(serializer.getFloat(fieldToColumnName[field])); - } - if (type.equals(SMALLINT)) { - return serializer.getShort(fieldToColumnName[field]); - } - if (type.equals(TIME_MILLIS)) { - return serializer.getTime(fieldToColumnName[field]).getTime(); - } - if (type.equals(TIMESTAMP_MILLIS)) { - return serializer.getTimestamp(fieldToColumnName[field]).getTime() * MICROSECONDS_PER_MILLISECOND; - } - if (type.equals(TINYINT)) { - return serializer.getByte(fieldToColumnName[field]); - } - throw new TrinoException(NOT_SUPPORTED, "Unsupported type " + getType(field)); - } - - @Override - public Object getObject(int field) - { - Type type = getType(field); - checkArgument(Types.isArrayType(type) || Types.isMapType(type), "Expected field %s to be a type of array or map but is %s", field, type); - - if (Types.isArrayType(type)) { - return serializer.getArray(fieldToColumnName[field], type); - } - - return serializer.getMap(fieldToColumnName[field], type); - } - - @Override - public Slice getSlice(int field) - { - Type type = getType(field); - if (type instanceof VarbinaryType) { - return Slices.wrappedBuffer(serializer.getVarbinary(fieldToColumnName[field])); - } - if (type instanceof VarcharType) { - return Slices.utf8Slice(serializer.getVarchar(fieldToColumnName[field])); - } - throw new TrinoException(NOT_SUPPORTED, "Unsupported type " + type); - } - - @Override - public void close() - { - scanner.close(); - } - - /** - * Gets a Boolean value indicating whether or not the scanner should only return row IDs. - *

- * This can occur in cases such as SELECT COUNT(*) or the table only has one column. - * Trino doesn't need the entire contents of the row to count them, - * so we can configure Accumulo to only give us the first key/value pair in the row - * - * @param rowIdName Row ID column name - * @return True if scanner should retriev eonly row IDs, false otherwise - */ - private boolean retrieveOnlyRowIds(String rowIdName) - { - return columnHandles.isEmpty() || (columnHandles.size() == 1 && columnHandles.get(0).name().equals(rowIdName)); - } - - /** - * Checks that the given field is one of the provided types. - * - * @param field Ordinal of the field - * @param expected An array of expected types - * @throws IllegalArgumentException If the given field does not match one of the types - */ - private void checkFieldType(int field, Type... expected) - { - Type actual = getType(field); - for (Type type : expected) { - if (actual.equals(type)) { - return; - } - } - - throw new IllegalArgumentException(format("Expected field %s to be a type of %s but is %s", field, Arrays.toString(expected), actual)); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSet.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSet.java deleted file mode 100644 index ddca17a53a12a..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSet.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.io; - -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import io.airlift.log.Logger; -import io.trino.plugin.accumulo.conf.AccumuloSessionProperties; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.model.AccumuloSplit; -import io.trino.plugin.accumulo.model.AccumuloTableHandle; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.RecordSet; -import io.trino.spi.type.Type; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.security.Authorizations; - -import java.util.List; -import java.util.Optional; - -import static io.trino.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -/** - * Implementation of a Trino RecordSet, responsible for returning the column types and the RecordCursor to the framework. - * - * @see AccumuloRecordCursor - * @see AccumuloRecordSetProvider - */ -public class AccumuloRecordSet - implements RecordSet -{ - private static final Logger LOG = Logger.get(AccumuloRecordSet.class); - private static final Splitter COMMA_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults(); - - private final List columnHandles; - private final List columnTypes; - private final AccumuloRowSerializer serializer; - private final BatchScanner scanner; - private final String rowIdName; - - public AccumuloRecordSet( - AccumuloClient client, - ConnectorSession session, - AccumuloSplit split, - String username, - AccumuloTableHandle table, - List columnHandles) - { - requireNonNull(session, "session is null"); - requireNonNull(split, "split is null"); - requireNonNull(username, "username is null"); - requireNonNull(table, "table is null"); - - rowIdName = table.getRowId(); - - serializer = table.getSerializerInstance(); - - // Save off the column handles and createa list of the Accumulo types - this.columnHandles = requireNonNull(columnHandles, "columnHandles is null"); - ImmutableList.Builder types = ImmutableList.builder(); - for (AccumuloColumnHandle column : columnHandles) { - types.add(column.type()); - } - this.columnTypes = types.build(); - - try { - // Create the BatchScanner and set the ranges from the split - scanner = client.createBatchScanner(table.getFullTableName(), getScanAuthorizations(session, table, client, username), 10); - scanner.setRanges(split.getRanges()); - } - catch (Exception e) { - throw new TrinoException(UNEXPECTED_ACCUMULO_ERROR, format("Failed to create batch scanner for table %s", table.getFullTableName()), e); - } - } - - /** - * Gets the scanner authorizations to use for scanning tables. - *

- * In order of priority: session username authorizations, then table property, then the default client auths. - * - * @param session Current session - * @param table Accumulo table - * @param client Accumulo client - * @param username Accumulo username - * @return Scan authorizations - * @throws AccumuloException If a generic Accumulo error occurs - * @throws AccumuloSecurityException If a security exception occurs - */ - private static Authorizations getScanAuthorizations(ConnectorSession session, AccumuloTableHandle table, AccumuloClient client, String username) - throws AccumuloException, AccumuloSecurityException - { - String sessionScanUser = AccumuloSessionProperties.getScanUsername(session); - if (sessionScanUser != null) { - Authorizations scanAuths = client.securityOperations().getUserAuthorizations(sessionScanUser); - LOG.debug("Using session scanner auths for user %s: %s", sessionScanUser, scanAuths); - return scanAuths; - } - - Optional scanAuths = table.getScanAuthorizations(); - if (scanAuths.isPresent()) { - Authorizations auths = new Authorizations(Iterables.toArray(COMMA_SPLITTER.split(scanAuths.get()), String.class)); - LOG.debug("scan_auths table property set: %s", auths); - return auths; - } - Authorizations auths = client.securityOperations().getUserAuthorizations(username); - LOG.debug("scan_auths table property not set, using user auths: %s", auths); - return auths; - } - - @Override - public List getColumnTypes() - { - return columnTypes; - } - - @Override - public RecordCursor cursor() - { - return new AccumuloRecordCursor(serializer, scanner, rowIdName, columnHandles); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSetProvider.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSetProvider.java deleted file mode 100644 index 56f896230d2c3..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/io/AccumuloRecordSetProvider.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.io; - -import com.google.inject.Inject; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.model.AccumuloSplit; -import io.trino.plugin.accumulo.model.AccumuloTableHandle; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorRecordSetProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.RecordSet; -import org.apache.accumulo.core.client.AccumuloClient; - -import java.util.List; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Objects.requireNonNull; - -/** - * Implementation of a ConnectorRecordSetProvider for Accumulo. Generates {@link AccumuloRecordSet} objects for a provided split. - * - * @see AccumuloRecordSet - * @see AccumuloRecordCursor - */ -public class AccumuloRecordSetProvider - implements ConnectorRecordSetProvider -{ - private final AccumuloClient client; - private final String username; - - @Inject - public AccumuloRecordSetProvider( - AccumuloClient client, - AccumuloConfig config) - { - this.client = requireNonNull(client, "client is null"); - this.username = config.getUsername(); - } - - @Override - public RecordSet getRecordSet(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List columns) - { - AccumuloSplit accSplit = (AccumuloSplit) split; - AccumuloTableHandle accTable = (AccumuloTableHandle) table; - - List accColumns = columns.stream() - .map(AccumuloColumnHandle.class::cast) - .collect(toImmutableList()); - - return new AccumuloRecordSet(client, session, accSplit, username, accTable, accColumns); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloTable.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloTable.java deleted file mode 100644 index 143e6a05da3aa..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloTable.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.metadata; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import io.trino.plugin.accumulo.index.Indexer; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.SchemaTableName; - -import java.lang.reflect.InvocationTargetException; -import java.util.List; -import java.util.Optional; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.spi.StandardErrorCode.NOT_FOUND; -import static java.util.Objects.requireNonNull; - -/** - * This class encapsulates metadata regarding an Accumulo table in Trino. - */ -public class AccumuloTable -{ - private final boolean external; - private final Integer rowIdOrdinal; - private final String schema; - private final String serializerClassName; - private final Optional scanAuthorizations; - private final List columnsMetadata; - private final boolean indexed; - private final List columns; - private final String rowId; - private final String table; - private final SchemaTableName schemaTableName; - - @JsonCreator - public AccumuloTable( - @JsonProperty("schema") String schema, - @JsonProperty("table") String table, - @JsonProperty("columns") List columns, - @JsonProperty("rowId") String rowId, - @JsonProperty("external") boolean external, - @JsonProperty("serializerClassName") String serializerClassName, - @JsonProperty("scanAuthorizations") Optional scanAuthorizations) - { - this.external = external; - this.rowId = requireNonNull(rowId, "rowId is null"); - this.schema = requireNonNull(schema, "schema is null"); - this.table = requireNonNull(table, "table is null"); - this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns are null")); - this.serializerClassName = requireNonNull(serializerClassName, "serializerClassName is null"); - this.scanAuthorizations = scanAuthorizations; - - boolean indexed = false; - Optional rowIdOrdinal = Optional.empty(); - - // Extract the ColumnMetadata from the handles for faster access - ImmutableList.Builder columnMetadataBuilder = ImmutableList.builder(); - for (AccumuloColumnHandle column : this.columns) { - columnMetadataBuilder.add(column.columnMetadata()); - indexed |= column.indexed(); - if (column.name().equals(this.rowId)) { - rowIdOrdinal = Optional.of(column.ordinal()); - } - } - - if (rowIdOrdinal.isPresent()) { - this.rowIdOrdinal = rowIdOrdinal.get(); - } - else { - throw new IllegalArgumentException("rowIdOrdinal is null, enable to locate rowId in given column list"); - } - - this.indexed = indexed; - this.columnsMetadata = columnMetadataBuilder.build(); - this.schemaTableName = new SchemaTableName(this.schema, this.table); - } - - @JsonProperty - public String getRowId() - { - return rowId; - } - - @JsonProperty - public String getSchema() - { - return schema; - } - - @JsonProperty - public String getTable() - { - return table; - } - - @JsonIgnore - public String getIndexTableName() - { - return Indexer.getIndexTableName(schema, table); - } - - @JsonIgnore - public String getMetricsTableName() - { - return Indexer.getMetricsTableName(schema, table); - } - - @JsonIgnore - public String getFullTableName() - { - return getFullTableName(schema, table); - } - - @JsonProperty - public List getColumns() - { - return columns; - } - - @JsonProperty - public Optional getScanAuthorizations() - { - return scanAuthorizations; - } - - @JsonProperty - public String getSerializerClassName() - { - return serializerClassName; - } - - @JsonIgnore - public List getColumnsMetadata() - { - return columnsMetadata; - } - - @JsonProperty - public boolean isExternal() - { - return external; - } - - @JsonIgnore - public boolean isIndexed() - { - return indexed; - } - - @JsonIgnore - public int getRowIdOrdinal() - { - return this.rowIdOrdinal; - } - - @JsonIgnore - public AccumuloRowSerializer getSerializerInstance() - { - try { - return (AccumuloRowSerializer) Class.forName(serializerClassName).getConstructor().newInstance(); - } - catch (ClassNotFoundException | InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - throw new TrinoException(NOT_FOUND, "Configured serializer class not found", e); - } - } - - @JsonIgnore - public static String getFullTableName(String schema, String table) - { - return schema.equals("default") ? table : schema + '.' + table; - } - - @JsonIgnore - public static String getFullTableName(SchemaTableName tableName) - { - return getFullTableName(tableName.getSchemaName(), tableName.getTableName()); - } - - @JsonIgnore - public SchemaTableName getSchemaTableName() - { - return schemaTableName; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("schemaName", schema) - .add("tableName", table) - .add("columns", columns) - .add("rowIdName", rowId) - .add("external", external) - .add("serializerClassName", serializerClassName) - .add("scanAuthorizations", scanAuthorizations) - .toString(); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloView.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloView.java deleted file mode 100644 index 0355b3b0a049b..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/AccumuloView.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.metadata; - -import io.trino.spi.connector.SchemaTableName; - -import static java.util.Objects.requireNonNull; - -/** - * This class encapsulates metadata regarding an Accumulo view in Trino. - */ -public record AccumuloView(String schema, String table, String data) -{ - public AccumuloView - { - requireNonNull(schema, "schema is null"); - requireNonNull(table, "table is null"); - requireNonNull(data, "data is null"); - } - - public SchemaTableName schemaTableName() - { - return new SchemaTableName(schema, table); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/ZooKeeperMetadataManager.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/ZooKeeperMetadataManager.java deleted file mode 100644 index 6c8550c5217b7..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/metadata/ZooKeeperMetadataManager.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.metadata; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import io.airlift.json.ObjectMapperProvider; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.base.TypeDeserializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import jakarta.annotation.PreDestroy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryForever; -import org.apache.zookeeper.KeeperException; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.trino.plugin.accumulo.AccumuloErrorCode.ZOOKEEPER_ERROR; -import static io.trino.plugin.base.util.JsonUtils.parseJson; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static org.apache.zookeeper.KeeperException.Code.NONODE; - -public class ZooKeeperMetadataManager -{ - public static final String DEFAULT_SCHEMA = "default"; - - private final CuratorFramework curator; - private final ObjectMapper mapper; - - @Inject - public ZooKeeperMetadataManager(AccumuloConfig config, TypeManager typeManager) - { - requireNonNull(typeManager, "typeManager is null"); - - // Create JSON deserializer for the AccumuloTable - ObjectMapperProvider objectMapperProvider = new ObjectMapperProvider(); - objectMapperProvider.setJsonDeserializers(ImmutableMap.of(Type.class, new TypeDeserializer(typeManager))); - mapper = objectMapperProvider.get(); - - String zkMetadataRoot = config.getZkMetadataRoot(); - String zookeepers = config.getZooKeepers(); - - // Create the connection to ZooKeeper to check if the metadata root exists - try (CuratorFramework checkRoot = CuratorFrameworkFactory.newClient(zookeepers, new RetryForever(1000))) { - checkRoot.start(); - // If the metadata root does not exist, create it - if (checkRoot.checkExists().forPath(zkMetadataRoot) == null) { - checkRoot.create().forPath(zkMetadataRoot); - } - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error checking metadata root", e); - } - - // Create the curator client framework to use for metadata management, set at the ZK root - curator = CuratorFrameworkFactory.newClient(zookeepers + zkMetadataRoot, new RetryForever(1000)); - curator.start(); - - try { - // Create default schema should it not exist - if (curator.checkExists().forPath("/" + DEFAULT_SCHEMA) == null) { - curator.create().forPath("/" + DEFAULT_SCHEMA); - } - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error checking/creating default schema", e); - } - } - - @PreDestroy - public void close() - { - curator.close(); - } - - public void createSchema(String schemaName) - { - try { - curator.create().forPath("/" + schemaName); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error creating schema: " + schemaName, e); - } - } - - public void dropSchema(String schemaName) - { - try { - curator.delete().forPath("/" + schemaName); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error deleting schema: " + schemaName, e); - } - } - - public Set getSchemaNames() - { - try { - Set schemas = new HashSet<>(); - schemas.addAll(curator.getChildren().forPath("/")); - return schemas; - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "Error fetching schemas", e); - } - } - - public Set getTableNames(String schema) - { - String schemaPath = getSchemaPath(schema); - try { - if (curator.checkExists().forPath(schemaPath) == null) { - return ImmutableSet.of(); - } - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "Error checking if schema exists", e); - } - - try { - return curator.getChildren().forPath(schemaPath).stream() - .filter(x -> isAccumuloTable(new SchemaTableName(schema, x))) - .collect(toImmutableSet()); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "Error fetching schemas", e); - } - } - - public AccumuloTable getTable(SchemaTableName stName) - { - try { - if (curator.checkExists().forPath(getTablePath(stName)) != null) { - return toAccumuloTable(curator.getData().forPath(getTablePath(stName))); - } - - return null; - } - catch (Exception e) { - // Capture race condition between checkExists and getData - if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) { - return null; - } - - throw new TrinoException(ZOOKEEPER_ERROR, "Error fetching table", e); - } - } - - public Set getViewNames(String schema) - { - String schemaPath = getSchemaPath(schema); - try { - if (curator.checkExists().forPath(schemaPath) == null) { - return ImmutableSet.of(); - } - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "Error checking if schema exists", e); - } - - try { - return curator.getChildren().forPath(schemaPath).stream() - .filter(x -> isAccumuloView(new SchemaTableName(schema, x))) - .collect(toImmutableSet()); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "Error fetching schemas", e); - } - } - - public AccumuloView getView(SchemaTableName stName) - { - try { - String tablePath = getTablePath(stName); - if (curator.checkExists().forPath(tablePath) != null) { - byte[] data = curator.getData().forPath(tablePath); - if (isAccumuloView(data)) { - return toAccumuloView(data); - } - } - - return null; - } - catch (Exception e) { - // Capture race condition between checkExists and getData - if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) { - return null; - } - - throw new TrinoException(ZOOKEEPER_ERROR, "Error fetching view", e); - } - } - - public void createTableMetadata(AccumuloTable table) - { - SchemaTableName tableName = table.getSchemaTableName(); - String tablePath = getTablePath(tableName); - try { - if (curator.checkExists().forPath(tablePath) != null) { - throw new IOException(format("Metadata for table %s already exists", tableName)); - } - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error when checking if table already exists", e); - } - - try { - curator.create().creatingParentsIfNeeded().forPath(tablePath, toJsonBytes(table)); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "Error creating table znode in ZooKeeper", e); - } - } - - public void deleteTableMetadata(SchemaTableName tableName) - { - try { - curator.delete().deletingChildrenIfNeeded().forPath(getTablePath(tableName)); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error when deleting table metadata", e); - } - } - - public void createViewMetadata(AccumuloView view) - { - SchemaTableName tableName = view.schemaTableName(); - String viewPath = getTablePath(tableName); - try { - if (curator.checkExists().forPath(viewPath) != null) { - throw new IOException(format("Metadata for view %s already exists", tableName)); - } - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error when checking if view already exists", e); - } - - try { - curator.create().creatingParentsIfNeeded().forPath(viewPath, toJsonBytes(view)); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "Error creating view znode in ZooKeeper", e); - } - } - - public void deleteViewMetadata(SchemaTableName tableName) - { - try { - curator.delete().deletingChildrenIfNeeded().forPath(getTablePath(tableName)); - } - catch (Exception e) { - throw new TrinoException(ZOOKEEPER_ERROR, "ZK error when deleting view metadata", e); - } - } - - private static String getSchemaPath(String schema) - { - return "/" + schema.toLowerCase(Locale.ENGLISH); - } - - private static String getSchemaPath(SchemaTableName tableName) - { - return getSchemaPath(tableName.getSchemaName()); - } - - private static String getTablePath(SchemaTableName tableName) - { - return getSchemaPath(tableName) + '/' + tableName.getTableName().toLowerCase(Locale.ENGLISH); - } - - private boolean isAccumuloTable(SchemaTableName tableName) - { - try { - String path = getTablePath(tableName); - return curator.checkExists().forPath(path) != null && isAccumuloTable(curator.getData().forPath(path)); - } - catch (Exception e) { - // Capture race condition between checkExists and getData - if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) { - return false; - } - - throw new TrinoException(ZOOKEEPER_ERROR, "Error checking if path is an AccumuloTable object", e); - } - } - - private boolean isAccumuloView(SchemaTableName tableName) - { - try { - String path = getTablePath(tableName); - return curator.checkExists().forPath(path) != null && isAccumuloView(curator.getData().forPath(path)); - } - catch (Exception e) { - // Capture race condition between checkExists and getData - if (e instanceof KeeperException && ((KeeperException) e).code() == NONODE) { - return false; - } - - throw new TrinoException(ZOOKEEPER_ERROR, "Error checking if path is an AccumuloView object", e); - } - } - - private boolean isAccumuloTable(byte[] data) - { - // AccumuloTable does not contain a 'data' node - return !parseJson(mapper, data, JsonNode.class).has("data"); - } - - private boolean isAccumuloView(byte[] data) - { - // AccumuloView contains a 'data' node - return parseJson(mapper, data, JsonNode.class).has("data"); - } - - private AccumuloTable toAccumuloTable(byte[] data) - { - return parseJson(mapper, data, AccumuloTable.class); - } - - private AccumuloView toAccumuloView(byte[] data) - throws IOException - { - return parseJson(mapper, data, AccumuloView.class); - } - - private byte[] toJsonBytes(Object obj) - throws IOException - { - return mapper.writeValueAsBytes(obj); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnConstraint.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnConstraint.java deleted file mode 100644 index a68bfd68e9b15..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnConstraint.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import io.trino.spi.predicate.Domain; - -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public record AccumuloColumnConstraint( - String name, - String family, - String qualifier, - Optional domain, - boolean indexed) -{ - public AccumuloColumnConstraint - { - requireNonNull(name, "name is null"); - requireNonNull(family, "family is null"); - requireNonNull(qualifier, "qualifier is null"); - requireNonNull(domain, "domain is null"); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnHandle.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnHandle.java deleted file mode 100644 index 81b2243cc4eb3..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloColumnHandle.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.Type; - -import java.util.Optional; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public record AccumuloColumnHandle( - String name, - Optional family, - Optional qualifier, - Type type, - int ordinal, - String extraInfo, - Optional comment, - boolean indexed) - implements ColumnHandle, Comparable -{ - public AccumuloColumnHandle - { - requireNonNull(name, "name is null"); - requireNonNull(family, "family is null"); - requireNonNull(qualifier, "qualifier is null"); - requireNonNull(type, "type is null"); - checkArgument(ordinal >= 0, "ordinal must be >= zero"); - requireNonNull(extraInfo, "extraInfo is null"); - requireNonNull(comment, "comment is null"); - } - - public ColumnMetadata columnMetadata() - { - return ColumnMetadata.builder() - .setName(name) - .setType(type) - .setExtraInfo(Optional.ofNullable(extraInfo)) - .setComment(comment) - .build(); - } - - @Override - public int compareTo(AccumuloColumnHandle obj) - { - return Integer.compare(this.ordinal(), obj.ordinal()); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloSplit.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloSplit.java deleted file mode 100644 index 85d019b99d666..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloSplit.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.slice.SizeOf; -import io.trino.spi.HostAddress; -import io.trino.spi.connector.ConnectorSplit; -import org.apache.accumulo.core.data.Range; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.airlift.slice.SizeOf.estimatedSizeOf; -import static io.airlift.slice.SizeOf.instanceSize; -import static io.airlift.slice.SizeOf.sizeOf; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.joining; - -public class AccumuloSplit - implements ConnectorSplit -{ - private static final int INSTANCE_SIZE = instanceSize(AccumuloSplit.class); - - private final Optional hostPort; - private final List addresses; - private final List ranges; - - @JsonCreator - public AccumuloSplit( - @JsonProperty("ranges") List ranges, - @JsonProperty("hostPort") Optional hostPort) - { - this.hostPort = requireNonNull(hostPort, "hostPort is null"); - this.ranges = ImmutableList.copyOf(requireNonNull(ranges, "ranges is null")); - - // Parse the host address into a list of addresses, this would be an Accumulo Tablet server or some localhost thing - if (hostPort.isPresent()) { - addresses = ImmutableList.of(HostAddress.fromString(hostPort.get())); - } - else { - addresses = ImmutableList.of(); - } - } - - @JsonProperty - public Optional getHostPort() - { - return hostPort; - } - - @JsonProperty("ranges") - public List getSerializedRanges() - { - return ranges; - } - - @JsonIgnore - public List getRanges() - { - return ranges.stream().map(SerializedRange::deserialize).collect(Collectors.toList()); - } - - @Override - public List getAddresses() - { - return addresses; - } - - @Override - public Map getSplitInfo() - { - return ImmutableMap.of("addresses", addresses.stream().map(HostAddress::toString).collect(joining(",")), "numRanges", String.valueOf(ranges.size())); - } - - @Override - public long getRetainedSizeInBytes() - { - return INSTANCE_SIZE - + sizeOf(hostPort, SizeOf::estimatedSizeOf) - + estimatedSizeOf(addresses, HostAddress::getRetainedSizeInBytes) - + estimatedSizeOf(ranges, SerializedRange::getRetainedSizeInBytes); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("addresses", addresses) - .add("numRanges", ranges.size()) - .add("hostPort", hostPort) - .toString(); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloTableHandle.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloTableHandle.java deleted file mode 100644 index c73b063c2084f..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/AccumuloTableHandle.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorOutputTableHandle; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.TupleDomain; - -import java.util.Objects; -import java.util.Optional; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.trino.spi.StandardErrorCode.NOT_FOUND; -import static java.util.Objects.requireNonNull; - -public final class AccumuloTableHandle - implements ConnectorInsertTableHandle, ConnectorOutputTableHandle, ConnectorTableHandle -{ - private final boolean external; - private final String rowId; - private final Optional scanAuthorizations; - private final String schema; - private final String serializerClassName; - private final String table; - private final TupleDomain constraint; - - public AccumuloTableHandle( - String schema, - String table, - String rowId, - boolean external, - String serializerClassName, - Optional scanAuthorizations) - { - this(schema, table, rowId, TupleDomain.all(), external, serializerClassName, scanAuthorizations); - } - - @JsonCreator - public AccumuloTableHandle( - @JsonProperty("schema") String schema, - @JsonProperty("table") String table, - @JsonProperty("rowId") String rowId, - @JsonProperty("constraint") TupleDomain constraint, - @JsonProperty("external") boolean external, - @JsonProperty("serializerClassName") String serializerClassName, - @JsonProperty("scanAuthorizations") Optional scanAuthorizations) - { - this.external = external; - this.rowId = requireNonNull(rowId, "rowId is null"); - this.scanAuthorizations = scanAuthorizations; - this.schema = requireNonNull(schema, "schema is null"); - this.serializerClassName = requireNonNull(serializerClassName, "serializerClassName is null"); - this.table = requireNonNull(table, "table is null"); - this.constraint = requireNonNull(constraint, "constraints is null"); - } - - @JsonProperty - public String getRowId() - { - return rowId; - } - - @JsonProperty - public Optional getScanAuthorizations() - { - return scanAuthorizations; - } - - @JsonProperty - public String getSchema() - { - return schema; - } - - @JsonProperty - public String getSerializerClassName() - { - return serializerClassName; - } - - @JsonIgnore - public AccumuloRowSerializer getSerializerInstance() - { - try { - return (AccumuloRowSerializer) Class.forName(serializerClassName).getConstructor().newInstance(); - } - catch (Exception e) { - throw new TrinoException(NOT_FOUND, "Configured serializer class not found", e); - } - } - - @JsonProperty - public String getTable() - { - return table; - } - - @JsonProperty - public boolean isExternal() - { - return external; - } - - @JsonProperty - public TupleDomain getConstraint() - { - return constraint; - } - - public SchemaTableName toSchemaTableName() - { - return new SchemaTableName(schema, table); - } - - @JsonIgnore - public String getFullTableName() - { - return AccumuloTable.getFullTableName(schema, table); - } - - @Override - public int hashCode() - { - return Objects.hash(schema, table, rowId, external, serializerClassName); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - - AccumuloTableHandle other = (AccumuloTableHandle) obj; - return Objects.equals(this.schema, other.schema) - && Objects.equals(this.table, other.table) - && Objects.equals(this.rowId, other.rowId) - && this.external == other.external - && Objects.equals(this.serializerClassName, other.serializerClassName) - && Objects.equals(this.scanAuthorizations, other.scanAuthorizations); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("schema", schema) - .add("table", table) - .add("rowId", rowId) - .add("internal", external) - .add("serializerClassName", serializerClassName) - .add("scanAuthorizations", scanAuthorizations) - .toString(); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Field.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Field.java deleted file mode 100644 index 74d3ac2cbe01e..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Field.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.google.common.primitives.Primitives; -import com.google.common.primitives.Shorts; -import com.google.common.primitives.SignedBytes; -import io.airlift.slice.Slice; -import io.trino.plugin.accumulo.Types; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.sql.Time; -import java.sql.Timestamp; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Preconditions.checkArgument; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static java.lang.Float.intBitsToFloat; -import static java.lang.Math.floorDiv; -import static java.lang.Math.toIntExact; -import static java.util.Objects.requireNonNull; - -public class Field -{ - private final Object value; - private final Type type; - private final boolean indexed; - - public Field(Object nativeValue, Type type) - { - this(nativeValue, type, false); - } - - public Field(Object nativeValue, Type type, boolean indexed) - { - this.value = convert(nativeValue, type); - this.type = requireNonNull(type, "type is null"); - this.indexed = indexed; - } - - public Type getType() - { - return type; - } - - public Block getArray() - { - return (Block) value; - } - - public Long getLong() - { - return (Long) value; - } - - public Boolean getBoolean() - { - return (Boolean) value; - } - - public Byte getByte() - { - return (Byte) value; - } - - public long getDate() - { - return (Long) value; - } - - public Double getDouble() - { - return (Double) value; - } - - public Float getFloat() - { - return (Float) value; - } - - public Integer getInt() - { - return (Integer) value; - } - - public SqlMap getMap() - { - return (SqlMap) value; - } - - public Object getObject() - { - return value; - } - - public Short getShort() - { - return (Short) value; - } - - public Timestamp getTimestamp() - { - return (Timestamp) value; - } - - public Time getTime() - { - return (Time) value; - } - - public byte[] getVarbinary() - { - return (byte[]) value; - } - - public String getVarchar() - { - return (String) value; - } - - public boolean isIndexed() - { - return indexed; - } - - public boolean isNull() - { - return value == null; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("value", value) - .add("type", type) - .toString(); - } - - /** - * Convert Trino native value (stack representation) of given type to Accumulo equivalent. - * - * @param value Object to convert - * @param type Destination Trino type - */ - private static Object convert(Object value, Type type) - { - if (value == null) { - return null; - } - - checkArgument(Primitives.wrap(type.getJavaType()).isInstance(value), "Invalid representation for %s: %s [%s]", type, value, value.getClass().getName()); - - // Array? Better be a block! - if (Types.isArrayType(type)) { - // Block - return value; - } - - // Map? Better be a block! - if (Types.isMapType(type)) { - // Block - return value; - } - - // And now for the plain types - if (type.equals(BIGINT)) { - // long - return value; - } - - if (type.equals(INTEGER)) { - return toIntExact((long) value); - } - - if (type.equals(BOOLEAN)) { - // boolean - return value; - } - - if (type.equals(DATE)) { - // long - return value; - } - - if (type.equals(DOUBLE)) { - // double - return value; - } - - if (type.equals(REAL)) { - return intBitsToFloat(toIntExact((long) value)); - } - - if (type.equals(SMALLINT)) { - return Shorts.checkedCast((long) value); - } - - if (type.equals(TIME_MILLIS)) { - return new Time((long) value); - } - - if (type.equals(TIMESTAMP_MILLIS)) { - return new Timestamp(floorDiv((Long) value, MICROSECONDS_PER_MILLISECOND)); - } - - if (type.equals(TINYINT)) { - return SignedBytes.checkedCast((long) value); - } - - if (type.equals(VARBINARY)) { - return ((Slice) value).getBytes(); - } - - if (type instanceof VarcharType) { - return ((Slice) value).toStringUtf8(); - } - - throw new TrinoException(NOT_SUPPORTED, "Unsupported Trino type: " + type); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Row.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Row.java deleted file mode 100644 index 3b5c674a5fc9d..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/Row.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import io.trino.spi.type.Type; - -import java.util.ArrayList; -import java.util.List; - -import static java.util.Objects.requireNonNull; - -public class Row -{ - private final List fields = new ArrayList<>(); - - public Row() {} - - public Row addField(Field field) - { - requireNonNull(field, "field is null"); - fields.add(field); - return this; - } - - public Row addField(Object nativeValue, Type type) - { - requireNonNull(type, "type is null"); - fields.add(new Field(nativeValue, type)); - return this; - } - - public Field getField(int i) - { - return fields.get(i); - } - - public int length() - { - return fields.size(); - } - - @Override - public String toString() - { - if (fields.isEmpty()) { - return "()"; - } - StringBuilder builder = new StringBuilder("("); - for (Field f : fields) { - builder.append(f).append(","); - } - builder.deleteCharAt(builder.length() - 1); - return builder.append(')').toString(); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/SerializedRange.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/SerializedRange.java deleted file mode 100644 index 602f4ead9ffe7..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/SerializedRange.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.io.ByteArrayDataOutput; -import com.google.common.io.ByteStreams; -import org.apache.accumulo.core.data.Range; - -import java.io.DataInput; -import java.io.IOException; -import java.io.UncheckedIOException; - -import static io.airlift.slice.SizeOf.instanceSize; -import static io.airlift.slice.SizeOf.sizeOf; -import static java.util.Objects.requireNonNull; - -public class SerializedRange -{ - private static final int INSTANCE_SIZE = instanceSize(SerializedRange.class); - - private final byte[] bytes; - - public static SerializedRange serialize(Range range) - { - ByteArrayDataOutput out = ByteStreams.newDataOutput(); - try { - range.write(out); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - return new SerializedRange(out.toByteArray()); - } - - @JsonCreator - public SerializedRange(@JsonProperty("data") byte[] bytes) - { - this.bytes = requireNonNull(bytes, "bytes is null"); - } - - @JsonProperty - public byte[] getBytes() - { - return bytes; - } - - public Range deserialize() - { - DataInput in = ByteStreams.newDataInput(bytes); - Range range = new Range(); - try { - range.readFields(in); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - return range; - } - - public long getRetainedSizeInBytes() - { - return INSTANCE_SIZE + sizeOf(bytes); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/TabletSplitMetadata.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/TabletSplitMetadata.java deleted file mode 100644 index 75b956014ac36..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/model/TabletSplitMetadata.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.google.common.collect.ImmutableList; -import org.apache.accumulo.core.data.Range; - -import java.util.List; -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public record TabletSplitMetadata(Optional hostPort, List ranges) -{ - public TabletSplitMetadata - { - requireNonNull(hostPort, "hostPort is null"); - ranges = ImmutableList.copyOf(requireNonNull(ranges, "ranges is null")); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/AccumuloRowSerializer.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/AccumuloRowSerializer.java deleted file mode 100644 index 53e1bf0f44e67..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/AccumuloRowSerializer.java +++ /dev/null @@ -1,620 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slice; -import io.trino.plugin.accumulo.Types; -import io.trino.spi.block.ArrayBlockBuilder; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.block.MapBlockBuilder; -import io.trino.spi.block.SqlMap; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.MapType; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeUtils; -import io.trino.spi.type.VarcharType; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.hadoop.io.Text; - -import java.sql.Time; -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import static io.trino.spi.block.MapValueBuilder.buildMapValue; - -/** - * Interface for deserializing the data in Accumulo into a Trino row. - *

- * Provides a means for end-users of the connector to customize how the data in an Accumulo row gets - * serialized and deserialized from/to a Trino row. - *

- * The workflow of how this class is called by the Accumulo connector for reading data is as - * follows: - *

    - *
  1. setRowIdName - Sets the Trino name which is the Accumulo row ID
    2. - *
  2. setRowOnly - True if only the row ID is going to be retrieved, false if more data is - * necessary.
    3. - *
  3. setMapping - Multiple calls for each Trino column, setting the mapping of Trino column name - * to Accumulo column family and qualifier
    4. - *
  4. deserialize - Called for each Accumulo entry in the same row. Implements should - * retrieve the Trino column value from the given key/value pair
    5. - *
  5. get* - Called to retrieve the data type for the given Trino column name
    6. - *
  6. reset - Begins a new Row, serializer is expected to clear any state
    7. - *
  7. If there are more entries left, go back to deserialize, else end!
    8. - *
- * - * @see LexicoderRowSerializer - * @see StringRowSerializer - */ -public interface AccumuloRowSerializer -{ - /** - * Gets the default AccumuloRowSerializer, {@link LexicoderRowSerializer}. - * - * @return Default serializer - */ - static AccumuloRowSerializer getDefault() - { - return new LexicoderRowSerializer(); - } - - /** - * Sets the Trino name which maps to the Accumulo row ID. - * - * @param name Trino column name - */ - void setRowIdName(String name); - - /** - * Sets the mapping for the Trino column name to Accumulo family and qualifier. - * - * @param name Trino name - * @param family Accumulo family - * @param qualifier Accumulo qualifier - */ - void setMapping(String name, String family, String qualifier); - - /** - * Sets a Boolean value indicating whether or not only the row ID is going to be retrieved from the serializer. - * - * @param rowOnly True if only the row ID is set, false otherwise - */ - void setRowOnly(boolean rowOnly); - - /** - * Reset the state of the serializer to prepare for a new set of entries with the same row ID. - */ - void reset(); - - /** - * Deserialize the given Accumulo entry, retrieving data for the Trino column. - * - * @param entry Entry to deserialize - */ - void deserialize(Entry entry); - - /** - * Gets a Boolean value indicating whether or not the Trino column is a null value. - * - * @param name Column name - * @return True if null, false otherwise. - */ - boolean isNull(String name); - - /** - * Gets the array Block of the given Trino column. - * - * @param name Column name - * @param type Array type - * @return True if null, false otherwise. - */ - Block getArray(String name, Type type); - - /** - * Encode the given array Block into the given Text object. - * - * @param text Text object to set - * @param type Array type - * @param block Array block - */ - void setArray(Text text, Type type, Block block); - - /** - * Gets the Boolean value of the given Trino column. - * - * @param name Column name - * @return Boolean value - */ - boolean getBoolean(String name); - - /** - * Encode the given Boolean value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setBoolean(Text text, Boolean value); - - /** - * Gets the Byte value of the given Trino column. - * - * @param name Column name - * @return Byte value - */ - byte getByte(String name); - - /** - * Encode the given Byte value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setByte(Text text, Byte value); - - /** - * Gets the Date value of the given Trino column. - * - * @param name Column name - * @return Date value - */ - long getDate(String name); - - /** - * Encode the given Date value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setDate(Text text, long value); - - /** - * Gets the Double value of the given Trino column. - * - * @param name Column name - * @return Double value - */ - double getDouble(String name); - - /** - * Encode the given Double value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setDouble(Text text, Double value); - - /** - * Gets the Float value of the given Trino column. - * - * @param name Column name - * @return Float value - */ - float getFloat(String name); - - /** - * Encode the given Float value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setFloat(Text text, Float value); - - /** - * Gets the Integer value of the given Trino column. - * - * @param name Column name - * @return Integer value - */ - int getInt(String name); - - /** - * Encode the given Integer value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setInt(Text text, Integer value); - - /** - * Gets the Long value of the given Trino column. - * - * @param name Column name - * @return Long value - */ - long getLong(String name); - - /** - * Encode the given Long value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setLong(Text text, Long value); - - /** - * Gets the Map value of the given Trino column and Map type. - * - * @param name Column name - * @param type Map type - * @return Map value - */ - SqlMap getMap(String name, Type type); - - /** - * Encode the given map Block into the given Text object. - */ - void setMap(Text text, Type type, SqlMap map); - - /** - * Gets the Short value of the given Trino column. - * - * @param name Column name - * @return Short value - */ - short getShort(String name); - - /** - * Encode the given Short value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setShort(Text text, Short value); - - /** - * Gets the Time value of the given Trino column. - * - * @param name Column name - * @return Time value - */ - Time getTime(String name); - - /** - * Encode the given Time value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setTime(Text text, Time value); - - /** - * Gets the Timestamp value of the given Trino column. - * - * @param name Column name - * @return Timestamp value - */ - Timestamp getTimestamp(String name); - - /** - * Encode the given Timestamp value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setTimestamp(Text text, Timestamp value); - - /** - * Gets the Varbinary value of the given Trino column. - * - * @param name Column name - * @return Varbinary value - */ - byte[] getVarbinary(String name); - - /** - * Encode the given byte[] value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setVarbinary(Text text, byte[] value); - - /** - * Gets the String value of the given Trino column. - * - * @param name Column name - * @return String value - */ - String getVarchar(String name); - - /** - * Encode the given String value into the given Text object. - * - * @param text Text object to set - * @param value Value to encode - */ - void setVarchar(Text text, String value); - - /** - * Encodes a Trino Java object to a byte array based on the given type. - *

- * Java Lists and Maps can be converted to Trino values using - * {@link AccumuloRowSerializer#getBlockFromArray(Type, java.util.List)} and - * {@link AccumuloRowSerializer#getSqlMapFromMap(Type, Map)} - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Type to EncodeExpected Java Object
ARRAYio.trino.spi.block.Block
BIGINTInteger or Long
BOOLEANBoolean
DATElong
DOUBLEDouble
INTEGERInteger
Mapio.trino.spi.block.Block
REALFloat
SMALLINTShort
TIMEjava.sql.Time, Long
TIMESTAMPjava.sql.Timestamp, Long
TINYINTByte
VARBINARYio.airlift.slice.Slice or byte[]
VARCHARio.airlift.slice.Slice or String
- * - * @param type The Trino {@link io.trino.spi.type.Type} - * @param value The Java object per the table in the method description - * @return Encoded bytes - */ - byte[] encode(Type type, Object value); - - /** - * Generic function to decode the given byte array to a Java object based on the given type. - *

- * Blocks from ARRAY and MAP types can be converted - * to Java Lists and Maps using {@link AccumuloRowSerializer#getArrayFromBlock(Type, Block)} - * and {@link AccumuloRowSerializer#getMapFromSqlMap(Type, SqlMap)} (Type, Block)} - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Encoded TypeReturned Java Object
ARRAYList<?>
BIGINTLong
BOOLEANBoolean
DATELong
DOUBLEDouble
MapMap<?,?>
REALDouble
SMALLINTLong
TIMELong
TIMESTAMPLong
TINYINTLong
VARBINARYbyte[]
VARCHARString
- * - * @param type The Trino {@link io.trino.spi.type.Type} - * @param value Encoded bytes to decode - * @param The Java type of the object that has been encoded to the given byte array - * @return The Java object per the table in the method description - */ - T decode(Type type, byte[] value); - - /** - * Given the array element type and Trino Block, decodes the Block into a list of values. - * - * @param elementType Array element type - * @param block Array block - * @return List of values - */ - static List getArrayFromBlock(Type elementType, Block block) - { - ImmutableList.Builder arrayBuilder = ImmutableList.builder(); - for (int i = 0; i < block.getPositionCount(); ++i) { - arrayBuilder.add(readObject(elementType, block, i)); - } - return arrayBuilder.build(); - } - - /** - * Given the map type and Trino SqlMap, decodes the SqlMap into a Java map of values. - */ - static Map getMapFromSqlMap(Type type, SqlMap sqlMap) - { - Map map = new HashMap<>(sqlMap.getSize()); - Type keyType = Types.getKeyType(type); - Type valueType = Types.getValueType(type); - - int rawOffset = sqlMap.getRawOffset(); - Block rawKeyBlock = sqlMap.getRawKeyBlock(); - Block rawValueBlock = sqlMap.getRawValueBlock(); - for (int i = 0; i < sqlMap.getSize(); i++) { - map.put(readObject(keyType, rawKeyBlock, rawOffset + i), readObject(valueType, rawValueBlock, rawOffset + i)); - } - return map; - } - - /** - * Encodes the given list into a Block. - * - * @param elementType Element type of the array - * @param array Array of elements to encode - * @return Trino Block - */ - static Block getBlockFromArray(Type elementType, List array) - { - BlockBuilder builder = elementType.createBlockBuilder(null, array.size()); - for (Object item : array) { - writeObject(builder, elementType, item); - } - return builder.build(); - } - - /** - * Encodes the given Java map into a SqlMap. - * - * @param type Trino type of the map - * @param map Map of key/value pairs to encode - * @return Trino SqlMap - */ - static SqlMap getSqlMapFromMap(Type type, Map map) - { - MapType mapType = (MapType) type; - Type keyType = mapType.getKeyType(); - Type valueType = mapType.getValueType(); - - return buildMapValue(mapType, map.size(), (keyBuilder, valueBuilder) -> { - map.forEach((key, value) -> { - writeObject(keyBuilder, keyType, key); - writeObject(valueBuilder, valueType, value); - }); - }); - } - - /** - * Recursive helper function used by {@link AccumuloRowSerializer#getBlockFromArray} and - * {@link AccumuloRowSerializer#getSqlMapFromMap} to add the given object to the given block - * builder. Supports nested complex types! - * - * @param builder Block builder - * @param type Trino type - * @param obj Object to write to the block builder - */ - static void writeObject(BlockBuilder builder, Type type, Object obj) - { - if (Types.isArrayType(type)) { - ((ArrayBlockBuilder) builder).buildEntry(elementBuilder -> { - Type elementType = Types.getElementType(type); - for (Object item : (List) obj) { - writeObject(elementBuilder, elementType, item); - } - }); - } - else if (type instanceof MapType mapType) { - ((MapBlockBuilder) builder).buildEntry((keyBuilder, valueBuilder) -> { - for (Entry entry : ((Map) obj).entrySet()) { - writeObject(keyBuilder, mapType.getKeyType(), entry.getKey()); - writeObject(valueBuilder, mapType.getValueType(), entry.getValue()); - } - }); - } - else { - TypeUtils.writeNativeValue(type, builder, obj); - } - } - - /** - * Recursive helper function used by {@link AccumuloRowSerializer#getArrayFromBlock} and - * {@link AccumuloRowSerializer#getMapFromSqlMap} to decode the Block into a Java type. - * - * @param type Trino type - * @param block Block to decode - * @param position Position in the block to get - * @return Java object from the Block - */ - static Object readObject(Type type, Block block, int position) - { - if (type instanceof ArrayType arrayType) { - Type elementType = arrayType.getElementType(); - return getArrayFromBlock(elementType, arrayType.getObject(block, position)); - } - if (type instanceof MapType mapType) { - return getMapFromSqlMap(type, mapType.getObject(block, position)); - } - if (type.getJavaType() == Slice.class) { - Slice slice = (Slice) TypeUtils.readNativeValue(type, block, position); - return type.equals(VarcharType.VARCHAR) ? slice.toStringUtf8() : slice.getBytes(); - } - - return TypeUtils.readNativeValue(type, block, position); - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/BooleanLexicoder.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/BooleanLexicoder.java deleted file mode 100644 index 47b9d666c84ab..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/BooleanLexicoder.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -import org.apache.accumulo.core.client.lexicoder.Lexicoder; - -/** - * Accumulo lexicoder for Booleans - */ -public class BooleanLexicoder - implements Lexicoder -{ - @Override - public byte[] encode(Boolean v) - { - return new byte[] {v ? (byte) 1 : 0}; - } - - @Override - public Boolean decode(byte[] b) - { - return b[0] != 0; - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/LexicoderRowSerializer.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/LexicoderRowSerializer.java deleted file mode 100644 index e1a24f69acfeb..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/LexicoderRowSerializer.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Table; -import io.airlift.slice.Slice; -import io.trino.plugin.accumulo.Types; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; -import org.apache.accumulo.core.client.lexicoder.BytesLexicoder; -import org.apache.accumulo.core.client.lexicoder.DoubleLexicoder; -import org.apache.accumulo.core.client.lexicoder.Lexicoder; -import org.apache.accumulo.core.client.lexicoder.ListLexicoder; -import org.apache.accumulo.core.client.lexicoder.LongLexicoder; -import org.apache.accumulo.core.client.lexicoder.StringLexicoder; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.hadoop.io.Text; - -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import static io.trino.plugin.accumulo.io.AccumuloPageSink.ROW_ID_COLUMN; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -/** - * Implementation of {@link AccumuloRowSerializer} that uses Accumulo lexicoders to serialize the values of the Trino columns. - */ -public class LexicoderRowSerializer - implements AccumuloRowSerializer -{ - private static final Map LEXICODER_MAP = new HashMap<>(); - private static final Map> LIST_LEXICODERS = new HashMap<>(); - private static final Map> MAP_LEXICODERS = new HashMap<>(); - - private final Table familyQualifierColumnMap = HashBasedTable.create(); - private final Map columnValues = new HashMap<>(); - private final Text rowId = new Text(); - private final Text family = new Text(); - private final Text qualifier = new Text(); - private final Text value = new Text(); - - private boolean rowOnly; - private String rowIdName; - - static { - LongLexicoder longLexicoder = new LongLexicoder(); - DoubleLexicoder doubleLexicoder = new DoubleLexicoder(); - LEXICODER_MAP.put(BIGINT, longLexicoder); - LEXICODER_MAP.put(BOOLEAN, new BooleanLexicoder()); - LEXICODER_MAP.put(DATE, longLexicoder); - LEXICODER_MAP.put(DOUBLE, doubleLexicoder); - LEXICODER_MAP.put(INTEGER, longLexicoder); - LEXICODER_MAP.put(REAL, doubleLexicoder); - LEXICODER_MAP.put(SMALLINT, longLexicoder); - LEXICODER_MAP.put(TIME_MILLIS, longLexicoder); - LEXICODER_MAP.put(TIMESTAMP_MILLIS, longLexicoder); - LEXICODER_MAP.put(TINYINT, longLexicoder); - LEXICODER_MAP.put(VARBINARY, new BytesLexicoder()); - LEXICODER_MAP.put(VARCHAR, new StringLexicoder()); - } - - @Override - public void setRowIdName(String name) - { - rowIdName = name; - } - - @Override - public void setRowOnly(boolean rowOnly) - { - this.rowOnly = rowOnly; - } - - @Override - public void setMapping(String name, String family, String qualifier) - { - columnValues.put(name, null); - familyQualifierColumnMap.put(family, qualifier, name); - } - - @Override - public void reset() - { - columnValues.clear(); - } - - @Override - public void deserialize(Entry entry) - { - if (!columnValues.containsKey(rowIdName)) { - entry.getKey().getRow(rowId); - columnValues.put(rowIdName, rowId.copyBytes()); - } - - if (rowOnly) { - return; - } - - entry.getKey().getColumnFamily(family); - entry.getKey().getColumnQualifier(qualifier); - - if (family.equals(ROW_ID_COLUMN) && qualifier.equals(ROW_ID_COLUMN)) { - return; - } - - value.set(entry.getValue().get()); - columnValues.put(familyQualifierColumnMap.get(family.toString(), qualifier.toString()), value.copyBytes()); - } - - @Override - public boolean isNull(String name) - { - return columnValues.get(name) == null; - } - - @Override - public Block getArray(String name, Type type) - { - Type elementType = Types.getElementType(type); - return AccumuloRowSerializer.getBlockFromArray(elementType, decode(type, getFieldValue(name))); - } - - @Override - public void setArray(Text text, Type type, Block block) - { - text.set(encode(type, block)); - } - - @Override - public boolean getBoolean(String name) - { - return decode(BOOLEAN, getFieldValue(name)); - } - - @Override - public void setBoolean(Text text, Boolean value) - { - text.set(encode(BOOLEAN, value)); - } - - @Override - public byte getByte(String name) - { - return ((Long) decode(TINYINT, getFieldValue(name))).byteValue(); - } - - @Override - public void setByte(Text text, Byte value) - { - text.set(encode(TINYINT, value)); - } - - @Override - public long getDate(String name) - { - return decode(BIGINT, getFieldValue(name)); - } - - @Override - public void setDate(Text text, long value) - { - text.set(encode(DATE, value)); - } - - @Override - public double getDouble(String name) - { - return decode(DOUBLE, getFieldValue(name)); - } - - @Override - public void setDouble(Text text, Double value) - { - text.set(encode(DOUBLE, value)); - } - - @Override - public float getFloat(String name) - { - return ((Double) decode(REAL, getFieldValue(name))).floatValue(); - } - - @Override - public void setFloat(Text text, Float value) - { - text.set(encode(REAL, value)); - } - - @Override - public int getInt(String name) - { - return ((Long) decode(INTEGER, getFieldValue(name))).intValue(); - } - - @Override - public void setInt(Text text, Integer value) - { - text.set(encode(INTEGER, value)); - } - - @Override - public long getLong(String name) - { - return decode(BIGINT, getFieldValue(name)); - } - - @Override - public void setLong(Text text, Long value) - { - text.set(encode(BIGINT, value)); - } - - @Override - public SqlMap getMap(String name, Type type) - { - return AccumuloRowSerializer.getSqlMapFromMap(type, decode(type, getFieldValue(name))); - } - - @Override - public void setMap(Text text, Type type, SqlMap map) - { - text.set(encode(type, map)); - } - - @Override - public short getShort(String name) - { - return ((Long) decode(SMALLINT, getFieldValue(name))).shortValue(); - } - - @Override - public void setShort(Text text, Short value) - { - text.set(encode(SMALLINT, value)); - } - - @Override - public Time getTime(String name) - { - return new Time(decode(BIGINT, getFieldValue(name))); - } - - @Override - public void setTime(Text text, Time value) - { - text.set(encode(TIME_MILLIS, value)); - } - - @Override - public Timestamp getTimestamp(String name) - { - return new Timestamp(decode(TIMESTAMP_MILLIS, getFieldValue(name))); - } - - @Override - public void setTimestamp(Text text, Timestamp value) - { - text.set(encode(TIMESTAMP_MILLIS, value)); - } - - @Override - public byte[] getVarbinary(String name) - { - return decode(VARBINARY, getFieldValue(name)); - } - - @Override - public void setVarbinary(Text text, byte[] value) - { - text.set(encode(VARBINARY, value)); - } - - @Override - public String getVarchar(String name) - { - return decode(VARCHAR, getFieldValue(name)); - } - - @Override - public void setVarchar(Text text, String value) - { - text.set(encode(VARCHAR, value)); - } - - private byte[] getFieldValue(String name) - { - return columnValues.get(name); - } - - @Override - public byte[] encode(Type type, Object value) - { - Object toEncode; - if (Types.isArrayType(type)) { - toEncode = AccumuloRowSerializer.getArrayFromBlock(Types.getElementType(type), (Block) value); - } - else if (Types.isMapType(type)) { - toEncode = AccumuloRowSerializer.getMapFromSqlMap(type, (SqlMap) value); - } - else if (type.equals(BIGINT) && value instanceof Integer) { - toEncode = ((Integer) value).longValue(); - } - else if (type.equals(DATE) && value instanceof Date) { - toEncode = MILLISECONDS.toDays(((Date) value).getTime()); - } - else if (type.equals(INTEGER) && value instanceof Integer) { - toEncode = ((Integer) value).longValue(); - } - else if (type.equals(REAL) && value instanceof Float) { - toEncode = ((Float) value).doubleValue(); - } - else if (type.equals(SMALLINT) && value instanceof Short) { - toEncode = ((Short) value).longValue(); - } - else if (type.equals(TIME_MILLIS) && value instanceof Time) { - toEncode = ((Time) value).getTime(); - } - else if (type.equals(TIMESTAMP_MILLIS) && value instanceof Timestamp) { - toEncode = ((Timestamp) value).getTime(); - } - else if (type.equals(TINYINT) && value instanceof Byte) { - toEncode = ((Byte) value).longValue(); - } - else if (type.equals(VARBINARY) && value instanceof Slice) { - toEncode = ((Slice) value).getBytes(); - } - else if (type instanceof VarcharType && value instanceof Slice) { - toEncode = ((Slice) value).toStringUtf8(); - } - else { - toEncode = value; - } - - return getLexicoder(type).encode(toEncode); - } - - @Override - public T decode(Type type, byte[] value) - { - return (T) getLexicoder(type).decode(value); - } - - public static Lexicoder getLexicoder(Type type) - { - if (Types.isArrayType(type)) { - return getListLexicoder(type); - } - if (Types.isMapType(type)) { - return getMapLexicoder(type); - } - if (type instanceof VarcharType) { - return LEXICODER_MAP.get(VARCHAR); - } - Lexicoder lexicoder = LEXICODER_MAP.get(type); - if (lexicoder == null) { - throw new TrinoException(NOT_SUPPORTED, "No lexicoder for type " + type); - } - return lexicoder; - } - - private static ListLexicoder getListLexicoder(Type elementType) - { - ListLexicoder listLexicoder = LIST_LEXICODERS.get(elementType); - if (listLexicoder == null) { - listLexicoder = new ListLexicoder(getLexicoder(Types.getElementType(elementType))); - LIST_LEXICODERS.put(elementType, listLexicoder); - } - return listLexicoder; - } - - private static MapLexicoder getMapLexicoder(Type type) - { - MapLexicoder mapLexicoder = MAP_LEXICODERS.get(type); - if (mapLexicoder == null) { - mapLexicoder = new MapLexicoder( - getLexicoder(Types.getKeyType(type)), - getLexicoder(Types.getValueType(type))); - MAP_LEXICODERS.put(type, mapLexicoder); - } - return mapLexicoder; - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/MapLexicoder.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/MapLexicoder.java deleted file mode 100644 index a841ad41629cf..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/MapLexicoder.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -import org.apache.accumulo.core.client.lexicoder.Lexicoder; - -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import static org.apache.accumulo.core.clientImpl.lexicoder.ByteUtils.concat; -import static org.apache.accumulo.core.clientImpl.lexicoder.ByteUtils.escape; -import static org.apache.accumulo.core.clientImpl.lexicoder.ByteUtils.split; -import static org.apache.accumulo.core.clientImpl.lexicoder.ByteUtils.unescape; - -/** - * Accumulo lexicoder for encoding a Java Map - * - * @param Key data type - * @param Value data type - */ -public class MapLexicoder - implements Lexicoder> -{ - private final Lexicoder keyLexicoder; - private final Lexicoder valueLexicoder; - - public MapLexicoder( - Lexicoder keyLexicoder, - Lexicoder valueLexicoder) - { - this.keyLexicoder = keyLexicoder; - this.valueLexicoder = valueLexicoder; - } - - @Override - public byte[] encode(Map v) - { - byte[][] elements = new byte[v.size() * 2][]; - int index = 0; - for (Entry entry : v.entrySet()) { - elements[index++] = escape(keyLexicoder.encode(entry.getKey())); - elements[index++] = escape(valueLexicoder.encode(entry.getValue())); - } - - return concat(elements); - } - - @Override - public Map decode(byte[] b) - { - byte[][] escapedElements = split(b); - Map decodedMap = new HashMap<>(); - for (int i = 0; i < escapedElements.length; i += 2) { - K key = keyLexicoder.decode(unescape(escapedElements[i])); - V value = valueLexicoder.decode(unescape(escapedElements[i + 1])); - decodedMap.put(key, value); - } - - return decodedMap; - } -} diff --git a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/StringRowSerializer.java b/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/StringRowSerializer.java deleted file mode 100644 index b368fabb83e40..0000000000000 --- a/plugin/trino-accumulo/src/main/java/io/trino/plugin/accumulo/serializers/StringRowSerializer.java +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Table; -import io.airlift.slice.Slice; -import io.trino.plugin.accumulo.Types; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.type.Type; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.hadoop.io.Text; - -import java.sql.Time; -import java.sql.Timestamp; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import static io.trino.plugin.accumulo.io.AccumuloPageSink.ROW_ID_COLUMN; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Implementation of {@link StringRowSerializer} that encodes and decodes Trino column values as human-readable String objects. - */ -public class StringRowSerializer - implements AccumuloRowSerializer -{ - private final Table familyQualifierColumnMap = HashBasedTable.create(); - private final Map columnValues = new HashMap<>(); - private final Text rowId = new Text(); - private final Text family = new Text(); - private final Text qualifier = new Text(); - private final Text value = new Text(); - - private boolean rowOnly; - private String rowIdName; - - @Override - public void setRowIdName(String name) - { - this.rowIdName = name; - } - - @Override - public void setRowOnly(boolean rowOnly) - { - this.rowOnly = rowOnly; - } - - @Override - public void setMapping(String name, String family, String qualifier) - { - columnValues.put(name, null); - familyQualifierColumnMap.put(family, qualifier, name); - } - - @Override - public void reset() - { - columnValues.clear(); - } - - @Override - public void deserialize(Entry entry) - { - if (!columnValues.containsKey(rowIdName)) { - entry.getKey().getRow(rowId); - columnValues.put(rowIdName, rowId.toString()); - } - - if (rowOnly) { - return; - } - - entry.getKey().getColumnFamily(family); - entry.getKey().getColumnQualifier(qualifier); - - if (family.equals(ROW_ID_COLUMN) && qualifier.equals(ROW_ID_COLUMN)) { - return; - } - - value.set(entry.getValue().get()); - columnValues.put(familyQualifierColumnMap.get(family.toString(), qualifier.toString()), value.toString()); - } - - @Override - public boolean isNull(String name) - { - return columnValues.get(name) == null; - } - - @Override - public Block getArray(String name, Type type) - { - throw new TrinoException(NOT_SUPPORTED, "arrays are not (yet?) supported for StringRowSerializer"); - } - - @Override - public void setArray(Text text, Type type, Block block) - { - throw new TrinoException(NOT_SUPPORTED, "arrays are not (yet?) supported for StringRowSerializer"); - } - - @Override - public boolean getBoolean(String name) - { - return Boolean.parseBoolean(getFieldValue(name)); - } - - @Override - public void setBoolean(Text text, Boolean value) - { - text.set(value.toString().getBytes(UTF_8)); - } - - @Override - public byte getByte(String name) - { - return Byte.parseByte(getFieldValue(name)); - } - - @Override - public void setByte(Text text, Byte value) - { - text.set(value.toString().getBytes(UTF_8)); - } - - @Override - public long getDate(String name) - { - return Long.parseLong(getFieldValue(name)); - } - - @Override - public void setDate(Text text, long value) - { - text.set(Long.toString(value).getBytes(UTF_8)); - } - - @Override - public double getDouble(String name) - { - return Double.parseDouble(getFieldValue(name)); - } - - @Override - public void setDouble(Text text, Double value) - { - text.set(value.toString().getBytes(UTF_8)); - } - - @Override - public float getFloat(String name) - { - return Float.parseFloat(getFieldValue(name)); - } - - @Override - public void setFloat(Text text, Float value) - { - text.set(value.toString().getBytes(UTF_8)); - } - - @Override - public int getInt(String name) - { - return Integer.parseInt(getFieldValue(name)); - } - - @Override - public void setInt(Text text, Integer value) - { - text.set(value.toString().getBytes(UTF_8)); - } - - @Override - public long getLong(String name) - { - return Long.parseLong(getFieldValue(name)); - } - - @Override - public void setLong(Text text, Long value) - { - text.set(value.toString().getBytes(UTF_8)); - } - - @Override - public SqlMap getMap(String name, Type type) - { - throw new TrinoException(NOT_SUPPORTED, "maps are not (yet?) supported for StringRowSerializer"); - } - - @Override - public void setMap(Text text, Type type, SqlMap map) - { - throw new TrinoException(NOT_SUPPORTED, "maps are not (yet?) supported for StringRowSerializer"); - } - - @Override - public short getShort(String name) - { - return Short.parseShort(getFieldValue(name)); - } - - @Override - public void setShort(Text text, Short value) - { - text.set(value.toString().getBytes(UTF_8)); - } - - @Override - public Time getTime(String name) - { - return new Time(Long.parseLong(getFieldValue(name))); - } - - @Override - public void setTime(Text text, Time value) - { - text.set(Long.toString(value.getTime()).getBytes(UTF_8)); - } - - @Override - public Timestamp getTimestamp(String name) - { - return new Timestamp(Long.parseLong(getFieldValue(name))); - } - - @Override - public void setTimestamp(Text text, Timestamp value) - { - text.set(Long.toString(value.getTime()).getBytes(UTF_8)); - } - - @Override - public byte[] getVarbinary(String name) - { - return getFieldValue(name).getBytes(UTF_8); - } - - @Override - public void setVarbinary(Text text, byte[] value) - { - text.set(value); - } - - @Override - public String getVarchar(String name) - { - return getFieldValue(name); - } - - @Override - public void setVarchar(Text text, String value) - { - text.set(value.getBytes(UTF_8)); - } - - private String getFieldValue(String name) - { - return columnValues.get(name).toString(); - } - - @Override - public byte[] encode(Type type, Object value) - { - Text text = new Text(); - if (Types.isArrayType(type)) { - throw new TrinoException(NOT_SUPPORTED, "arrays are not (yet?) supported for StringRowSerializer"); - } - if (Types.isMapType(type)) { - throw new TrinoException(NOT_SUPPORTED, "maps are not (yet?) supported for StringRowSerializer"); - } - if (type.equals(BIGINT) && value instanceof Integer) { - setLong(text, ((Integer) value).longValue()); - } - else if (type.equals(BIGINT) && value instanceof Long) { - setLong(text, (Long) value); - } - else if (type.equals(BOOLEAN)) { - setBoolean(text, value.equals(Boolean.TRUE)); - } - else if (type.equals(DATE)) { - setDate(text, (long) value); - } - else if (type.equals(DOUBLE)) { - setDouble(text, (Double) value); - } - else if (type.equals(INTEGER) && value instanceof Integer) { - setInt(text, (Integer) value); - } - else if (type.equals(INTEGER) && value instanceof Long) { - setInt(text, ((Long) value).intValue()); - } - else if (type.equals(REAL)) { - setFloat(text, (Float) value); - } - else if (type.equals(SMALLINT)) { - setShort(text, (Short) value); - } - else if (type.equals(TIME_MILLIS)) { - setTime(text, (Time) value); - } - else if (type.equals(TIMESTAMP_MILLIS)) { - setTimestamp(text, (Timestamp) value); - } - else if (type.equals(TINYINT)) { - setByte(text, (Byte) value); - } - else if (type.equals(VARBINARY) && value instanceof byte[]) { - setVarbinary(text, (byte[]) value); - } - else if (type.equals(VARBINARY) && value instanceof Slice) { - setVarbinary(text, ((Slice) value).getBytes()); - } - else if (type.equals(VARCHAR) && value instanceof String) { - setVarchar(text, ((String) value)); - } - else if (type.equals(VARCHAR) && value instanceof Slice) { - setVarchar(text, ((Slice) value).toStringUtf8()); - } - else { - throw new TrinoException(NOT_SUPPORTED, format("StringLexicoder does not support encoding type %s, object class is %s", type, value.getClass())); - } - - return text.copyBytes(); - } - - @SuppressWarnings("unchecked") - @Override - public T decode(Type type, byte[] value) - { - String strValue = new String(value, UTF_8); - if (type.equals(BIGINT)) { - return (T) (Long) Long.parseLong(strValue); - } - if (type.equals(BOOLEAN)) { - return (T) (Boolean) Boolean.parseBoolean(strValue); - } - if (type.equals(DATE)) { - return (T) (Long) Long.parseLong(strValue); - } - if (type.equals(DOUBLE)) { - return (T) (Double) Double.parseDouble(strValue); - } - if (type.equals(INTEGER)) { - return (T) (Long) ((Integer) Integer.parseInt(strValue)).longValue(); - } - if (type.equals(REAL)) { - return (T) (Double) ((Float) Float.parseFloat(strValue)).doubleValue(); - } - if (type.equals(SMALLINT)) { - return (T) (Long) ((Short) Short.parseShort(strValue)).longValue(); - } - if (type.equals(TIME_MILLIS)) { - return (T) (Long) Long.parseLong(strValue); - } - if (type.equals(TIMESTAMP_MILLIS)) { - return (T) (Long) Long.parseLong(strValue); - } - if (type.equals(TINYINT)) { - return (T) (Long) ((Byte) Byte.parseByte(strValue)).longValue(); - } - if (type.equals(VARBINARY)) { - return (T) value; - } - if (type.equals(VARCHAR)) { - return (T) strValue; - } - throw new TrinoException(NOT_SUPPORTED, "Unsupported type " + type); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/AccumuloQueryRunner.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/AccumuloQueryRunner.java deleted file mode 100644 index 3787e46a6c114..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/AccumuloQueryRunner.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedSet; -import io.airlift.log.Level; -import io.airlift.log.Logger; -import io.airlift.log.Logging; -import io.trino.Session; -import io.trino.metadata.QualifiedObjectName; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.accumulo.serializers.LexicoderRowSerializer; -import io.trino.plugin.base.util.Closables; -import io.trino.plugin.tpch.TpchPlugin; -import io.trino.testing.DistributedQueryRunner; -import io.trino.testing.QueryRunner; -import io.trino.tpch.TpchTable; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.hadoop.io.Text; -import org.intellij.lang.annotations.Language; - -import java.util.Map; - -import static io.airlift.units.Duration.nanosSince; -import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.testing.TestingSession.testSessionBuilder; -import static java.lang.String.format; -import static java.util.concurrent.TimeUnit.SECONDS; - -public final class AccumuloQueryRunner -{ - static { - Logging logging = Logging.initialize(); - logging.setLevel("org.apache.accumulo", Level.OFF); - logging.setLevel("org.apache.zookeeper", Level.OFF); - logging.setLevel("org.apache.curator", Level.OFF); - } - - private static final Logger LOG = Logger.get(AccumuloQueryRunner.class); - - private static boolean tpchLoaded; - - private AccumuloQueryRunner() {} - - public static Builder builder() - { - return new Builder(); - } - - public static class Builder - extends DistributedQueryRunner.Builder - { - protected Builder() - { - super(testSessionBuilder() - .setCatalog("accumulo") - .setSchema("tpch") - .build()); - } - - @Override - public DistributedQueryRunner build() - throws Exception - { - DistributedQueryRunner queryRunner = super.build(); - try { - queryRunner.installPlugin(new TpchPlugin()); - queryRunner.createCatalog("tpch", "tpch"); - - TestingAccumuloServer server = TestingAccumuloServer.getInstance(); - queryRunner.installPlugin(new AccumuloPlugin()); - Map accumuloProperties = - ImmutableMap.builder() - .put(AccumuloConfig.INSTANCE, server.getInstanceName()) - .put(AccumuloConfig.ZOOKEEPERS, server.getZooKeepers()) - .put(AccumuloConfig.USERNAME, server.getUser()) - .put(AccumuloConfig.PASSWORD, server.getPassword()) - .put(AccumuloConfig.ZOOKEEPER_METADATA_ROOT, "/trino-accumulo-test") - .buildOrThrow(); - - queryRunner.createCatalog("accumulo", "accumulo", accumuloProperties); - - if (!tpchLoaded) { - queryRunner.execute("CREATE SCHEMA accumulo.tpch"); - copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), TpchTable.getTables()); - try (AccumuloClient client = server.createClient()) { - client.tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L)))); - } - tpchLoaded = true; - } - } - catch (Throwable e) { - Closables.closeAllSuppress(e, queryRunner); - throw e; - } - return queryRunner; - } - } - - private static void copyTpchTables( - QueryRunner queryRunner, - String sourceCatalog, - String sourceSchema, - Session session, - Iterable> tables) - { - LOG.info("Loading data from %s.%s...", sourceCatalog, sourceSchema); - long startTime = System.nanoTime(); - for (TpchTable table : tables) { - copyTable(queryRunner, sourceCatalog, session, sourceSchema, table); - } - LOG.info("Loading from %s.%s complete in %s", sourceCatalog, sourceSchema, nanosSince(startTime).toString(SECONDS)); - } - - private static void copyTable( - QueryRunner queryRunner, - String catalog, - Session session, - String schema, - TpchTable table) - { - QualifiedObjectName source = new QualifiedObjectName(catalog, schema, table.getTableName()); - String target = table.getTableName(); - - @Language("SQL") - String sql; - switch (target) { - case "customer": - sql = format("CREATE TABLE %s WITH (index_columns = 'mktsegment') AS SELECT * FROM %s", target, source); - break; - case "lineitem": - sql = format("CREATE TABLE %s WITH (index_columns = 'quantity,discount,returnflag,shipdate,receiptdate,shipinstruct,shipmode') AS SELECT cast(uuid() AS varchar) AS uuid, * FROM %s", target, source); - break; - case "orders": - sql = format("CREATE TABLE %s WITH (index_columns = 'orderdate') AS SELECT * FROM %s", target, source); - break; - case "part": - sql = format("CREATE TABLE %s WITH (index_columns = 'brand,type,size,container') AS SELECT * FROM %s", target, source); - break; - case "partsupp": - sql = format("CREATE TABLE %s WITH (index_columns = 'partkey') AS SELECT cast(uuid() AS varchar) AS uuid, * FROM %s", target, source); - break; - case "supplier": - sql = format("CREATE TABLE %s WITH (index_columns = 'name') AS SELECT * FROM %s", target, source); - break; - default: - sql = format("CREATE TABLE %s AS SELECT * FROM %s", target, source); - break; - } - - LOG.info("Running import for %s%n%s", target, sql); - long start = System.nanoTime(); - long rows = queryRunner.execute(session, sql).getUpdateCount().getAsLong(); - LOG.info("Imported %s rows for %s in %s", rows, target, nanosSince(start)); - } - - public static Session createSession() - { - return testSessionBuilder().setCatalog("accumulo").setSchema("tpch").build(); - } - - public static void main(String[] args) - throws Exception - { - QueryRunner queryRunner = builder() - .addCoordinatorProperty("http-server.http.port", "8080") - .build(); - Logger log = Logger.get(AccumuloQueryRunner.class); - log.info("======== SERVER STARTED ========"); - log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloConnectorTest.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloConnectorTest.java deleted file mode 100644 index d6abec4898dfb..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloConnectorTest.java +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import io.trino.testing.BaseConnectorTest; -import io.trino.testing.MaterializedResult; -import io.trino.testing.QueryRunner; -import io.trino.testing.TestingConnectorBehavior; -import io.trino.testing.sql.TestTable; -import org.intellij.lang.annotations.Language; -import org.junit.jupiter.api.Test; - -import java.util.Optional; -import java.util.OptionalInt; -import java.util.regex.Pattern; - -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.testing.MaterializedResult.resultBuilder; -import static java.util.regex.Pattern.DOTALL; -import static java.util.regex.Pattern.MULTILINE; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assumptions.abort; - -/** - * Accumulo requires a unique identifier for the rows. - * Any row that has a duplicate row ID is effectively an update, - * overwriting existing values of the row with whatever the new values are. - * For the lineitem and partsupp tables, there is no unique identifier, - * so a generated UUID is used in order to prevent overwriting rows of data. - * This is the same for any test cases that were creating tables with duplicate rows, - * so some test cases are overridden from the base class and slightly modified to add an additional UUID column. - */ -public class TestAccumuloConnectorTest - extends BaseConnectorTest -{ - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - return AccumuloQueryRunner.builder().build(); - } - - @Override - protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) - { - return switch (connectorBehavior) { - case SUPPORTS_ADD_COLUMN, - SUPPORTS_COMMENT_ON_COLUMN, - SUPPORTS_COMMENT_ON_TABLE, - SUPPORTS_CREATE_MATERIALIZED_VIEW, - SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT, - SUPPORTS_DELETE, - SUPPORTS_DROP_SCHEMA_CASCADE, - SUPPORTS_MERGE, - SUPPORTS_NOT_NULL_CONSTRAINT, - SUPPORTS_RENAME_SCHEMA, - SUPPORTS_RENAME_TABLE_ACROSS_SCHEMAS, - SUPPORTS_ROW_TYPE, - SUPPORTS_SET_COLUMN_TYPE, - SUPPORTS_TOPN_PUSHDOWN, - SUPPORTS_UPDATE -> false; - default -> super.hasBehavior(connectorBehavior); - }; - } - - @Override - protected TestTable createTableWithDefaultColumns() - { - return abort("Accumulo connector does not support column default values"); - } - - @Test - @Override - public void testCreateTableAsSelect() - { - // This test is overridden due to Function "UUID" not found errors - // Some test cases from the base class are removed - - // TODO some test cases from overridden method succeed to create table, but with wrong number or rows. - - assertUpdate("CREATE TABLE test_create_table_as_if_not_exists (a bigint, b double)"); - assertThat(getQueryRunner().tableExists(getSession(), "test_create_table_as_if_not_exists")).isTrue(); - assertTableColumnNames("test_create_table_as_if_not_exists", "a", "b"); - - assertUpdate("CREATE TABLE IF NOT EXISTS test_create_table_as_if_not_exists AS SELECT cast(uuid() AS uuid) AS uuid, orderkey, discount FROM lineitem", 0); - assertThat(getQueryRunner().tableExists(getSession(), "test_create_table_as_if_not_exists")).isTrue(); - assertTableColumnNames("test_create_table_as_if_not_exists", "a", "b"); - - assertUpdate("DROP TABLE test_create_table_as_if_not_exists"); - assertThat(getQueryRunner().tableExists(getSession(), "test_create_table_as_if_not_exists")).isFalse(); - - this.assertCreateTableAsSelect( - "SELECT orderstatus, sum(totalprice) x FROM orders GROUP BY orderstatus", - "SELECT count(DISTINCT orderstatus) FROM orders"); - - this.assertCreateTableAsSelect( - "SELECT * FROM orders WITH DATA", - "SELECT * FROM orders", - "SELECT count(*) FROM orders"); - - this.assertCreateTableAsSelect( - "SELECT * FROM orders WITH NO DATA", - "SELECT * FROM orders LIMIT 0", - "SELECT 0"); - } - - @Override - protected OptionalInt maxTableNameLength() - { - return OptionalInt.of(1024); - } - - @Override - protected OptionalInt maxSchemaNameLength() - { - return OptionalInt.of(1024); - } - - @Override - protected void verifyTableNameLengthFailurePermissible(Throwable e) - { - assertThat(e).hasMessageContaining("Table name exceeds a maximum length"); - } - - @Override - protected void verifySchemaNameLengthFailurePermissible(Throwable e) - { - assertThat(e).hasMessageContaining("Namespace name exceeds a maximum length of 1024"); - } - - @Test - @Override - public void testCreateSchemaWithLongName() - { - // Despite that create operating fails, invalid schema can be listed - assertThatThrownBy(super::testCreateSchemaWithLongName) - .isInstanceOf(AssertionError.class) - .hasMessageMatching(Pattern.compile(".*Expecting.*not\\sto\\scontain.*", MULTILINE | DOTALL)); - } - - @Test - @Override - public void testInsert() - { - @Language("SQL") String query = "SELECT cast(uuid() AS varchar) AS uuid, orderdate, orderkey FROM orders"; - - assertUpdate("CREATE TABLE test_insert AS " + query + " WITH NO DATA", 0); - assertQuery("SELECT count(*) FROM test_insert", "SELECT 0"); - - assertUpdate("INSERT INTO test_insert " + query, "SELECT count(*) FROM orders"); - - assertQuery("SELECT orderdate, orderkey FROM test_insert", "SELECT orderdate, orderkey FROM orders"); - // Override because base class error: Cannot insert null row ID - assertUpdate("INSERT INTO test_insert (uuid, orderkey) VALUES ('000000', -1)", 1); - assertUpdate("INSERT INTO test_insert (uuid, orderdate) VALUES ('000001', DATE '2001-01-01')", 1); - assertUpdate("INSERT INTO test_insert (uuid, orderkey, orderdate) VALUES ('000002', -2, DATE '2001-01-02')", 1); - assertUpdate("INSERT INTO test_insert (uuid, orderdate, orderkey) VALUES ('000003', DATE '2001-01-03', -3)", 1); - - assertQuery("SELECT orderdate, orderkey FROM test_insert", - "SELECT orderdate, orderkey FROM orders" - + " UNION ALL SELECT null, -1" - + " UNION ALL SELECT DATE '2001-01-01', null" - + " UNION ALL SELECT DATE '2001-01-02', -2" - + " UNION ALL SELECT DATE '2001-01-03', -3"); - - // UNION query produces columns in the opposite order - // of how they are declared in the table schema - assertUpdate( - "INSERT INTO test_insert (uuid, orderkey, orderdate) " + - "SELECT cast(uuid() AS varchar) AS uuid, orderkey, orderdate FROM orders " + - "UNION ALL " + - "SELECT cast(uuid() AS varchar) AS uuid, orderkey, orderdate FROM orders", - "SELECT 2 * count(*) FROM orders"); - - assertUpdate("DROP TABLE test_insert"); - } - - @Test - @Override // Overridden because we currently do not support arrays with null elements - public void testInsertArray() - { - assertUpdate("CREATE TABLE test_insert_array (a ARRAY, b ARRAY)"); - - // assertUpdate("INSERT INTO test_insert_array (a) VALUES (ARRAY[null])", 1); TODO support ARRAY with null elements - - assertUpdate("INSERT INTO test_insert_array (a, b) VALUES (ARRAY[1.23E1], ARRAY[1.23E1])", 1); - assertQuery("SELECT a[1], b[1] FROM test_insert_array", "VALUES (12.3, 12)"); - - assertUpdate("DROP TABLE test_insert_array"); - } - - @Test - public void testInsertDuplicateRows() - { - // This test case tests the Accumulo connectors override capabilities - // When a row is inserted into a table where a row with the same row ID already exists, - // the cells of the existing row are overwritten with the new values - try { - assertUpdate("CREATE TABLE test_insert_duplicate AS SELECT 1 a, 2 b, '3' c", 1); - assertQuery("SELECT a, b, c FROM test_insert_duplicate", "SELECT 1, 2, '3'"); - assertUpdate("INSERT INTO test_insert_duplicate (a, c) VALUES (1, '4')", 1); - assertUpdate("INSERT INTO test_insert_duplicate (a, b) VALUES (1, 3)", 1); - assertQuery("SELECT a, b, c FROM test_insert_duplicate", "SELECT 1, 3, '4'"); - } - finally { - assertUpdate("DROP TABLE test_insert_duplicate"); - } - } - - @Test - @Override - public void testShowColumns() - { - // Override base class because table descriptions for Accumulo connector include extra info - MaterializedResult actual = computeActual("SHOW COLUMNS FROM orders"); - - assertThat(actual.getMaterializedRows().get(0).getField(0)).isEqualTo("orderkey"); - assertThat(actual.getMaterializedRows().get(0).getField(1)).isEqualTo("bigint"); - assertThat(actual.getMaterializedRows().get(1).getField(0)).isEqualTo("custkey"); - assertThat(actual.getMaterializedRows().get(1).getField(1)).isEqualTo("bigint"); - assertThat(actual.getMaterializedRows().get(2).getField(0)).isEqualTo("orderstatus"); - assertThat(actual.getMaterializedRows().get(2).getField(1)).isEqualTo("varchar(1)"); - assertThat(actual.getMaterializedRows().get(3).getField(0)).isEqualTo("totalprice"); - assertThat(actual.getMaterializedRows().get(3).getField(1)).isEqualTo("double"); - assertThat(actual.getMaterializedRows().get(4).getField(0)).isEqualTo("orderdate"); - assertThat(actual.getMaterializedRows().get(4).getField(1)).isEqualTo("date"); - assertThat(actual.getMaterializedRows().get(5).getField(0)).isEqualTo("orderpriority"); - assertThat(actual.getMaterializedRows().get(5).getField(1)).isEqualTo("varchar(15)"); - assertThat(actual.getMaterializedRows().get(6).getField(0)).isEqualTo("clerk"); - assertThat(actual.getMaterializedRows().get(6).getField(1)).isEqualTo("varchar(15)"); - assertThat(actual.getMaterializedRows().get(7).getField(0)).isEqualTo("shippriority"); - assertThat(actual.getMaterializedRows().get(7).getField(1)).isEqualTo("integer"); - assertThat(actual.getMaterializedRows().get(8).getField(0)).isEqualTo("comment"); - assertThat(actual.getMaterializedRows().get(8).getField(1)).isEqualTo("varchar(79)"); - } - - @Test - public void testMultiInBelowCardinality() - { - assertQuery("SELECT COUNT(*) FROM partsupp WHERE partkey = 1", "SELECT 4"); - assertQuery("SELECT COUNT(*) FROM partsupp WHERE partkey = 2", "SELECT 4"); - assertQuery("SELECT COUNT(*) FROM partsupp WHERE partkey IN (1, 2)", "SELECT 8"); - } - - @Test - public void testSelectNullValue() - { - try { - assertUpdate("CREATE TABLE test_select_null_value AS SELECT 1 a, 2 b, CAST(NULL AS BIGINT) c", 1); - assertQuery("SELECT * FROM test_select_null_value", "SELECT 1, 2, NULL"); - assertQuery("SELECT a, c FROM test_select_null_value", "SELECT 1, NULL"); - } - finally { - assertUpdate("DROP TABLE test_select_null_value"); - } - } - - @Test - public void testCreateTableEmptyColumns() - { - try { - assertUpdate("CREATE TABLE test_create_table_empty_columns WITH (column_mapping = 'a:a:a,b::b,c:c:,d::', index_columns='a,b,c,d') AS SELECT 1 id, 2 a, 3 b, 4 c, 5 d", 1); - assertQuery("SELECT * FROM test_create_table_empty_columns", "SELECT 1, 2, 3, 4, 5"); - assertQuery("SELECT * FROM test_create_table_empty_columns WHERE a = 2", "SELECT 1, 2, 3, 4, 5"); - assertQuery("SELECT * FROM test_create_table_empty_columns WHERE b = 3", "SELECT 1, 2, 3, 4, 5"); - assertQuery("SELECT * FROM test_create_table_empty_columns WHERE c = 4", "SELECT 1, 2, 3, 4, 5"); - assertQuery("SELECT * FROM test_create_table_empty_columns WHERE d = 5", "SELECT 1, 2, 3, 4, 5"); - } - finally { - assertUpdate("DROP TABLE test_create_table_empty_columns"); - } - } - - @Override - protected MaterializedResult getDescribeOrdersResult() - { - return resultBuilder(getSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR) - .row("orderkey", "bigint", "Accumulo row ID", "") - .row("custkey", "bigint", "Accumulo column custkey:custkey. Indexed: false", "") - .row("orderstatus", "varchar(1)", "Accumulo column orderstatus:orderstatus. Indexed: false", "") - .row("totalprice", "double", "Accumulo column totalprice:totalprice. Indexed: false", "") - .row("orderdate", "date", "Accumulo column orderdate:orderdate. Indexed: true", "") - .row("orderpriority", "varchar(15)", "Accumulo column orderpriority:orderpriority. Indexed: false", "") - .row("clerk", "varchar(15)", "Accumulo column clerk:clerk. Indexed: false", "") - .row("shippriority", "integer", "Accumulo column shippriority:shippriority. Indexed: false", "") - .row("comment", "varchar(79)", "Accumulo column comment:comment. Indexed: false", "") - .build(); - } - - @Test - @Override - public void testInsertSameValues() - { - assertThatThrownBy(super::testInsertSameValues) - .isInstanceOf(AssertionError.class) - .hasMessageContaining("not equal"); - } - - @Override - protected Optional filterDataMappingSmokeTestData(DataMappingTestSetup dataMappingTestSetup) - { - String typeName = dataMappingTestSetup.getTrinoTypeName(); - if (typeName.startsWith("decimal(") - || typeName.equals("time(6)") - || typeName.equals("timestamp(6)") - || typeName.equals("timestamp(3) with time zone") - || typeName.equals("timestamp(6) with time zone") - || typeName.startsWith("char(")) { - return Optional.of(dataMappingTestSetup.asUnsupported()); - } - - return Optional.of(dataMappingTestSetup); - } - - @Override - protected Optional filterCaseSensitiveDataMappingTestData(DataMappingTestSetup dataMappingTestSetup) - { - String typeName = dataMappingTestSetup.getTrinoTypeName(); - if (typeName.equals("char(1)")) { - return Optional.of(dataMappingTestSetup.asUnsupported()); - } - return Optional.of(dataMappingTestSetup); - } - - @Test - @Override - public void testCharVarcharComparison() - { - assertThatThrownBy(super::testCharVarcharComparison) - .hasMessage("Unsupported Trino type: char(3)"); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloMetadataManager.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloMetadataManager.java deleted file mode 100644 index 301b462748dd7..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestAccumuloMetadataManager.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import com.google.common.collect.ImmutableList; -import io.trino.plugin.accumulo.conf.AccumuloConfig; -import io.trino.plugin.accumulo.conf.AccumuloTableProperties; -import io.trino.plugin.accumulo.index.ColumnCardinalityCache; -import io.trino.plugin.accumulo.index.IndexLookup; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.metadata.ZooKeeperMetadataManager; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.SchemaTableName; -import org.apache.accumulo.core.client.AccumuloClient; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; - -@TestInstance(PER_CLASS) -public class TestAccumuloMetadataManager -{ - private AccumuloClient client; - private AccumuloMetadataManager metadataManager; - private ZooKeeperMetadataManager zooKeeperMetadataManager; - - @BeforeAll - public void setUp() - throws Exception - { - TestingAccumuloServer server = TestingAccumuloServer.getInstance(); - AccumuloConfig config = new AccumuloConfig() - .setZooKeepers(server.getZooKeepers()) - .setInstance(server.getInstanceName()) - .setUsername("root") - .setPassword("secret"); - client = server.createClient(); - zooKeeperMetadataManager = new ZooKeeperMetadataManager(config, TESTING_TYPE_MANAGER); - metadataManager = new AccumuloMetadataManager(client, config, zooKeeperMetadataManager, new AccumuloTableManager(client), new IndexLookup(client, new ColumnCardinalityCache(client, config))); - } - - @AfterAll - public void tearDown() - { - zooKeeperMetadataManager.close(); - zooKeeperMetadataManager = null; - metadataManager = null; - client.close(); - client = null; - } - - @Test - public void testCreateTableEmptyAccumuloColumn() - { - SchemaTableName tableName = new SchemaTableName("default", "test_create_table_empty_accumulo_column"); - - try { - List columns = ImmutableList.of( - new ColumnMetadata("id", BIGINT), - new ColumnMetadata("a", BIGINT), - new ColumnMetadata("b", BIGINT), - new ColumnMetadata("c", BIGINT), - new ColumnMetadata("d", BIGINT)); - - Map properties = new HashMap<>(); - new AccumuloTableProperties().getTableProperties().forEach(meta -> properties.put(meta.getName(), meta.getDefaultValue())); - properties.put("external", true); - properties.put("column_mapping", "a:a:a,b::b,c:c:,d::"); - metadataManager.createTable(new ConnectorTableMetadata(tableName, columns, properties)); - assertThat(metadataManager.getTable(tableName)).isNotNull(); - } - finally { - AccumuloTable table = zooKeeperMetadataManager.getTable(tableName); - if (table != null) { - metadataManager.dropTable(table); - } - } - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestingAccumuloServer.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestingAccumuloServer.java deleted file mode 100644 index f9b32de3c49d0..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/TestingAccumuloServer.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo; - -import io.trino.testing.TestingProperties; -import io.trino.testing.containers.junit.ReportLeakedContainers; -import org.apache.accumulo.core.client.Accumulo; -import org.apache.accumulo.core.client.AccumuloClient; -import org.testcontainers.containers.FixedHostPortGenericContainer; -import org.testcontainers.containers.wait.strategy.Wait; - -import java.time.Duration; - -import static java.lang.String.format; -import static org.testcontainers.utility.MountableFile.forClasspathResource; - -public class TestingAccumuloServer -{ - private static final int ACCUMULO_MASTER_PORT = 9999; - private static final int ACCUMULO_TSERVER_PORT = 9997; - private static final int ACCUMULO_MONITOR_UI_PORT = 9995; - private static final int ZOOKEEPER_PORT = 2181; - - private static final String ACCUMULO_EXT_PATH = "/usr/local/lib/accumulo/lib"; - private static final String ITERATORS_JAR = "trino-accumulo-iterators.jar"; - - private static final TestingAccumuloServer instance = new TestingAccumuloServer(); - - private final FixedHostPortGenericContainer accumuloContainer; - - public static TestingAccumuloServer getInstance() - { - return instance; - } - - private TestingAccumuloServer() - { - accumuloContainer = new FixedHostPortGenericContainer<>("ghcr.io/trinodb/testing/accumulo:" + TestingProperties.getDockerImagesVersion()); - accumuloContainer.withFixedExposedPort(ACCUMULO_MASTER_PORT, ACCUMULO_MASTER_PORT); - accumuloContainer.withFixedExposedPort(ACCUMULO_TSERVER_PORT, ACCUMULO_TSERVER_PORT); - accumuloContainer.withFixedExposedPort(ACCUMULO_MONITOR_UI_PORT, ACCUMULO_MONITOR_UI_PORT); - accumuloContainer.withExposedPorts(ZOOKEEPER_PORT); - accumuloContainer.withCreateContainerCmdModifier(cmd -> cmd - .withHostName("localhost") - .withEnv("ADDRESS=0.0.0.0") - .withEntrypoint("supervisord", "-c", "/etc/supervisord.conf")); - accumuloContainer.waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofMinutes(10))); - accumuloContainer.withCopyFileToContainer(forClasspathResource(ITERATORS_JAR), ACCUMULO_EXT_PATH + "/" + ITERATORS_JAR); - - // No need for an explicit stop since this server is a singleton - // and the container will be stopped by TestContainers on shutdown - // TODO Change this class to not be a singleton - // https://github.com/trinodb/trino/issues/5842 - accumuloContainer.start(); - ReportLeakedContainers.ignoreContainerId(accumuloContainer.getContainerId()); - } - - public String getInstanceName() - { - return "default"; - } - - public String getZooKeepers() - { - return format("%s:%s", accumuloContainer.getHost(), accumuloContainer.getMappedPort(ZOOKEEPER_PORT)); - } - - public String getUser() - { - return "root"; - } - - public String getPassword() - { - return "secret"; - } - - public AccumuloClient createClient() - { - return Accumulo.newClient().to(getInstanceName(), getZooKeepers()) - .as(getUser(), getPassword()) - .build(); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/conf/TestAccumuloConfig.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/conf/TestAccumuloConfig.java deleted file mode 100644 index 2c7614ce09632..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/conf/TestAccumuloConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.conf; - -import com.google.common.collect.ImmutableMap; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MINUTES; - -class TestAccumuloConfig -{ - @Test - void testDefaults() - { - assertRecordedDefaults(recordDefaults(AccumuloConfig.class) - .setInstance(null) - .setZooKeepers(null) - .setUsername(null) - .setPassword(null) - .setZkMetadataRoot("/trino-accumulo") - .setCardinalityCacheSize(100_000) - .setCardinalityCacheExpiration(new Duration(5, MINUTES))); - } - - @Test - void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("accumulo.instance", "accumulo") - .put("accumulo.zookeepers", "zookeeper.example.com:2181") - .put("accumulo.username", "user") - .put("accumulo.password", "password") - .put("accumulo.zookeeper.metadata.root", "/trino-accumulo-metadata") - .put("accumulo.cardinality.cache.size", "999") - .put("accumulo.cardinality.cache.expire.duration", "1h") - .buildOrThrow(); - - AccumuloConfig expected = new AccumuloConfig() - .setInstance("accumulo") - .setZooKeepers("zookeeper.example.com:2181") - .setUsername("user") - .setPassword("password") - .setZkMetadataRoot("/trino-accumulo-metadata") - .setCardinalityCacheSize(999) - .setCardinalityCacheExpiration(new Duration(1, HOURS)); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/index/TestIndexer.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/index/TestIndexer.java deleted file mode 100644 index d9e7a6b3e9ff2..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/index/TestIndexer.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.index; - -import com.google.common.collect.ImmutableList; -import com.google.common.io.Files; -import io.trino.plugin.accumulo.metadata.AccumuloTable; -import io.trino.plugin.accumulo.model.AccumuloColumnHandle; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.plugin.accumulo.serializers.LexicoderRowSerializer; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.Type; -import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.minicluster.MiniAccumuloCluster; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.IOException; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.Optional; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; - -@TestInstance(PER_CLASS) -@Execution(CONCURRENT) -public class TestIndexer -{ - private static final LexicoderRowSerializer SERIALIZER = new LexicoderRowSerializer(); - private static final AccumuloColumnHandle c1 = new AccumuloColumnHandle("id", Optional.empty(), Optional.empty(), VARCHAR, 0, "", Optional.empty(), false); - private static final AccumuloColumnHandle c2 = new AccumuloColumnHandle("age", Optional.of("cf"), Optional.of("age"), BIGINT, 1, "", Optional.empty(), true); - private static final AccumuloColumnHandle c3 = new AccumuloColumnHandle("firstname", Optional.of("cf"), Optional.of("firstname"), VARCHAR, 2, "", Optional.empty(), true); - private static final AccumuloColumnHandle c4 = new AccumuloColumnHandle("arr", Optional.of("cf"), Optional.of("arr"), new ArrayType(VARCHAR), 3, "", Optional.empty(), true); - - private static byte[] encode(Type type, Object v) - { - return SERIALIZER.encode(type, v); - } - - private static final byte[] AGE = bytes("age"); - private static final byte[] CF = bytes("cf"); - private static final byte[] FIRSTNAME = bytes("firstname"); - private static final byte[] SENDERS = bytes("arr"); - - private static final byte[] M1_ROWID = encode(VARCHAR, "row1"); - private static final byte[] AGE_VALUE = encode(BIGINT, 27L); - private static final byte[] M1_FNAME_VALUE = encode(VARCHAR, "alice"); - private static final byte[] M1_ARR_VALUE = encode(new ArrayType(VARCHAR), AccumuloRowSerializer.getBlockFromArray(VARCHAR, ImmutableList.of("abc", "def", "ghi"))); - - private static final byte[] M2_ROWID = encode(VARCHAR, "row2"); - private static final byte[] M2_FNAME_VALUE = encode(VARCHAR, "bob"); - private static final byte[] M2_ARR_VALUE = encode(new ArrayType(VARCHAR), AccumuloRowSerializer.getBlockFromArray(VARCHAR, ImmutableList.of("ghi", "mno", "abc"))); - - private Mutation m1; - private Mutation m2; - private Mutation m1v; - private Mutation m2v; - - private MiniAccumuloCluster cluster; - - @BeforeAll - public void setupClass() - throws IOException, InterruptedException - { - m1 = new Mutation(M1_ROWID); - m1.put(CF, AGE, AGE_VALUE); - m1.put(CF, FIRSTNAME, M1_FNAME_VALUE); - m1.put(CF, SENDERS, M1_ARR_VALUE); - - m2 = new Mutation(M2_ROWID); - m2.put(CF, AGE, AGE_VALUE); - m2.put(CF, FIRSTNAME, M2_FNAME_VALUE); - m2.put(CF, SENDERS, M2_ARR_VALUE); - - ColumnVisibility visibility1 = new ColumnVisibility("private"); - ColumnVisibility visibility2 = new ColumnVisibility("moreprivate"); - m1v = new Mutation(M1_ROWID); - m1v.put(CF, AGE, visibility1, AGE_VALUE); - m1v.put(CF, FIRSTNAME, visibility1, M1_FNAME_VALUE); - m1v.put(CF, SENDERS, visibility2, M1_ARR_VALUE); - - m2v = new Mutation(M2_ROWID); - m2v.put(CF, AGE, visibility1, AGE_VALUE); - m2v.put(CF, FIRSTNAME, visibility2, M2_FNAME_VALUE); - m2v.put(CF, SENDERS, visibility2, M2_ARR_VALUE); - - cluster = new MiniAccumuloCluster(Files.createTempDir().getAbsoluteFile(), "rootpassword"); - cluster.start(); - } - - @AfterAll - public void close() - throws IOException, InterruptedException - { - cluster.stop(); - } - - @Test - public void testMutationIndex() - throws Exception - { - AccumuloTable table = new AccumuloTable("default", "index_test_mutation_index", ImmutableList.of(c1, c2, c3, c4), "id", true, LexicoderRowSerializer.class.getCanonicalName(), Optional.empty()); - AccumuloClient client = cluster.createAccumuloClient("root", new PasswordToken("rootpassword")); - client.tableOperations().create(table.getFullTableName()); - client.tableOperations().create(table.getIndexTableName()); - client.tableOperations().create(table.getMetricsTableName()); - - for (IteratorSetting s : Indexer.getMetricIterators(table)) { - client.tableOperations().attachIterator(table.getMetricsTableName(), s); - } - - Indexer indexer = new Indexer(client, new Authorizations(), table, new BatchWriterConfig()); - indexer.index(m1); - indexer.flush(); - - Scanner scan = client.createScanner(table.getIndexTableName(), new Authorizations()); - scan.setRange(new Range()); - - Iterator> iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", ""); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", ""); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - - scan = client.createScanner(table.getMetricsTableName(), new Authorizations()); - scan.setRange(new Range()); - - iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "1"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "1"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row1"); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "1"); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - - indexer.index(m2); - indexer.close(); - - scan = client.createScanner(table.getIndexTableName(), new Authorizations()); - scan.setRange(new Range()); - iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", ""); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", ""); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", ""); - assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", ""); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", ""); - assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", ""); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - - scan = client.createScanner(table.getMetricsTableName(), new Authorizations()); - scan.setRange(new Range()); - - iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "2"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "2"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row2"); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "2"); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1"); - assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "2"); - assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "1"); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - } - - @Test - public void testMutationIndexWithVisibilities() - throws Exception - { - AccumuloTable table = new AccumuloTable("default", "index_test_mutation_index_visibility", ImmutableList.of(c1, c2, c3, c4), "id", true, LexicoderRowSerializer.class.getCanonicalName(), Optional.empty()); - AccumuloClient client = cluster.createAccumuloClient("root", new PasswordToken("rootpassword")); - Authorizations authorizations = new Authorizations("private", "moreprivate"); - client.securityOperations().changeUserAuthorizations("root", authorizations); - - client.tableOperations().create(table.getFullTableName()); - client.tableOperations().create(table.getIndexTableName()); - client.tableOperations().create(table.getMetricsTableName()); - - for (IteratorSetting s : Indexer.getMetricIterators(table)) { - client.tableOperations().attachIterator(table.getMetricsTableName(), s); - } - - Indexer indexer = new Indexer(client, new Authorizations(), table, new BatchWriterConfig()); - indexer.index(m1); - indexer.index(m1v); - indexer.flush(); - - Scanner scan = client.createScanner(table.getIndexTableName(), authorizations); - scan.setRange(new Range()); - - Iterator> iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", ""); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "private", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "moreprivate", ""); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", ""); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "private", ""); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "moreprivate", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "moreprivate", ""); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - - scan = client.createScanner(table.getMetricsTableName(), authorizations); - scan.setRange(new Range()); - - iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "1"); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "private", "1"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "2"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row1"); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "moreprivate", "1"); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1"); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "private", "1"); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "moreprivate", "1"); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "moreprivate", "1"); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - - indexer.index(m2); - indexer.index(m2v); - indexer.close(); - - scan = client.createScanner(table.getIndexTableName(), authorizations); - scan.setRange(new Range()); - iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", ""); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "private", ""); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", ""); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", "private", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "moreprivate", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", ""); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", "moreprivate", ""); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", ""); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "private", ""); - assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", ""); - assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", "moreprivate", ""); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "moreprivate", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "moreprivate", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", ""); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", "moreprivate", ""); - assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", ""); - assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", "moreprivate", ""); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - - scan = client.createScanner(table.getMetricsTableName(), authorizations); - scan.setRange(new Range()); - - iter = scan.iterator(); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "2"); - assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "private", "2"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "4"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1"); - assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row2"); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "2"); - assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "moreprivate", "2"); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1"); - assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "private", "1"); - assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "1"); - assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "moreprivate", "1"); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "moreprivate", "1"); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "2"); - assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "moreprivate", "2"); - assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "1"); - assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "moreprivate", "1"); - assertThat(iter.hasNext()).isFalse(); - - scan.close(); - } - - private static void assertKeyValuePair(Entry e, byte[] row, String cf, String cq, String value) - { - assertThat(row).isEqualTo(e.getKey().getRow().copyBytes()); - assertThat(cf).isEqualTo(e.getKey().getColumnFamily().toString()); - assertThat(cq).isEqualTo(e.getKey().getColumnQualifier().toString()); - assertThat(value).isEqualTo(e.getValue().toString()); - } - - private static void assertKeyValuePair(Entry e, byte[] row, String cf, String cq, String cv, String value) - { - assertThat(row).isEqualTo(e.getKey().getRow().copyBytes()); - assertThat(cf).isEqualTo(e.getKey().getColumnFamily().toString()); - assertThat(cq).isEqualTo(e.getKey().getColumnQualifier().toString()); - assertThat(cv).isEqualTo(e.getKey().getColumnVisibility().toString()); - assertThat(value).isEqualTo(e.getValue().toString()); - } - - private static byte[] bytes(String s) - { - return s.getBytes(UTF_8); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestAccumuloSplit.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestAccumuloSplit.java deleted file mode 100644 index 926f775cce6ac..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestAccumuloSplit.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.google.common.collect.ImmutableList; -import io.airlift.json.JsonCodec; -import org.apache.accumulo.core.data.Range; -import org.junit.jupiter.api.Test; - -import java.util.Optional; -import java.util.stream.Collectors; - -import static org.assertj.core.api.Assertions.assertThat; - -public class TestAccumuloSplit -{ - private final JsonCodec codec = JsonCodec.jsonCodec(AccumuloSplit.class); - - @Test - public void testJsonRoundTrip() - { - AccumuloSplit expected = new AccumuloSplit( - ImmutableList.of(new Range(), new Range("bar", "foo"), new Range("bar", false, "baz", false)).stream().map(SerializedRange::serialize).collect(Collectors.toList()), - Optional.of("localhost:9000")); - - String json = codec.toJson(expected); - AccumuloSplit actual = codec.fromJson(json); - assertSplit(actual, expected); - } - - @Test - public void testJsonRoundTripEmptyThings() - { - AccumuloSplit expected = new AccumuloSplit( - ImmutableList.of(), - Optional.empty()); - - String json = codec.toJson(expected); - AccumuloSplit actual = codec.fromJson(json); - assertSplit(actual, expected); - } - - private static void assertSplit(AccumuloSplit actual, AccumuloSplit expected) - { - assertThat(actual.getAddresses()).isEqualTo(expected.getAddresses()); - assertThat(actual.getHostPort()).isEqualTo(expected.getHostPort()); - assertThat(actual.getRanges()).isEqualTo(expected.getRanges()); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestField.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestField.java deleted file mode 100644 index d22ea29149ed6..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestField.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.airlift.slice.Slices; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.block.Block; -import io.trino.spi.block.SqlMap; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeSignatureParameter; -import org.junit.jupiter.api.Test; - -import java.sql.Time; -import java.sql.Timestamp; -import java.time.LocalDate; -import java.util.GregorianCalendar; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.lang.Float.floatToIntBits; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class TestField -{ - @Test - public void testTypeIsNull() - { - assertThatThrownBy(() -> new Field(null, null)) - .isInstanceOf(NullPointerException.class) - .hasMessage("type is null"); - } - - @Test - public void testArray() - { - Type type = new ArrayType(VARCHAR); - Block expected = AccumuloRowSerializer.getBlockFromArray(VARCHAR, ImmutableList.of("a", "b", "c")); - Field f1 = new Field(expected, type); - assertThat(f1.getArray()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testBoolean() - { - Type type = BOOLEAN; - Field f1 = new Field(true, type); - assertThat(f1.getBoolean().booleanValue()).isEqualTo(true); - assertThat(f1.getObject()).isEqualTo(true); - assertThat(f1.getType()).isEqualTo(type); - - f1 = new Field(false, type); - assertThat(f1.getBoolean().booleanValue()).isEqualTo(false); - assertThat(f1.getObject()).isEqualTo(false); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testDate() - { - Type type = DATE; - long expected = LocalDate.parse("1999-01-01").toEpochDay(); - Field f1 = new Field(10592L, type); - assertThat(f1.getDate()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testDouble() - { - Type type = DOUBLE; - Double expected = 123.45678; - Field f1 = new Field(expected, type); - assertThat(f1.getDouble()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testFloat() - { - Type type = REAL; - Float expected = 123.45678f; - Field f1 = new Field((long) floatToIntBits(expected), type); - assertThat(f1.getFloat()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testInt() - { - Type type = INTEGER; - Integer expected = 12345678; - Field f1 = new Field((long) expected, type); - assertThat(f1.getInt()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testLong() - { - Type type = BIGINT; - Long expected = 12345678L; - Field f1 = new Field(expected, type); - assertThat(f1.getLong()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testMap() - { - Type type = TESTING_TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of( - TypeSignatureParameter.typeParameter(VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(BIGINT.getTypeSignature()))); - SqlMap expected = AccumuloRowSerializer.getSqlMapFromMap(type, ImmutableMap.of("a", 1L, "b", 2L, "c", 3L)); - Field f1 = new Field(expected, type); - assertThat(f1.getMap()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testSmallInt() - { - Type type = SMALLINT; - Short expected = 12345; - Field f1 = new Field((long) expected, type); - assertThat(f1.getShort()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testTime() - { - Type type = TIME_MILLIS; - Time expected = new Time(new GregorianCalendar(1970, 0, 1, 12, 30, 0).getTime().getTime()); - Field f1 = new Field(70200000L, type); - assertThat(f1.getTime()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testTimestamp() - { - Type type = TIMESTAMP_MILLIS; - Timestamp expected = new Timestamp(new GregorianCalendar(1999, 0, 1, 12, 30, 0).getTime().getTime()); - Field f1 = new Field(915_219_000_000_000L, type); - assertThat(f1.getTimestamp()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testTinyInt() - { - Type type = TINYINT; - Byte expected = 123; - Field f1 = new Field((long) expected, type); - assertThat(f1.getByte()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testVarbinary() - { - Type type = VARBINARY; - byte[] expected = "O'Leary".getBytes(UTF_8); - Field f1 = new Field(Slices.wrappedBuffer(expected.clone()), type); - assertThat(f1.getVarbinary()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } - - @Test - public void testVarchar() - { - Type type = VARCHAR; - String expected = "O'Leary"; - Field f1 = new Field(utf8Slice(expected), type); - assertThat(f1.getVarchar()).isEqualTo(expected); - assertThat(f1.getObject()).isEqualTo(expected); - assertThat(f1.getType()).isEqualTo(type); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestRow.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestRow.java deleted file mode 100644 index 2d7d182e8315b..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestRow.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import com.google.common.collect.ImmutableList; -import io.airlift.slice.Slices; -import io.trino.plugin.accumulo.serializers.AccumuloRowSerializer; -import io.trino.spi.type.ArrayType; -import org.junit.jupiter.api.Test; - -import java.sql.Timestamp; -import java.time.LocalDateTime; -import java.util.GregorianCalendar; - -import static io.airlift.slice.Slices.utf8Slice; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.lang.Float.floatToIntBits; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -public class TestRow -{ - @Test - public void testRow() - { - Row r1 = new Row(); - r1.addField(new Field(AccumuloRowSerializer.getBlockFromArray(VARCHAR, ImmutableList.of("a", "b", "c")), new ArrayType(VARCHAR))); - r1.addField(true, BOOLEAN); - r1.addField(new Field(10592L, DATE)); - r1.addField(123.45678, DOUBLE); - r1.addField(new Field((long) floatToIntBits(123.45678f), REAL)); - r1.addField(12345678L, INTEGER); - r1.addField(new Field(12345678L, BIGINT)); - r1.addField(new Field(12345L, SMALLINT)); - r1.addField(new GregorianCalendar(1970, 0, 1, 12, 30, 0).getTime().getTime(), TIME_MILLIS); - r1.addField(new Field(Timestamp.valueOf(LocalDateTime.of(1999, 1, 1, 12, 30, 0)).getTime(), TIMESTAMP_MILLIS)); - r1.addField((long) 123, TINYINT); - r1.addField(new Field(Slices.wrappedBuffer("O'Leary".getBytes(UTF_8)), VARBINARY)); - r1.addField(utf8Slice("O'Leary"), VARCHAR); - r1.addField(null, VARCHAR); - - assertThat(r1.length()).isEqualTo(14); - } - - @Test - public void testRowTypeIsNull() - { - Row r1 = new Row(); - assertThatThrownBy(() -> r1.addField(VARCHAR, null)) - .isInstanceOf(NullPointerException.class) - .hasMessage("type is null"); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestSerializedRange.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestSerializedRange.java deleted file mode 100644 index dcb3804e93769..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/model/TestSerializedRange.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.model; - -import org.apache.accumulo.core.data.Range; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class TestSerializedRange -{ - @Test - public void testJsonRoundTrip() - { - Range exact = new Range("foo"); - Range range = new Range("bar", "foo"); - Range exclusiveRange = new Range("asiago", false, "bagel", false); - - assertThat(SerializedRange.serialize(exact).deserialize()).isEqualTo(exact); - assertThat(SerializedRange.serialize(range).deserialize()).isEqualTo(range); - assertThat(SerializedRange.serialize(exclusiveRange).deserialize()).isEqualTo(exclusiveRange); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/AbstractTestAccumuloRowSerializer.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/AbstractTestAccumuloRowSerializer.java deleted file mode 100644 index 5f7877b5555d7..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/AbstractTestAccumuloRowSerializer.java +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.trino.spi.type.ArrayType; -import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeSignatureParameter; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.junit.jupiter.api.Test; - -import java.sql.Time; -import java.sql.Timestamp; -import java.time.LocalDate; -import java.util.AbstractMap.SimpleImmutableEntry; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.TIME_MILLIS; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; - -public abstract class AbstractTestAccumuloRowSerializer -{ - private final Class serializerClass; - private static final String COLUMN_NAME = "foo"; - - protected AbstractTestAccumuloRowSerializer(Class serializerClass) - { - this.serializerClass = serializerClass; - } - - @Test - public void testArray() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = new ArrayType(VARCHAR); - List expected = ImmutableList.of("a", "b", "c"); - byte[] data = serializer.encode(type, AccumuloRowSerializer.getBlockFromArray(VARCHAR, expected)); - List actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = AccumuloRowSerializer.getArrayFromBlock(VARCHAR, serializer.getArray(COLUMN_NAME, type)); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testBoolean() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = BOOLEAN; - byte[] data = serializer.encode(type, true); - boolean actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(true); - - deserializeData(serializer, data); - actual = serializer.getBoolean(COLUMN_NAME); - assertThat(actual).isEqualTo(true); - - data = serializer.encode(type, false); - actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(false); - - deserializeData(serializer, data); - actual = serializer.getBoolean(COLUMN_NAME); - assertThat(actual).isEqualTo(false); - } - - @Test - public void testDate() - throws Exception - { - long expected = LocalDate.parse("2001-02-03").toEpochDay(); - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - byte[] data = serializer.encode(DATE, expected); - - deserializeData(serializer, data); - long actual = serializer.getDate(COLUMN_NAME); - - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testDouble() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = DOUBLE; - Double expected = 123.45678; - byte[] data = serializer.encode(type, expected); - Double actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getDouble(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testFloat() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = REAL; - Float expected = 123.45678f; - byte[] data = serializer.encode(type, expected); - Float actual = ((Double) serializer.decode(type, data)).floatValue(); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getFloat(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testInt() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = INTEGER; - Integer expected = 123456; - byte[] data = serializer.encode(type, expected); - @SuppressWarnings("unchecked") - Integer actual = ((Long) serializer.decode(type, data)).intValue(); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getInt(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testLong() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = BIGINT; - Long expected = 123456L; - byte[] data = serializer.encode(type, expected); - Long actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getLong(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testMap() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = TESTING_TYPE_MANAGER.getParameterizedType(StandardTypes.MAP, ImmutableList.of( - TypeSignatureParameter.typeParameter(VARCHAR.getTypeSignature()), - TypeSignatureParameter.typeParameter(BIGINT.getTypeSignature()))); - Map expected = ImmutableMap.of("a", 1L, "b", 2L, "3", 3L); - byte[] data = serializer.encode(type, AccumuloRowSerializer.getSqlMapFromMap(type, expected)); - Map actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = AccumuloRowSerializer.getMapFromSqlMap(type, serializer.getMap(COLUMN_NAME, type)); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testSmallInt() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = SMALLINT; - Short expected = 12345; - byte[] data = serializer.encode(type, expected); - Short actual = ((Long) serializer.decode(type, data)).shortValue(); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getShort(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testTime() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = TIME_MILLIS; - Time expected = new Time(new java.util.Date().getTime()); - byte[] data = serializer.encode(type, expected); - Time actual = new Time(serializer.decode(type, data)); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getTime(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testTimestamp() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = TIMESTAMP_MILLIS; - Timestamp expected = new Timestamp(new java.util.Date().getTime()); - byte[] data = serializer.encode(type, expected); - Timestamp actual = new Timestamp(serializer.decode(type, data)); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getTimestamp(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testTinyInt() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = TINYINT; - Byte expected = 123; - byte[] data = serializer.encode(type, expected); - Byte actual = ((Long) serializer.decode(type, data)).byteValue(); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getByte(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testVarbinary() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = VARBINARY; - byte[] expected = b(UUID.randomUUID().toString()); - byte[] data = serializer.encode(type, expected); - byte[] actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getVarbinary(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - @Test - public void testVarchar() - throws Exception - { - AccumuloRowSerializer serializer = serializerClass.getConstructor().newInstance(); - Type type = VARCHAR; - String expected = UUID.randomUUID().toString(); - byte[] data = serializer.encode(type, expected); - String actual = serializer.decode(type, data); - assertThat(actual).isEqualTo(expected); - - deserializeData(serializer, data); - actual = serializer.getVarchar(COLUMN_NAME); - assertThat(actual).isEqualTo(expected); - } - - protected void deserializeData(AccumuloRowSerializer serializer, byte[] data) - { - Mutation m = new Mutation("row"); - m.put(b("a"), b("a"), data); - Key key = new Key(b("row"), b("a"), b("b"), b(), 0, false); - Value value = new Value(data); - serializer.setMapping(COLUMN_NAME, "a", "b"); - serializer.deserialize(new SimpleImmutableEntry<>(key, value)); - } - - protected static byte[] b(String str) - { - return str.getBytes(UTF_8); - } - - protected static byte[] b() - { - return new byte[0]; - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestLexicoderRowSerializer.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestLexicoderRowSerializer.java deleted file mode 100644 index d62627e8bb648..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestLexicoderRowSerializer.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -public class TestLexicoderRowSerializer - extends AbstractTestAccumuloRowSerializer -{ - public TestLexicoderRowSerializer() - { - super(LexicoderRowSerializer.class); - } -} diff --git a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestStringRowSerializer.java b/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestStringRowSerializer.java deleted file mode 100644 index 99433934cbdc4..0000000000000 --- a/plugin/trino-accumulo/src/test/java/io/trino/plugin/accumulo/serializers/TestStringRowSerializer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.accumulo.serializers; - -import org.junit.jupiter.api.Test; - -public class TestStringRowSerializer - extends AbstractTestAccumuloRowSerializer -{ - public TestStringRowSerializer() - { - super(StringRowSerializer.class); - } - - @Test - @Override - public void testArray() - { - // Arrays are not supported by StringRowSerializer - } - - @Test - @Override - public void testMap() - { - // Maps are not supported by StringRowSerializer - } -} diff --git a/pom.xml b/pom.xml index 1b77fa8ec3f66..abd73342c2e42 100644 --- a/pom.xml +++ b/pom.xml @@ -59,8 +59,6 @@ lib/trino-parquet lib/trino-plugin-toolkit lib/trino-record-decoder - plugin/trino-accumulo - plugin/trino-accumulo-iterators plugin/trino-base-jdbc plugin/trino-bigquery plugin/trino-blackhole @@ -182,8 +180,6 @@ ${air.test.jvm.additional-arguments.default} - 2.7.7-1 - 3.0.0 275 2.9.6 4.13.2 diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java index 0950b188a0c6f..2d02d12691f24 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java @@ -46,8 +46,6 @@ public void extendEnvironment(Environment.Builder builder) { // blackhole, jmx, tpch are already configured in Standard base env List.of( - // TODO accumulo needs to connect to ZooKeeper, it won't start otherwise - //"accumulo", "bigquery", "cassandra", "clickhouse", diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/accumulo.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/accumulo.properties deleted file mode 100644 index 449733b168877..0000000000000 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/accumulo.properties +++ /dev/null @@ -1,5 +0,0 @@ -connector.name=accumulo -accumulo.instance=host1.invalid -accumulo.zookeepers=host2.invalid -accumulo.username=username -accumulo.password=password