Support skip archive for Glue in Iceberg

plugin/trino-iceberg/pom.xml

@@ -431,6 +431,7 @@
                                 <exclude>**/TestIcebergGlueCatalogAccessOperations.java</exclude>
                                 <exclude>**/TestIcebergGlueCatalogMaterializedViewTest.java</exclude>
                                 <exclude>**/TestIcebergGlueTableOperationsInsertFailure.java</exclude>
+                                <exclude>**/TestIcebergGlueCatalogSkipArchive.java</exclude>
                                 <exclude>**/TestIcebergGcsConnectorSmokeTest.java</exclude>
                             </excludes>
                         </configuration>
@@ -454,6 +455,7 @@
                                 <include>**/TestIcebergGlueCatalogAccessOperations.java</include>
                                 <include>**/TestIcebergGlueCatalogMaterializedViewTest.java</include>
                                 <include>**/TestIcebergGlueTableOperationsInsertFailure.java</include>
+                                <include>**/TestIcebergGlueCatalogSkipArchive.java</include>
                                 <include>**/TestIcebergGcsConnectorSmokeTest.java</include>
                             </includes>
                         </configuration>

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueClientProvider.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.glue;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.glue.AWSGlueAsync;
+import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig;
+import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+
+import java.util.Optional;
+
+import static io.trino.plugin.hive.metastore.glue.GlueHiveMetastore.createAsyncGlueClient;
+import static java.util.Objects.requireNonNull;
+
+public class GlueClientProvider
+        implements Provider<AWSGlueAsync>
+{
+    private final GlueMetastoreStats stats;
+    private final AWSCredentialsProvider credentialsProvider;
+    private final GlueHiveMetastoreConfig glueConfig; // TODO do not keep mutable config instance on a field
+    private final boolean skipArchive;
+
+    @Inject
+    public GlueClientProvider(
+            GlueMetastoreStats stats,
+            AWSCredentialsProvider credentialsProvider,
+            GlueHiveMetastoreConfig glueConfig,
+            IcebergGlueCatalogConfig icebergGlueConfig)
+    {
+        this.stats = requireNonNull(stats, "stats is null");
+        this.credentialsProvider = requireNonNull(credentialsProvider, "credentialsProvider is null");
+        this.glueConfig = glueConfig;
+        this.skipArchive = icebergGlueConfig.isSkipArchive();
+    }
+
+    @Override
+    public AWSGlueAsync get()
+    {
+        return createAsyncGlueClient(glueConfig, credentialsProvider, Optional.of(new SkipArchiveRequestHandler(skipArchive)), stats.newRequestMetricsCollector());
+    }
+}

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/GlueIcebergTableOperationsProvider.java

@@ -13,10 +13,8 @@
  */
 package io.trino.plugin.iceberg.catalog.glue;
 
-import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.glue.AWSGlueAsync;
 import io.trino.filesystem.TrinoFileSystemFactory;
-import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig;
 import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats;
 import io.trino.plugin.iceberg.catalog.IcebergTableOperations;
 import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider;
@@ -27,7 +25,6 @@
 
 import java.util.Optional;
 
-import static io.trino.plugin.hive.metastore.glue.GlueHiveMetastore.createAsyncGlueClient;
 import static java.util.Objects.requireNonNull;
 
 public class GlueIcebergTableOperationsProvider
@@ -41,14 +38,11 @@ public class GlueIcebergTableOperationsProvider
     public GlueIcebergTableOperationsProvider(
             TrinoFileSystemFactory fileSystemFactory,
             GlueMetastoreStats stats,
-            GlueHiveMetastoreConfig glueConfig,
-            AWSCredentialsProvider credentialsProvider)
+            AWSGlueAsync glueClient)
     {
         this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null");
         this.stats = requireNonNull(stats, "stats is null");
-        requireNonNull(glueConfig, "glueConfig is null");
-        requireNonNull(credentialsProvider, "credentialsProvider is null");
-        this.glueClient = createAsyncGlueClient(glueConfig, credentialsProvider, Optional.empty(), stats.newRequestMetricsCollector());
+        this.glueClient = requireNonNull(glueClient, "glueClient is null");
     }
 
     @Override

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/IcebergGlueCatalogConfig.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.glue;
+
+import io.airlift.configuration.Config;
+import io.airlift.configuration.ConfigDescription;
+
+public class IcebergGlueCatalogConfig
+{
+    private boolean skipArchive;
+
+    public boolean isSkipArchive()
+    {
+        return skipArchive;
+    }
+
+    @Config("iceberg.glue.skip-archive")
+    @ConfigDescription("Skip archiving an old table version when creating a new version in a commit")
+    public IcebergGlueCatalogConfig setSkipArchive(boolean skipArchive)
+    {
+        this.skipArchive = skipArchive;
+        return this;
+    }
+}

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/IcebergGlueCatalogModule.java

@@ -14,6 +14,7 @@
 package io.trino.plugin.iceberg.catalog.glue;
 
 import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.glue.AWSGlueAsync;
 import com.google.inject.Binder;
 import com.google.inject.Scopes;
 import io.airlift.configuration.AbstractConfigurationAwareModule;
@@ -33,7 +34,9 @@ public class IcebergGlueCatalogModule
     protected void setup(Binder binder)
     {
         configBinder(binder).bindConfig(GlueHiveMetastoreConfig.class);
+        configBinder(binder).bindConfig(IcebergGlueCatalogConfig.class);
         binder.bind(GlueMetastoreStats.class).in(Scopes.SINGLETON);
+        binder.bind(AWSGlueAsync.class).toProvider(GlueClientProvider.class).in(Scopes.SINGLETON);
         newExporter(binder).export(GlueMetastoreStats.class).withGeneratedName();
         binder.bind(AWSCredentialsProvider.class).toProvider(GlueCredentialsProvider.class).in(Scopes.SINGLETON);
         binder.bind(IcebergTableOperationsProvider.class).to(GlueIcebergTableOperationsProvider.class).in(Scopes.SINGLETON);

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/SkipArchiveRequestHandler.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.glue;
+
+import com.amazonaws.AmazonWebServiceRequest;
+import com.amazonaws.handlers.RequestHandler2;
+import com.amazonaws.services.glue.model.CreateDatabaseRequest;
+import com.amazonaws.services.glue.model.CreateTableRequest;
+import com.amazonaws.services.glue.model.DeleteDatabaseRequest;
+import com.amazonaws.services.glue.model.DeleteTableRequest;
+import com.amazonaws.services.glue.model.GetDatabaseRequest;
+import com.amazonaws.services.glue.model.GetDatabasesRequest;
+import com.amazonaws.services.glue.model.GetTableRequest;
+import com.amazonaws.services.glue.model.GetTablesRequest;
+import com.amazonaws.services.glue.model.UpdateTableRequest;
+
+public class SkipArchiveRequestHandler
+        extends RequestHandler2
+{
+    private final boolean skipArchive;
+
+    public SkipArchiveRequestHandler(boolean skipArchive)
+    {
+        this.skipArchive = skipArchive;
+    }
+
+    @Override
+    public AmazonWebServiceRequest beforeExecution(AmazonWebServiceRequest request)
+    {
+        if (request instanceof UpdateTableRequest updateTableRequest) {
+            return updateTableRequest.withSkipArchive(skipArchive);
+        }
+        if (request instanceof CreateDatabaseRequest ||
+                request instanceof DeleteDatabaseRequest ||
+                request instanceof GetDatabasesRequest ||
+                request instanceof GetDatabaseRequest ||
+                request instanceof CreateTableRequest ||
+                request instanceof DeleteTableRequest ||
+                request instanceof GetTablesRequest ||
+                request instanceof GetTableRequest) {
+            return request;
+        }
+        throw new IllegalArgumentException("Unsupported request: " + request);
+    }
+}

plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java

@@ -13,7 +13,6 @@
  */
 package io.trino.plugin.iceberg.catalog.glue;
 
-import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.glue.AWSGlueAsync;
 import io.trino.filesystem.TrinoFileSystemFactory;
 import io.trino.plugin.base.CatalogName;
@@ -33,7 +32,6 @@
 
 import java.util.Optional;
 
-import static io.trino.plugin.hive.metastore.glue.GlueHiveMetastore.createAsyncGlueClient;
 import static java.util.Objects.requireNonNull;
 
 public class TrinoGlueCatalogFactory
@@ -57,18 +55,17 @@ public TrinoGlueCatalogFactory(
             IcebergTableOperationsProvider tableOperationsProvider,
             NodeVersion nodeVersion,
             GlueHiveMetastoreConfig glueConfig,
-            AWSCredentialsProvider credentialsProvider,
             IcebergConfig icebergConfig,
-            GlueMetastoreStats stats)
+            GlueMetastoreStats stats,
+            AWSGlueAsync glueClient)
     {
         this.catalogName = requireNonNull(catalogName, "catalogName is null");
         this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null");
         this.typeManager = requireNonNull(typeManager, "typeManager is null");
         this.tableOperationsProvider = requireNonNull(tableOperationsProvider, "tableOperationsProvider is null");
         this.trinoVersion = nodeVersion.toString();
         this.defaultSchemaLocation = glueConfig.getDefaultWarehouseDir();
-        requireNonNull(credentialsProvider, "credentialsProvider is null");
-        this.glueClient = createAsyncGlueClient(glueConfig, credentialsProvider, Optional.empty(), stats.newRequestMetricsCollector());
+        this.glueClient = requireNonNull(glueClient, "glueClient is null");
         this.isUniqueTableLocation = icebergConfig.isUniqueTableLocation();
         this.stats = requireNonNull(stats, "stats is null");
     }

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogConfig.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.iceberg.catalog.glue;
+
+import com.google.common.collect.ImmutableMap;
+import org.testng.annotations.Test;
+
+import java.util.Map;
+
+import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
+import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
+import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
+
+public class TestIcebergGlueCatalogConfig
+{
+    @Test
+    public void testDefaults()
+    {
+        assertRecordedDefaults(recordDefaults(IcebergGlueCatalogConfig.class)
+                .setSkipArchive(false));
+    }
+
+    @Test
+    public void testExplicitPropertyMapping()
+    {
+        Map<String, String> properties = ImmutableMap.<String, String>builder()
+                .put("iceberg.glue.skip-archive", "true")
+                .buildOrThrow();
+
+        IcebergGlueCatalogConfig expected = new IcebergGlueCatalogConfig()
+                .setSkipArchive(true);
+
+        assertFullMapping(properties, expected);
+    }
+}