[feature][plugin][s3writer] add pathStyleAccessEnabled config (#1186)

Fix error 400 when setting pathStyleAccessEnabled to true for MinIO
wgzhao · Nov 25, 2024 · 50440b9 · 50440b9
1 parent 93c1938
commit 50440b9
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 18 deletions.
diff --git a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Key.java b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Key.java
@@ -20,4 +20,6 @@ public class S3Key extends Key
     public static final String MAX_FILE_SIZE = "maxFileSize";
 
     public static final String DEFAULT_SUFFIX = "defaultSuffix";
+
+    public static final String PATH_STYLE_ACCESS_ENABLED = "pathStyleAccessEnabled";
 }
diff --git a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Util.java b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Util.java
@@ -18,18 +18,23 @@ public static S3Client initS3Client(Configuration conf) {
         Region region = Region.of(regionStr);
         String accessId = conf.getString(S3Key.ACCESS_ID);
         String accessKey = conf.getString(S3Key.ACCESS_KEY);
+        String pathStyleAccessEnabled =conf.getString(S3Key.PATH_STYLE_ACCESS_ENABLED,"");
 
-        return initS3Client(conf.getString(S3Key.ENDPOINT), region, accessId, accessKey);
+        return initS3Client(conf.getString(S3Key.ENDPOINT), region, accessId, accessKey ,pathStyleAccessEnabled);
 
     }
 
-    public static S3Client initS3Client(String endpoint, Region region, String accessId, String accessKey) {
+    public static S3Client initS3Client(String endpoint, Region region, String accessId, String accessKey ,String pathStyleAccessEnabled) {
         if (null == region) {
             region = Region.of("ap-northeast-1");
         }
         try {
             AwsBasicCredentials awsCreds = AwsBasicCredentials.create(accessId, accessKey);
-            return S3Client.builder()
+            return S3Client.builder().serviceConfiguration(e -> {
+                        if("true".equals(pathStyleAccessEnabled)) {
+                            e.pathStyleAccessEnabled(true);
+                        }
+                    })
                     .credentialsProvider(StaticCredentialsProvider.create(awsCreds))
                     .region(region)
                     .endpointOverride(URI.create(endpoint))

diff --git a/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Writer.java b/plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Writer.java
@@ -307,22 +307,26 @@ public void startWrite(RecordReceiver lineReceiver)
                 completedParts.add(completedPart);
                 outputStream.reset();
             }
-            // Finally, call completeMultipartUpload operation to tell S3 to merge all uploaded
-            // parts and finish the multipart operation.
-            CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder()
-                    .parts(completedParts)
-                    .build();
+            if(!completedParts.isEmpty()) {
+                // Finally, call completeMultipartUpload operation to tell S3 to merge all uploaded
+                // parts and finish the multipart operation.
+                CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder()
+                        .parts(completedParts)
+                        .build();
 
-            CompleteMultipartUploadRequest completeMultipartUploadRequest =
-                    CompleteMultipartUploadRequest.builder()
-                            .bucket(bucket)
-                            .key(object)
-                            .uploadId(uploadId)
-                            .multipartUpload(completedMultipartUpload)
-                            .build();
+                CompleteMultipartUploadRequest completeMultipartUploadRequest =
+                        CompleteMultipartUploadRequest.builder()
+                                .bucket(bucket)
+                                .key(object)
+                                .uploadId(uploadId)
+                                .multipartUpload(completedMultipartUpload)
+                                .build();
 
-            s3Client.completeMultipartUpload(completeMultipartUploadRequest);
-            LOG.info("end do write");
+                s3Client.completeMultipartUpload(completeMultipartUploadRequest);
+                LOG.info("end do write");
+            } else {
+                LOG.info("no content do write");
+            }
         }
 
         private String record2String(Record record)

diff --git a/pom.xml b/pom.xml
@@ -29,7 +29,7 @@
     <properties>
         <!-- basic components -->
         <aircompressor.version>0.25</aircompressor.version>
-        <awssdk.version>2.20.5</awssdk.version>
+        <awssdk.version>2.29.20</awssdk.version>
         <avro.version>1.11.3</avro.version>
         <commons.beanutils.version>1.9.4</commons.beanutils.version>
         <commons.configuration.version>1.10</commons.configuration.version>