fix test

apache · May 14, 2024 · ba291d8 · ba291d8
1 parent 4947e5b
commit ba291d8
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
@@ -41,7 +41,6 @@
 import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
 import org.apache.hudi.common.table.view.TableFileSystemView;
 import org.apache.hudi.common.util.Option;
-import org.apache.hudi.common.util.ValidationUtils;
 import org.apache.hudi.exception.HoodieException;
 
 import org.apache.hadoop.fs.FileSystem;
@@ -373,8 +372,6 @@ public static String createMarkerFile(String basePath, String partitionPath, Str
 
   public static String createMarkerFile(String basePath, String partitionPath, String commitInstant,
       String instantTime, String fileId, IOType ioType, String writeToken) throws IOException {
-    // with HUDI-1517, this method is used for creating marker files for data files. So append type is not allowed
-    ValidationUtils.checkArgument(ioType != IOType.APPEND);
     Path parentPath = Paths.get(basePath, HoodieTableMetaClient.TEMPFOLDER_NAME, instantTime, partitionPath);
     Files.createDirectories(parentPath);
     Path markerFilePath = parentPath.resolve(dataFileMarkerFileName(instantTime, fileId, ioType, BASE_FILE_EXTENSION, writeToken));

diff --git a/...asource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java b/...asource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java
@@ -18,6 +18,7 @@
 
 package org.apache.hudi.sink.bucket;
 
+import org.apache.hudi.common.fs.FSUtils;
 import org.apache.hudi.common.model.HoodieCommitMetadata;
 import org.apache.hudi.common.model.IOType;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -113,13 +114,17 @@ private static void doDeleteCommit(String tablePath, boolean isCow) throws Excep
     Path path = new Path(metaClient.getMetaPath() + StoragePath.SEPARATOR + filename);
     fs.delete(path);
 
+    // marker types are different for COW and MOR
+    IOType ioType = isCow ? IOType.CREATE : IOType.APPEND;
+
     commitMetadata.getFileIdAndRelativePaths().forEach((fileId, relativePath) -> {
       // hacky way to reconstruct markers ¯\_(ツ)_/¯
       String[] partitionFileNameSplit = relativePath.split("/");
+      String fileInstant = FSUtils.getCommitTime(partitionFileNameSplit[1]);
       String partition = partitionFileNameSplit[0];
-      String fileName = partitionFileNameSplit[1];
+      String writeToken = isCow ? getWriteToken(partitionFileNameSplit[1]) : FSUtils.getWriteTokenFromLogPath(new Path(relativePath));
       try {
-        FileCreateUtils.createFileMarkerByFileName(tablePath, partition, commitInstant, fileName, IOType.CREATE);
+        FileCreateUtils.createMarkerFile(tablePath, partition, commitInstant, fileInstant, fileId, ioType, writeToken);
       } catch (IOException e) {
         throw new RuntimeException(e);
       }