From 9443358f5810640accacbd578dace289dd3b6807 Mon Sep 17 00:00:00 2001 From: hailin0 Date: Mon, 29 May 2023 17:41:24 +0800 Subject: [PATCH] [Hotfix][Zeta] Fix completePendingCheckpoint concurrent action This operation does not allow concurrent execution --- config/seatunnel.yaml | 2 +- docs/en/seatunnel-engine/checkpoint-storage.md | 8 ++++---- .../src/test/resources/seatunnel.yaml | 2 +- .../src/test/resources/seatunnel.yaml | 2 +- .../src/main/resources/seatunnel.yaml | 2 +- .../common/config/YamlSeaTunnelConfigParserTest.java | 2 +- .../src/test/resources/seatunnel.yaml | 2 +- .../engine/server/checkpoint/CheckpointCoordinator.java | 2 +- .../src/test/resources/seatunnel.yaml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config/seatunnel.yaml b/config/seatunnel.yaml index 86eaee6a071..7e496ca39ad 100644 --- a/config/seatunnel.yaml +++ b/config/seatunnel.yaml @@ -26,7 +26,7 @@ seatunnel: checkpoint: interval: 10000 timeout: 60000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs diff --git a/docs/en/seatunnel-engine/checkpoint-storage.md b/docs/en/seatunnel-engine/checkpoint-storage.md index d1f8d974662..a88f301439e 100644 --- a/docs/en/seatunnel-engine/checkpoint-storage.md +++ b/docs/en/seatunnel-engine/checkpoint-storage.md @@ -59,7 +59,7 @@ seatunnel: checkpoint: interval: 6000 timeout: 7000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs @@ -94,7 +94,7 @@ seatunnel: checkpoint: interval: 6000 timeout: 7000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs @@ -119,7 +119,7 @@ seatunnel: checkpoint: interval: 6000 timeout: 7000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs @@ -160,7 +160,7 @@ seatunnel: checkpoint: interval: 6000 timeout: 7000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/seatunnel.yaml b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/seatunnel.yaml index 164d1285bca..16b9f55c30d 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/seatunnel.yaml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/seatunnel.yaml @@ -25,7 +25,7 @@ seatunnel: checkpoint: interval: 300000 timeout: 10000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: localfile diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/resources/seatunnel.yaml b/seatunnel-engine/seatunnel-engine-client/src/test/resources/seatunnel.yaml index 555d5865641..ea5b5ac2307 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/resources/seatunnel.yaml +++ b/seatunnel-engine/seatunnel-engine-client/src/test/resources/seatunnel.yaml @@ -24,7 +24,7 @@ seatunnel: checkpoint: interval: 6000 timeout: 7000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml b/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml index cab12e5dcac..e5d92281da7 100644 --- a/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml +++ b/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml @@ -25,7 +25,7 @@ seatunnel: checkpoint: interval: 300000 timeout: 10000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs diff --git a/seatunnel-engine/seatunnel-engine-common/src/test/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelConfigParserTest.java b/seatunnel-engine/seatunnel-engine-common/src/test/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelConfigParserTest.java index 1263a572325..4c199b352ef 100644 --- a/seatunnel-engine/seatunnel-engine-common/src/test/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelConfigParserTest.java +++ b/seatunnel-engine/seatunnel-engine-common/src/test/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelConfigParserTest.java @@ -54,7 +54,7 @@ public void testSeaTunnelConfig() { 7000, config.getEngineConfig().getCheckpointConfig().getCheckpointTimeout()); Assertions.assertEquals( - 5, config.getEngineConfig().getCheckpointConfig().getMaxConcurrentCheckpoints()); + 1, config.getEngineConfig().getCheckpointConfig().getMaxConcurrentCheckpoints()); Assertions.assertEquals( 2, config.getEngineConfig().getCheckpointConfig().getTolerableFailureCheckpoints()); diff --git a/seatunnel-engine/seatunnel-engine-common/src/test/resources/seatunnel.yaml b/seatunnel-engine/seatunnel-engine-common/src/test/resources/seatunnel.yaml index d70dfc5d303..4f6ce5f4ef1 100644 --- a/seatunnel-engine/seatunnel-engine-common/src/test/resources/seatunnel.yaml +++ b/seatunnel-engine/seatunnel-engine-common/src/test/resources/seatunnel.yaml @@ -25,7 +25,7 @@ seatunnel: checkpoint: interval: 6000 timeout: 7000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointCoordinator.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointCoordinator.java index 091fe99bef7..40c880c82a3 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointCoordinator.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointCoordinator.java @@ -608,7 +608,7 @@ protected void acknowledgeTask(TaskAcknowledgeOperation ackOperation) { : SubtaskStatus.RUNNING); } - public void completePendingCheckpoint(CompletedCheckpoint completedCheckpoint) { + public synchronized void completePendingCheckpoint(CompletedCheckpoint completedCheckpoint) { LOG.debug( "pending checkpoint({}/{}@{}) completed! cost: {}, trigger: {}, completed: {}", completedCheckpoint.getCheckpointId(), diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/resources/seatunnel.yaml b/seatunnel-engine/seatunnel-engine-server/src/test/resources/seatunnel.yaml index e8303b99451..8f22b0613ca 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/resources/seatunnel.yaml +++ b/seatunnel-engine/seatunnel-engine-server/src/test/resources/seatunnel.yaml @@ -25,7 +25,7 @@ seatunnel: checkpoint: interval: 6000 timeout: 7000 - max-concurrent: 5 + max-concurrent: 1 tolerable-failure: 2 storage: type: hdfs