Skip to content

Commit

Permalink
Wait for the node to start before scheduling & Move jar to parent pom
Browse files Browse the repository at this point in the history
  • Loading branch information
@sunxiaojian
sunxiaojian committed May 13, 2023
1 parent cac6ab4 commit 9b371c6
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 50 deletions.
40 changes: 39 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,12 @@
<checker.qual.version>3.10.0</checker.qual.version>
<awaitility.version>4.2.0</awaitility.version>
<e2e.dependency.skip>true</e2e.dependency.skip>

<!-- Imap storage dependency package -->
<hadoop-aliyun.version>3.0.0</hadoop-aliyun.version>
<json-smart.version>2.4.7</json-smart.version>
<hadoop-aws.version>3.1.4</hadoop-aws.version>
<netty-buffer.version>4.1.60.Final</netty-buffer.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -446,8 +452,40 @@
<scope>provided</scope>
</dependency>

</dependencies>
<!-- Imap storage dependency package -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aliyun</artifactId>
<version>${hadoop-aliyun.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
<version>${json-smart.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop-aws.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>${netty-buffer.version}</version>
</dependency>

</dependencies>
</dependencyManagement>

<dependencies>
Expand Down
35 changes: 12 additions & 23 deletions seatunnel-dist/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,14 @@
<tablestore.version>5.13.9</tablestore.version>
<teradata.version>17.20.00.12</teradata.version>
<redshift.version>2.1.0.9</redshift.version>

<!-- Imap storage dependency package -->
<hadoop-aliyun.version>3.0.0</hadoop-aliyun.version>
<json-smart.version>2.4.7</json-smart.version>
<hadoop-aws.version>3.1.4</hadoop-aws.version>
<aws-java-sdk.version>1.11.271</aws-java-sdk.version>
<netty-buffer.version>4.1.89.Final</netty-buffer.version>

</properties>
<dependencies>
<!-- starters -->
Expand Down Expand Up @@ -567,44 +575,25 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>4.1.89.Final</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.89.Final</version>
<version>${netty-buffer.version}</version>
<scope>provided</scope>
</dependency>

<!-- hadoop jar -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aliyun</artifactId>
<version>3.0.0</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>2.8.3</version>
<version>${hadoop-aliyun.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>3.1.4</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bundle</artifactId>
<version>1.11.271</version>
<version>${hadoop-aws.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>seatunnel-hadoop3-3.1.4-uber</artifactId>
Expand Down
14 changes: 13 additions & 1 deletion ...-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.commons.collections4.CollectionUtils;

import com.google.common.collect.Lists;
import com.hazelcast.instance.impl.NodeState;
import com.hazelcast.internal.metrics.DynamicMetricsProvider;
import com.hazelcast.internal.metrics.MetricDescriptor;
import com.hazelcast.internal.metrics.MetricsCollectionContext;
Expand Down Expand Up @@ -127,6 +128,8 @@ public class TaskExecutionService implements DynamicMetricsProvider {

private final ScheduledExecutorService scheduledExecutorService;

private CountDownLatch waitClusterStarted;

public TaskExecutionService(NodeEngineImpl nodeEngine, HazelcastProperties properties) {
seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig();
this.hzInstanceName = nodeEngine.getHazelcastInstance().getName();
Expand All @@ -142,7 +145,7 @@ public TaskExecutionService(NodeEngineImpl nodeEngine, HazelcastProperties prope
scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
scheduledExecutorService.scheduleAtFixedRate(
this::updateMetricsContextInImap,
30, // Wait for MapStore loading to complete, wait 30s
0,
seaTunnelConfig.getEngineConfig().getJobMetricsBackupInterval(),
TimeUnit.SECONDS);
}
Expand Down Expand Up @@ -455,6 +458,15 @@ private synchronized void updateMetricsContextInImap() {
contextMap.putAll(finishedExecutionContexts);
contextMap.putAll(executionContexts);
try {
if (!nodeEngine.getNode().getState().equals(NodeState.ACTIVE)) {
logger.warning(
String.format(
"The Node is not ready yet, Node state %s,looking forward to the next "
+ "scheduling",
nodeEngine.getNode().getState()));
return;
}

IMap<TaskLocation, SeaTunnelMetricsContext> map =
nodeEngine.getHazelcastInstance().getMap(Constant.IMAP_RUNNING_JOB_METRICS);
contextMap.forEach(
Expand Down
24 changes: 2 additions & 22 deletions seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,6 @@
<artifactId>imap-storage-file</artifactId>
<name>SeaTunnel : Engine : Storage : IMap Storage Plugins : File</name>

<properties>
<hadoop-aliyun.version>3.0.0</hadoop-aliyun.version>
<hadoop-aws.version>3.1.4</hadoop-aws.version>
<aws.java.sdk.version>1.11.271</aws.java.sdk.version>
<netty-buffer.version>4.1.89.Final</netty-buffer.version>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.seatunnel</groupId>
Expand Down Expand Up @@ -74,34 +67,21 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aliyun</artifactId>
<version>${hadoop-aliyun.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>2.8.3</version>
<scope>provided</scope>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop-aws.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bundle</artifactId>
<version>${aws.java.sdk.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>${netty-buffer.version}</version>
<scope>provided</scope>
</dependency>

Expand Down
9 changes: 6 additions & 3 deletions tools/dependencies/known-dependencies.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ protostuff-collectionschema-1.8.0.jar
protostuff-core-1.8.0.jar
protostuff-runtime-1.8.0.jar
scala-library-2.11.12.jar
seatunnel-jackson-2.3.1-SNAPSHOT-optional.jar
seatunnel-guava-2.3.1-SNAPSHOT-optional.jar
seatunnel-jackson-2.3.2-SNAPSHOT-optional.jar
seatunnel-guava-2.3.2-SNAPSHOT-optional.jar
slf4j-api-1.7.25.jar
jsqlparser-4.5.jar
animal-sniffer-annotations-1.17.jar
Expand All @@ -34,4 +34,7 @@ j2objc-annotations-1.1.jar
jsr305-1.3.9.jar
jsr305-3.0.0.jar
jsr305-3.0.2.jar
listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
accessors-smart-2.4.7.jar
asm-9.1.jar
json-smart-2.4.7.jar

0 comments on commit 9b371c6

Please sign in to comment.