[Improve][build] The e2e module don't depend on the connector*-dist m…

…odule (#2702) * [Improve][build] The e2e module don't depend on the connector*-dist module * fix connector type error * fix flink sql
apache · Sep 9, 2022 · 7f9f202 · 7f9f202
1 parent 5039752
commit 7f9f202
Show file tree

Hide file tree

Showing 30 changed files with 197 additions and 61 deletions.
diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-assert-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-assert-flink-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-assert</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-datahub</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-fake-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-fake-flink-e2e/pom.xml
@@ -34,12 +34,20 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-fake</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-console</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-file-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-file-flink-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-file-local</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-iotdb-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-iotdb-flink-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-iotdb</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-jdbc-flink-e2e/pom.xml
@@ -34,12 +34,28 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-console</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-jdbc</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+
+        <!-- jdbc drivers -->
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>postgresql</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-mongodb-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-mongodb-flink-e2e/pom.xml
@@ -35,11 +35,19 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-mongodb</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-console</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-redis-flink-e2e/pom.xml
@@ -34,12 +34,20 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-redis</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-assert</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml
@@ -45,18 +45,6 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.seatunnel</groupId>
-            <artifactId>seatunnel-connectors-v2-dist</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>*</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>org.awaitility</groupId>
             <artifactId>awaitility</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-e2e/pom.xml
@@ -42,18 +42,6 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.seatunnel</groupId>
-            <artifactId>seatunnel-connectors-flink-dist</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>*</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
     </dependencies>
 
 </project>
diff --git a/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-assert-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-assert-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-connector-flink-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>seatunnel-connector-flink-assert</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-clickhouse-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-clickhouse-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-connector-flink-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>seatunnel-connector-flink-clickhouse</artifactId>

diff --git a/...connector-flink-e2e-base/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java b/...connector-flink-e2e-base/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java
@@ -38,7 +38,7 @@ protected String getStartShellName() {
 
     @Override
     protected String getConnectorType() {
-        return "seatunnel";
+        return "flink";
     }
 
     @Override

diff --git a/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-fake-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-fake-e2e/pom.xml
@@ -34,12 +34,20 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- connectors -->
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>seatunnel-connector-flink-fake</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-connector-flink-console</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-file-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-file-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-connector-flink-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>seatunnel-connector-flink-file</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-http-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-e2e/seatunnel-connector-flink-http-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-connector-flink-console</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>seatunnel-connector-flink-http</artifactId>

diff --git a/seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-sql-e2e/pom.xml
@@ -38,17 +38,5 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.seatunnel</groupId>
-            <artifactId>seatunnel-connectors-flink-sql-dist</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>*</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
     </dependencies>
 </project>
diff --git a/...r-flink-sql-e2e-base/src/test/java/org/apache/seatunnel/e2e/flink/sql/FlinkContainer.java b/...r-flink-sql-e2e-base/src/test/java/org/apache/seatunnel/e2e/flink/sql/FlinkContainer.java
@@ -17,8 +17,14 @@
 
 package org.apache.seatunnel.e2e.flink.sql;
 
+import static org.apache.seatunnel.e2e.common.ContainerUtil.copyConfigFileToContainer;
+
 import org.apache.seatunnel.e2e.common.AbstractFlinkContainer;
 
+import org.testcontainers.containers.Container;
+
+import java.io.IOException;
+
 /**
  * This class is the base class of FlinkEnvironment test.
  * The before method will create a Flink cluster, and after method will close the Flink cluster.
@@ -50,4 +56,9 @@ protected String getConnectorModulePath() {
     protected String getConnectorNamePrefix() {
         return "flink-sql-connector-";
     }
+
+    public Container.ExecResult executeSeaTunnelFlinkJob(String confFile) throws IOException, InterruptedException {
+        final String confInContainerPath = copyConfigFileToContainer(jobManager, confFile);
+        return executeCommand(jobManager, confInContainerPath);
+    }
 }
diff --git a/seatunnel-e2e/seatunnel-flink-sql-e2e/setunnel-connector-flink-sql-fake-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-sql-e2e/setunnel-connector-flink-sql-fake-e2e/pom.xml
@@ -34,6 +34,7 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+        <!-- Flink has built-in DataGen and Print sql connectors -->
     </dependencies>
 
 </project>
diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-datahub</artifactId>

diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-fake-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-fake-spark-e2e/pom.xml
@@ -34,12 +34,20 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-fake</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-console</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-file-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-file-spark-e2e/pom.xml
@@ -34,6 +34,14 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <!-- SeaTunnel connectors -->
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-fake</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.seatunnel</groupId>
             <artifactId>connector-file-local</artifactId>