[FLINK-30064][hive] Move Hive connector to external repo

apache · Dec 8, 2023 · 323631a · 323631a
1 parent 785660f
commit 323631a
Show file tree

Hide file tree

Showing 33 changed files with 2,821 additions and 167 deletions.
diff --git a/.github/workflows/weekly.yml → .github/workflows/nightly.yml b/.github/workflows/weekly.yml → .github/workflows/nightly.yml
@@ -22,10 +22,29 @@ on:
     - cron: "0 0 * * 0"
 jobs:
   compile_and_test:
-    if: github.repository_owner == 'apache'
+    # if: github.repository_owner == 'apache'
     strategy:
       matrix:
-        flink: [1.16-SNAPSHOT, 1.17-SNAPSHOT]
+        flink: [ 1.18.0, 1.18-SNAPSHOT, 1.19-SNAPSHOT ]
+        jdk: [ 8, 11 ]
+        # empty means test against hive2
+        mvn_profile: [ "", hive3 ]
+        exclude:
+          - jdk: 11
+            mvn_profile: hive3
+        include:
+          - jdk: 11
+            flink: 1.18.0
+            mvn_profile: "hive3,skipE2ETests"
+          - jdk: 11
+            flink: 1.18-SNAPSHOT
+            mvn_profile: "hive3,skipE2ETests"
+          - jdk: 11
+            flink: 1.19-SNAPSHOT
+            mvn_profile: "hive3,skipE2ETests"
+
     uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
     with:
       flink_version: ${{ matrix.flink }}
+      jdk_version: ${{ matrix.jdk }}
+      optional_maven_profiles: ${{ matrix.mvn_profile }}
diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml
@@ -23,6 +23,25 @@ concurrency:
   cancel-in-progress: true
 jobs:
   compile_and_test:
+    strategy:
+      matrix:
+        flink: [ 1.18-SNAPSHOT, 1.19-SNAPSHOT ]
+        jdk: [ 8, 11 ]
+        # empty means test against hive2
+        mvn_profile: [ "", hive3 ]
+        exclude:
+          - jdk: 11
+            mvn_profile: hive3
+        include:
+          - jdk: 11
+            flink: 1.18-SNAPSHOT
+            mvn_profile: "hive3,skipE2ETests"
+          - jdk: 11
+            flink: 1.19-SNAPSHOT
+            mvn_profile: "hive3,skipE2ETests"
+
     uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
     with:
-      flink_version: 1.16.0
+      flink_version: ${{ matrix.flink }}
+      jdk_version: ${{ matrix.jdk }}
+      optional_maven_profiles: ${{ matrix.mvn_profile }}
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/flink-connector-hive-e2e-tests/pom.xml b/flink-connector-hive-e2e-tests/pom.xml
@@ -21,45 +21,78 @@ under the License.
 		 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
+	<modelVersion>4.0.0</modelVersion>
 	<parent>
 		<groupId>org.apache.flink</groupId>
-		<artifactId>flink-end-to-end-tests</artifactId>
-		<version>1.19-SNAPSHOT</version>
+		<artifactId>flink-connector-hive-parent</artifactId>
+		<version>3.0-SNAPSHOT</version>
 	</parent>
-	<modelVersion>4.0.0</modelVersion>
+
 
 	<properties>
+		<okhttp.version>3.14.9</okhttp.version>
 		<!--
 		Hive requires Hadoop 2 to avoid
 		java.lang.NoClassDefFoundError: org/apache/hadoop/metrics/Updater errors
 		Using this dedicated property avoids CI failures with the Hadoop 3 profile
 		-->
 		<hive.hadoop.version>2.10.2</hive.hadoop.version>
+		<skip.e2e.tests>false</skip.e2e.tests>
 	</properties>
 
 
 
-	<artifactId>flink-end-to-end-tests-hive</artifactId>
+	<artifactId>flink-connector-hive-e2e-tests</artifactId>
 	<name>Flink : E2E Tests : Hive</name>
 
 	<dependencies>
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-end-to-end-tests-common</artifactId>
-			<version>${project.version}</version>
+			<artifactId>flink-test-utils-junit</artifactId>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-test-utils</artifactId>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+		</dependency>
+		<dependency>
+			<!-- API bridge between log4j 1 and 2 -->
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-1.2-api</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-api</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-slf4j-impl</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.squareup.okhttp3</groupId>
+			<artifactId>okhttp</artifactId>
+			<version>${okhttp.version}</version>
 		</dependency>
-
 		<dependency>
 			<groupId>org.apache.flink</groupId>
 			<artifactId>flink-sql-connector-hive-${hive.version}_${scala.binary.version}</artifactId>
-			<version>${project.version}</version>
+			<version>${flink.version}</version>
 		</dependency>
 
 		<!-- hadoop dependencies for end-to-end test -->
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-common</artifactId>
-			<version>${flink.hadoop.version}</version>
+			<version>${hadoop.version}</version>
 			<scope>test</scope>
 			<exclusions>
 				<exclusion>
@@ -150,7 +183,7 @@ under the License.
 									<artifactId>
 										flink-sql-connector-hive-${hive.version}_${scala.binary.version}
 									</artifactId>
-									<version>${project.version}</version>
+									<version>${flink.version}</version>
 									<destFileName>
 										sql-hive-${hive.version}_${scala.binary.version}.jar
 									</destFileName>
@@ -174,6 +207,13 @@ under the License.
 					</execution>
 				</executions>
 			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<configuration>
+					<skip>${skip.e2e.tests}</skip>
+				</configuration>
+			</plugin>
 		</plugins>
 	</build>
 
@@ -183,25 +223,42 @@ under the License.
 			<id>hive3</id>
 			<properties>
 				<hive.version>3.1.3</hive.version>
+				<netty.version>4.1.46.Final</netty.version>
 			</properties>
-
-			<dependencies>
-				<dependency>
-					<!-- Bump arrow netty dependency -->
-					<groupId>io.netty</groupId>
-					<artifactId>netty-buffer</artifactId>
-					<version>4.1.46.Final</version>
-					<scope>provided</scope>
-				</dependency>
-
-				<dependency>
-					<!-- Bump arrow netty dependency -->
-					<groupId>io.netty</groupId>
-					<artifactId>netty-common</artifactId>
-					<version>4.1.46.Final</version>
-					<scope>provided</scope>
-				</dependency>
-			</dependencies>
+		</profile>
+		<profile>
+			<id>skipE2ETests</id>
+			<properties>
+				<skip.e2e.tests>true</skip.e2e.tests>
+			</properties>
+		</profile>
+		<profile>
+			<id>run-end-to-end-tests</id>
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-surefire-plugin</artifactId>
+						<executions>
+							<execution>
+								<id>end-to-end-tests</id>
+								<phase>integration-test</phase>
+								<goals>
+									<goal>test</goal>
+								</goals>
+								<configuration>
+									<includes>
+										<include>**/*.*</include>
+									</includes>
+									<systemPropertyVariables>
+										<moduleDir>${project.basedir}</moduleDir>
+									</systemPropertyVariables>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+				</plugins>
+			</build>
 		</profile>
 	</profiles>