Fix hdfs integration test (#30458)

* Fix hdfs integration test debug only skip yaml test * Add back azureIntegrationTest get removed when teardown Py37 support
apache · Mar 6, 2024 · 8d7adba · 8d7adba
1 parent f0f1cd3
commit 8d7adba
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 4 deletions.
diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -504,6 +504,7 @@ tasks.register("python38PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:hdfsIntegrationTest")
+  dependsOn(":sdks:python:test-suites:direct:py38:azureIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
   // TODO: https://github.com/apache/beam/issues/22651
   // The default container uses Python 3.8. The goal here is to
@@ -516,7 +517,6 @@ tasks.register("python38PostCommit") {
 tasks.register("python39PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
   // TODO (https://github.com/apache/beam/issues/23966)
   // Move this to Python 3.10 test suite once tfx-bsl has python 3.10 wheel.
@@ -526,7 +526,6 @@ tasks.register("python39PostCommit") {
 tasks.register("python310PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
   dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
 }
 

diff --git a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
@@ -20,7 +20,7 @@ version: '3'
 services:
   # HDFS namenode.
   namenode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.10.2
     hostname: namenode
     environment:
       - CLUSTER_NAME=test
@@ -30,7 +30,7 @@ services:
 
   # HDFS datanode.
   datanode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.10.2
     hostname: datanode
     environment:
       - CLUSTER_NAME=test

diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
@@ -182,6 +182,9 @@ tasks.register("directRunnerIT") {
 }
 
 tasks.register("hdfsIntegrationTest") {
+  // needs generated srcs in sdks/python
+  dependsOn ':sdks:python:sdist'
+
   doLast {
     exec {
       executable 'sh'
@@ -191,6 +194,9 @@ tasks.register("hdfsIntegrationTest") {
 }
 
 tasks.register("azureIntegrationTest") {
+  // needs generated srcs in sdks/python
+  dependsOn ':sdks:python:sdist'
+
   doLast {
     exec {
       executable 'sh'