Fix hdfs integration test

debug only skip yaml test
apache · Mar 5, 2024 · 2141d13 · 2141d13
1 parent 89366bb
commit 2141d13
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 17 deletions.
diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -501,40 +501,40 @@ tasks.register("pythonFormatterPreCommit") {
 }
 
 tasks.register("python38PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
+  /// dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
   // TODO: https://github.com/apache/beam/issues/22651
   // The default container uses Python 3.8. The goal here is to
   // duild Docker images for TensorRT tests during run time for python versions
   // other than 3.8 and add these tests in other python postcommit suites.
-  dependsOn(":sdks:python:test-suites:dataflow:py38:inferencePostCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py38:inferencePostCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py38:inferencePostCommitIT")
+  /// dependsOn(":sdks:python:test-suites:direct:py38:inferencePostCommitIT")
 }
 
 tasks.register("python39PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
+  /// dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
+  /// dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
   // TODO (https://github.com/apache/beam/issues/23966)
   // Move this to Python 3.10 test suite once tfx-bsl has python 3.10 wheel.
-  dependsOn(":sdks:python:test-suites:direct:py39:inferencePostCommitIT")
+  /// dependsOn(":sdks:python:test-suites:direct:py39:inferencePostCommitIT")
 }
 
 tasks.register("python310PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
+  /// dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
+  /// dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
 }
 
 tasks.register("python311PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
+  /// dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
+  /// dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
 }
 
 tasks.register("portablePythonPreCommit") {

diff --git a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
@@ -20,7 +20,7 @@ version: '3'
 services:
   # HDFS namenode.
   namenode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.10.2
     hostname: namenode
     environment:
       - CLUSTER_NAME=test
@@ -30,7 +30,7 @@ services:
 
   # HDFS datanode.
   datanode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.10.2
     hostname: datanode
     environment:
       - CLUSTER_NAME=test

diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
@@ -182,6 +182,9 @@ tasks.register("directRunnerIT") {
 }
 
 tasks.register("hdfsIntegrationTest") {
+  // needs generated srcs in sdks/python
+  dependsOn ':sdks:python:sdist'
+
   doLast {
     exec {
       executable 'sh'