From 2141d13fc0b0257c70ba9740ebd470271fbf1c0f Mon Sep 17 00:00:00 2001
From: Yi Hu <yathu@google.com>
Date: Thu, 29 Feb 2024 12:43:51 -0500
Subject: [PATCH] Fix hdfs integration test

debug only

skip yaml test
---
 .../trigger_files/beam_PostCommit_Python.json |  0
 build.gradle.kts                              | 30 +++++++++----------
 .../hdfs_integration_test/docker-compose.yml  |  4 +--
 sdks/python/test-suites/direct/common.gradle  |  3 ++
 4 files changed, 20 insertions(+), 17 deletions(-)
 delete mode 100644 .github/trigger_files/beam_PostCommit_Python.json

diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/build.gradle.kts b/build.gradle.kts
index 788d9a48dc04e..7d0371a7282ab 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -501,40 +501,40 @@ tasks.register("pythonFormatterPreCommit") {
 }
 
 tasks.register("python38PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py38:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
+  /// dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
   // TODO: https://github.com/apache/beam/issues/22651
   // The default container uses Python 3.8. The goal here is to
   // duild Docker images for TensorRT tests during run time for python versions
   // other than 3.8 and add these tests in other python postcommit suites.
-  dependsOn(":sdks:python:test-suites:dataflow:py38:inferencePostCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py38:inferencePostCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py38:inferencePostCommitIT")
+  /// dependsOn(":sdks:python:test-suites:direct:py38:inferencePostCommitIT")
 }
 
 tasks.register("python39PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
+  /// dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
+  /// dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
   // TODO (https://github.com/apache/beam/issues/23966)
   // Move this to Python 3.10 test suite once tfx-bsl has python 3.10 wheel.
-  dependsOn(":sdks:python:test-suites:direct:py39:inferencePostCommitIT")
+  /// dependsOn(":sdks:python:test-suites:direct:py39:inferencePostCommitIT")
 }
 
 tasks.register("python310PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
+  /// dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
+  /// dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
 }
 
 tasks.register("python311PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
+  /// dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
+  /// dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
+  /// dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
 }
 
 tasks.register("portablePythonPreCommit") {
diff --git a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
index 41410028ca34c..8248f43bc558a 100644
--- a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
+++ b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
@@ -20,7 +20,7 @@ version: '3'
 services:
   # HDFS namenode.
   namenode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.10.2
     hostname: namenode
     environment:
       - CLUSTER_NAME=test
@@ -30,7 +30,7 @@ services:
 
   # HDFS datanode.
   datanode:
-    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.8.1
+    image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.10.2
     hostname: datanode
     environment:
       - CLUSTER_NAME=test
diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
index b5680c2e1e9a8..0350fbf1c4150 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -182,6 +182,9 @@ tasks.register("directRunnerIT") {
 }
 
 tasks.register("hdfsIntegrationTest") {
+  // needs generated srcs in sdks/python
+  dependsOn ':sdks:python:sdist'
+
   doLast {
     exec {
       executable 'sh'