Fix Dataproc cleanup race condition (#30154)

.github/workflows/README.md

@@ -79,6 +79,7 @@ jobs:
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
+      github.event_name == 'workflow_dispatch' ||
       startsWith(github.event.comment.body, 'Run Job With Matrix')
     steps:
       - uses: actions/checkout@v3

.github/workflows/beam_CleanUpDataprocResources.yml

@@ -17,7 +17,7 @@ name: Cleanup Dataproc Resources
 
 on:
   schedule:
-    - cron: '0 */6 * * *'
+    - cron: '0 0 * * *'
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

.test-infra/dataproc/cleanup.sh

@@ -44,8 +44,8 @@ function filterClusters(){
 
 		elapsedHours=$((($currentDate - $clusterStartTime)/3600))
 
-		#Most of the jobs commonly finish in less than one hour
-		if [[ $elapsedHours -ge 2 ]]; then 
+		# teardown clusters >= 6 hr old
+		if [[ $elapsedHours -ge 6 ]]; then
 			for name in ${generatedResources[@]}; do
 				# Only resources generated by the groovy jobs set are queued for deletion
 				if [[ "$cluster" == *${name}* && ! ("$cluster" =~ nokill) ]]; then