E2E: Add additional check to verify if the components are ready

Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com>

test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh

@@ -65,10 +65,40 @@ echo "Deploying Katib"
 cd ../../../../../ && WITH_DATABASE_TYPE=$WITH_DATABASE_TYPE make deploy && cd -
 
 # Wait until all Katib pods is running.
+MAX_RETRY_COUNT=10
+COMPONENTS_NUM=3
+if "${DEPLOY_KATIB_UI}"; then
+  COMPONENTS_NUM=4
+fi
+until [ "$(kubectl get pods -n kubeflow -l "katib.kubeflow.org/component in ($WITH_DATABASE_TYPE,controller,ui,db-manager)" -oname | wc -l)" = "${COMPONENTS_NUM}" ]; do
+  echo -e "Waiting for katib components to be created\n"
+  kubectl get pods -n kubeflow || :
+
+  sleep 2
+  MAX_RETRY_COUNT=$((MAX_RETRY_COUNT - 1))
+  if [ "${MAX_RETRY_COUNT}" = "0" ]; then
+    echo "Failed to create katib components"
+    exit 1
+  fi
+done
+
 TIMEOUT=120s
 kubectl wait --for=condition=ready --timeout=${TIMEOUT} -l "katib.kubeflow.org/component in ($WITH_DATABASE_TYPE,controller,db-manager,ui)" -n kubeflow pod ||
   (kubectl get pods -n kubeflow && kubectl describe pods -n kubeflow && exit 1)
 
+MAX_RETRY_COUNT=10
+until [ "$(kubectl get pod -n kubeflow -l katib.kubeflow.org/component=controller -ojson | jq '.items[].status.conditions[] | select(.type=="ContainersReady").status')" = "True" ]; do
+  echo -e "Waiting for Katib controller to be ready\n"
+  kubectl get pods -n kubeflow || :
+
+  sleep 2
+  MAX_RETRY_COUNT=$((MAX_RETRY_COUNT - 1))
+  if [ "${MAX_RETRY_COUNT}" = "0" ]; then
+    echo "Failed to set up Katib controller"
+    exit 1
+  fi
+done
+
 echo "All Katib components are running."
 echo "Katib deployments"
 kubectl -n kubeflow get deploy