E2E: Add additional check to verify if the components are ready

Signed-off-by: Yuki Iwai <[email protected]>

Author: tenzen-y

test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh

@@ -65,8 +65,27 @@ echo "Deploying Katib"
 cd ../../../../../ && WITH_DATABASE_TYPE=$WITH_DATABASE_TYPE make deploy && cd -
 
 # Wait until all Katib pods is running.
+MAX_RETRY_COUNT=24
+COMPONENTS_NUM=3
+if "${DEPLOY_KATIB_UI}"; then
+  COMPONENTS_NUM=4
+fi
+until [ "$(kubectl get pods -n kubeflow -l "katib.kubeflow.org/component in ($WITH_DATABASE_TYPE,controller,ui,db-manager)" -oname | wc -l | tr -d ' ')" = "${COMPONENTS_NUM}" ]; do
+  echo -e "\nWaiting for katib components to be created"
+  kubectl get pods -n kubeflow || :
+  kubectl describe replicasets -n kubeflow || :
+
+  sleep 5
+  MAX_RETRY_COUNT=$((MAX_RETRY_COUNT - 1))
+  if [ "${MAX_RETRY_COUNT}" = "0" ]; then
+    echo "Failed to create katib components"
+    exit 1
+  fi
+done
+
 TIMEOUT=120s
-kubectl wait --for=condition=ready --timeout=${TIMEOUT} -l "katib.kubeflow.org/component in ($WITH_DATABASE_TYPE,controller,db-manager,ui)" -n kubeflow pod ||
+
+kubectl wait --for=condition=ContainersReady=True --timeout=${TIMEOUT} -l "katib.kubeflow.org/component in ($WITH_DATABASE_TYPE,controller,db-manager,ui)" -n kubeflow pod ||
   (kubectl get pods -n kubeflow && kubectl describe pods -n kubeflow && exit 1)
 
 echo "All Katib components are running."