Verify profiles installation for capd cluster

test/acceptance/test/cli_add_delete.go

@@ -466,7 +466,7 @@ func DescribeCliAddDelete(gitopsTestRunner GitopsTestRunner) {
 				})
 
 				By(fmt.Sprintf("And I verify %s capd cluster is healthy and profiles are installed)", clusterName), func() {
-					verifyCapiClusterHealth(kubeconfigPath, GITOPS_DEFAULT_NAMESPACE)
+					verifyCapiClusterHealth(kubeconfigPath, clusterName, []string{}, GITOPS_DEFAULT_NAMESPACE)
 				})
 
 				clusterName2 := capdClusterNames[1]

test/acceptance/test/ui_templates.go

@@ -715,10 +715,9 @@ func DescribeTemplates(gitopsTestRunner GitopsTestRunner) {
 				})
 
 				By("Then AWS test-role-identity credential can be selected", func() {
-
-					Expect(createPage.Credentials.Click()).To(Succeed())
-					// FIXME - credentials may or may no be filtered
-					// Expect(pages.GetCredentials(webDriver).Count()).Should(Equal(4), "Credentials count in the cluster should be '3' excluding 'None")
+					Eventually(createPage.Credentials.Click).Should(Succeed())
+					// Credentials are not filtered for selected template
+					Eventually(pages.GetCredentials(webDriver).Count).Should(Equal(4), "Credentials count in the cluster should be '4' including 'None")
 					Expect(pages.GetCredential(webDriver, "test-role-identity").Click()).To(Succeed())
 				})
 
@@ -1032,6 +1031,10 @@ func DescribeTemplates(gitopsTestRunner GitopsTestRunner) {
 				setParameterValues(createPage, paramSection)
 				pages.ScrollWindow(webDriver, 0, 4000)
 
+				// Temporaroary FIX - Authenticating before profile selection. Gitlab authentication redirect resets the profiles section
+				AuthenticateWithGitProvider(webDriver, gitProviderEnv.Type)
+				pages.ScrollWindow(webDriver, 0, 4000)
+
 				By("And select the podinfo profile to install", func() {
 					Eventually(createPage.ProfileSelect.Click).Should(Succeed())
 					Eventually(createPage.SelectProfile("podinfo").Click).Should(Succeed())
@@ -1145,7 +1148,9 @@ func DescribeTemplates(gitopsTestRunner GitopsTestRunner) {
 				})
 
 				By(fmt.Sprintf("And I verify %s capd cluster is healthy and profiles are installed)", clusterName), func() {
-					verifyCapiClusterHealth(downloadedKubeconfigPath, GITOPS_DEFAULT_NAMESPACE)
+					// List of Profiles in order of layering
+					profiles := []string{"observability", "podinfo"}
+					verifyCapiClusterHealth(downloadedKubeconfigPath, clusterName, profiles, GITOPS_DEFAULT_NAMESPACE)
 				})
 
 				By("Then I should select the cluster to create the delete pull request", func() {

test/acceptance/test/utils_gitops.go

@@ -22,11 +22,13 @@ func waitForResource(resourceType string, resourceName string, namespace string,
 	}
 
 	timeoutInSeconds := int(timeout.Seconds())
+	cmd := fmt.Sprintf("kubectl %s get %s %s -n %s", kubeconfig, resourceType, resourceName, namespace)
+	logger.Trace(cmd)
 	for i := pollInterval; i < timeoutInSeconds; i += pollInterval {
 		logger.Tracef("Waiting for %s in namespace: %s... : %d second(s) passed of %d seconds timeout", resourceType+"/"+resourceName, namespace, i, timeoutInSeconds)
-		err := runCommandPassThroughWithoutOutput("sh", "-c", fmt.Sprintf("kubectl %s get %s %s -n %s", kubeconfig, resourceType, resourceName, namespace))
+		err := runCommandPassThroughWithoutOutput("sh", "-c", cmd)
 		if err == nil {
-			stdOut, _ := runCommandAndReturnStringOutput(fmt.Sprintf("kubectl %s get %s %s -n %s", kubeconfig, resourceType, resourceName, namespace))
+			stdOut, _ := runCommandAndReturnStringOutput(cmd)
 
 			noResourcesFoundMessage := ""
 			if namespace == "default" {
@@ -45,18 +47,19 @@ func waitForResource(resourceType string, resourceName string, namespace string,
 	return fmt.Errorf("error: Failed to find the resource %s of type %s, timeout reached", resourceName, resourceType)
 }
 
-func waitForResourceState(state string, resourceName string, nameSpace string, selector string, kubeconfig string) {
+func waitForResourceState(state string, statusCondition string, resourceName string, nameSpace string, selector string, kubeconfig string) {
 	if kubeconfig != "" {
 		kubeconfig = "--kubeconfig=" + kubeconfig
 	}
 
 	if selector != "" {
-		selector = "--selector=" + selector
+		selector = fmt.Sprintf("--selector='%s'", selector)
 	}
 
 	logger.Tracef("Waiting for %s '%s' state in namespace: %s", resourceName, state, nameSpace)
 
-	cmd := fmt.Sprintf(" kubectl wait --for=condition=%s --timeout=180s %s -n %s --all %s %s", state, resourceName, nameSpace, selector, kubeconfig)
+	cmd := fmt.Sprintf(" kubectl wait --for=condition=%s=%s --timeout=180s %s -n %s --all %s %s", state, statusCondition, resourceName, nameSpace, selector, kubeconfig)
+	logger.Trace(cmd)
 	_, stdErr := runCommandAndReturnStringOutput(cmd, ASSERTION_3MINUTE_TIME_OUT)
 	Expect(stdErr).Should(BeEmpty(), fmt.Sprintf("%s resource has failed to become %s.", resourceName, state))
 }
@@ -71,7 +74,7 @@ func verifyCoreControllers(namespace string) {
 	Expect(waitForResource("pods", "", namespace, "", ASSERTION_2MINUTE_TIME_OUT))
 
 	By("And I wait for the gitops core controllers to be ready", func() {
-		waitForResourceState("Ready", "pod", namespace, "app!=wego-app", "")
+		waitForResourceState("Ready", "true", "pod", namespace, "app!=wego-app", "")
 	})
 }
 
@@ -86,7 +89,7 @@ func verifyEnterpriseControllers(releaseName string, mccpPrefix, namespace strin
 	Expect(waitForResource("pods", "", namespace, "", ASSERTION_2MINUTE_TIME_OUT))
 
 	By("And I wait for the gitops enterprise controllers to be ready", func() {
-		waitForResourceState("Ready", "pod", namespace, "app!=wego-app", "")
+		waitForResourceState("Ready", "true", "pod", namespace, "app!=wego-app", "")
 	})
 }
 
@@ -101,36 +104,33 @@ func runWegoAddCommand(repoAbsolutePath string, addCommand string, namespace str
 }
 
 func verifyWegoAddCommand(appName string, namespace string) {
-	waitForResourceState("Ready", "GitRepositories", namespace, "", "")
+	waitForResourceState("Ready", "true", "GitRepositories", namespace, "", "")
 	Expect(waitForResource("GitRepositories", appName, namespace, "", ASSERTION_5MINUTE_TIME_OUT)).To(Succeed())
 }
 
 func installAndVerifyGitops(gitopsNamespace string, manifestRepoURL string) {
-
-	// Deploy key secret should not exist already
-	deleteGitopsDeploySecret(gitopsNamespace)
-
 	cmdInstall := fmt.Sprintf("%s install --config-repo %s --namespace=%s --auto-merge", gitops_bin_path, manifestRepoURL, gitopsNamespace)
-	By(fmt.Sprintf("And I run '%s'", cmdInstall), func() {
+	logger.Info(cmdInstall)
+
+	verifyGitRepositories := false
+	for i := 1; i < 5; i++ {
+		// Deploy key secret should not exist already
+		deleteGitopsDeploySecret(gitopsNamespace)
+		deleteGitopsGitRepository(gitopsNamespace)
 		_, stdErr := runCommandAndReturnStringOutput(cmdInstall, ASSERTION_5MINUTE_TIME_OUT)
 		Expect(stdErr).Should(BeEmpty())
 		verifyCoreControllers(gitopsNamespace)
 
 		// Check if GitRepository resource is Ready
-		cmdGitRepository := fmt.Sprintf(" kubectl wait --for=condition=Ready --timeout=120s -n %s GitRepositories --all", gitopsNamespace)
-		_, stdErr = runCommandAndReturnStringOutput(cmdGitRepository, ASSERTION_3MINUTE_TIME_OUT)
-
-		if stdErr != "" {
-			// Here we will do one more try to make the GitRepository Ready; maybe gitops install needs to do more error checking
-			deleteGitopsDeploySecret(gitopsNamespace)
-			deleteGitopsGitRepository(gitopsNamespace)
-			_, stdErr := runCommandAndReturnStringOutput(cmdInstall, ASSERTION_5MINUTE_TIME_OUT)
-			Expect(stdErr).Should(BeEmpty())
-			verifyCoreControllers(gitopsNamespace)
-
-			waitForResourceState("Ready", "GitRepositories", gitopsNamespace, "", "")
+		logger.Tracef("Waiting for GitRepositories 'Ready' state in namespace: %s", gitopsNamespace)
+		cmdGitRepository := fmt.Sprintf(" kubectl wait --for=condition=Ready --timeout=90s -n %s GitRepositories --all", gitopsNamespace)
+		_, stdErr = runCommandAndReturnStringOutput(cmdGitRepository, ASSERTION_2MINUTE_TIME_OUT)
+		if stdErr == "" {
+			verifyGitRepositories = true
+			break
 		}
-	})
+	}
+	Expect(verifyGitRepositories).Should(BeTrue(), "GitRepositories resource has failed to become READY.")
 }
 
 func removeGitopsCapiClusters(appName string, clusternames []string, nameSpace string) {
@@ -241,11 +241,25 @@ func verifyCapiClusterKubeconfig(kubeconfigPath string, capiCluster string) {
 	}
 }
 
-func verifyCapiClusterHealth(kubeconfigPath string, namespace string) {
-
-	Expect(waitForResource("nodes", "", "default", kubeconfigPath, ASSERTION_3MINUTE_TIME_OUT))
-	waitForResourceState("Ready", "nodes", "default", "", kubeconfigPath)
-
-	Expect(waitForResource("pods", "", namespace, kubeconfigPath, ASSERTION_3MINUTE_TIME_OUT))
-	waitForResourceState("Ready", "pods", namespace, "", kubeconfigPath)
+func verifyCapiClusterHealth(kubeconfigPath string, capiCluster string, profiles []string, namespace string) {
+
+	Expect(waitForResource("nodes", "", "default", kubeconfigPath, ASSERTION_2MINUTE_TIME_OUT))
+	waitForResourceState("Ready", "true", "nodes", "default", "", kubeconfigPath)
+
+	Expect(waitForResource("pods", "", namespace, kubeconfigPath, ASSERTION_2MINUTE_TIME_OUT))
+	waitForResourceState("Ready", "true", "pods", namespace, "", kubeconfigPath)
+
+	for _, profile := range profiles {
+		// Check all profiles are installed in layering order
+		switch profile {
+		case "observability":
+			Expect(waitForResource("deploy", capiCluster+"-observability-grafana", namespace, kubeconfigPath, ASSERTION_2MINUTE_TIME_OUT))
+			Expect(waitForResource("deploy", capiCluster+"-observability-kube-state-metrics", namespace, kubeconfigPath, ASSERTION_2MINUTE_TIME_OUT))
+			Expect(waitForResource("deploy", capiCluster+"-operator", namespace, kubeconfigPath, ASSERTION_2MINUTE_TIME_OUT))
+			waitForResourceState("Ready", "true", "pods", namespace, "release="+capiCluster+"-observability", kubeconfigPath)
+		case "podinfo":
+			Expect(waitForResource("deploy", capiCluster+"-podinfo ", namespace, kubeconfigPath, ASSERTION_2MINUTE_TIME_OUT))
+			waitForResourceState("Ready", "true", "pods", namespace, "app.kubernetes.io/name="+capiCluster+"-podinfo", kubeconfigPath)
+		}
+	}
 }

test/utils/data/capi-gitops-cluster-bootstrap-config.yaml

@@ -11,7 +11,7 @@ spec:
     generateName: "run-gitops-{{ .ObjectMeta.Name }}"
     spec:
       containers:
-        - image: ghcr.io/fluxcd/flux-cli:v0.26.3
+        - image: ghcr.io/fluxcd/flux-cli:v0.27.3
           name: flux-bootstrap
           resources: {}
           volumeMounts: