Small fixes and tweeks for nightly cluster cleanup (#796)

Author: saeedfazal

.github/workflows/acceptance-test.yaml

@@ -48,6 +48,11 @@ on:
         description: "Type of management cluster e.g. kind, eks or gke"
         required: true
         type: string
+      capi_provider:
+        description: "Capi provider e.g. capd, capa or capg"
+        default: "capd"
+        required: false
+        type: string
       gitops-bin-path:
         description: "Path to gitops binary"
         default: "/usr/local/bin/gitops"
@@ -161,6 +166,7 @@ env:
   MANAGEMENT_CLUSTER_CNAME: weave.gitops.enterprise.com
   UPGRADE_MANAGEMENT_CLUSTER_CNAME: weave.gitops.upgrade.enterprise.com
   MANAGEMENT_CLUSTER_KIND: ${{ inputs.management-cluster-kind }}
+  CAPI_PROVIDER: ${{ inputs.capi_provider }}
   GITOPS_BIN_PATH: ${{ inputs.gitops-bin-path }}
   ACCEPTANCE_TESTS_DATABASE_TYPE: ${{ inputs.database-type }}
   EKS_LEAF_KUBECONFIG: ${{ inputs.eks-leaf-kubeconfig }}

.github/workflows/deploy.yaml

@@ -91,6 +91,7 @@ jobs:
       git-provider_hostname: gitlab.git.dev.weave.works
       cluster_resource_set: true
       management-cluster-kind: kind
+      capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
       database-type: sqlite
       artifacts-base-dir: "/tmp/acceptance-test-gitlab-on-prem"

.github/workflows/nightly.yaml

@@ -37,6 +37,7 @@ jobs:
       git-provider_hostname: github.com
       cluster_resource_set: false
       management-cluster-kind: eks
+      capi_provider: capa
       gitops-bin-path: /usr/local/bin/gitops
       database-type: postgres
       eks-leaf-kubeconfig: "/tmp/eks-leaf-kubeconfig"
@@ -75,6 +76,7 @@ jobs:
       git-provider_hostname: gitlab.com
       cluster_resource_set: false
       management-cluster-kind: gke
+      capi_provider: capg
       gitops-bin-path: /usr/local/bin/gitops
       database-type: postgres
       gce-leaf-kubeconfig: "/tmp/gce-leaf-kubeconfig"

.github/workflows/test.yaml

@@ -237,6 +237,7 @@ jobs:
       git-provider_hostname: github.com
       cluster_resource_set: true
       management-cluster-kind: kind
+      capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
       database-type: sqlite
       artifacts-base-dir: "/tmp/smoke-test-github"
@@ -265,6 +266,7 @@ jobs:
       git-provider_hostname: gitlab.com
       cluster_resource_set: true
       management-cluster-kind: kind
+      capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
       database-type: sqlite
       artifacts-base-dir: "/tmp/smoke-test-gitlab"
@@ -295,6 +297,7 @@ jobs:
       git-provider_hostname: gitlab.git.dev.weave.works
       cluster_resource_set: true
       management-cluster-kind: kind
+      capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
       database-type: sqlite
       artifacts-base-dir: "/tmp/smoke-test-gitlab-on-prem"
@@ -331,6 +334,7 @@ jobs:
       git-provider_hostname: github.com
       cluster_resource_set: true
       management-cluster-kind: kind
+      capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
       database-type: sqlite
       artifacts-base-dir: "/tmp/acceptance-test-github"

docs/acceptance_testing.md

@@ -55,6 +55,7 @@ export SELENIUM_DEBUG=true
 export GITOPS_BIN_PATH=/usr/local/bin/gitops
 export ARTIFACTS_BASE_DIR=/tmp/acceptance-tests
 export MANAGEMENT_CLUSTER_KIND=kind
+export CAPI_PROVIDER=capd
 export ACCEPTANCE_TESTS_DATABASE_TYPE=sqlite
 export EXP_CLUSTER_RESOURCE_SET=true
 export UI_NODEPORT=30080
@@ -64,12 +65,13 @@ export CLUSTER_REPOSITORY=gitops-testing
 export LOGIN_USER_TYPE=cluster-user
 ```
 
-You can either set 'LOGIN_USER_TYPE' to `oidc` if oidc user authentication is desired or `cluster-user` if cluster user i.e. `admin` account authentication is desired to run the tests.
+You can either set 'LOGIN_USER_TYPE' to `oidc` if oidc user authentication is desired or `cluster-user` if cluster user i.e. `wego-admin` account authentication is desired to run the tests.
+You can also set 'CAPI_PROVIDER' to 'capa' or 'capg' as per requirements.
 
 **User login**
 ```
-export CLUSTER_ADMIN_PASSWORD=<Your chosen password for admin account>
-export CLUSTER_ADMIN_PASSWORD_HASH=<Bcrypt hash for your chosen password>
+export CLUSTER_ADMIN_PASSWORD=wego-enterprise
+export CLUSTER_ADMIN_PASSWORD_HASH='$2b$12$1mxK92n85K.Zga8eNe2j.eqhtC2HnSrvbOk9obSVKbUgJm4SCMwFm'
 export OIDC_ISSUER_URL='https://dex-01.wge.dev.weave.works'
 export DEX_CLIENT_ID='weave-gitops-enterprise'
 export DEX_CLIENT_SECRET='2JPIcb5IvO1isJ3Zii7jvjqbUtLtTC'

test/acceptance/test/ui_templates.go

@@ -849,6 +849,7 @@ func DescribeTemplates(gitopsTestRunner GitopsTestRunner) {
 				}
 
 				setParameterValues(createPage, paramSection)
+				pages.ScrollWindow(webDriver, 0, 4000)
 
 				By("Then I should see PR preview containing identity reference added in the template", func() {
 					Eventually(createPage.PreviewPR.Click).Should(Succeed())
@@ -958,6 +959,7 @@ func DescribeTemplates(gitopsTestRunner GitopsTestRunner) {
 				}
 
 				setParameterValues(createPage, paramSection)
+				pages.ScrollWindow(webDriver, 0, 4000)
 
 				By("Then I should see PR preview without identity reference added to the template", func() {
 					Expect(createPage.PreviewPR.Click()).To(Succeed())

test/acceptance/test/utils.go

@@ -34,6 +34,7 @@ var (
 	git_repository_url   string
 	selenium_service_url string
 	gitops_bin_path      string
+	capi_provider        string
 	capi_endpoint_url    string
 	test_ui_url          string
 	artifacts_base_dir   string
@@ -126,6 +127,7 @@ func SetupTestEnvironment() {
 	test_ui_url = fmt.Sprintf(`https://%s:%s`, GetEnv("MANAGEMENT_CLUSTER_CNAME", "localhost"), GetEnv("UI_NODEPORT", "30080"))
 	capi_endpoint_url = fmt.Sprintf(`https://%s:%s`, GetEnv("MANAGEMENT_CLUSTER_CNAME", "localhost"), GetEnv("UI_NODEPORT", "30080"))
 	gitops_bin_path = GetEnv("GITOPS_BIN_PATH", "/usr/local/bin/gitops")
+	capi_provider = GetEnv("CAPI_PROVIDER", "capd")
 	artifacts_base_dir = GetEnv("ARTIFACTS_BASE_DIR", "/tmp/gitops-test/")
 
 	gitProviderEnv = initGitProviderData()
@@ -190,7 +192,7 @@ func InitializeWebdriver(wgeURL string) {
 			a["enableNetwork"] = true
 			chromeDriver := agouti.ChromeDriver(
 				agouti.ChromeOptions("w3c", false),
-				agouti.ChromeOptions("args", []string{"--disable-gpu", "--no-sandbox", "window-size=1500,2000", "--disable-blink-features=AutomationControlled", "--ignore-ssl-errors=yes", "--ignore-certificate-errors"}),
+				agouti.ChromeOptions("args", []string{"--disable-gpu", "--no-sandbox", "window-size=1800,2500", "--disable-blink-features=AutomationControlled", "--ignore-ssl-errors=yes", "--ignore-certificate-errors"}),
 				agouti.ChromeOptions("excludeSwitches", []string{"enable-automation"}))
 			err = chromeDriver.Start()
 			Expect(err).NotTo(HaveOccurred())
@@ -201,7 +203,7 @@ func InitializeWebdriver(wgeURL string) {
 			webDriver, err = agouti.NewPage(selenium_service_url, agouti.Debug, agouti.Desired(agouti.Capabilities{
 				"acceptInsecureCerts": true,
 				"chromeOptions": map[string]interface{}{
-					"args":            []string{"--disable-gpu", "--no-sandbox", "window-size=1500,2000", "--disable-blink-features=AutomationControlled"},
+					"args":            []string{"--disable-gpu", "--no-sandbox", "window-size=1800,2500", "--disable-blink-features=AutomationControlled"},
 					"w3c":             false,
 					"excludeSwitches": []string{"enable-automation"},
 				}}))

test/acceptance/test/utils_runner.go

@@ -344,17 +344,21 @@ func (b RealGitopsTestRunner) RestartDeploymentPods(appName string, namespace st
 func (b RealGitopsTestRunner) CreateIPCredentials(infrastructureProvider string) {
 	testDataPath := path.Join(getCheckoutRepoPath(), "test", "utils", "data")
 	if infrastructureProvider == "AWS" {
-		By("Install AWSClusterStaticIdentity CRD", func() {
-			_, _ = runCommandAndReturnStringOutput(fmt.Sprintf("kubectl apply -f %s/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml", testDataPath))
-			_, _ = runCommandAndReturnStringOutput("kubectl wait --for=condition=established --timeout=90s crd/awsclusterstaticidentities.infrastructure.cluster.x-k8s.io", ASSERTION_2MINUTE_TIME_OUT)
-		})
-
-		By("Install AWSClusterRoleIdentity CRD", func() {
-			_, _ = runCommandAndReturnStringOutput(fmt.Sprintf("kubectl apply -f %s/infrastructure.cluster.x-k8s.io_awsclusterroleidentities.yaml", testDataPath))
-			_, _ = runCommandAndReturnStringOutput("kubectl wait --for=condition=established --timeout=90s crd/awsclusterroleidentities.infrastructure.cluster.x-k8s.io", ASSERTION_2MINUTE_TIME_OUT)
-		})
+		// CAPA installs the AWS identity crds
+		if capi_provider != "capa" {
+			By("Install AWSClusterStaticIdentity CRD", func() {
+				_, _ = runCommandAndReturnStringOutput(fmt.Sprintf("kubectl apply -f %s/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml", testDataPath))
+				_, _ = runCommandAndReturnStringOutput("kubectl wait --for=condition=established --timeout=90s crd/awsclusterstaticidentities.infrastructure.cluster.x-k8s.io", ASSERTION_2MINUTE_TIME_OUT)
+			})
+
+			By("Install AWSClusterRoleIdentity CRD", func() {
+				_, _ = runCommandAndReturnStringOutput(fmt.Sprintf("kubectl apply -f %s/infrastructure.cluster.x-k8s.io_awsclusterroleidentities.yaml", testDataPath))
+				_, _ = runCommandAndReturnStringOutput("kubectl wait --for=condition=established --timeout=90s crd/awsclusterroleidentities.infrastructure.cluster.x-k8s.io", ASSERTION_2MINUTE_TIME_OUT)
+			})
+		}
 
 		By("Create AWS Secret, AWSClusterStaticIdentity and AWSClusterRoleIdentity)", func() {
+			_, _ = runCommandAndReturnStringOutput("kubectl create namespace capa-system")
 			_, _ = runCommandAndReturnStringOutput(fmt.Sprintf("kubectl apply -f %s/aws_cluster_credentials.yaml", testDataPath), ASSERTION_30SECONDS_TIME_OUT)
 		})
 
@@ -375,9 +379,13 @@ func (b RealGitopsTestRunner) DeleteIPCredentials(infrastructureProvider string)
 	testDataPath := path.Join(getCheckoutRepoPath(), "test", "utils", "data")
 	if infrastructureProvider == "AWS" {
 		By("Delete AWS identities and CRD", func() {
+			// Identity crds are installed as part of CAPA installation
 			_ = b.KubectlDelete([]string{}, fmt.Sprintf("%s/aws_cluster_credentials.yaml", testDataPath))
-			_ = b.KubectlDelete([]string{}, fmt.Sprintf("%s/infrastructure.cluster.x-k8s.io_awsclusterroleidentities.yaml", testDataPath))
-			_ = b.KubectlDelete([]string{}, fmt.Sprintf("%s/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml", testDataPath))
+			if capi_provider != "capa" {
+				_ = b.KubectlDelete([]string{}, fmt.Sprintf("%s/infrastructure.cluster.x-k8s.io_awsclusterroleidentities.yaml", testDataPath))
+				_ = b.KubectlDelete([]string{}, fmt.Sprintf("%s/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml", testDataPath))
+				_, _ = runCommandAndReturnStringOutput("kubectl delete namespace capa-system")
+			}
 		})
 
 	} else if infrastructureProvider == "AZURE" {

test/utils/data/aws_cluster_credentials.yaml

@@ -1,9 +1,3 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: capa-system
-
 ---
 apiVersion: v1
 kind: Secret

test/utils/scripts/wego-enterprise.sh

@@ -215,7 +215,7 @@ function setup {
   # enable cluster resource sets
   export EXP_CLUSTER_RESOURCE_SET=true
   # Install capi infrastructure provider
-  if [ "$MANAGEMENT_CLUSTER_KIND" == "eks" ]; then
+  if [ "$CAPI_PROVIDER" == "capa" ]; then
     aws cloudformation describe-stacks --stack-name wge-capi-cluster-api-provider-aws-sigs-k8s-io --region us-east-1
     if [ $? -ne 0 ]; then
       clusterawsadm bootstrap iam create-cloudformation-stack --config aws_bootstrap_config.yaml --region=us-east-1
@@ -226,7 +226,7 @@ function setup {
       aws ec2 create-key-pair --key-name weave-gitops-pesto --region us-east-1 --output text > ~/.ssh/weave-gitops-pesto.pem
     fi
     clusterctl init --infrastructure aws
-  elif [ "$MANAGEMENT_CLUSTER_KIND" == "gke" ]; then
+  elif [ "$CAPI_PROVIDER" == "capg" ]; then
     export GCP_B64ENCODED_CREDENTIALS=$( echo ${GCP_SA_KEY} | base64 | tr -d '\n' )
     clusterctl init --infrastructure gcp
   else
@@ -278,8 +278,6 @@ function reset {
   # Delete postgres db 
   kubectl delete deployment postgres
   kubectl delete service postgres
-  # Delete namespaces and their respective resources
-  kubectl delete namespaces wkp-agent
   # Delete flux system from the management cluster
   flux uninstall --silent
   # Delete any orphan resources
@@ -288,11 +286,19 @@ function reset {
   kubectl delete secret my-pat
   kubectl delete ClusterResourceSet --all
   kubectl delete configmap calico-crs-configmap
-  kubectl delete rolebinding read-templates
-  kubectl delete rolebinding clusters-service-secrets-role
+  kubectl delete ClusterRoleBinding clusters-service-impersonator
+  kubectl delete ClusterRole clusters-service-impersonator-role 
   # Delete policy agent
   kubectl delete ValidatingWebhookConfiguration policy-agent
   kubectl delete namespaces policy-system  
+  # Delete capi provider
+  if [ "$CAPI_PROVIDER" == "capa" ]; then
+    clusterctl delete --infrastructure aws
+  elif [ "$CAPI_PROVIDER" == "capg" ]; then
+    clusterctl delete --infrastructure gcp
+  else
+    clusterctl delete --infrastructure docker    
+  fi
 }
 
 function reset_controllers {