Schedule acceptance tests for CI pipelines

.github/workflows/acceptance-test.yaml

@@ -58,8 +58,8 @@ on:
         default: "/usr/local/bin/gitops"
         required: false
         type: string
-      artifacts-base-dir:
-        description: "Directory for test artifacts"
+      test-artifact-name:
+        description: "Test artifacts name"
         required: true
         type: string
       eks-leaf-kubeconfig:
@@ -82,7 +82,7 @@ on:
         required: true
       WGE_GIT_PROVIDER_PRIVATE_KEY:
         description: "Key for accessing git provider e.g. github or gitlab"
-        required: false
+        required: false      
       WGE_GITHUB_TOKEN:
         description: "Token for accessing GitHub"
         required: false
@@ -116,6 +116,9 @@ on:
       WGE_GITLAB_CLIENT_SECRET:
         description: "Gitlab oauth pplication secret"
         required: false
+      WGE_SLACK_BOT_TOKEN:
+        description: "Token for posting messages to slack"
+        required: false
       GCP_PROJECT_ID:
         description: "Project id of the Google Cloud project"
         required: false
@@ -156,7 +159,8 @@ env:
   MANAGEMENT_CLUSTER_KIND: ${{ inputs.management-cluster-kind }}
   CAPI_PROVIDER: ${{ inputs.capi_provider }}
   GITOPS_BIN_PATH: ${{ inputs.gitops-bin-path }}
-  ARTIFACTS_BASE_DIR: ${{ inputs.artifacts-base-dir }}
+  ARTIFACTS_BASE_DIR: "/tmp/acceptance-test-artifact"
+  TEST_ARTIFACT_NAME: ${{ inputs.test-artifact-name }}  
   EXP_CLUSTER_RESOURCE_SET: ${{ inputs.cluster_resource_set }}
   USE_GKE_GCLOUD_AUTH_PLUGIN: True
   SELENIUM_DEBUG: true
@@ -358,17 +362,13 @@ jobs:
             export GITLAB_HOSTNAME=${{ env.GIT_PROVIDER_HOSTNAME }}
           fi
 
-          # Set the ACCEPTANCE_TEST_ARTIFACT_NAME as environment variable for subsequent steps
-          ACCEPTANCE_TEST_ARTIFACT_NAME=$(basename ${{ env.ARTIFACTS_BASE_DIR }})-artifacts
-          echo "ACCEPTANCE_TEST_ARTIFACT_NAME=$ACCEPTANCE_TEST_ARTIFACT_NAME" >> $GITHUB_ENV
-
           cd test/acceptance/test/
-          go test ${{ inputs.label-filter }} -ginkgo.v -ginkgo.junit-report=${{ env.ARTIFACTS_BASE_DIR }}/acceptance-test-results.xml --timeout=99999s
+          go test ${{ inputs.label-filter }} -ginkgo.v -ginkgo.junit-report=${{ env.ARTIFACTS_BASE_DIR }}/${{ env.TEST_ARTIFACT_NAME }}.xml --timeout=99999s
       - name: Store test results
         if: ${{ always() }}
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ env.ACCEPTANCE_TEST_ARTIFACT_NAME }}
+          name: ${{ env.TEST_ARTIFACT_NAME }}
           path: ${{ env.ARTIFACTS_BASE_DIR }}
           retention-days: 7
       - name: Reset management cluster

.github/workflows/deploy.yaml

@@ -75,23 +75,81 @@ jobs:
       runs-on: ubuntu-latest
       ref: main
 
-  acceptance-tests-gitlab-on-prem:
+  smoke-tests-github:
     needs: [build, coverage, gitops-binary]
     uses: ./.github/workflows/acceptance-test.yaml
     with:
       runs-on: ubuntu-latest
       os-name: linux
-      timeout-minutes: 90
-      label-filter: "-ginkgo.label-filter='!(upgrade)'"
-      kubectl-version: "v1.23.3"
+      timeout-minutes: 60
+      label-filter: "-ginkgo.label-filter='smoke'"
+      kubectl-version: "v1.21.1"
+      login_user_type: "oidc"
+      git-provider: github
+      git-provider_hostname: github.com
+      cluster_resource_set: true
+      management-cluster-kind: kind
+      capi_provider: capd
+      gitops-bin-path: /usr/local/bin/gitops
+      test-artifact-name: smoke-tests-github
+    secrets:
+      WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
+      WGE_CLUSTER_ADMIN_PASSWORD_HASH: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD_HASH }}"
+      WGE_DEX_CLIENT_SECRET: ${{ secrets.WGE_DEX_CLIENT_SECRET }}
+      WGE_GIT_PROVIDER_PRIVATE_KEY: ${{ secrets.WGE_GITHUB_PRIVATE_KEY }}
+      WGE_GITHUB_TOKEN: ${{ secrets.WGE_GITHUB_TOKEN }}
+      WGE_GITHUB_ORG: wge-org-test
+      WGE_GITHUB_USER: ${{ secrets.WGE_GITHUB_USER }}
+      WGE_GITHUB_PASSWORD: ${{ secrets.WGE_GITHUB_PASSWORD }}
+      WGE_GITHUB_MFA_KEY: ${{ secrets.WGE_GITHUB_MFA_KEY }}
+
+  smoke-tests-gitlab:
+    needs: [build, coverage, gitops-binary]
+    uses: ./.github/workflows/acceptance-test.yaml
+    with:
+      runs-on: ubuntu-latest
+      os-name: linux
+      timeout-minutes: 60
+      label-filter: "-ginkgo.label-filter='smoke'"
+      kubectl-version: "v1.22.0"
       login_user_type: "cluster-user"
       git-provider: gitlab
+      git-provider_hostname: gitlab.com
+      cluster_resource_set: true
+      management-cluster-kind: kind
+      capi_provider: capd
+      gitops-bin-path: /usr/local/bin/gitops
+      test-artifact-name: smoke-tests-gitlab
+    secrets:
+      WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
+      WGE_CLUSTER_ADMIN_PASSWORD_HASH: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD_HASH }}"
+      WGE_DEX_CLIENT_SECRET: ${{ secrets.WGE_DEX_CLIENT_SECRET }}
+      WGE_GITHUB_TOKEN: ${{ secrets.WGE_GITHUB_TOKEN }}
+      WGE_GIT_PROVIDER_PRIVATE_KEY: ${{ secrets.WGE_SAS_GITLAB_PRIVATE_KEY }}
+      WGE_GITLAB_TOKEN: ${{ secrets.WGE_SAS_GITLAB_TOKEN }}
+      WGE_GITLAB_ORG: ${{ secrets.WGE_SAS_GITLAB_ORG }}
+      WGE_GITLAB_USER: ${{ secrets.WGE_SAS_GITLAB_USER }}
+      WGE_GITLAB_PASSWORD: ${{ secrets.WGE_SAS_GITLAB_PASSWORD }}
+      WGE_GITLAB_CLIENT_ID: ${{ secrets.WGE_SAS_GITLAB_CLIENT_ID }}
+      WGE_GITLAB_CLIENT_SECRET: ${{ secrets.WGE_SAS_GITLAB_CLIENT_SECRET }}
+
+  smoke-tests-gitlab-on-prem:
+    needs: [build, coverage, gitops-binary]
+    uses: ./.github/workflows/acceptance-test.yaml
+    with:
+      runs-on: ubuntu-latest
+      os-name: linux
+      timeout-minutes: 60
+      label-filter: "-ginkgo.label-filter='smoke'"
+      kubectl-version: "v1.23.3"
+      login_user_type: "oidc"
+      git-provider: gitlab
       git-provider_hostname: gitlab.git.dev.weave.works
       cluster_resource_set: true
       management-cluster-kind: kind
       capi_provider: capd
       gitops-bin-path: /usr/local/bin/gitops
-      artifacts-base-dir: "/tmp/acceptance-test-gitlab-on-prem"
+      test-artifact-name: smoke-tests-gitlab-on-prem
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
       WGE_CLUSTER_ADMIN_PASSWORD_HASH: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD_HASH }}"
@@ -105,6 +163,17 @@ jobs:
       WGE_GITLAB_CLIENT_ID: ${{ secrets.WGE_ON_PREM_GITLAB_CLIENT_ID }}
       WGE_GITLAB_CLIENT_SECRET: ${{ secrets.WGE_ON_PREM_GITLAB_CLIENT_SECRET }}
 
+  smoke-test-results:
+    needs: [smoke-tests-github, smoke-tests-gitlab, smoke-tests-gitlab-on-prem]
+    uses: ./.github/workflows/publish-test-results.yaml
+    with:
+      runs-on: ubuntu-latest
+      junit-results-glob-pattern: "*smoke-tests*.xml"
+      check-name: "Smoke Tests"
+      slack-notification: false
+    secrets:
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
   snyk-monitor:
     runs-on: ubuntu-latest
     steps:

.github/workflows/nightly.yaml

@@ -8,7 +8,7 @@ jobs:
   build:
     uses: ./.github/workflows/build.yaml
     with:
-      helmrepo: "nightly/charts-v3"
+      helmrepo: "charts-v3"
     secrets:
       BUILD_BOT_PERSONAL_ACCESS_TOKEN: ${{ secrets.BUILD_BOT_PERSONAL_ACCESS_TOKEN }}
       WGE_DOCKER_IO_USER: ${{ secrets.WGE_DOCKER_IO_USER }}
@@ -23,23 +23,53 @@ jobs:
       runs-on: ubuntu-latest
       ref: main
 
+  acceptance-tests-kind-gitlab-on-prem:
+    needs: [build, gitops-binary-ubuntu]
+    uses: ./.github/workflows/acceptance-test.yaml
+    with:
+      runs-on: ubuntu-latest
+      os-name: linux
+      timeout-minutes: 90
+      label-filter: "-ginkgo.label-filter='!(smoke, upgrade)'"
+      kubectl-version: "v1.23.3"
+      login_user_type: "cluster-user"
+      git-provider: gitlab
+      git-provider_hostname: gitlab.git.dev.weave.works
+      cluster_resource_set: true
+      management-cluster-kind: kind
+      capi_provider: capd
+      gitops-bin-path: /usr/local/bin/gitops      
+      test-artifact-name: acceptance-tests-kind-gitlab-on-prem
+    secrets:
+      WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
+      WGE_CLUSTER_ADMIN_PASSWORD_HASH: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD_HASH }}"
+      WGE_DEX_CLIENT_SECRET: ${{ secrets.WGE_DEX_CLIENT_SECRET }}
+      WGE_GITHUB_TOKEN: ${{ secrets.WGE_GITHUB_TOKEN }}
+      WGE_GIT_PROVIDER_PRIVATE_KEY: ${{ secrets.WGE_ON_PREM_GITLAB_PRIVATE_KEY }}
+      WGE_GITLAB_TOKEN: ${{ secrets.WGE_ON_PREM_GITLAB_TOKEN }}
+      WGE_GITLAB_ORG: ${{ secrets.WGE_ON_PREM_GITLAB_ORG }}
+      WGE_GITLAB_USER: ${{ secrets.WGE_ON_PREM_GITLAB_USER }}
+      WGE_GITLAB_PASSWORD: ${{ secrets.WGE_ON_PREM_GITLAB_PASSWORD }}
+      WGE_GITLAB_CLIENT_ID: ${{ secrets.WGE_ON_PREM_GITLAB_CLIENT_ID }}
+      WGE_GITLAB_CLIENT_SECRET: ${{ secrets.WGE_ON_PREM_GITLAB_CLIENT_SECRET }}
+
   acceptance-tests-eks-github:
     needs: [build, gitops-binary-ubuntu]
     uses: ./.github/workflows/acceptance-test.yaml
     with:
       runs-on: ubuntu-latest
       os-name: linux
       timeout-minutes: 90
-      label-filter: "-ginkgo.label-filter='!(capd, kind-gitops-cluster, leaf-policy, leaf-violation)'"
+      label-filter: "-ginkgo.label-filter='!(capd, upgrade, kind-gitops-cluster, leaf-policy, leaf-violation)'"
       kubectl-version: "v1.23.6"
       login_user_type: "oidc"
       git-provider: github
       git-provider_hostname: github.com
       cluster_resource_set: false
       management-cluster-kind: eks
       capi_provider: capa
-      gitops-bin-path: /usr/local/bin/gitops
-      artifacts-base-dir: "/tmp/acceptance-test-eks-github"
+      gitops-bin-path: /usr/local/bin/gitops      
+      test-artifact-name: acceptance-tests-eks-github
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
       WGE_CLUSTER_ADMIN_PASSWORD_HASH: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD_HASH }}"
@@ -75,7 +105,7 @@ jobs:
       management-cluster-kind: gke
       capi_provider: capg
       gitops-bin-path: /usr/local/bin/gitops
-      artifacts-base-dir: "/tmp/acceptance-test-gke-gitlab"
+      test-artifact-name: acceptance-tests-gke-gitlab
     secrets:
       WGE_CLUSTER_ADMIN_PASSWORD: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD }}"
       WGE_CLUSTER_ADMIN_PASSWORD_HASH: "${{ secrets.WGE_CLUSTER_ADMIN_PASSWORD_HASH }}"
@@ -90,3 +120,14 @@ jobs:
       WGE_GITLAB_CLIENT_SECRET: ${{ secrets.WGE_SAS_GITLAB_CLIENT_SECRET }}
       GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
       GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+
+  acceptance-test-results:
+    needs: [acceptance-tests-kind-gitlab-on-prem, acceptance-tests-eks-github, acceptance-tests-gke-gitlab]
+    uses: ./.github/workflows/publish-test-results.yaml
+    with:
+      runs-on: ubuntu-latest
+      junit-results-glob-pattern: "*acceptance-tests*.xml"
+      check-name: "Acceptance Tests"
+      slack-notification: true
+    secrets:
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} 

.github/workflows/publish-test-results.yaml

@@ -0,0 +1,50 @@
+name: "Test result summary"
+on:
+  workflow_call:
+    inputs:
+      runs-on:
+        description: "The type of machine to run the job on e.g macOS-latest or ubuntu-latest"
+        required: true
+        type: string      
+      junit-results-glob-pattern:
+        description: "Glob expression to junit report paths"
+        required: true
+        type: string
+      check-name:
+        description: "Check run name"
+        required: true
+        type: string
+      slack-notification:
+        description: "Enable slack push notification for test results"
+        default: false
+        required: false
+        type: boolean 
+    secrets:
+      SLACK_BOT_TOKEN:
+        description: "Slack bot secret for push notification"
+        required: false
+env:
+  TEST_RESULTS_PATH: "/tmp/test-results"
+jobs:
+  tests:
+    runs-on: ${{ inputs.runs-on }}
+    steps:
+      - name: Download test artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.TEST_RESULTS_PATH }}
+      - name: Publish Test Report      
+        id: test-report
+        uses: mikepenz/action-junit-report@v3
+        if: always()
+        with:
+          report_paths: ${{ env.TEST_RESULTS_PATH }}/**/${{ inputs.junit-results-glob-pattern }}
+          check_name: ${{ inputs.check-name }}
+      - name: Notify slack of test results
+        if: ${{ (inputs.slack-notification) && (github.event_name != 'workflow_dispatch') }}
+        uses: skarlso/slack-notification-action@28f5c1166e2fba7ed5a1d9ceaeb4f2c89a4e5bc5
+        with:
+          token: ${{ secrets.SLACK_BOT_TOKEN }}
+          channel: C03FDNVE7RC
+          message: "*Smoke Tests*\n>Tests :test_tube:\t\t\tPassed :check:\t\t\tSkipped :arrow_right_hook:\t\t\tFailed :x:\n>${{steps.test-report.outputs.total}} runs\t\t\t ${{steps.test-report.outputs.passed}} passed\t\t\t ${{steps.test-report.outputs.skipped}} skipped\t\t\t  ${{steps.test-report.outputs.failed}} failed"
+