[DMS-818] Relational flattening performance spike #2003
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | |
| # Licensed to the Ed-Fi Alliance under one or more agreements. | |
| # The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. | |
| # See the LICENSE and NOTICES files in the project root for more information. | |
| name: On DMS Pull Request | |
| on: | |
| push: | |
| # Running on push to main to support CodeQL on C# | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CONFIGURATION: "Release" | |
| permissions: read-all | |
| jobs: | |
| scan-actions-bidi: | |
| name: Scan Actions, scan all files for BIDI Trojan Attacks | |
| uses: Ed-Fi-Alliance-OSS/Ed-Fi-Actions/.github/workflows/repository-scanner.yml@main | |
| with: | |
| config-file-path: ./.github/workflows/bidi-config.json | |
| run-unit-tests: | |
| name: Run Unit Tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| defaults: | |
| run: | |
| shell: pwsh | |
| steps: | |
| - name: Checkout the Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Cache Nuget packages | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 | |
| with: | |
| path: ~/.nuget/packages | |
| key: ${{ runner.os }}-nuget-${{ hashFiles('**/Directory.Packages.props') }} | |
| restore-keys: | | |
| ${{ runner.os }}-nuget- | |
| - name: Setup dotnet | |
| uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2 | |
| with: | |
| dotnet-version: "8.0.x" | |
| - name: Build | |
| run: ./build-dms.ps1 Build -Configuration ${{ env.CONFIGURATION }} | |
| - name: Install Coverlet Reference | |
| if: success() | |
| run: | | |
| dotnet tool install --global coverlet.console | |
| dotnet tool install --global dotnet-reportgenerator-globaltool | |
| - name: Run Unit Tests | |
| run: ./build-dms.ps1 UnitTest -Configuration ${{ env.CONFIGURATION }} | |
| - name: Generate Coverage Report | |
| if: ${{ hashFiles('coverage.cobertura.xml') != '' }} | |
| run: ./build-dms.ps1 Coverage | |
| - name: Upload Coverage Report | |
| if: always() | |
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5 | |
| with: | |
| name: Coverage Report | |
| path: coveragereport | |
| - name: Upload Test Results | |
| uses: dorny/test-reporter@31a54ee7ebcacc03a09ea97a7e5465a47b84aea5 # v1.9.1 | |
| if: (success() || failure()) | |
| with: | |
| name: Unit Tests | |
| path: "**/*.trx" | |
| path-replace-backslashes: "true" | |
| reporter: dotnet-trx | |
| run-integration-tests: | |
| name: Run Integration Tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| defaults: | |
| run: | |
| shell: pwsh | |
| services: | |
| postgres: | |
| image: postgres:16.3-alpine | |
| options: >- | |
| --health-cmd="pg_isready" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=5 | |
| ports: | |
| - 5432:5432 | |
| env: | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| steps: | |
| - name: Checkout the Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Cache Nuget packages | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 | |
| with: | |
| path: ~/.nuget/packages | |
| key: ${{ runner.os }}-nuget-${{ hashFiles('**/Directory.Packages.props') }} | |
| restore-keys: | | |
| ${{ runner.os }}-nuget- | |
| - name: Setup dotnet | |
| uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2 | |
| with: | |
| dotnet-version: "8.0.x" | |
| - name: Build | |
| run: ./build-dms.ps1 Build -Configuration ${{ env.CONFIGURATION }} | |
| - name: Run Integration Tests | |
| if: success() | |
| run: ./build-dms.ps1 IntegrationTest -Configuration ${{ env.CONFIGURATION }} | |
| - name: Upload Integration Test Results | |
| uses: dorny/test-reporter@31a54ee7ebcacc03a09ea97a7e5465a47b84aea5 # v1.9.1 | |
| if: (success() || failure()) | |
| with: | |
| name: Integration Tests | |
| path: "**/*.trx" | |
| path-replace-backslashes: "true" | |
| reporter: dotnet-trx | |
| run-e2e-tests: | |
| name: Run ${{matrix.queryhandler}} E2E Tests (IdentityProvider ${{matrix.identityprovider}}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| identityprovider: [keycloak, self-contained] | |
| queryhandler: [opensearch, postgresql] | |
| permissions: | |
| checks: write | |
| defaults: | |
| run: | |
| shell: pwsh | |
| steps: | |
| - name: Checkout the Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0 | |
| - name: Cache Docker layers | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ hashFiles('src/Directory.Packages.props', 'src/dms/Dockerfile', 'src/dms/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx- | |
| - name: Build DMS Docker image | |
| uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 | |
| with: | |
| load: true | |
| context: ./src/dms | |
| file: ./src/dms/Dockerfile | |
| build-contexts: | | |
| parentdir=./src | |
| push: false | |
| tags: dms-local-dms:latest | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| cache-to: type=local,dest=/tmp/.buildx-cache | |
| - name: Cache Nuget packages | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 | |
| with: | |
| path: ~/.nuget/packages | |
| key: ${{ runner.os }}-nuget-${{ hashFiles('**/Directory.Packages.props') }} | |
| restore-keys: | | |
| ${{ runner.os }}-nuget- | |
| - name: Setup dotnet | |
| uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2 | |
| with: | |
| dotnet-version: "8.0.x" | |
| - name: Build | |
| run: ./build-dms.ps1 Build -Configuration ${{ env.CONFIGURATION }} -IdentityProvider ${{matrix.identityprovider}} | |
| - name: Add Kafka hostname to /etc/hosts | |
| run: echo "127.0.0.1 dms-kafka1" | sudo tee -a /etc/hosts | |
| - name: Run ${{matrix.queryhandler}} - IdentityProvider (${{matrix.identityprovider}}) E2E Tests | |
| if: success() | |
| env: | |
| KAFKA_BOOTSTRAP_SERVERS: dms-kafka1:9092 | |
| run: | | |
| if ("${{ matrix.queryhandler }}" -eq "postgresql") { | |
| $env:DMS_QUERYHANDLER="postgresql" | |
| # TODO: We're starting up OpenSearch's Docker container to keep both scenarios | |
| # as similar as possible so that we can compare their performance. | |
| } | |
| ./build-dms.ps1 E2ETest -EnableOpenSearch -Configuration ${{ env.CONFIGURATION }} -SkipDockerBuild -IdentityProvider ${{matrix.identityprovider}} | |
| - name: Export Docker Logs | |
| if: failure() | |
| run: | | |
| $logDir = "${{ github.workspace }}/logs" | |
| New-Item -ItemType Directory -Path $logDir -Force | Out-Null | |
| docker ps --format "{{.Names}}" | ForEach-Object { | |
| $container = $_ | |
| $logPath = Join-Path $logDir "$container.log" | |
| docker logs $container > $logPath | |
| } | |
| shell: pwsh | |
| - name: Upload Logs | |
| if: failure() | |
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5 | |
| with: | |
| name: ${{matrix.queryhandler}}-test-logs | |
| path: | | |
| ${{ github.workspace }}/logs | |
| retention-days: 10 | |
| - name: Upload Test Results | |
| uses: dorny/test-reporter@31a54ee7ebcacc03a09ea97a7e5465a47b84aea5 # v1.9.1 | |
| if: (success() || failure()) | |
| with: | |
| name: E2E Test Results | |
| path: "**/*.trx" | |
| path-replace-backslashes: "true" | |
| reporter: dotnet-trx | |
| build-and-start-dms: | |
| name: Build and Start DMS, Download OpenAPI Specs | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: pwsh | |
| steps: | |
| - name: Checkout the Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Cache Nuget packages | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 | |
| with: | |
| path: ~/.nuget/packages | |
| key: ${{ runner.os }}-nuget-${{ hashFiles('**/Directory.Packages.props') }} | |
| restore-keys: | | |
| ${{ runner.os }}-nuget- | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0 | |
| - name: Cache Docker layers | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ hashFiles('src/Directory.Packages.props', 'src/dms/Dockerfile', 'src/dms/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx- | |
| - name: Build DMS Docker image | |
| uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0 | |
| with: | |
| load: true | |
| context: ./src/dms | |
| file: ./src/dms/Dockerfile | |
| build-contexts: | | |
| parentdir=./src | |
| push: false | |
| tags: dms-local-dms:latest | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| cache-to: type=local,dest=/tmp/.buildx-cache | |
| - name: Setup dotnet | |
| uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2 | |
| with: | |
| dotnet-version: "8.0.x" | |
| - name: Build | |
| run: ./build-dms.ps1 Build -Configuration ${{ env.CONFIGURATION }} | |
| - name: Start DMS Docker | |
| run: | | |
| ./start-local-dms.ps1 -EnvironmentFile "./.env.e2e" -EnableConfig -r | |
| working-directory: eng/docker-compose/ | |
| - name: Download OpenAPI specs | |
| run: | | |
| mkdir specs | |
| Invoke-WebRequest -Uri "http://localhost:8080/metadata/specifications/resources-spec.json" -OutFile "specs/resources-spec.json" | |
| Invoke-WebRequest -Uri "http://localhost:8080/metadata/specifications/descriptors-spec.json" -OutFile "specs/descriptors-spec.json" | |
| Invoke-WebRequest -Uri "http://localhost:8080/metadata/specifications/discovery-spec.json" -OutFile "specs/discovery-spec.json" | |
| - name: Upload Resources Spec | |
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5 | |
| with: | |
| name: resources-spec | |
| path: specs/resources-spec.json | |
| - name: Upload Descriptors Spec | |
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5 | |
| with: | |
| name: descriptors-spec | |
| path: specs/descriptors-spec.json | |
| - name: Upload Discovery Spec | |
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5 | |
| with: | |
| name: discovery-spec | |
| path: specs/discovery-spec.json | |
| validate-resources-spec: | |
| name: Validate Resources Spec | |
| runs-on: ubuntu-latest | |
| needs: build-and-start-dms | |
| steps: | |
| - name: Download Resources Spec | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8 | |
| with: | |
| name: resources-spec | |
| path: specs | |
| - name: Validate Resources Spec | |
| uses: swaggerexpert/swagger-editor-validate@54b85e2f5c1fcdee85308dd57cad8c8ec19d3970 | |
| with: | |
| definition-file: specs/resources-spec.json | |
| default-timeout: 45000 # 45 seconds | |
| - name: Download Resources Spec | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8 | |
| with: | |
| name: descriptors-spec | |
| path: specs | |
| - name: Validate Descriptors Spec | |
| uses: swaggerexpert/swagger-editor-validate@54b85e2f5c1fcdee85308dd57cad8c8ec19d3970 | |
| with: | |
| definition-file: specs/descriptors-spec.json | |
| default-timeout: 45000 | |
| validate-descriptors-spec: | |
| name: Validate Descriptors Spec | |
| runs-on: ubuntu-latest | |
| needs: build-and-start-dms | |
| steps: | |
| - name: Download Descriptors Spec | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8 | |
| with: | |
| name: descriptors-spec | |
| path: specs | |
| - name: Validate Descriptors Spec | |
| uses: swaggerexpert/swagger-editor-validate@54b85e2f5c1fcdee85308dd57cad8c8ec19d3970 | |
| with: | |
| definition-file: specs/descriptors-spec.json | |
| default-timeout: 45000 | |
| validate-discovery-spec: | |
| name: Validate Discovery Spec | |
| runs-on: ubuntu-latest | |
| needs: build-and-start-dms | |
| steps: | |
| - name: Download Discovery Spec | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4.1.8 | |
| with: | |
| name: discovery-spec | |
| path: specs | |
| - name: Validate Discovery Spec | |
| uses: swaggerexpert/swagger-editor-validate@54b85e2f5c1fcdee85308dd57cad8c8ec19d3970 | |
| with: | |
| definition-file: specs/discovery-spec.json | |
| default-timeout: 45000 # 45 seconds | |
| event_file: | |
| name: Upload Event File | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| run-unit-tests, | |
| run-e2e-tests, | |
| run-integration-tests, | |
| validate-resources-spec, | |
| validate-descriptors-spec, | |
| validate-discovery-spec, | |
| ] | |
| if: always() | |
| steps: | |
| - name: Upload | |
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5 | |
| with: | |
| name: Event File | |
| path: ${{ github.event_path }} |