fix(benches): bench sequentially
#33
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Foundry Benchmarks | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: "PR number to comment on (optional)" | |
| required: false | |
| type: string | |
| versions: | |
| description: "Comma-separated list of Foundry versions to benchmark (e.g., stable,nightly,v1.0.0)" | |
| required: false | |
| type: string | |
| default: "stable,nightly" | |
| repos: | |
| description: "Comma-separated list of repos to benchmark (e.g., ithacaxyz/account:main,Vectorized/solady)" | |
| required: false | |
| type: string | |
| default: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22" | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| env: | |
| ITHACAXYZ_ACCOUNT: "ithacaxyz/account:v0.3.2" | |
| VECTORIZED_SOLADY: "Vectorized/solady:v0.1.22" | |
| DEFAULT_REPOS: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22" | |
| jobs: | |
| run-benchmarks: | |
| name: Run All Benchmarks | |
| runs-on: foundry-runner | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential pkg-config | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: | | |
| ./ | |
| - name: Setup Foundry | |
| env: | |
| FOUNDRY_DIR: ${{ github.workspace }}/.foundry | |
| run: | | |
| ./.github/scripts/setup-foundryup.sh | |
| echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH | |
| - name: Build benchmark binary | |
| run: cargo build --release --bin foundry-bench | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24" | |
| - name: Install hyperfine | |
| run: | | |
| curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz | |
| sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/ | |
| rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu | |
| - name: Run forge test benchmarks | |
| env: | |
| FOUNDRY_DIR: ${{ github.workspace }}/.foundry | |
| run: | | |
| VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" | |
| REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}" | |
| ./target/release/foundry-bench --output-dir ./benches --force-install \ | |
| --versions $VERSIONS \ | |
| --repos $REPOS \ | |
| --benchmarks forge_test,forge_fuzz_test \ | |
| --output-file forge_test_bench.md | |
| - name: Run forge build benchmarks | |
| env: | |
| FOUNDRY_DIR: ${{ github.workspace }}/.foundry | |
| run: | | |
| VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" | |
| REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}" | |
| ./target/release/foundry-bench --output-dir ./benches --force-install \ | |
| --versions $VERSIONS \ | |
| --repos $REPOS \ | |
| --benchmarks forge_build_no_cache,forge_build_with_cache \ | |
| --output-file forge_build_bench.md | |
| - name: Run forge coverage benchmarks | |
| env: | |
| FOUNDRY_DIR: ${{ github.workspace }}/.foundry | |
| run: | | |
| VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" | |
| # Coverage only runs on ithacaxyz/account:v0.3.2 | |
| ./target/release/foundry-bench --output-dir ./benches --force-install \ | |
| --versions $VERSIONS \ | |
| --repos ${{ env.ITHACAXYZ_ACCOUNT }} \ | |
| --benchmarks forge_coverage \ | |
| --output-file forge_coverage_bench.md | |
| - name: Combine benchmark results | |
| run: ./.github/scripts/combine-benchmarks.sh benches | |
| - name: Commit and read benchmark results | |
| id: benchmark_results | |
| env: | |
| GITHUB_HEAD_REF: ${{ github.head_ref }} | |
| run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}" | |
| - name: Upload benchmark results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| benches/forge_test_bench.md | |
| benches/forge_build_bench.md | |
| benches/forge_coverage_bench.md | |
| benches/LATEST.md | |
| outputs: | |
| branch_name: ${{ steps.benchmark_results.outputs.branch_name }} | |
| pr_comment: ${{ steps.benchmark_results.outputs.pr_comment }} | |
| publish-results: | |
| name: Publish Results | |
| needs: run-benchmarks | |
| runs-on: foundry-runner | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Download benchmark results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: benches/ | |
| - name: Push branch for manual runs | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| git push origin "${{ needs.run-benchmarks.outputs.branch_name }}" | |
| echo "Pushed branch: ${{ needs.run-benchmarks.outputs.branch_name }}" | |
| - name: Create PR for manual runs | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const branchName = '${{ needs.run-benchmarks.outputs.branch_name }}'; | |
| const prComment = `${{ needs.run-benchmarks.outputs.pr_comment }}`; | |
| // Create the pull request | |
| const { data: pr } = await github.rest.pulls.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: 'chore(bench): update benchmark results', | |
| head: branchName, | |
| base: 'master', | |
| body: `## Benchmark Results Update | |
| This PR contains the latest benchmark results from a manual workflow run. | |
| ${prComment} | |
| --- | |
| 🤖 This PR was automatically generated by the [Foundry Benchmarks workflow](https://github.com/${{ github.repository }}/actions).` | |
| }); | |
| console.log(`Created PR #${pr.number}: ${pr.html_url}`); | |
| - name: Comment on PR | |
| if: github.event.inputs.pr_number != '' || github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const prNumber = ${{ github.event.inputs.pr_number || github.event.pull_request.number }}; | |
| const prComment = `${{ needs.run-benchmarks.outputs.pr_comment }}`; | |
| const comment = `${prComment} | |
| --- | |
| 🤖 This comment was automatically generated by the [Foundry Benchmarks workflow](https://github.com/${{ github.repository }}/actions). | |
| To run benchmarks manually: Go to [Actions](https://github.com/${{ github.repository }}/actions/workflows/benchmarks.yml) → "Run workflow"`; | |
| github.rest.issues.createComment({ | |
| issue_number: prNumber, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); |