diff --git a/.github/workflows/bencher.yaml b/.github/workflows/bencher.yaml new file mode 100644 index 000000000..58da5a951 --- /dev/null +++ b/.github/workflows/bencher.yaml @@ -0,0 +1,200 @@ +name: Bencher + +on: + workflow_run: + workflows: [Run and Cache Benchmarks] + types: + - completed + +jobs: + track_sv1_criterion_with_bencher: + name: Track sv1 criterion benchmarks with Bencher + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + env: + BENCHER_PROJECT: stratum + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_ADAPTER: rust_criterion + BENCHER_TESTBED: sv1 + steps: + - name: Download Benchmark Results + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "criterion_sv1_benchmarks" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/criterion_sv1_benchmarks.zip`, Buffer.from(download.data)); + + - name: Unzip Benchmark Results + run: unzip criterion_sv1_benchmarks.zip + + - name: Setup Bencher + #uses: bencherdev/bencher@main + uses: baptiste0928/cargo-install@v2 + with: + crate: bencher_cli + git: https://github.com/bencherdev/bencher + branch: devel + + - name: Track Benchmarks with Bencher + run: | + cat criterion_sv1_benchmarks.txt | bencher run \ + --err \ + --github-actions ${{ secrets.GITHUB_TOKEN }} + + track_sv2_criterion_with_bencher: + name: Track sv1 criterion benchmarks with Bencher + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + env: + BENCHER_PROJECT: stratum + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_ADAPTER: rust_criterion + BENCHER_TESTBED: sv2 + steps: + - name: Download Benchmark Results + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "criterion_sv2_benchmarks" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/criterion_sv2_benchmarks.zip`, Buffer.from(download.data)); + + - name: Unzip Benchmark Results + run: unzip criterion_sv2_benchmarks.zip + + - name: Setup Bencher + #uses: bencherdev/bencher@main + uses: baptiste0928/cargo-install@v2 + with: + crate: bencher_cli + git: https://github.com/bencherdev/bencher + branch: devel + + - name: Track Benchmarks with Bencher + run: | + cat criterion_sv2_benchmarks.txt | bencher run \ + --err \ + --github-actions ${{ secrets.GITHUB_TOKEN }} + + track_sv1_iai_with_bencher: + name: Track sv1 iai benchmarks with Bencher + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + env: + BENCHER_PROJECT: stratum + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_ADAPTER: rust_iai + BENCHER_TESTBED: sv1 + steps: + - name: Download Benchmark Results + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "iai_sv1_benchmarks" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/iai_sv1_benchmarks.zip`, Buffer.from(download.data)); + + - name: Unzip Benchmark Results + run: unzip iai_sv1_benchmarks.zip + + - name: Setup Bencher + #uses: bencherdev/bencher@main + uses: baptiste0928/cargo-install@v2 + with: + crate: bencher_cli + git: https://github.com/bencherdev/bencher + branch: devel + + - name: Track Benchmarks with Bencher + run: | + cat iai_sv1_benchmarks.txt | bencher run \ + --err \ + --github-actions ${{ secrets.GITHUB_TOKEN }} + + track_sv2_iai_with_bencher: + name: Track sv2 iai benchmarks with Bencher + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + env: + BENCHER_PROJECT: stratum + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_ADAPTER: rust_iai + BENCHER_TESTBED: sv2 + steps: + - name: Download Benchmark Results + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "iai_sv2_benchmarks" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/iai_sv2_benchmarks.zip`, Buffer.from(download.data)); + + - name: Unzip Benchmark Results + run: unzip iai_sv2_benchmarks.zip + + - name: Setup Bencher + #uses: bencherdev/bencher@main + uses: baptiste0928/cargo-install@v2 + with: + crate: bencher_cli + git: https://github.com/bencherdev/bencher + branch: devel + + - name: Track Benchmarks with Bencher + run: | + cat iai_sv2_benchmarks.txt | bencher run \ + --err \ + --github-actions ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 000000000..f71b4a073 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,110 @@ +name: Run and Cache Benchmarks + +on: pull_request + +jobs: + benchmark_sv1_criterion: + name: Run and cache criterion sv1 benchmarks + runs-on: ubuntu-latest + steps: + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.70.0 + override: true + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Benchmark + run: | + cd benches + cargo bench --bench criterion_sv1_benchmark > criterion_sv1_benchmarks.txt + + - uses: actions/upload-artifact@v3 + with: + name: criterion_sv1_benchmarks.txt + path: ./benches/criterion_sv1_benchmarks.txt + + benchmark_sv2_criterion: + name: Run and cache criterion sv2 benchmarks + runs-on: ubuntu-latest + steps: + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.70.0 + override: true + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Benchmark + run: | + cd benches + cargo bench --bench criterion_sv2_benchmark > criterion_sv2_benchmarks.txt + + - uses: actions/upload-artifact@v3 + with: + name: criterion_sv2_benchmarks.txt + path: ./benches/criterion_sv2_benchmarks.txt + + benchmark_sv1_iai: + name: Run and cache iai sv1 benchmarks + runs-on: ubuntu-latest + steps: + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.70.0 + override: true + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Valgrind + run: | + sudo apt-get update + sudo apt-get install -y valgrind=1:3.18.1-1ubuntu2 + + - name: Benchmark + run: | + cd benches + cargo bench --bench iai_sv1_benchmark > iai_sv1_benchmarks.txt + + - uses: actions/upload-artifact@v3 + with: + name: iai_sv1_benchmarks.txt + path: ./benches/iai_sv1_benchmarks.txt + + benchmark_sv2_iai: + name: Run and cache iai sv2 benchmarks + runs-on: ubuntu-latest + steps: + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.70.0 + override: true + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Valgrind + run: | + sudo apt-get update + sudo apt-get install -y valgrind=1:3.18.1-1ubuntu2 + + - name: Benchmark + run: | + cd benches + cargo bench --bench iai_sv2_benchmark > iai_sv2_benchmarks.txt + + - uses: actions/upload-artifact@v3 + with: + name: iai_sv2_benchmarks.txt + path: ./benches/iai_sv2_benchmarks.txt