Merge pull request #7 from GitGab19/bencher_testing

Testing bencher fix
rrybarczyk · Sep 14, 2023 · 4502b5d · 4502b5d
2 parents 7572e59 + 23c67a5
commit 4502b5d
Show file tree

Hide file tree

Showing 2 changed files with 310 additions and 0 deletions.
diff --git a/.github/workflows/bencher.yaml b/.github/workflows/bencher.yaml
@@ -0,0 +1,200 @@
+name: Bencher
+
+on:
+  workflow_run:
+    workflows: [Run and Cache Benchmarks]
+    types:
+    - completed
+
+jobs:
+    track_sv1_criterion_with_bencher:
+      name: Track sv1 criterion benchmarks with Bencher
+      if: ${{ github.event.workflow_run.conclusion == 'success' }}
+      runs-on: ubuntu-latest
+      env:
+        BENCHER_PROJECT: stratum
+        BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+        BENCHER_ADAPTER: rust_criterion
+        BENCHER_TESTBED: sv1
+      steps:
+        - name: Download Benchmark Results
+          uses: actions/github-script@v6
+          with:
+            script: |
+                let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.payload.workflow_run.id,
+                });
+                let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                return artifact.name == "criterion_sv1_benchmarks"
+                })[0];
+                let download = await github.rest.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+                });
+                let fs = require('fs');
+                fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/criterion_sv1_benchmarks.zip`, Buffer.from(download.data));
+
+        - name: Unzip Benchmark Results
+          run: unzip criterion_sv1_benchmarks.zip
+
+        - name: Setup Bencher
+          #uses: bencherdev/bencher@main
+          uses: baptiste0928/cargo-install@v2
+          with:
+            crate: bencher_cli
+            git: https://github.com/bencherdev/bencher
+            branch: devel
+
+        - name: Track Benchmarks with Bencher
+          run: |
+            cat criterion_sv1_benchmarks.txt | bencher run \
+            --err \
+            --github-actions ${{ secrets.GITHUB_TOKEN }}
+
+    track_sv2_criterion_with_bencher:
+        name: Track sv1 criterion benchmarks with Bencher
+        if: ${{ github.event.workflow_run.conclusion == 'success' }}
+        runs-on: ubuntu-latest
+        env:
+            BENCHER_PROJECT: stratum
+            BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+            BENCHER_ADAPTER: rust_criterion
+            BENCHER_TESTBED: sv2
+        steps:
+            - name: Download Benchmark Results
+              uses: actions/github-script@v6
+              with:
+                script: |
+                    let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    run_id: context.payload.workflow_run.id,
+                    });
+                    let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                    return artifact.name == "criterion_sv2_benchmarks"
+                    })[0];
+                    let download = await github.rest.actions.downloadArtifact({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    artifact_id: matchArtifact.id,
+                    archive_format: 'zip',
+                    });
+                    let fs = require('fs');
+                    fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/criterion_sv2_benchmarks.zip`, Buffer.from(download.data));
+    
+            - name: Unzip Benchmark Results
+              run: unzip criterion_sv2_benchmarks.zip
+
+            - name: Setup Bencher
+              #uses: bencherdev/bencher@main
+              uses: baptiste0928/cargo-install@v2
+              with:
+                crate: bencher_cli
+                git: https://github.com/bencherdev/bencher
+                branch: devel
+
+            - name: Track Benchmarks with Bencher
+              run: |
+                cat criterion_sv2_benchmarks.txt | bencher run \
+                --err \
+                --github-actions ${{ secrets.GITHUB_TOKEN }}
+
+    track_sv1_iai_with_bencher:
+        name: Track sv1 iai benchmarks with Bencher
+        if: ${{ github.event.workflow_run.conclusion == 'success' }}
+        runs-on: ubuntu-latest
+        env:
+            BENCHER_PROJECT: stratum
+            BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+            BENCHER_ADAPTER: rust_iai
+            BENCHER_TESTBED: sv1
+        steps:
+            - name: Download Benchmark Results
+              uses: actions/github-script@v6
+              with:
+                script: |
+                    let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    run_id: context.payload.workflow_run.id,
+                    });
+                    let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                    return artifact.name == "iai_sv1_benchmarks"
+                    })[0];
+                    let download = await github.rest.actions.downloadArtifact({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    artifact_id: matchArtifact.id,
+                    archive_format: 'zip',
+                    });
+                    let fs = require('fs');
+                    fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/iai_sv1_benchmarks.zip`, Buffer.from(download.data));
+    
+            - name: Unzip Benchmark Results
+              run: unzip iai_sv1_benchmarks.zip
+
+            - name: Setup Bencher
+              #uses: bencherdev/bencher@main
+              uses: baptiste0928/cargo-install@v2
+              with:
+               crate: bencher_cli
+               git: https://github.com/bencherdev/bencher
+               branch: devel
+
+            - name: Track Benchmarks with Bencher
+              run: |
+                cat iai_sv1_benchmarks.txt | bencher run \
+                --err \
+                --github-actions ${{ secrets.GITHUB_TOKEN }}
+
+    track_sv2_iai_with_bencher:
+        name: Track sv2 iai benchmarks with Bencher
+        if: ${{ github.event.workflow_run.conclusion == 'success' }}
+        runs-on: ubuntu-latest
+        env:
+            BENCHER_PROJECT: stratum
+            BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+            BENCHER_ADAPTER: rust_iai
+            BENCHER_TESTBED: sv2
+        steps:
+            - name: Download Benchmark Results
+              uses: actions/github-script@v6
+              with:
+                script: |
+                    let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    run_id: context.payload.workflow_run.id,
+                    });
+                    let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                    return artifact.name == "iai_sv2_benchmarks"
+                    })[0];
+                    let download = await github.rest.actions.downloadArtifact({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    artifact_id: matchArtifact.id,
+                    archive_format: 'zip',
+                    });
+                    let fs = require('fs');
+                    fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/iai_sv2_benchmarks.zip`, Buffer.from(download.data));
+    
+            - name: Unzip Benchmark Results
+              run: unzip iai_sv2_benchmarks.zip
+
+            - name: Setup Bencher
+              #uses: bencherdev/bencher@main
+              uses: baptiste0928/cargo-install@v2
+              with:
+                crate: bencher_cli
+                git: https://github.com/bencherdev/bencher
+                branch: devel
+
+            - name: Track Benchmarks with Bencher
+              run: |
+                cat iai_sv2_benchmarks.txt | bencher run \
+                --err \
+                --github-actions ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
@@ -0,0 +1,110 @@
+name: Run and Cache Benchmarks
+
+on: pull_request
+
+jobs:
+  benchmark_sv1_criterion:
+    name: Run and cache criterion sv1 benchmarks 
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.70.0
+          override: true
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Benchmark 
+        run: |
+          cd benches 
+          cargo bench --bench criterion_sv1_benchmark > criterion_sv1_benchmarks.txt
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: criterion_sv1_benchmarks.txt
+          path: ./benches/criterion_sv1_benchmarks.txt
+
+  benchmark_sv2_criterion:
+    name: Run and cache criterion sv2 benchmarks 
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.70.0
+          override: true
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Benchmark 
+        run: |
+          cd benches 
+          cargo bench --bench criterion_sv2_benchmark > criterion_sv2_benchmarks.txt
+      
+      - uses: actions/upload-artifact@v3
+        with:
+          name: criterion_sv2_benchmarks.txt
+          path: ./benches/criterion_sv2_benchmarks.txt
+
+  benchmark_sv1_iai:
+    name: Run and cache iai sv1 benchmarks 
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.70.0
+          override: true
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind=1:3.18.1-1ubuntu2 
+          
+      - name: Benchmark 
+        run: |
+          cd benches 
+          cargo bench --bench iai_sv1_benchmark > iai_sv1_benchmarks.txt
+      
+      - uses: actions/upload-artifact@v3
+        with:
+          name: iai_sv1_benchmarks.txt
+          path: ./benches/iai_sv1_benchmarks.txt
+
+  benchmark_sv2_iai:
+    name: Run and cache iai sv2 benchmarks 
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.70.0
+          override: true
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind=1:3.18.1-1ubuntu2 
+          
+      - name: Benchmark 
+        run: |
+          cd benches 
+          cargo bench --bench iai_sv2_benchmark > iai_sv2_benchmarks.txt
+      
+      - uses: actions/upload-artifact@v3
+        with:
+          name: iai_sv2_benchmarks.txt
+          path: ./benches/iai_sv2_benchmarks.txt