diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index cae72e0..114dd30 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -5,26 +5,40 @@ on: # Runs at 08:00 UTC every day - cron: "0 8 * * *" workflow_dispatch: + inputs: + max_parallel: + description: 'Maximum number of parallel jobs (1-20, use 1 for sequential execution)' + required: true + default: '10' + type: string jobs: - run-benchmarks: + pre-warm: runs-on: ubuntu-latest + steps: + - name: Pre-warm audio instance + run: curl -f -X POST "https://ai-benchmarks.fly.dev/bench?mode=audio&max_tokens=20&num_requests=1" + run-benchmarks: + needs: pre-warm + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + region: [sea, iad, cdg, nrt] + medium: [text, tools, image, audio] + max-parallel: ${{ fromJson(github.event.inputs.max_parallel || '10') }} + steps: - - name: Send Benchmark Requests - run: | - # Pre-warm any spun-down instances (only audio right now) - curl -f -X POST "https://ai-benchmarks.fly.dev/bench?mode=audio&max_tokens=20&num_requests=1" - # Run the benchmarks - declare -A max_tokens=( - ["tools"]=100 - ) - default_max_tokens=20 - regions=("sea" "iad" "cdg") - media=("text" "tools" "image" "audio") - for region in "${regions[@]}"; do - for medium in "${media[@]}"; do - max_tokens=${max_tokens[$medium]:-$default_max_tokens} - curl -f -X POST "https://ai-benchmarks.fly.dev/bench?mode=$medium&max_tokens=$max_tokens&spread=30&store" -H "fly-prefer-region: $region" - done - done + - name: Send Benchmark Request + env: + MAX_TOKENS: ${{ (matrix.medium == 'tools' && 100) || 20 }} + uses: nick-fields/retry@v3 + with: + timeout_minutes: 15 + max_attempts: 3 + retry_on: error + command: | + echo "Running benchmark for ${{ matrix.medium }} in region ${{ matrix.region }}" + curl -f -X POST "https://ai-benchmarks.fly.dev/bench?mode=${{ matrix.medium }}&max_tokens=${{ env.MAX_TOKENS }}&spread=30&store" -H "fly-prefer-region: ${{ matrix.region }}" + echo "Benchmark completed for ${{ matrix.medium }} in region ${{ matrix.region }}" \ No newline at end of file diff --git a/utils/GenerateLatestData.js b/utils/GenerateLatestData.js index 0082c7f..6bbb75a 100644 --- a/utils/GenerateLatestData.js +++ b/utils/GenerateLatestData.js @@ -1,5 +1,5 @@ // Define the regions to search for files -const regions = ["cdg", "sea", "iad"]; +const regions = ["cdg", "sea", "iad", "nrt"]; // Function to fetch data from a given URL async function fetchData(url) { diff --git a/website/src/components/DataGrid.astro b/website/src/components/DataGrid.astro index ff80bb9..4ff258c 100644 --- a/website/src/components/DataGrid.astro +++ b/website/src/components/DataGrid.astro @@ -71,6 +71,15 @@ import "ag-grid-community/styles/ag-theme-quartz.css"; /> +
+ + +