Ecosystem testing (#320)

Workflow for testing a package upgrade against the ecosystem.

See for example
mosuem/i18n#2 (comment)

---

- [x] I’ve reviewed the contributor guide and applied the relevant
portions to this PR.

<details>
  <summary>Contribution guidelines:</summary><br>

- See our [contributor
guide](https://github.com/dart-lang/.github/blob/main/CONTRIBUTING.md)
for general expectations for PRs.
- Larger or significant changes should be discussed in an issue before
creating a PR.
- Contributions to our repos should follow the [Dart style
guide](https://dart.dev/guides/language/effective-dart) and use `dart
format`.
- Most changes should add an entry to the changelog and may need to [rev
the pubspec package
version](https://github.com/dart-lang/sdk/blob/main/docs/External-Package-Maintenance.md#making-a-change).
- Changes to packages require [corresponding
tests](https://github.com/dart-lang/.github/blob/main/CONTRIBUTING.md#Testing).

Note that many Dart repos have a weekly cadence for reviewing PRs -
please allow for some latency before initial review feedback.
</details>

.github/test_repos/repos.json

@@ -0,0 +1,12 @@
+{
+    "$schema": "../../pkgs/quest/schema.json",
+    "https://github.com/mosuem/my_app_old_web": {
+        "level": "analyze"
+    },
+    "https://github.com/mosuem/my_app_new_web": {
+        "level": "test",
+        "packages": {
+            "exclude": "intl4x"
+        }
+    }
+}

.github/workflows/ecosystem_test.yaml

@@ -0,0 +1,66 @@
+name: Ecosystem test
+
+on:
+  workflow_call:
+    inputs:
+      repos_file:
+        description: 'Path to the file containing the list of repository names'
+        type: string
+        required: true
+      local_debug:
+        description: Whether to use a local version of ecosystem testing - only for debug
+        default: false
+        type: boolean
+        required: false
+
+jobs:
+  update_and_test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: subosito/flutter-action@44ac965b96f18d999802d4b807e3256d5a3f9fa1
+        with:
+          channel: main
+
+      - run: echo "${{ toJSON(github.event.pull_request.labels.*.name) }}"
+
+      - name: Install firehose
+        run: dart pub global activate -s path pkgs/quest
+        if:  ${{ inputs.local_debug }}
+
+      - run: dart pub global activate -s git https://github.com/dart-lang/ecosystem.git --git-ref main --git-path pkgs/quest
+        if:  ${{ !inputs.local_debug }}
+
+      - name: Update package and test
+        run: |
+          dart pub global run quest ${{ inputs.repos_file }} ${{ github.repositoryUrl }} ${{ github.head_ref || github.ref_name }} "${{ toJSON(github.event.pull_request.labels.*.name) }}"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Output issue number
+        run: |
+          mkdir -p output
+          echo ${{ github.event.number }} > output/issueNumber
+
+      - name: Find Comment
+        uses: peter-evans/find-comment@90e9b82d6319a7ae3f32bc0b434db48d6c376e55
+        id: fc
+        with:
+          issue-number: ${{ github.event.number }}
+          comment-author: github-actions[bot]
+          body-includes: '## Ecosystem testing'
+
+      - name: Write comment id to file
+        if: ${{ steps.fc.outputs.comment-id != 0 }} 
+        run: echo ${{ steps.fc.outputs.comment-id }} >> output/commentId
+
+      - name: Upload markdown
+        if: success() || failure()
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
+        with:
+          name: output
+          path: output/

.github/workflows/ecosystem_test_internal.yaml

@@ -0,0 +1,13 @@
+name: Ecosystem test:Internal
+
+on:
+  pull_request:
+    branches: [ main ]
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+
+jobs:
+  test_ecosystem:
+    uses: ./.github/workflows/ecosystem_test.yaml
+    with:
+      repos_file: .github/test_repos/repos.json
+      local_debug: true

.github/workflows/post_summaries.yaml

@@ -8,6 +8,7 @@ on:
     workflows:
       - Publish:Internal
       - Health:Internal
+      - Ecosystem test:Internal
     types:
       - completed
 
@@ -45,6 +46,12 @@ jobs:
             fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
       - run: unzip comment.zip
 
+      - name: 'Print the comment'
+        run: |
+          if [ -f "./comment.md" ]; then
+            cat ./comment.md
+          fi
+
 
       # Create the comment, or update the existing one, with the markdown
       # generated in the Health workflow.

.github/workflows/quest.yml

@@ -0,0 +1,41 @@
+name: package:quest
+
+permissions: read-all
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/quest.yml'
+      - 'pkgs/quest/**'
+  push:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/quest.yml'
+      - 'pkgs/quest/**'
+  schedule:
+    - cron: '0 0 * * 0' # weekly
+
+defaults:
+  run:
+    working-directory: pkgs/quest
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: subosito/flutter-action@44ac965b96f18d999802d4b807e3256d5a3f9fa1
+        with:
+          channel: main
+
+      - run: dart pub get
+
+      - run: dart analyze --fatal-infos
+
+      - run: dart format --output=none --set-exit-if-changed .
+
+      - run: dart test

pkgs/quest/.gitignore

@@ -0,0 +1,3 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/

pkgs/quest/LICENSE

@@ -0,0 +1,27 @@
+Copyright 2024, the Dart project authors.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google LLC nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkgs/quest/README.md

@@ -0,0 +1,79 @@
+# Quest: Ecosystem Testing for Dart Packages
+Embark your package on a quest of testing against a suite of applications. This helps identify potential breaking changes introduced by package updates, ensuring seamless integration across the ecosystem.
+
+## What does it do?
+It checks if your package upgrade would result in failures in the ecosystem. This is achieved by running the following pseudocode:
+```dart
+for (final app in applicationSuite) {
+  if (app.dependencies.contains(package)) {
+    pubGet(app);
+    analyze(app);
+    test(app);
+
+    upgradePackage(app);
+
+    pubGet(app);
+    analyze(app);
+    test(app);
+  }
+}
+```
+
+## How do I use it?
+1. Create a suite of repositories to test against at `.github/test_repos/repos.json`. Follow the schema specified [here](schema.json).
+```json
+{
+    "https://github.com/mosuem/my_app_old_web": {
+        "level": "analyze"
+    },
+    "https://github.com/mosuem/my_app_new_web": {
+        "level": "test",
+        "packages": {
+            "exclude": "intl4x"
+        }
+    }
+}
+```
+
+2. Add a workflow file with the following contents:
+```yaml
+name: Ecosystem test
+
+on:
+  pull_request:
+    branches: [ main ]
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+
+jobs:
+  test_ecosystem:
+    uses: dart-lang/ecosystem/.github/workflows/ecosystem_test.yaml@main
+    with:
+      repos_file: .github/test_repos/repos.json
+```
+
+3. To show the markdown result as a comment, also add a workflow file
+```yaml
+name: Comment on the pull request
+
+on:
+  # Trigger this workflow after the Health workflow completes. This workflow will have permissions to
+  # do things like create comments on the PR, even if the original workflow couldn't.
+  workflow_run:
+    workflows: 
+      - Health
+      - Publish
+      - Ecosystem test
+    types:
+      - completed
+
+jobs:
+  upload:
+    uses: dart-lang/ecosystem/.github/workflows/post_summaries.yaml@main
+    permissions:
+      pull-requests: write
+```
+
+4. Profit!
+
+# Contributing
+Contributions are welcome! Please see the [contribution guidelines](../../CONTRIBUTING.md).

pkgs/quest/analysis_options.yaml

@@ -0,0 +1,5 @@
+include: package:dart_flutter_team_lints/analysis_options.yaml
+
+linter:
+  rules:
+    - prefer_final_locals