debug #82
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check for changes in OL | |
| # Assumption here is that there will be a change that touches spec or spark integration | |
| on: push | |
| # actual trigger for daily run of workflow | |
| #on: | |
| # schedule: | |
| # - cron: "0 5 * * 0" | |
| permissions: | |
| id-token: write | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| initialize_workflow: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| changes_in_spec: ${{steps.check-changes.outputs.changes_found}} | |
| ol_release: ${{ steps.get-release.outputs.openlineage_release }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: create dir for ol spec and report | |
| id: dir_for_spec | |
| shell: bash | |
| run: | | |
| mkdir -p openlineage | |
| - name: get latest OL spec | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: OpenLineage/OpenLineage | |
| path: openlineage | |
| - name: check for changes in spec | |
| id: check-changes | |
| run: | | |
| echo "changes_found=false" >> $GITHUB_OUTPUT | |
| python scripts/compare_spec_versions.py \ | |
| --spec_base=./openlineage/spec \ | |
| --versions_path=./generated-files/spec_versions.json \ | |
| --new_versions_path=./updated_spec_versions.json | |
| if [ -f updated_spec_versions.json ]; then | |
| echo "changes_found=true" >> $GITHUB_OUTPUT | |
| fi | |
| - uses: actions/upload-artifact@v4 | |
| if: steps.check-changes.outputs.changes_found == 'true' | |
| with: | |
| name: spec-versions | |
| path: generated-files/updated_spec_versions.json | |
| retention-days: 1 | |
| - name: get openlineage release | |
| if: steps.check-changes.outputs.changes_found == 'true' | |
| id: get-release | |
| run: | | |
| openlineage_release=$(cat generated-files/releases.json | jq -c '.[] | select(.name | contains("openlineage")) | .latest_version ' -r) | |
| echo "openlineage_release=${openlineage_release}" >> $GITHUB_OUTPUT | |
| ######## COMPONENT VALIDATION ######## | |
| # check if scenarios are still compatible with the new spec | |
| scenarios_check: | |
| needs: | |
| - initialize_workflow | |
| if: ${{ success() && needs.initialize_workflow.outputs.changes_in_spec == 'true' }} | |
| uses: ./.github/workflows/check_scenarios.yml | |
| with: | |
| release: ${{ needs.initialize_workflow.outputs.ol_release }} | |
| get-latest-snapshots: 'true' | |
| fail-for-failures: false | |
| # check if producers still produce compatible events | |
| spark-dataproc: | |
| needs: | |
| - initialize_workflow | |
| if: ${{ success() && needs.initialize_workflow.outputs.changes_in_spec == 'true' }} | |
| uses: ./.github/workflows/producer_spark_dataproc.yml | |
| secrets: | |
| gcpKey: ${{ secrets.GCP_SA_KEY }} | |
| with: | |
| release: ${{ needs.initialize_workflow.outputs.ol_release }} | |
| get-latest-snapshots: 'true' | |
| ######## COLLECTION OF REPORTS AND EXECUTE APPROPRIATE ACTIONS ######## | |
| collect-and-compare-reports: | |
| needs: | |
| - initialize_workflow | |
| - scenarios_check | |
| - spark-dataproc | |
| uses: ./.github/workflows/collect_and_compare_reports.yml | |
| notify-maintainers: | |
| needs: collect-and-compare-reports | |
| uses: ./.github/workflows/notify_maintainers.yml | |