warehouse-meltano-ecosystems-ost.yml

# This is a test meltano pipeline
name: meltano-ecosystem-ost
env:
  X_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PASSWORD: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PASSWORD }}
  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_HOST: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_HOST }}
  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_DATABASE: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_DATABASE }}
  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PORT: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PORT }}
  TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_USERNAME: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_USERNAME }}
  TARGET_BIGQUERY_PROJECT: ${{ vars.TARGET_BIGQUERY_PROJECT }}

on:
  workflow_dispatch:
  schedule:
    # Run every 30 mins
    - cron: '*/30 * * * *'

jobs:
  meltano-ecosystem-ost:
    name: meltano-ecosystem-ost
    environment: indexer
    runs-on: ubuntu-latest

    permissions:
      contents: 'read'
      id-token: 'write'

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - uses: 'google-github-actions/auth@v2'
        with:
          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_JSON }}'
          create_credentials_file: true

      - name: Run meltano
        uses: ./.github/workflows/meltano
        with:
          tap: tap-ecosystems-ost
          target: target-bigquery
          meltano_path: warehouse/meltano-setup
          db_user: ${{ secrets.METADATA_DB_USER }}
          db_name: ${{ secrets.METADATA_DB_NAME }}
          db_password: ${{ secrets.METADATA_DB_PASSWORD }}
          db_instance_id: ${{ secrets.METADATA_DB_INSTANCE_ID }}
          meltano_state_backend_uri: ${{ secrets.MELTANO_STATE_BACKEND_URI }}
        env:
          TARGET_BIGQUERY_CREDENTIALS_PATH: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
          TARGET_BIGQUERY_BUCKET: oso-dataset-transfer-bucket
          TARGET_BIGQUERY_DATASET: ecosystems_ost