Skip to content

meltano-ecosystem-ost #404

meltano-ecosystem-ost

meltano-ecosystem-ost #404

# This is a test meltano pipeline
name: meltano-ecosystem-ost
env:
X_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PASSWORD: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PASSWORD }}
TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_HOST: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_HOST }}
TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_DATABASE: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_DATABASE }}
TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PORT: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_PORT }}
TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_USERNAME: ${{ secrets.TAP_ECOSYSTEMS_OST_AIRBYTE_CONFIG_USERNAME }}
TARGET_BIGQUERY_PROJECT: ${{ vars.TARGET_BIGQUERY_PROJECT }}
on:
workflow_dispatch:
schedule:
# Run every 30 mins
- cron: '*/30 * * * *'
jobs:
meltano-ecosystem-ost:
name: meltano-ecosystem-ost
environment: indexer
runs-on: ubuntu-latest
permissions:
contents: 'read'
id-token: 'write'
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: 'google-github-actions/auth@v2'
with:
credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_JSON }}'
create_credentials_file: true
- name: Run meltano
uses: ./.github/workflows/meltano
with:
tap: tap-ecosystems-ost
target: target-bigquery
meltano_path: warehouse/meltano-setup
db_user: ${{ secrets.METADATA_DB_USER }}
db_name: ${{ secrets.METADATA_DB_NAME }}
db_password: ${{ secrets.METADATA_DB_PASSWORD }}
db_instance_id: ${{ secrets.METADATA_DB_INSTANCE_ID }}
meltano_state_backend_uri: ${{ secrets.MELTANO_STATE_BACKEND_URI }}
env:
TARGET_BIGQUERY_CREDENTIALS_PATH: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
TARGET_BIGQUERY_BUCKET: oso-dataset-transfer-bucket
TARGET_BIGQUERY_DATASET: ecosystems_ost