scrap_sitemap_and_ingest_db.yml

name: db_ingestion_by_scraping_sitemap

on:
  workflow_dispatch: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch

  schedule: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
    - cron:  '00 2 * * *'  # every day at 02:00 UTC

env:
  PYTHON_VERSION: '3.10'
  POETRY_VERSION: '1.2.1'

jobs:
  db_ingestion_by_scraping_sitemamp:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - uses: actions/checkout@v2
      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          version: ${{ env.POETRY_VERSION }}
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true
      - name: Verify poetry.lock agrees with pyproject.toml
        run: poetry lock --check
      - name: Install dependencies
        run: poetry install --no-interaction --no-ansi
      - name: RunSitemapScrappingAndIngestScript
        run: |
          poetry run python quotaclimat/data_ingestion/ingest_db/ingest_sitemap_in_db.py -p ${{ secrets.D4G_VM_DB_PWD }}