update_parts_database.yml

---
name: "Update parts database"
on:
  schedule:  # 2 hours after jlcparts updates their database
    - cron: "0 5 * * *"
  workflow_dispatch:  # allow for manually trigger workflow
jobs:
  build_and_update:
    name: "Update component database and frontend"
    runs-on: ubuntu-22.04
    environment: github-pages
    steps:
      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            python3 python3-pip wget zip unzip p7zip-full sqlite3
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        with:
          root-reserve-mb: 512
          swap-size-mb: 1024
          remove-dotnet: 'true'
      - name: Free more Space, Found in https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
      - name: Checkout
        uses: actions/checkout@v4
      - name: Show versions
        run: |
          sqlite3 --version
          python3 --version
          python3 -c "import sqlite3; import pprint; db = sqlite3.connect(':memory:'); cursor = db.execute('PRAGMA COMPILE_OPTIONS'); pprint.pprint(cursor.fetchall())"
      - name: Install python dependencies
        run: |
          pip install humanize
      - name: Update database
        run: |
          set -x

          mkdir -p db_build
          cd db_build

          wget -q https://yaqwsx.github.io/jlcparts/data/cache.zip

          VOLUMES=$(7z l cache.zip | grep "Volume Index = " | grep -Eoh "[0-9]+")

          for seq in $(seq 1 $VOLUMES); do
            CACHE=$(printf '%02d' $seq)
            wget -q https://yaqwsx.github.io/jlcparts/data/cache.z$CACHE || true
          done

          7z x cache.zip
          rm -rf cache.z*

          ls -lah
          cd ..
          # creates the converted database in 80M split-zip files in folder db_build
          python3 jlcparts_db_convert.py
          # remove the source db, as we don't want to package that one
          rm db_build/cache.sqlite3
          # some info output for sanity check
          ls -lah db_build
          du -cslh db_build

      - name: Upload pages artifact
        uses: actions/upload-pages-artifact@v3
        with:
          name: github-pages
          path: db_build
  deploy:
    name: "Deploy"
    runs-on: ubuntu-20.04
    needs: build_and_update
    permissions:
      actions: write
      contents: write
      pages: write
      id-token: write
    # Deploy to the github-pages environment
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    if: github.ref == 'refs/heads/main'
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4
      # delete-artifact
      - name: Clean
        uses: geekyeggo/delete-artifact@v4
        with:
          name: github-pages