Update parts database #462
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: "Update parts database" | |
| on: | |
| schedule: # 2 hours after jlcparts updates their database | |
| - cron: "0 5 * * *" | |
| workflow_dispatch: # allow for manually trigger workflow | |
| jobs: | |
| build_and_update: | |
| name: "Update component database and frontend" | |
| runs-on: ubuntu-22.04 | |
| environment: github-pages | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| python3 python3-pip wget zip unzip p7zip-full sqlite3 | |
| - name: Show versions | |
| run: | | |
| sqlite3 --version | |
| python3 --version | |
| python3 -c "import sqlite3; import pprint; db = sqlite3.connect(':memory:'); cursor = db.execute('PRAGMA COMPILE_OPTIONS'); pprint.pprint(cursor.fetchall())" | |
| - name: Install python dependencies | |
| run: | | |
| pip install humanize | |
| - name: Update database | |
| run: | | |
| set -x | |
| mkdir -p db_build | |
| cd db_build | |
| wget -q https://yaqwsx.github.io/jlcparts/data/cache.zip | |
| VOLUMES=$(7z l cache.zip | grep "Volume Index = " | grep -Eoh "[0-9]+") | |
| for seq in $(seq 1 $VOLUMES); do | |
| CACHE=$(printf '%02d' $seq) | |
| wget -q https://yaqwsx.github.io/jlcparts/data/cache.z$CACHE || true | |
| done | |
| 7z x cache.zip | |
| rm -rf cache.z* | |
| ls -lah | |
| cd .. | |
| # creates the converted database in 80M split-zip files in folder db_build | |
| python3 jlcparts_db_convert.py | |
| # remove the source db, as we don't want to package that one | |
| rm db_build/cache.sqlite3 | |
| # some info output for sanity check | |
| ls -lah db_build | |
| du -cslh db_build | |
| - name: Upload pages artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| name: github-pages | |
| path: db_build | |
| deploy: | |
| name: "Deploy" | |
| runs-on: ubuntu-20.04 | |
| needs: build_and_update | |
| permissions: | |
| actions: write | |
| contents: write | |
| pages: write | |
| id-token: write | |
| # Deploy to the github-pages environment | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v4 | |
| # delete-artifact | |
| - name: Clean | |
| uses: geekyeggo/delete-artifact@v4 | |
| with: | |
| name: github-pages |