Caching zaken

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:14
+        image: postgis/postgis:14-3.4
         env:
           POSTGRES_HOST_AUTH_METHOD: trust
         ports:
@@ -45,7 +45,7 @@ jobs:
       - name: Set up backend environment
         uses: maykinmedia/[email protected]
         with:
-          apt-packages: 'gettext postgresql-client'
+          apt-packages: 'gettext postgresql-client libgdal-dev gdal-bin'
           python-version: '3.12'
           optimize-postgres: 'yes'
           pg-service: 'postgres'
@@ -111,7 +111,7 @@ jobs:
      - name: Set up backend environment
        uses: maykinmedia/setup-django-backend@v1
        with:
-         apt-packages: 'gettext postgresql-client'
+         apt-packages: 'gettext postgresql-client  libgdal-dev gdal-bin'
          python-version: '3.12'
          setup-node: 'no'
          working-directory: open-archiefbeheer/backend

.github/workflows/code_quality.yml

@@ -122,7 +122,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:14
+        image: postgis/postgis:14-3.4
         env:
           POSTGRES_HOST_AUTH_METHOD: trust
         ports:
@@ -136,10 +136,10 @@ jobs:
       - name: Set up backend environment
         uses: maykinmedia/[email protected]
         with:
+          apt-packages: 'gettext postgresql-client libgdal-dev gdal-bin'
           python-version: '3.12'
           setup-node: 'no'
           working-directory: backend
-          nvmrc-custom-dir: backend
       - name: Run makemigrations to check for missing migrations
         working-directory: backend
         run: |

backend/Dockerfile

@@ -62,6 +62,7 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco
         shared-mime-info \
         # lxml deps
         # libxslt \
+        gdal-bin \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app

backend/bin/celery_worker.sh

@@ -9,12 +9,10 @@ QUEUE=${1:-${CELERY_WORKER_QUEUE:=celery}}
 WORKER_NAME=${2:-${CELERY_WORKER_NAME:="${QUEUE}"@%n}}
 
 echo "Starting celery worker $WORKER_NAME with queue $QUEUE"
-exec celery worker \
-    --app openarchiefbeheer \
+exec celery --workdir src --app openarchiefbeheer.celery worker \
     -Q $QUEUE \
     -n $WORKER_NAME \
     -l $LOGLEVEL \
-    --workdir src \
     -O fair \
     -c $CONCURRENCY
 

backend/requirements/base.in

@@ -19,11 +19,15 @@ djangorestframework
 drf-spectacular
 djangorestframework-camel-case
 django-cors-headers
+django-filter
 
 # WSGI servers & monitoring - production oriented
 uwsgi
 sentry-sdk  # error monitoring
 elastic-apm  # Elastic APM integration
 
+# Queueing
+celery
+
 # Additional libraries
 zgw-consumers

backend/requirements/base.txt

@@ -4,6 +4,8 @@
 #
 #    ./bin/compile_dependencies.sh
 #
+amqp==5.2.0
+    # via kombu
 ape-pie==0.1.0
     # via zgw-consumers
 asgiref==3.8.1
@@ -16,8 +18,12 @@ attrs==23.2.0
     # via
     #   jsonschema
     #   referencing
+billiard==4.2.0
+    # via celery
 cbor2==5.6.3
     # via webauthn
+celery==5.4.0
+    # via -r requirements/base.in
 certifi==2024.2.2
     # via
     #   elastic-apm
@@ -27,6 +33,18 @@ cffi==1.16.0
     # via cryptography
 charset-normalizer==3.3.2
     # via requests
+click==8.1.7
+    # via
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+click-didyoumean==0.3.1
+    # via celery
+click-plugins==1.1.1
+    # via celery
+click-repl==0.3.0
+    # via celery
 cryptography==42.0.5
     # via
     #   django-simple-certmanager
@@ -39,6 +57,7 @@ django==4.2.11
     #   django-appconf
     #   django-axes
     #   django-cors-headers
+    #   django-filter
     #   django-formtools
     #   django-hijack
     #   django-otp
@@ -64,6 +83,8 @@ django-axes==6.4.0
     # via -r requirements/base.in
 django-cors-headers==4.3.1
     # via -r requirements/base.in
+django-filter==24.2
+    # via -r requirements/base.in
 django-formtools==2.5.1
     # via django-two-factor-auth
 django-hijack==3.4.5
@@ -114,6 +135,8 @@ jsonschema==4.21.1
     # via drf-spectacular
 jsonschema-specifications==2023.12.1
     # via jsonschema
+kombu==5.3.7
+    # via celery
 maykin-2fa==1.0.0
     # via -r requirements/base.in
 orderedmultidict==1.0.1
@@ -124,6 +147,8 @@ pillow==10.3.0
     # via -r requirements/base.in
 polib==1.2.0
     # via django-rosetta
+prompt-toolkit==3.0.43
+    # via click-repl
 psycopg2==2.9.9
     # via -r requirements/base.in
 pycparser==2.22
@@ -135,7 +160,9 @@ pyopenssl==24.1.0
 pypng==0.20220715.0
     # via qrcode
 python-dateutil==2.9.0.post0
-    # via django-relativedelta
+    # via
+    #   celery
+    #   django-relativedelta
 python-decouple==3.8
     # via -r requirements/base.in
 python-dotenv==1.0.1
@@ -172,6 +199,8 @@ typing-extensions==4.11.0
     # via
     #   qrcode
     #   zgw-consumers
+tzdata==2024.1
+    # via celery
 uritemplate==4.1.1
     # via drf-spectacular
 urllib3==2.2.1
@@ -181,6 +210,13 @@ urllib3==2.2.1
     #   sentry-sdk
 uwsgi==2.0.25.1
     # via -r requirements/base.in
+vine==5.1.0
+    # via
+    #   amqp
+    #   celery
+    #   kombu
+wcwidth==0.2.13
+    # via prompt-toolkit
 webauthn==2.1.0
     # via django-two-factor-auth
 wrapt==1.14.1

backend/requirements/ci.txt

@@ -6,6 +6,11 @@
 #
 alabaster==0.7.16
     # via sphinx
+amqp==5.2.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   kombu
 ape-pie==0.1.0
     # via
     #   -c requirements/base.txt
@@ -34,13 +39,22 @@ babel==2.14.0
     # via sphinx
 beautifulsoup4==4.12.3
     # via webtest
+billiard==4.2.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   celery
 black==24.4.2
     # via -r requirements/test-tools.in
 cbor2==5.6.3
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   webauthn
+celery==5.4.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 certifi==2024.2.2
     # via
     #   -c requirements/base.txt
@@ -59,7 +73,29 @@ charset-normalizer==3.3.2
     #   -r requirements/base.txt
     #   requests
 click==8.1.7
-    # via black
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   black
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+click-didyoumean==0.3.1
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   celery
+click-plugins==1.1.1
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   celery
+click-repl==0.3.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   celery
 coverage==4.5.4
     # via -r requirements/test-tools.in
 cryptography==42.0.5
@@ -81,6 +117,7 @@ django==4.2.11
     #   django-appconf
     #   django-axes
     #   django-cors-headers
+    #   django-filter
     #   django-formtools
     #   django-hijack
     #   django-jenkins
@@ -116,6 +153,10 @@ django-cors-headers==4.3.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+django-filter==24.2
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 django-formtools==2.5.1
     # via
     #   -c requirements/base.txt
@@ -253,6 +294,11 @@ jsonschema-specifications==2023.12.1
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   jsonschema
+kombu==5.3.7
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   celery
 lxml==5.2.1
     # via pyquery
 markupsafe==2.1.5
@@ -299,6 +345,11 @@ polib==1.2.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   django-rosetta
+prompt-toolkit==3.0.43
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   click-repl
 psycopg2==2.9.9
     # via
     #   -c requirements/base.txt
@@ -339,6 +390,7 @@ python-dateutil==2.9.0.post0
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+    #   celery
     #   django-relativedelta
     #   faker
     #   freezegun
@@ -439,6 +491,11 @@ typing-extensions==4.11.0
     #   -r requirements/base.txt
     #   qrcode
     #   zgw-consumers
+tzdata==2024.1
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   celery
 uritemplate==4.1.1
     # via
     #   -c requirements/base.txt
@@ -455,8 +512,20 @@ uwsgi==2.0.25.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+vine==5.1.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   amqp
+    #   celery
+    #   kombu
 waitress==3.0.0
     # via webtest
+wcwidth==0.2.13
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   prompt-toolkit
 webauthn==2.1.0
     # via
     #   -c requirements/base.txt