Upgrade Python dependencies

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.9"
           cache: "pip"
       - name: Setup, build, install
         run: |
@@ -31,12 +31,18 @@ jobs:
         run: black . --check
 
   test:
-    name: Test
     runs-on: ubuntu-22.04
     timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        postgis_versions:
+          - 12-3.4
+          - 15-3.4
+    name: Test with PostGIS v. ${{ matrix.postgis_versions }}
     steps:
       - uses: actions/checkout@v4
       - name: Start docker stack
-        run: docker compose up db -d
+        run: POSTGIS_VERSION=${{ matrix.postgis_versions }} docker compose up db -d
       - name: Running tests
-        run: docker compose run qgis_tester
+        run: POSTGIS_VERSION=${{ matrix.postgis_versions }} docker compose run qgis_tester

.gitignore

@@ -5,4 +5,5 @@
 .vscode/settings.json
 *.nix
 *.log
-**/__pycache__/
+**/__pycache__/
+test_outputs/

comptages/__init__.py

@@ -46,7 +46,7 @@ def prepare_django(default_db=None, **additional_settings):
         USE_TZ=True,
         TIME_ZONE="Europe/Zurich",
         SECRET_KEY="09n+dhzh+02+_#$!1+8h-&(s-wbda#0*2mrv@lx*y#&fzlv&l)",
-        **additional_settings
+        **additional_settings,
     )
     django.setup()
 

comptages/test/test_import.py

@@ -1,5 +1,5 @@
 import pytz
-from datetime import datetime
+from datetime import datetime, timezone, tzinfo
 from django.test import TransactionTestCase
 from django.core.management import call_command
 
@@ -42,14 +42,17 @@ def test_import_vbv1(self):
         importer.import_file(utils.test_data_path("00056520.V01"), count)
 
         self.assertEqual(models.CountDetail.objects.count(), 18114)
+        first = models.CountDetail.objects.first()
+        last = models.CountDetail.objects.last()
+        assert first
+        assert last
 
-        tz = pytz.timezone("Europe/Zurich")
+        zurich_timezone = pytz.timezone("Europe/Zurich")
+        first = first.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
+        last = last.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
 
-        first = tz.normalize(models.CountDetail.objects.first().timestamp)
-        last = tz.normalize(models.CountDetail.objects.last().timestamp)
-
-        self.assertEqual(first, tz.localize(datetime(2021, 10, 15, 9, 46, 43, 500000)))
-        self.assertEqual(last, tz.localize(datetime(2021, 10, 15, 23, 59, 54, 600000)))
+        self.assertEqual(first, datetime(2021, 10, 15, 9, 46, 43, 500000))
+        self.assertEqual(last, datetime(2021, 10, 15, 23, 59, 54, 600000))
 
     def test_import_mc(self):
         model = models.Model.objects.all()[0]
@@ -76,13 +79,18 @@ def test_import_mc(self):
 
         self.assertEqual(models.CountDetail.objects.count(), 25867)
 
-        tz = pytz.timezone("Europe/Zurich")
+        first = models.CountDetail.objects.first()
+        last = models.CountDetail.objects.last()
+        zurich_timezone = pytz.timezone("Europe/Zurich")
+
+        assert first
+        assert last
 
-        first = tz.normalize(models.CountDetail.objects.first().timestamp)
-        last = tz.normalize(models.CountDetail.objects.last().timestamp)
+        first = first.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
+        last = last.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
 
-        self.assertEqual(first, tz.localize(datetime(2021, 9, 10, 4, 16, 16)))
-        self.assertEqual(last, tz.localize(datetime(2021, 9, 21, 8, 2, 15)))
+        self.assertEqual(first, datetime(2021, 9, 10, 4, 16, 16))
+        self.assertEqual(last, datetime(2021, 9, 21, 8, 2, 15))
 
     def test_import_int2(self):
         model = models.Model.objects.all()[0]
@@ -107,13 +115,17 @@ def test_import_int2(self):
 
         importer.import_file(utils.test_data_path("10020260.A01"), count)
 
-        tz = pytz.timezone("Europe/Zurich")
+        first = models.CountDetail.objects.first()
+        last = models.CountDetail.objects.last()
+        assert first
+        assert last
 
-        first = tz.normalize(models.CountDetail.objects.first().timestamp)
-        last = tz.normalize(models.CountDetail.objects.last().timestamp)
+        zurich_timezone = pytz.timezone("Europe/Zurich")
+        first = first.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
+        last = last.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
 
-        self.assertEqual(first, tz.localize(datetime(2018, 4, 13, 12, 0)))
-        self.assertEqual(last, tz.localize(datetime(2018, 5, 1, 13, 0)))
+        self.assertEqual(first, datetime(2018, 4, 13, 12, 0))
+        self.assertEqual(last, datetime(2018, 5, 1, 13, 0))
 
     def test_import_simple_int2(self):
         model = models.Model.objects.all()[0]
@@ -142,13 +154,17 @@ def test_import_simple_int2(self):
 
         self.assertEqual(models.CountDetail.objects.count(), 52)
 
-        tz = pytz.timezone("Europe/Zurich")
+        first = models.CountDetail.objects.first()
+        last = models.CountDetail.objects.last()
+        assert first
+        assert last
 
-        first = tz.normalize(models.CountDetail.objects.first().timestamp)
-        last = tz.normalize(models.CountDetail.objects.last().timestamp)
+        zurich_timezone = pytz.timezone("Europe/Zurich")
+        first = first.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
+        last = last.timestamp.astimezone(zurich_timezone).replace(tzinfo=None)
 
-        self.assertEqual(first, tz.localize(datetime(2018, 9, 24, 0, 0)))
-        self.assertEqual(last, tz.localize(datetime(2018, 9, 24, 1, 0)))
+        self.assertEqual(first, datetime(2018, 9, 24, 0, 0))
+        self.assertEqual(last, datetime(2018, 9, 24, 1, 0))
 
         speed20 = models.CountDetail.objects.filter(speed=20)
         self.assertEqual(speed20[0].times, 3)

docker-compose.yml

@@ -26,7 +26,7 @@ services:
       - ${PWD}/testoutputs:/OpenComptage/testoutputs
 
   db:
-    image: postgis/postgis:12-2.5
+    image: postgis/postgis:${POSTGIS_VERSION}
     ports:
       - 5432:5432
     environment:

docs/development.md

@@ -1,4 +1,19 @@
 # Development
+
+## Upgrade dependencies
+
+__Disclaimer__: This section presupposes that you are using a version of `pip-tools` matching your Python interpreter. In other words, if you want to upgrade or pin dependencies for Python 3.9, `pip-tools` has to be built for Python 3.9 -- which in turn means that if you install it through `pip`, you have to use `pip` built for Python 3.9. (You could use the python:3.9 Docker image available from Docker Hub to meet all these requirements with minimal footwork.)
+
+Once `pip-tools` is installed, to upgrade the dependencies run
+
+    pip-compile --upgrade
+
+If it fails you can re-generate `requirements.txt`, which amounts to pinning the dependencies described in `requirements.in` and constrained by `pyproject.toml` to a version matching the project's configuration:
+
+    pip-compile -o requirements.txt pyproject.toml
+
+Now `pip-compile --upgrade` should work.
+
 ## Data model
 The data model has been created to easily allow to add functionality to the product
 e.g. adding new vehicle classes and to be as simple as possible and easily
@@ -18,10 +33,13 @@ run the script `create_data_model_sql_script.sh`.
    | Directory  | Content                                                                                                                                                    |
    |------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
    | .docker/   | docker-compose files to run a development environment (i.e. PostGIS db) or a test environment (i.e. PostGIS db and QGIS to allow using QGIS API in Travis) |
-   | comptages/ | root directory of the QGIS plugin (for more details see the "Code Structure" section)                                                                      |
+   | comptages/ | root directory of the QGIS plugin (for more details see the "Code Structure" section)                                                                 |
    | db/        | contains the sql files to create the database structure and a dump of the initial data                                                                     |
-   | docs/      | contains the user manual that is published online using Github Pages                                                                                       |
-   | scripts/   | contains useful scripts to launch docker enviroments, run tests, create the database, etc                                                                  |
+   | docs/      | contains the user manual that is published online using Github Pages                                                                    |
+   | scripts/   | contains useful scripts to launch docker enviroments, run tests, create the database, etc                                                                      |
+   | pyproject.toml       | configuration file (entry point for packaging and building)      | 
+   | requirements.txt     | Python dependencies, including transitive dependencies, pinned to an exact version matching the project's requirements                                                             |
+   | requiremnts.in       | Python dependencies, not including transitive dependencies, unpinned                                                   
 
 ## Code structure
 
@@ -44,3 +62,4 @@ The code of the plugin (directory =comptages=) is structured in the following wa
    | ui/                 | contains QT's files with the definition of the user interface dialogs              |
    | comptages.py        | plugin main module                                                                 |
    | metadata.txt        | plugin metadata                                                                    |
+

pyproject.toml

@@ -3,6 +3,7 @@ requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [project]
+requires-python = ">=3.9.0,<=3.9.18"
 name = "comptages"
 version = "0.1"
 dynamic = ["dependencies"]
@@ -11,7 +12,7 @@ dynamic = ["dependencies"]
 check = ["pyright>=1.1.36", "black>=23.11.0", "qgis-plugin-ci>=2.8.1"]
 
 [tool.setuptools.dynamic]
-dependencies = { file = ["requirements.txt"] }
+dependencies = { file = ["requirements.in"] }
 
 [tool.qgis-plugin-ci]
 plugin_path = "comptages"

requirements.in

@@ -0,0 +1,8 @@
+django
+icalendar
+nose2
+numpy
+openpyxl
+pandas
+plotly==4.14.3
+psycopg2-binary

requirements.txt

@@ -1,7 +1,49 @@
-nose2==0.8.0
-psycopg2-binary==2.8.6 # on Debian derivates this requires `libpq-dev`
-icalendar==4.0.3
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --output-file=requirements.txt pyproject.toml
+#
+asgiref==3.7.2
+    # via django
+django==4.2.9
+    # via comptages (pyproject.toml)
+et-xmlfile==1.1.0
+    # via openpyxl
+icalendar==5.0.11
+    # via comptages (pyproject.toml)
+nose2==0.14.0
+    # via comptages (pyproject.toml)
+numpy==1.26.3
+    # via
+    #   comptages (pyproject.toml)
+    #   pandas
 openpyxl==3.1.2
-django==3.2.15
-plotly==5.3.1
-pandas==1.3.4
+    # via comptages (pyproject.toml)
+pandas==2.1.4
+    # via comptages (pyproject.toml)
+plotly==4.14.3
+    # via comptages (pyproject.toml)
+psycopg2-binary==2.9.9
+    # via comptages (pyproject.toml)
+python-dateutil==2.8.2
+    # via
+    #   icalendar
+    #   pandas
+pytz==2023.3.post1
+    # via
+    #   icalendar
+    #   pandas
+retrying==1.3.4
+    # via plotly
+six==1.16.0
+    # via
+    #   plotly
+    #   python-dateutil
+    #   retrying
+sqlparse==0.4.4
+    # via django
+typing-extensions==4.9.0
+    # via asgiref
+tzdata==2023.4
+    # via pandas