Exclut le type autoroutier des tronçons JOP (#899)

* Exclut le type autoroutier des tronçons JOP

* Fix ogc_fid column name

Makefile

@@ -81,6 +81,12 @@ bdtopo_migration: ## Generate new db migration for bdtopo
 bdtopo_migrate: ## Run db migrations for bdtopo
 	${BIN_CONSOLE} doctrine:migrations:migrate -n --all-or-nothing --configuration ./config/packages/bdtopo/doctrine_migrations.yaml ${ARGS}
 
+bdtopo_migrate_redo: ## Revert db migrations for bdtopo and run them again
+	# Revert to first migration which creates the postgis extension
+	make bdtopo_migrate ARGS="App\\\Infrastructure\\\Persistence\\\Doctrine\\\BdTopoMigrations\\\Version20240320122522"
+	# Re-run migrations from there
+	make bdtopo_migrate
+
 dbshell: ## Connect to the database
 	docker-compose exec database psql postgresql://dialog:dialog@database:5432/dialog
 

docs/tools/bdtopo.md

@@ -48,6 +48,28 @@ Pour intégrer une nouvelle table, mettez à jour le fichier `tools/bdtopo_updat
 
 De même, pour retirer une table qui n'est plus utilisée, retirez-la du fichier de configuration puis mettez à jour les données.
 
+### Intégrer de nouvelles colonnes dans la BD TOPO
+
+Pour les plus grosses tables comme `troncon_de_route`, on n'intègre que les colonnes dont on a besoin afin de limiter la taille de la BD TOPO sur le disque.
+
+Si vous avez besoin d'intégrer une nouvelle colonne, mettez à jour `tools/bdtopo_update.config.json`, puis [mettez à jour les données](#mettre-à-jour-les-données) mais ajoutez l'option `--overwrite` lors de l'exécution du script de mise à jour :
+
+```bash
+./tools/bdtopo_update ~/path/to/bdtopo --prod --overwrite
+```
+
+Cette option va supprimer les tables et refaire un import de zéro.
+
+Il faut ensuite recréer les indexes en réexecutant les migrations :
+
+```bash
+make bdtopo_migrate_redo
+```
+
+Ensuite elle réexécute les migrations BD TOPO afin de recréer les indexes.
+
+Cet import de zéro est nécessaire car sinon la nouvelle colonne sera ignorée, seul le contenu des colonnes existantes sera mis à jour.
+
 ### Configurer des indexes
 
 La création d'indexes judicieux sur les tables BD TOPO peut permettre d'accélérer les requêtes.

src/Application/RoadGeocoderInterface.php

@@ -8,6 +8,8 @@
 
 interface RoadGeocoderInterface
 {
+    public const HIGHWAY = 'HIGHWAY';
+
     public function computeRoadLine(string $roadName, string $inseeCode): string;
 
     public function findRoads(string $search, string $administrator): array;
@@ -25,5 +27,5 @@ public function computeReferencePoint(
 
     public function findRoadNames(string $search, string $cityCode): array;
 
-    public function findSectionsInArea(string $areaGeometry): string;
+    public function findSectionsInArea(string $areaGeometry, array $excludeTypes = []): string;
 }

src/Infrastructure/Adapter/BdTopoRoadGeocoder.php

@@ -10,6 +10,7 @@
 use App\Application\IntersectionGeocoderInterface;
 use App\Application\RoadGeocoderInterface;
 use App\Domain\Geography\Coordinates;
+use Doctrine\DBAL\ArrayParameterType;
 use Doctrine\DBAL\Connection;
 
 final class BdTopoRoadGeocoder implements RoadGeocoderInterface, IntersectionGeocoderInterface
@@ -321,16 +322,33 @@ public function computeIntersection(string $roadName, string $otherRoadName, str
         return Coordinates::fromLonLat((float) $x, (float) $y);
     }
 
-    public function findSectionsInArea(string $areaGeometry): string
+    public function findSectionsInArea(string $areaGeometry, array $excludeTypes = []): string
     {
+        $bdTopoExcludeTypes = [];
+
+        foreach ($excludeTypes as $type) {
+            $bdTopoExcludeTypes[] = match ($type) {
+                $this::HIGHWAY => 'Type autoroutier',
+                default => $type,
+            };
+        }
+
         try {
             $row = $this->bdtopoConnection->fetchAssociative(
-                'SELECT ST_AsGeoJSON(ST_Force2D(ST_Collect(t.geometrie))) AS geom
-                FROM troncon_de_route AS t
-                WHERE ST_Intersects(t.geometrie, :areaGeometry)
-                ',
+                sprintf(
+                    'SELECT ST_AsGeoJSON(ST_Force2D(ST_Collect(t.geometrie))) AS geom
+                    FROM troncon_de_route AS t
+                    WHERE ST_Intersects(t.geometrie, :areaGeometry)
+                    %s
+                    ',
+                    $bdTopoExcludeTypes ? 'AND t.nature NOT IN (:types)' : '',
+                ),
                 [
                     'areaGeometry' => $areaGeometry,
+                    'types' => $bdTopoExcludeTypes,
+                ],
+                [
+                    'types' => ArrayParameterType::STRING,
                 ],
             );
         } catch (\Exception $exc) {

src/Infrastructure/JOP/JOPTransformer.php

@@ -105,7 +105,10 @@ public function transform(array $geoJSON, Organization $organization): ImportJOP
 
             // CRS is missing in the data but required by PostGIS
             $areaGeometry['crs'] = ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']];
-            $sectionsGeometryCollection = $this->roadGeocoder->findSectionsInArea(json_encode($areaGeometry));
+            $sectionsGeometryCollection = $this->roadGeocoder->findSectionsInArea(
+                json_encode($areaGeometry),
+                excludeTypes: [$this->roadGeocoder::HIGHWAY],
+            );
 
             $locationCommand = new SaveLocationCommand();
             $locationCommand->roadType = RoadTypeEnum::RAW_GEOJSON->value;

src/Infrastructure/Persistence/Doctrine/BdTopoMigrations/Version20240415125716.php

@@ -28,6 +28,6 @@ public function down(Schema $schema): void
     {
         $this->addSql('DROP INDEX IF EXISTS voie_nommee_nom_minuscule_search_idx');
         $this->addSql('DROP INDEX IF EXISTS voie_nommee_code_insee_idx');
-        $this->addSql('ALTER TABLE voie_nommee DROP COLUMN nom_minuscule_search');
+        $this->addSql('ALTER TABLE voie_nommee DROP COLUMN IF EXISTS nom_minuscule_search');
     }
 }

src/Infrastructure/Persistence/Doctrine/BdTopoMigrations/Version20240730134819.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Persistence\Doctrine\BdTopoMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20240730134819 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return 'Add index on troncon_de_route.nature';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql('CREATE INDEX IF NOT EXISTS troncon_de_route_nature_idx ON troncon_de_route (nature);');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('DROP INDEX IF EXISTS troncon_de_route_nature_idx');
+    }
+}

tests/Integration/Infrastructure/Adapter/BdTopoRoadGeocoderTest.php

@@ -279,4 +279,40 @@ public function testComputeIntersectionError(string $roadName, string $otherRoad
 
         $this->roadGeocoder->computeIntersection($roadName, $otherRoadName, $cityCode);
     }
+
+    public function testFindSectionsInArea(): void
+    {
+        // This area is a rectangle, it contains:
+        // * A portion of Rue de Vaudherlant near Le Fayel (60680).
+        // * A portion of Autoroute A1 (highway)
+        // * A portion of "bretelle" (access to A1 from Rue de Vaudherlant)
+        $northWest = [2.70299, 49.375841];
+        $northEast = [2.705149, 49.375841];
+        $southEast = [2.705149, 49.374397];
+        $southWest = [2.70299, 49.374397];
+
+        $area = json_encode([
+            'type' => 'Polygon',
+            'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']],
+            'coordinates' => [
+                // Note: according to GeoJSON standard, a polygon is made of "linear rings"
+                // which must be closed (end point = start point)
+                [$northWest, $northEast, $southEast, $southWest, $northWest],
+            ],
+        ]);
+
+        // By default all types of sections are included.
+        $geometry = $this->roadGeocoder->findSectionsInArea($area);
+        $this->assertSame(
+            '{"type":"MultiLineString","coordinates":[[[2.704663706,49.375064848],[2.705508168,49.374958144],[2.706102966,49.374938917],[2.706678394,49.374936719],[2.707757531,49.37495641]],[[2.704047618,49.379991827],[2.704002207,49.37998812],[2.703927917,49.37997895],[2.703874323,49.37996174],[2.703838642,49.379941878],[2.703823638,49.379917572],[2.703819668,49.379889697],[2.703829635,49.379832194],[2.703868123,49.379603078],[2.703908085,49.379356887],[2.704045651,49.378176115],[2.704074021,49.377790571],[2.704148908,49.377215479],[2.704347775,49.376145415],[2.70449489,49.375697239],[2.704644914,49.375221205],[2.704661989,49.375124169],[2.704663706,49.375064848]],[[2.703931583,49.375025305],[2.703972588,49.373877549],[2.703977585,49.373726551],[2.703973356,49.373024523],[2.703911925,49.368140794]],[[2.702812091,49.375080986],[2.703003456,49.37507786],[2.703226487,49.375073913],[2.703234746,49.375073934],[2.703444002,49.37507175],[2.703720699,49.37507153],[2.703928546,49.375074735],[2.704122628,49.375077907],[2.70422448,49.375080853],[2.704291927,49.375081917],[2.704411689,49.375082209],[2.704450238,49.375081405],[2.704547996,49.375078048],[2.704663706,49.375064848]],[[2.704141355,49.370375059],[2.704154607,49.371424076],[2.704175827,49.373723441],[2.704157915,49.374446088],[2.704127042,49.37502848]],[[2.704119365,49.375166887],[2.70408568,49.375759159],[2.703941665,49.37710531],[2.70390435,49.377370385]],[[2.704127042,49.37502848],[2.704124118,49.375058136],[2.704122628,49.375077907],[2.704119365,49.375166887]],[[2.703926664,49.37516282],[2.703928546,49.375074735],[2.703931583,49.375025305]],[[2.703813123,49.376475788],[2.703820301,49.376424571],[2.703926664,49.37516282]]]}',
+            $geometry,
+        );
+
+        // When excluding highways, the result contains only Rue de Vaudherlant and the "bretelle".
+        $geometry = $this->roadGeocoder->findSectionsInArea($area, excludeTypes: [$this->roadGeocoder::HIGHWAY]);
+        $this->assertSame(
+            '{"type":"MultiLineString","coordinates":[[[2.704663706,49.375064848],[2.705508168,49.374958144],[2.706102966,49.374938917],[2.706678394,49.374936719],[2.707757531,49.37495641]],[[2.704047618,49.379991827],[2.704002207,49.37998812],[2.703927917,49.37997895],[2.703874323,49.37996174],[2.703838642,49.379941878],[2.703823638,49.379917572],[2.703819668,49.379889697],[2.703829635,49.379832194],[2.703868123,49.379603078],[2.703908085,49.379356887],[2.704045651,49.378176115],[2.704074021,49.377790571],[2.704148908,49.377215479],[2.704347775,49.376145415],[2.70449489,49.375697239],[2.704644914,49.375221205],[2.704661989,49.375124169],[2.704663706,49.375064848]],[[2.702812091,49.375080986],[2.703003456,49.37507786],[2.703226487,49.375073913],[2.703234746,49.375073934],[2.703444002,49.37507175],[2.703720699,49.37507153],[2.703928546,49.375074735],[2.704122628,49.375077907],[2.70422448,49.375080853],[2.704291927,49.375081917],[2.704411689,49.375082209],[2.704450238,49.375081405],[2.704547996,49.375078048],[2.704663706,49.375064848]]]}',
+            $geometry,
+        );
+    }
 }

tests/Unit/Infrastructure/JOP/JOPTransformerTest.php

@@ -75,7 +75,10 @@ public function testTransform(): void
         $roadGeocoder
             ->expects(self::once())
             ->method('findSectionsInArea')
-            ->with('{"type":"Polygon","coordinates":"<coords1>","crs":{"type":"name","properties":{"name":"EPSG:4326"}}}')
+            ->with(
+                '{"type":"Polygon","coordinates":"<coords1>","crs":{"type":"name","properties":{"name":"EPSG:4326"}}}',
+                [RoadGeocoderInterface::HIGHWAY],
+            )
             ->willReturn('<sectionsGeometry1>');
 
         $translator

tools/bdtopo_update

@@ -1,13 +1,11 @@
 #!/usr/bin/env python3
 import argparse
 import json
-import re
 import signal
-import string
 import subprocess
 import sys
 from contextlib import ExitStack
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 from urllib.parse import urlparse
 
@@ -73,7 +71,9 @@ class Config:
     tables: list[Table]
 
 
-def main(config: Config, directory: Path, url: str | None, yes: bool) -> int:
+def main(
+    config: Config, directory: Path, url: str | None, overwrite: bool, yes: bool
+) -> int:
     if not directory.exists():
         print(f"ERROR: directory {directory} does not exist", file=sys.stderr)
         return 1
@@ -157,8 +157,9 @@ def main(config: Config, directory: Path, url: str | None, yes: bool) -> int:
         # are provided, such as '?sslmode=prefer' by Scalingo.
         database_url = database_url.replace("postgres://", "postgresql://")
 
-        print("===> Will import into:")
-        print(database_url)
+        print("===> Import info:")
+        print("Database URL:", database_url)
+        print("Import mode:", "overwrite" if overwrite else "append")
 
         if not yes and input("------> Proceed? (y/N) ") != "y":
             return 1
@@ -185,8 +186,8 @@ def main(config: Config, directory: Path, url: str | None, yes: bool) -> int:
                     # (Geopackages in BD TOPO may not all use the same projection)
                     "-t_srs",
                     "EPSG:4326",
-                    # Append on subsequent calls
-                    "-append",
+                    # Append to existing data, or overwrite for faster re-ingestion
+                    "-overwrite" if overwrite else "-append",
                     # Use provided SQL statement, if any, to only import certain columns
                     # (In this case, progress bar won't be available due a limitation of the ogr2ogr Postgres driver)
                     *(["-sql", table.select_sql] if table.select_sql else []),
@@ -195,6 +196,9 @@ def main(config: Config, directory: Path, url: str | None, yes: bool) -> int:
                     "--config",
                     "PG_USE_COPY",
                     "YES",
+                    # Explicitly set the name of the FID column to create
+                    "-lco",
+                    "FID=ogc_fid",
                 ]
 
                 if table.name not in tables_seen:
@@ -247,6 +251,12 @@ def main(config: Config, directory: Path, url: str | None, yes: bool) -> int:
             )
             result.check_returncode()
             print("------> VACUUM ANALYZE ran successfully!")
+
+            if overwrite:
+                print(
+                    "===> NOTE: run 'make bdtopo_migrate_redo' "
+                    "to recreate indexes"
+                )
         except KeyboardInterrupt:
             # ogr2ogr already manages most of the cleanup for us, nothing to do.
             return 2
@@ -280,6 +290,9 @@ if __name__ == "__main__":
         help="Deploy to a PostgreSQL database identified by this database URL",
         default=None,
     )
+    parser.add_argument(
+        "--overwrite", action="store_true", help="Recreate tables instead of appending"
+    )
     parser.add_argument("-y", "--yes", action="store_true", help="Accept all prompts")
     parser.add_argument(
         "-c",
@@ -312,6 +325,7 @@ if __name__ == "__main__":
             config,
             directory=directory,
             url=args.url,
+            overwrite=args.overwrite,
             yes=args.yes,
         )
     )

tools/bdtopo_update.config.json

@@ -12,7 +12,7 @@
         {
             "name": "troncon_de_route",
             "select_sql": [
-                "SELECT identifiant_voie_1_gauche, sens_de_circulation, geometrie",
+                "SELECT identifiant_voie_1_gauche, sens_de_circulation, nature, geometrie",
                 "FROM troncon_de_route"
             ]
         }