Improve search autocomplete (#727)

src/Application/GeocoderInterface.php

@@ -12,7 +12,5 @@ public function computeCoordinates(string $address, string $cityCode): Coordinat
 
     public function computeJunctionCoordinates(string $address, string $roadName, string $cityCode): Coordinates;
 
-    public function findRoadNames(string $search, string $cityCode): array;
-
     public function findCities(string $search): array;
 }

src/Application/RoadGeocoderInterface.php

@@ -22,4 +22,6 @@ public function computeReferencePoint(
         string $side,
         int $abscissa,
     ): Coordinates;
+
+    public function findRoadNames(string $search, string $cityCode): array;
 }

src/Infrastructure/Adapter/APIAdresseGeocoder.php

@@ -107,45 +107,6 @@ public function computeJunctionCoordinates(string $address, string $roadName, st
         return $this->computeCoordinates($roadName . ' / ' . $address, $cityCode, type: 'poi');
     }
 
-    public function findRoadNames(string $search, string $cityCode): array
-    {
-        if (\strlen($search) < 3) {
-            // APIAdresse returns error if search string has length strictly less than 3.
-            return [];
-        }
-
-        $response = $this->apiAdresseClient->request('GET', '/search/', [
-            'headers' => [
-                'Accept' => 'application/json',
-            ],
-            'query' => [
-                'q' => $search,
-                'autocomplete' => '1',
-                'limit' => 7,
-                'type' => 'street',
-                'citycode' => $cityCode,
-            ],
-        ]);
-
-        try {
-            $data = $response->toArray(throw: true);
-            $roadNames = [];
-
-            foreach ($data['features'] as $feature) {
-                $roadNames[] = [
-                    'value' => $feature['properties']['name'],
-                    'label' => $feature['properties']['label'],
-                ];
-            }
-
-            return $roadNames;
-        } catch (\Exception $exc) {
-            \Sentry\captureException($exc);
-
-            return [];
-        }
-    }
-
     public function findCities(string $search): array
     {
         if (\strlen($search) < 3) {

src/Infrastructure/Adapter/BdTopoRoadGeocoder.php

@@ -177,4 +177,54 @@ public function computeReferencePoint(
             return Coordinates::fromLonLat($coordinates[0], $coordinates[1]);
         }
     }
+
+    public function findRoadNames(string $search, string $cityCode): array
+    {
+        // Build search query
+        // https://www.postgresql.org/docs/current/datatype-textsearch.html#DATATYPE-TSQUERY
+        $query = str_replace(' ', ' & ', trim($search)) . ':*';
+
+        try {
+            $rows = $this->bdtopoConnection->fetchAllAssociative(
+                "
+                    SELECT array_to_string(
+                        -- BD TOPO contains lowercase road names. We capitalize non-stopwords.
+                        -- Example: 'rue de france' -> 'Rue de France'. (Better than INITCAP('rue de france') -> 'Rue De France')
+                        array(
+                            SELECT CASE WHEN cardinality(t.lexemes) > 0 THEN INITCAP(t.token) ELSE t.token END
+                            FROM ts_debug('french', nom_minuscule) AS t
+                            WHERE t.alias NOT IN ('asciihword')
+                        ),
+                        ''
+                    ) AS road_name
+                    FROM voie_nommee
+                    WHERE (
+                        nom_minuscule_search @@ to_tsquery('french', :query::text)
+                        OR :search % ANY(STRING_TO_ARRAY(f_bdtopo_voie_nommee_normalize_nom_minuscule(nom_minuscule), ' '))
+                    )
+                    AND code_insee = :cityCode
+                    ORDER BY ts_rank(nom_minuscule_search, to_tsquery('french', :query::text)) DESC
+                    LIMIT 7
+                ",
+                [
+                    'cityCode' => $cityCode,
+                    'query' => $query,
+                    'search' => $search,
+                ],
+            );
+        } catch (\Exception $exc) {
+            throw new GeocodingFailureException(sprintf('Road names query has failed: %s', $exc->getMessage()), previous: $exc);
+        }
+
+        $roadNames = [];
+
+        foreach ($rows as $row) {
+            $roadNames[] = [
+                'value' => $row['road_name'],
+                'label' => $row['road_name'],
+            ];
+        }
+
+        return $roadNames;
+    }
 }

src/Infrastructure/Controller/Regulation/Fragments/GetAddressCompletionFragmentController.php

@@ -4,7 +4,7 @@
 
 namespace App\Infrastructure\Controller\Regulation\Fragments;
 
-use App\Application\GeocoderInterface;
+use App\Application\RoadGeocoderInterface;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;
@@ -13,7 +13,7 @@
 final class GetAddressCompletionFragmentController
 {
     public function __construct(
-        private GeocoderInterface $geocoder,
+        private RoadGeocoderInterface $roadGeocoder,
         private \Twig\Environment $twig,
     ) {
     }
@@ -32,7 +32,7 @@ public function __invoke(Request $request): Response
             throw new BadRequestHttpException();
         }
 
-        $roadNames = $this->geocoder->findRoadNames($search, $cityCode);
+        $roadNames = $this->roadGeocoder->findRoadNames($search, $cityCode);
 
         return new Response(
             $this->twig->render(

src/Infrastructure/Persistence/Doctrine/BdTopoMigrations/Version20240415125716.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Persistence\Doctrine\BdTopoMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20240415125716 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql("ALTER TABLE voie_nommee ADD COLUMN nom_minuscule_search tsvector GENERATED ALWAYS AS (to_tsvector('french', nom_minuscule)) STORED");
+        $this->addSql('CREATE INDEX voie_nommee_nom_minuscule_search_idx ON voie_nommee USING GIN(nom_minuscule_search)');
+        $this->addSql('CREATE INDEX voie_nommee_code_insee_idx ON voie_nommee (code_insee)');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('DROP INDEX IF EXISTS voie_nommee_nom_minuscule_search_idx');
+        $this->addSql('DROP INDEX IF EXISTS voie_nommee_code_insee_idx');
+        $this->addSql('ALTER TABLE voie_nommee DROP COLUMN nom_minuscule_search');
+    }
+}

src/Infrastructure/Persistence/Doctrine/BdTopoMigrations/Version20240417143817.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Infrastructure\Persistence\Doctrine\BdTopoMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20240417143817 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql('CREATE EXTENSION pg_trgm');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('DROP EXTENSION pg_trgm');
+    }
+}

templates/regulation/fragments/_measure_form.html.twig

@@ -203,7 +203,8 @@
                 <div
                     class="fr-x-autocomplete-wrapper"
                     data-controller="autocomplete"
-                    data-autocomplete-url-value="{{ path('fragment_road_number_completion') }}" data-autocomplete-query-param-value="search"
+                    data-autocomplete-url-value="{{ path('fragment_road_number_completion') }}"
+                    data-autocomplete-query-param-value="search"
                     data-autocomplete-extra-query-params-value="{{ {administrator: '#' ~ form.administrator.vars.id}|json_encode }}"
                     data-autocomplete-min-length-value="2"
                     data-autocomplete-delay-value="500"

tests/Mock/BdTopoRoadGeocoderMock.php

@@ -81,4 +81,17 @@ public function computeReferencePoint(string $lineGeometry, string $administrato
             default => throw new RoadGeocodingFailureException(),
         };
     }
+
+    public function findRoadNames(string $search, string $cityCode): array
+    {
+        return match ($search) {
+            'Rue Eugène Berthoud' => [
+                [
+                    'value' => 'Rue Eugène Berthoud',
+                    'label' => 'Rue Eugène Berthoud, 93400 Saint-Ouen-sur-Seine',
+                ],
+            ],
+            default => [],
+        };
+    }
 }

tests/Unit/Infrastructure/Adapter/APIAdresseGeocoderTest.php

@@ -92,105 +92,6 @@ public function testComputeCoordinatesDecodeError(string $body, string $pattern)
         $geocoder->computeCoordinates($this->address, $this->cityCode);
     }
 
-    public function testFindRoadNames(): void
-    {
-        $expectedRequests = [
-            function ($method, $url, $options) {
-                $this->assertSame('GET', $method);
-                $this->assertEmpty(
-                    array_diff(
-                        ['scheme' => 'http', 'host' => 'testserver', 'path' => '/search/'],
-                        parse_url($url),
-                    ),
-                );
-                $this->assertContains('Accept: application/json', $options['headers']);
-                $this->assertSame('Rue Eugene', $options['query']['q']);
-                $this->assertSame('1', $options['query']['autocomplete']);
-                $this->assertSame(7, $options['query']['limit']);
-                $this->assertSame('street', $options['query']['type']);
-                $this->assertSame($this->cityCode, $options['query']['citycode']);
-
-                return new MockResponse(
-                    json_encode([
-                        'features' => [
-                            [
-                                'properties' => [
-                                    'name' => 'Rue Eugene Berthoud',
-                                    'label' => 'Rue Eugene Berthoud 75018 Paris',
-                                ],
-                            ],
-                        ],
-                    ]),
-                    ['http_code' => 200],
-                );
-            },
-        ];
-
-        $http = new MockHttpClient($expectedRequests, 'http://testserver');
-
-        $geocoder = new APIAdresseGeocoder($http);
-        $addresses = $geocoder->findRoadNames('Rue Eugene', $this->cityCode);
-        $this->assertEquals([['value' => 'Rue Eugene Berthoud', 'label' => 'Rue Eugene Berthoud 75018 Paris']], $addresses);
-    }
-
-    public function testfindRoadNamesIncompleteFeature(): void
-    {
-        $body = json_encode(['features' => [[]]]);
-        $response = new MockResponse($body, ['http_code' => 200]);
-        $http = new MockHttpClient([$response]);
-
-        $geocoder = new APIAdresseGeocoder($http);
-        $addresses = $geocoder->findRoadNames('Test', $this->cityCode);
-        $this->assertEquals([], $addresses);
-    }
-
-    public function testfindRoadNamesAPIFailure(): void
-    {
-        $response = new MockResponse('...', ['http_code' => 500]);
-        $http = new MockHttpClient([$response]);
-
-        $geocoder = new APIAdresseGeocoder($http);
-        $addresses = $geocoder->findRoadNames('Test', $this->cityCode);
-        $this->assertEquals([], $addresses);
-    }
-
-    public function testfindRoadNamesInvalidJSON(): void
-    {
-        $response = new MockResponse('{"blah', ['http_code' => 200]);
-        $http = new MockHttpClient([$response]);
-
-        $geocoder = new APIAdresseGeocoder($http);
-        $addresses = $geocoder->findRoadNames('Test', $this->cityCode);
-        $this->assertEquals([], $addresses);
-    }
-
-    public function testfindRoadNamesSearchTooShort(): void
-    {
-        $response = new MockResponse(
-            json_encode([
-                'features' => [
-                    [
-                        'properties' => [
-                            'name' => 'Rue Eugene Berthoud',
-                            'label' => 'Rue Eugene Berthoud 75018 Paris',
-                        ],
-                    ],
-                ],
-            ]),
-            ['http_code' => 200],
-        );
-        $http = new MockHttpClient([$response]);
-        $geocoder = new APIAdresseGeocoder($http);
-
-        $addresses = $geocoder->findRoadNames('aa', $this->cityCode);
-        $this->assertEquals([], $addresses);
-        $this->assertEquals(0, $http->getRequestsCount());
-
-        $addresses = $geocoder->findRoadNames('aaa', $this->cityCode);
-        $this->assertEquals([['value' => 'Rue Eugene Berthoud', 'label' => 'Rue Eugene Berthoud 75018 Paris']], $addresses);
-        $this->assertEquals(1, $http->getRequestsCount());
-    }
-
     public function testFindCities(): void
     {
         $expectedRequests = [

tests/Unit/Infrastructure/Adapter/BdTopoRoadGeocoderTest.php

@@ -231,4 +231,33 @@ public function testComputeReferencePointUnexpectedError(): void
 
         $this->roadGeocoder->computeReferencePoint('geom', 'Ardennes', 'D32', '1', 'U', 0);
     }
+
+    public function testFindRoadNames(): void
+    {
+        $this->conn
+            ->expects(self::once())
+            ->method('fetchAllAssociative')
+            ->willReturn([[
+                'road_name' => 'Rue Eugène Berthoud',
+            ]]);
+
+        $this->assertEquals([
+            [
+                'value' => 'Rue Eugène Berthoud',
+                'label' => 'Rue Eugène Berthoud',
+            ],
+        ], $this->roadGeocoder->findRoadNames('Rue Eugène Berthoud', '93070'));
+    }
+
+    public function testFindRoadNamesUnexpectedError(): void
+    {
+        $this->expectException(GeocodingFailureException::class);
+
+        $this->conn
+            ->expects(self::once())
+            ->method('fetchAllAssociative')
+            ->willThrowException(new \RuntimeException('Some network error'));
+
+        $this->roadGeocoder->findRoadNames('Rue Eugène Berthoud', '93070');
+    }
 }