From ba686e681a3bb25efd1bec08bae452d770784b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Tue, 29 Oct 2024 12:23:16 +0100 Subject: [PATCH 1/4] API datasets: exclure les datasets experimentaux See #4275. --- apps/transport/lib/db/dataset.ex | 6 ++ .../api/controllers/datasets_controller.ex | 3 + .../api/datasets_controller_test.exs | 77 +++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/apps/transport/lib/db/dataset.ex b/apps/transport/lib/db/dataset.ex index ef9319cf29..afda43e0c3 100644 --- a/apps/transport/lib/db/dataset.ex +++ b/apps/transport/lib/db/dataset.ex @@ -25,6 +25,7 @@ defmodule DB.Dataset do @licences_ouvertes ["fr-lo", "lov2"] @licence_mobilités_tag "licence-mobilités" @hidden_dataset_custom_tag_value "masqué" + @experimental_tag "experimental" typed_schema "dataset" do field(:datagouv_id, :string) @@ -1125,4 +1126,9 @@ defmodule DB.Dataset do @spec full_logo(__MODULE__.t()) :: binary() def full_logo(%__MODULE__{full_logo: full_logo, custom_full_logo: custom_full_logo}), do: custom_full_logo || full_logo + + def reject_experimental_datasets(queryable) do + queryable + |> where([d], @experimental_tag not in d.tags) + end end diff --git a/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex b/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex index a8aee030cd..5226479af8 100644 --- a/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex +++ b/apps/transport/lib/transport_web/api/controllers/datasets_controller.ex @@ -79,6 +79,7 @@ defmodule TransportWeb.API.DatasetController do def by_id(%Plug.Conn{} = conn, %{"id" => datagouv_id}) do dataset = Dataset + |> Dataset.reject_experimental_datasets() |> preload([:resources, :aom, :region, :communes, :legal_owners_aom, :legal_owners_region]) |> Repo.get_by(datagouv_id: datagouv_id) @@ -95,6 +96,7 @@ defmodule TransportWeb.API.DatasetController do @spec geojson_by_id(Plug.Conn.t(), map) :: Plug.Conn.t() def geojson_by_id(%Plug.Conn{} = conn, %{"id" => id}) do Dataset + |> Dataset.reject_experimental_datasets() |> Repo.get_by(datagouv_id: id) |> Repo.preload([:aom, :region, :communes]) |> case do @@ -374,6 +376,7 @@ defmodule TransportWeb.API.DatasetController do %{} |> Dataset.list_datasets() + |> Dataset.reject_experimental_datasets() |> preload([:resources, :aom, :region, :communes, :legal_owners_aom, :legal_owners_region]) |> Repo.all() |> Enum.map(fn dataset -> diff --git a/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs b/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs index 7d634303f8..a9ddd4a822 100644 --- a/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/api/datasets_controller_test.exs @@ -256,6 +256,54 @@ defmodule TransportWeb.API.DatasetControllerTest do assert_schema(json, "DatasetsResponse", TransportWeb.API.Spec.spec()) end + test "GET /api/datasets without the experimental tagged datasets", %{conn: conn} do + aom = insert(:aom, nom: "Angers Métropole", siren: "siren") + + insert(:resource, + dataset: + insert(:dataset, + custom_title: "TC", + type: "public-transit", + licence: "lov2", + datagouv_id: "datagouv-1", + slug: "slug-1", + is_active: true, + created_at: ~U[2021-12-23 13:30:40.000000Z], + aom: aom, + tags: ["netex"] + ), + url: "https://link.to/gbfs.json", + datagouv_id: "1", + type: "main", + format: "gbfs" + ) + + insert(:resource, + dataset: + insert(:dataset, + custom_title: "Tarifs (expérimental)", + type: "public-transit", + licence: "lov2", + datagouv_id: "datagouv-2", + slug: "slug-2", + is_active: true, + created_at: ~U[2021-12-23 13:30:40.000000Z], + aom: aom, + tags: ["netex", "experimental"] + ), + url: "https://link.to/gbfs.json", + datagouv_id: "2", + type: "main", + format: "gbfs" + ) + + path = Helpers.dataset_path(conn, :datasets) + + json = conn |> get(path) |> json_response(200) + + assert [%{"title" => "TC"}] = json + end + test "GET /api/datasets/:id *without* history, multi_validation and resource_metadata", %{conn: conn} do aom = insert(:aom, nom: "Angers Métropole", siren: "siren", id: 4242) region = DB.Region |> Ecto.Query.where(insee: "52") |> DB.Repo.one!() @@ -506,6 +554,35 @@ defmodule TransportWeb.API.DatasetControllerTest do |> json_response(200) end + test "GET /api/datasets/:id with a dataset tagged 'experimental'", %{conn: conn} do + setup_empty_history_resources() + + %DB.Dataset{datagouv_id: visible_dataset_datagouv_id} = + insert(:dataset, + datagouv_id: "datagouv-1", + is_active: true, + created_at: ~U[2021-12-23 13:30:40.000000Z], + tags: ["netex"] + ) + + %DB.Dataset{datagouv_id: experimental_dataset_datagouv_id} = + insert(:dataset, + datagouv_id: "datagouv-2", + is_active: true, + created_at: ~U[2021-12-23 13:30:40.000000Z], + tags: ["netex", "experimental"] + ) + + assert %{"datagouv_id" => ^visible_dataset_datagouv_id} = + conn + |> get(Helpers.dataset_path(conn, :by_id, visible_dataset_datagouv_id)) + |> json_response(200) + + conn + |> get(Helpers.dataset_path(conn, :by_id, experimental_dataset_datagouv_id)) + |> json_response(404) + end + test "gtfs-rt features are filled", %{conn: conn} do dataset_1 = insert(:dataset, datagouv_id: datagouv_id_1 = Ecto.UUID.generate()) resource_1 = insert(:resource, dataset_id: dataset_1.id, format: "gtfs-rt") From bdd3a1c1ed3eb63691c95e9f8e8fc658d65eb7a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Tue, 29 Oct 2024 12:11:43 +0100 Subject: [PATCH 2/4] Datasets: label experimental Fixes #4275. --- apps/transport/lib/db/dataset.ex | 8 ++++++++ .../lib/transport_web/live/backoffice/custom_tags_live.ex | 4 ++++ .../lib/transport_web/templates/dataset/_banner.html.heex | 6 ++++++ apps/transport/lib/transport_web/views/dataset_view.ex | 1 + .../priv/gettext/en/LC_MESSAGES/page-dataset-details.po | 4 ++++ .../priv/gettext/fr/LC_MESSAGES/page-dataset-details.po | 4 ++++ apps/transport/priv/gettext/page-dataset-details.pot | 4 ++++ 7 files changed, 31 insertions(+) diff --git a/apps/transport/lib/db/dataset.ex b/apps/transport/lib/db/dataset.ex index afda43e0c3..9728258da6 100644 --- a/apps/transport/lib/db/dataset.ex +++ b/apps/transport/lib/db/dataset.ex @@ -1127,6 +1127,14 @@ defmodule DB.Dataset do def full_logo(%__MODULE__{full_logo: full_logo, custom_full_logo: custom_full_logo}), do: custom_full_logo || full_logo + @doc """ + iex> experimental?(%DB.Dataset{custom_tags: ["experimental", "foo"]}) + true + iex> experimental?(%DB.Dataset{custom_tags: ["foo"]}) + false + """ + def experimental?(%__MODULE__{} = dataset), do: has_custom_tag?(dataset, @experimental_tag) + def reject_experimental_datasets(queryable) do queryable |> where([d], @experimental_tag not in d.tags) diff --git a/apps/transport/lib/transport_web/live/backoffice/custom_tags_live.ex b/apps/transport/lib/transport_web/live/backoffice/custom_tags_live.ex index c647902e38..bdb1f2a2b3 100644 --- a/apps/transport/lib/transport_web/live/backoffice/custom_tags_live.ex +++ b/apps/transport/lib/transport_web/live/backoffice/custom_tags_live.ex @@ -72,6 +72,10 @@ defmodule TransportWeb.CustomTagsLive do %{ name: "authentification_requise", doc: "Indique sur la page du JDD qu'il est nécessaire de s'authentifier pour accéder aux données." + }, + %{ + name: "experimental", + doc: "Ajoute sur la page du JDD une bannière indiquant que le jeu est expérimental" } ] end diff --git a/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex b/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex index 02aa63f1e3..1558516c12 100644 --- a/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex +++ b/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex @@ -10,3 +10,9 @@ "The producer requires authentication to access the data. Consequently, some features on transport.data.gouv.fr, such as data availability, validations, and metadata, are unavailable for this dataset. Please follow the producer's instructions to gain access to the data." ) %>

+

+ ℹ️ <%= dgettext( + "page-dataset-details", + "This unofficial dataset is provided experimentally. Do not use it for travel information purpose." + ) %> +

diff --git a/apps/transport/lib/transport_web/views/dataset_view.ex b/apps/transport/lib/transport_web/views/dataset_view.ex index 8a9e9701b9..32efdb5b3b 100644 --- a/apps/transport/lib/transport_web/views/dataset_view.ex +++ b/apps/transport/lib/transport_web/views/dataset_view.ex @@ -8,6 +8,7 @@ defmodule TransportWeb.DatasetView do # ~H expects a variable named `assigns`, so wrapping the calls to `~H` inside # a helper function would be cleaner and more future-proof to avoid conflicts at some point. import Phoenix.Component, only: [sigil_H: 2, live_render: 3] + import DB.Dataset, only: [experimental?: 1] import DB.MultiValidation, only: [get_metadata_info: 2, get_metadata_info: 3] alias Shared.DateTimeDisplay alias Transport.Validators.GTFSTransport diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/page-dataset-details.po b/apps/transport/priv/gettext/en/LC_MESSAGES/page-dataset-details.po index 15e8eea31c..f6facddd66 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/page-dataset-details.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/page-dataset-details.po @@ -717,3 +717,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Data under the responsibility of" msgstr "" + +#, elixir-autogen, elixir-format +msgid "This unofficial dataset is provided experimentally. Do not use it for travel information purpose." +msgstr "" diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/page-dataset-details.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/page-dataset-details.po index 8a335ccb0b..c9a8e17447 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/page-dataset-details.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/page-dataset-details.po @@ -717,3 +717,7 @@ msgstr "Le producteur requiert une authentification pour accéder aux données. #, elixir-autogen, elixir-format msgid "Data under the responsibility of" msgstr "Données sous la responsabilité de" + +#, elixir-autogen, elixir-format +msgid "This unofficial dataset is provided experimentally. Do not use it for travel information purpose." +msgstr "Ce jeu de données non officiel est publié à titre expérimental. Veuillez à ne pas le réutiliser à des fins d'information voyageur." diff --git a/apps/transport/priv/gettext/page-dataset-details.pot b/apps/transport/priv/gettext/page-dataset-details.pot index 661cd167d9..43d6d175dd 100644 --- a/apps/transport/priv/gettext/page-dataset-details.pot +++ b/apps/transport/priv/gettext/page-dataset-details.pot @@ -717,3 +717,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Data under the responsibility of" msgstr "" + +#, elixir-autogen, elixir-format +msgid "This unofficial dataset is provided experimentally. Do not use it for travel information purpose." +msgstr "" From 7ffad6257c91f409c9bf148c23724bbf75c32945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Tue, 29 Oct 2024 12:11:43 +0100 Subject: [PATCH 3/4] =?UTF-8?q?Emoji=20avertissement=20et=20affichage=20en?= =?UTF-8?q?=20priorit=C3=A9.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/dataset/_banner.html.heex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex b/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex index 1558516c12..f36a856ad8 100644 --- a/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex +++ b/apps/transport/lib/transport_web/templates/dataset/_banner.html.heex @@ -1,3 +1,9 @@ +

+ ⚠️ <%= dgettext( + "page-dataset-details", + "This unofficial dataset is provided experimentally. Do not use it for travel information purpose." + ) %> +

ℹ️ <%= dgettext( "page-dataset-details", @@ -10,9 +16,3 @@ "The producer requires authentication to access the data. Consequently, some features on transport.data.gouv.fr, such as data availability, validations, and metadata, are unavailable for this dataset. Please follow the producer's instructions to gain access to the data." ) %>

-

- ℹ️ <%= dgettext( - "page-dataset-details", - "This unofficial dataset is provided experimentally. Do not use it for travel information purpose." - ) %> -

From ac65e0c131a6caca8f4b80ba9621051ac6a9eb72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Wed, 30 Oct 2024 16:46:07 +0100 Subject: [PATCH 4/4] =?UTF-8?q?Test=20de=20l'affichage=20de=20la=20bani?= =?UTF-8?q?=C3=A8re=20"exp=C3=A9rimental"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/dataset_controller_test.exs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/transport/test/transport_web/controllers/dataset_controller_test.exs b/apps/transport/test/transport_web/controllers/dataset_controller_test.exs index d8eb3be7d5..ca3377687e 100644 --- a/apps/transport/test/transport_web/controllers/dataset_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/dataset_controller_test.exs @@ -679,7 +679,7 @@ defmodule TransportWeb.DatasetControllerTest do ] == content |> Floki.find("#quality-indicators table") end - describe "information banners are displayed" do + describe "information & warning banners are displayed" do test "a seasonal dataset", %{conn: conn} do dataset = insert(:dataset, is_active: true, custom_tags: ["saisonnier", "foo"]) assert TransportWeb.DatasetView.seasonal_warning?(dataset) @@ -702,6 +702,17 @@ defmodule TransportWeb.DatasetControllerTest do "Le producteur requiert une authentification pour accéder aux données" ) end + + test "an experimental dataset", %{conn: conn} do + dataset = insert(:dataset, is_active: true, custom_tags: ["experimental", "foo"]) + assert DB.Dataset.experimental?(dataset) + + dataset_has_banner_with_text( + conn, + dataset, + "Ce jeu de données non officiel est publié à titre expérimental. Veuillez à ne pas le réutiliser à des fins d'information voyageur." + ) + end end test "custom logo is displayed when set", %{conn: conn} do