diff --git a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/entities/alerts/type/AlertTypeMapping.kt b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/entities/alerts/type/AlertTypeMapping.kt index e7e97202a8..d6f5aaca68 100644 --- a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/entities/alerts/type/AlertTypeMapping.kt +++ b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/entities/alerts/type/AlertTypeMapping.kt @@ -32,6 +32,10 @@ enum class AlertTypeMapping( clazz = MissingFAR48HoursAlert::class.java, alertName = "Non-emission de message \"FAR\" en 48h", ), + SUSPICION_OF_UNDER_DECLARATION_ALERT( + clazz = SuspicionOfUnderDeclarationAlert::class.java, + alertName = "Suspicion de sous-déclaration", + ), ; override fun getImplementation(): Class { diff --git a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/entities/alerts/type/SuspicionOfUnderDeclarationAlert.kt b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/entities/alerts/type/SuspicionOfUnderDeclarationAlert.kt new file mode 100644 index 0000000000..cc8374380f --- /dev/null +++ b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/entities/alerts/type/SuspicionOfUnderDeclarationAlert.kt @@ -0,0 +1,7 @@ +package fr.gouv.cnsp.monitorfish.domain.entities.alerts.type + +class SuspicionOfUnderDeclarationAlert( + override var seaFront: String? = null, + override var dml: String? = null, + var riskFactor: Double? = null, +) : AlertType(AlertTypeMapping.SUSPICION_OF_UNDER_DECLARATION_ALERT, seaFront, dml, 27689) diff --git a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/alert/GetPendingAlerts.kt b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/alert/GetPendingAlerts.kt index aab14bf8cd..eaca16376b 100644 --- a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/alert/GetPendingAlerts.kt +++ b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/alert/GetPendingAlerts.kt @@ -23,6 +23,7 @@ class GetPendingAlerts( AlertTypeMapping.TWELVE_MILES_FISHING_ALERT, AlertTypeMapping.MISSING_DEP_ALERT, AlertTypeMapping.MISSING_FAR_48_HOURS_ALERT, + AlertTypeMapping.SUSPICION_OF_UNDER_DECLARATION_ALERT, ), ).map { pendingAlert -> pendingAlert.value.natinfCode?.let { diff --git a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/reporting/ArchiveOutdatedReportings.kt b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/reporting/ArchiveOutdatedReportings.kt index dce68d0c6c..1d624ebd77 100644 --- a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/reporting/ArchiveOutdatedReportings.kt +++ b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/reporting/ArchiveOutdatedReportings.kt @@ -21,7 +21,8 @@ class ArchiveOutdatedReportings(private val reportingRepository: ReportingReposi reportingCandidatesToArchive.filter { it.second.type == AlertTypeMapping.MISSING_FAR_ALERT || it.second.type == AlertTypeMapping.THREE_MILES_TRAWLING_ALERT || - it.second.type == AlertTypeMapping.MISSING_DEP_ALERT + it.second.type == AlertTypeMapping.MISSING_DEP_ALERT || + it.second.type == AlertTypeMapping.SUSPICION_OF_UNDER_DECLARATION_ALERT }.map { it.first } logger.info("Found ${filteredReportingIdsToArchive.size} reportings to archive.") diff --git a/backend/src/test/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/GetPendingAlertsUTests.kt b/backend/src/test/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/GetPendingAlertsUTests.kt index 144c9b6aea..558810fc7e 100644 --- a/backend/src/test/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/GetPendingAlertsUTests.kt +++ b/backend/src/test/kotlin/fr/gouv/cnsp/monitorfish/domain/use_cases/GetPendingAlertsUTests.kt @@ -67,6 +67,7 @@ class GetPendingAlertsUTests { AlertTypeMapping.TWELVE_MILES_FISHING_ALERT, AlertTypeMapping.MISSING_DEP_ALERT, AlertTypeMapping.MISSING_FAR_48_HOURS_ALERT, + AlertTypeMapping.SUSPICION_OF_UNDER_DECLARATION_ALERT, ), ) Mockito.verify(infractionRepository, Mockito.times(1)).findInfractionByNatinfCode(eq(7059)) diff --git a/datascience/src/pipeline/entities/alerts.py b/datascience/src/pipeline/entities/alerts.py index 1c2e179b1e..b22eec7b1e 100644 --- a/datascience/src/pipeline/entities/alerts.py +++ b/datascience/src/pipeline/entities/alerts.py @@ -8,3 +8,4 @@ class AlertType(Enum): MISSING_DEP_ALERT = "MISSING_DEP_ALERT" MISSING_FAR_ALERT = "MISSING_FAR_ALERT" MISSING_FAR_48_HOURS_ALERT = "MISSING_FAR_48_HOURS_ALERT" + SUSPICION_OF_UNDER_DECLARATION = "SUSPICION_OF_UNDER_DECLARATION" diff --git a/datascience/src/pipeline/flows/suspicions_of_under_declaration_alerts.py b/datascience/src/pipeline/flows/suspicions_of_under_declaration_alerts.py new file mode 100644 index 0000000000..3173a698d1 --- /dev/null +++ b/datascience/src/pipeline/flows/suspicions_of_under_declaration_alerts.py @@ -0,0 +1,52 @@ +from pathlib import Path + +from prefect import Flow, case, task +from prefect.executors import LocalDaskExecutor + +from src.pipeline.entities.alerts import AlertType +from src.pipeline.generic_tasks import extract +from src.pipeline.shared_tasks.alerts import ( + extract_active_reportings, + extract_silenced_alerts, + filter_alerts, + load_alerts, + make_alerts, +) +from src.pipeline.shared_tasks.control_flow import check_flow_not_running + + +@task(checkpoint=False) +def extract_suspicions_of_under_declaration(): + return extract( + db_name="monitorfish_remote", + query_filepath="monitorfish/suspicions_of_under_declaration.sql", + ) + + +with Flow("Suspicions of under-declaration", executor=LocalDaskExecutor()) as flow: + flow_not_running = check_flow_not_running() + with case(flow_not_running, True): + vessels_with_suspicions_of_under_declaration = ( + extract_suspicions_of_under_declaration() + ) + + alerts = make_alerts( + vessels_with_suspicions_of_under_declaration, + AlertType.SUSPICION_OF_UNDER_DECLARATION.value, + AlertType.SUSPICION_OF_UNDER_DECLARATION.value, + ) + silenced_alerts = extract_silenced_alerts( + AlertType.SUSPICION_OF_UNDER_DECLARATION.value + ) + active_reportings = extract_active_reportings( + AlertType.SUSPICION_OF_UNDER_DECLARATION.value + ) + filtered_alerts = filter_alerts(alerts, silenced_alerts, active_reportings) + + # Load + load_alerts( + filtered_alerts, + alert_config_name=AlertType.SUSPICION_OF_UNDER_DECLARATION.value, + ) + +flow.file_name = Path(__file__).name diff --git a/datascience/src/pipeline/queries/monitorfish/suspicions_of_under_declaration.sql b/datascience/src/pipeline/queries/monitorfish/suspicions_of_under_declaration.sql new file mode 100644 index 0000000000..d305ea7b96 --- /dev/null +++ b/datascience/src/pipeline/queries/monitorfish/suspicions_of_under_declaration.sql @@ -0,0 +1,63 @@ +WITH fishing_efforts AS ( + SELECT + p.internal_reference_number AS cfr, + d.dml, + COALESCE(p.flag_state, v.flag_state) AS flag_state, + v.power * EXTRACT(epoch FROM SUM(p.time_since_previous_position)) / 3600 AS fishing_effort_kwh + FROM positions p + LEFT JOIN vessels v + ON v.cfr = p.internal_reference_number + LEFT JOIN districts d + ON d.district_code = v.district_code + WHERE + p.date_time >= DATE_TRUNC('day', CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - INTERVAL '7 days' + AND p.date_time < DATE_TRUNC('day', CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + AND p.internal_reference_number IS NOT NULL + AND p.flag_state = 'FR' + AND v.length >= 12 + AND v.logbook_equipment_status = 'Equipé' + AND p.is_fishing + GROUP BY 1, 2, 3, v.power + -- Minimum number of days with fishing activity + HAVING COUNT(DISTINCT DATE_TRUNC('day', date_time)) >= 2 +), + +catches AS ( + SELECT + lb.cfr, + COALESCE(SUM(weight), 0) AS weight + FROM logbook_reports lb + LEFT JOIN jsonb_array_elements(lb.value->'hauls') haul ON true + LEFT JOIN LATERAL ( + SELECT + SUM((catch->>'weight')::DOUBLE PRECISION) AS weight + FROM jsonb_array_elements(haul->'catches') catch + ) catch_weight ON true + WHERE + lb.operation_datetime_utc >= DATE_TRUNC('day', CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - INTERVAL '7 days' + AND lb.activity_datetime_utc >= DATE_TRUNC('day', CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - INTERVAL '7 days' + AND lb.log_type = 'FAR' + GROUP BY 1 +) + +SELECT + fe.cfr, + lp.external_immatriculation, + lp.ircs, + lp.vessel_id, + lp.vessel_identifier, + lp.vessel_name, + f.facade, + fe.dml, + fe.flag_state, + lp.risk_factor, + lp.latitude, + lp.longitude +FROM fishing_efforts fe +JOIN catches c +ON fe.cfr = c.cfr +LEFT JOIN last_positions lp +ON lp.cfr = fe.cfr +LEFT JOIN facade_areas_subdivided f +ON ST_Intersects(ST_SetSRID(ST_Point(lp.longitude, lp.latitude), 4326), f.geometry) +WHERE c.weight < 0.015 * COALESCE(fe.fishing_effort_kWh, 0) \ No newline at end of file diff --git a/datascience/tests/test_data/remote_database/V666.0__Reset_test_vessels.sql b/datascience/tests/test_data/remote_database/V666.0__Reset_test_vessels.sql index 853a490540..9fc985c4b8 100644 --- a/datascience/tests/test_data/remote_database/V666.0__Reset_test_vessels.sql +++ b/datascience/tests/test_data/remote_database/V666.0__Reset_test_vessels.sql @@ -7,7 +7,7 @@ INSERT INTO public.vessels ( declared_fishing_gears, nav_licence_expiration_date, vessel_emails, vessel_phones, proprietor_name, proprietor_phones, proprietor_emails, operator_name, operator_phones, under_charter, - operator_mobile_phone, vessel_mobile_phone, vessel_telex, vessel_fax, operator_fax, operator_email + operator_mobile_phone, vessel_mobile_phone, vessel_telex, vessel_fax, operator_fax, operator_email, logbook_equipment_status ) VALUES ( 1, @@ -16,7 +16,7 @@ INSERT INTO public.vessels ( '{GNS,GTR,LLS}', (NOW() AT TIME ZONE 'UTC')::TIMESTAMP + INTERVAL '2 months', '{}', '{}', NULL, '{}', '{}', 'Le pêcheur de poissons', '{1234567890,"06 06 06 06 06"}', false, - null, null, null, null, null, 'write_to_me@gmail.com' + null, null, null, null, null, 'write_to_me@gmail.com', 'Equipé' ), ( 2, @@ -25,7 +25,7 @@ INSERT INTO public.vessels ( '{DRB,PS1}', (NOW() AT TIME ZONE 'UTC')::TIMESTAMP + INTERVAL '3 months', '{figure@conscience.fr, figure2@conscience.fr}', '{}', NULL, '{}', '{}', 'Le pêcheur de crevettes', '{9876543210}', true, - '0600000000', null, null, '0100000000', '0200000000', 'address@email.bzh' + '0600000000', null, null, '0100000000', '0200000000', 'address@email.bzh', 'Equipé' ), ( 3, @@ -34,7 +34,7 @@ INSERT INTO public.vessels ( '{OTM,OTB,OTT}', NULL, '{}', '{}', NULL, '{}', '{}', 'Le pêcheur de fonds', '{0000000000}', false, - null, null, null, null, null, 'address@email.nl' + null, null, null, null, null, 'address@email.nl', 'Equipé' ), ( 4, @@ -43,7 +43,7 @@ INSERT INTO public.vessels ( '{OTM,OTB,OTT}', NULL, '{}', '{}', NULL, '{}', '{}', 'Le pêcheur', '{11111111111}', false, - null, '0111111111', null, null, null, 'pecheur@poissecaille.fr' + null, '0111111111', null, null, null, 'pecheur@poissecaille.fr', 'Equipé' ), ( 5, @@ -52,7 +52,7 @@ INSERT INTO public.vessels ( '{OTT}', NULL, '{}', '{}', NULL, '{}', '{}', 'Le pêcheur qui se cache', '{2222222222}', false, - null, null, null, null, null, 'discrete@cache-cache.fish' + null, null, null, null, null, 'discrete@cache-cache.fish', 'Equipé' ), ( 6, @@ -61,7 +61,7 @@ INSERT INTO public.vessels ( '{OTT}', NULL, '{}', '{}', NULL, '{}', '{}', 'Le pêcheur qui se fait ses 4h reports', '{3333333333}', false, - null, null, null, null, null, 'reglo@bateau.fr' + null, null, null, null, null, 'reglo@bateau.fr', 'Equipé' ), ( 7, @@ -70,6 +70,6 @@ INSERT INTO public.vessels ( '{LLS}', NULL, '{}', '{}', NULL, '{}', '{}', 'Pêchou', '{9546458753}', false, - null, null, null, null, null, 'target@me' + null, null, null, null, null, 'target@me', 'Equipé' ) ; \ No newline at end of file diff --git a/datascience/tests/test_data/remote_database/V666.1__Reset_test_positions.sql b/datascience/tests/test_data/remote_database/V666.1__Reset_test_positions.sql index b3719df7c4..75ac7a8356 100644 --- a/datascience/tests/test_data/remote_database/V666.1__Reset_test_positions.sql +++ b/datascience/tests/test_data/remote_database/V666.1__Reset_test_positions.sql @@ -10,6 +10,8 @@ INSERT INTO positions ( ( 13639240, 'ABC000055481', 'AS761555', NULL, 'IL2468', 'PLACE SPECTACLE SUBIR', 'NL', 'FR', NULL, NULL, 53.4279999999999973, 5.55299999999999994, 2, 31, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '1 day 1 hour', 'VMS', false, false, 2500, INTERVAL '30 minutes', 2.69, true, INTERVAL '1 day 2 hours 30 minutes'), ( 13640592, 'ABC000055481', 'AS761555', NULL, 'IL2468', 'PLACE SPECTACLE SUBIR', 'NL', 'FR', NULL, NULL, 53.4239999999999995, 5.54900000000000038, 2, 338, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '1 day 30 minutes', 'VMS', false, false, 2500, INTERVAL '30 minutes', 2.69, true, INTERVAL '1 day 3 hours'), ( 13641745, 'ABC000055481', 'AS761555', NULL, 'IL2468', 'PLACE SPECTACLE SUBIR', 'NL', 'FR', NULL, NULL, 53.4350000000000023, 5.55299999999999994, 2, 356, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '1 day', 'VMS', false, false, 2500, INTERVAL '30 minutes', 2.69, NULL, INTERVAL '1 day 3 hours 30 minutes'), +( 13634203, 'ABC000306959', 'RV348407', NULL, 'LLUK', 'ÉTABLIR IMPRESSION LORSQUE', 'FR', 'FR', NULL, NULL, 49.6069999999999993, -0.744999999999999996, 1, 343, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '50 hours 10 minutes', 'VMS', false, true, 2050, INTERVAL '24 hours', 1.107, true, INTERVAL '0 hour'), +( 13634204, 'ABC000306959', 'RV348407', NULL, 'LLUK', 'ÉTABLIR IMPRESSION LORSQUE', 'FR', 'FR', NULL, NULL, 49.6069999999999993, -0.744999999999999996, 1, 343, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '26 hours 10 minutes', 'VMS', false, true, 2050, INTERVAL '24 hours', 1.107, true, INTERVAL '0 hour'), ( 13634205, 'ABC000306959', 'RV348407', NULL, 'LLUK', 'ÉTABLIR IMPRESSION LORSQUE', 'FR', 'FR', NULL, NULL, 49.6069999999999993, -0.744999999999999996, 1, 343, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '2 hours 10 minutes', 'VMS', false, true, 2050, INTERVAL '1 hour', 1.107, true, INTERVAL '0 hour'), ( 13637054, 'ABC000306959', 'RV348407', NULL, 'LLUK', 'ÉTABLIR IMPRESSION LORSQUE', 'FR', 'FR', NULL, NULL, 49.6060000000000016, -0.735999999999999988, 1.5, 351, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '1 hour 10 minutes', 'VMS', false, false, 2050, INTERVAL '1 hour', 1.107, NULL, INTERVAL '0 hour'), ( 13639642, 'ABC000306959', 'RV348407', NULL, 'LLUK', 'ÉTABLIR IMPRESSION LORSQUE', 'FR', 'FR', NULL, NULL, 49.6099999999999994, -0.739999999999999991, 1, 302, (NOW() AT TIME ZONE 'UTC')::TIMESTAMP - INTERVAL '10 minutes', 'VMS', false, NULL, NULL, NULL, NULL, NULL, NULL), diff --git a/datascience/tests/test_data/remote_database/V666.5__Reset_test_logbook.sql b/datascience/tests/test_data/remote_database/V666.5__Reset_test_logbook.sql index a77719a584..36708aba4a 100644 --- a/datascience/tests/test_data/remote_database/V666.5__Reset_test_logbook.sql +++ b/datascience/tests/test_data/remote_database/V666.5__Reset_test_logbook.sql @@ -118,22 +118,6 @@ VALUES ((now() AT TIME ZONE 'UTC') - INTERVAL '1 month 3 days 23 hours 48 minutes')::TIMESTAMP, NULL, 'ERS' ); -UPDATE logbook_reports -SET value = jsonb_set( - value, - '{departureDatetimeUtc}', - ('"' || to_char(CURRENT_TIMESTAMP - INTERVAL '1 month 5 days', 'YYYY-MM-DD') || 'T' || to_char(CURRENT_TIMESTAMP - INTERVAL '1 week 5 days', 'HH24:MI:SS') || 'Z"')::jsonb -) -WHERE operation_number = '3'; - -UPDATE logbook_reports -SET value = jsonb_set( - value, - '{departureDatetimeUtc}', - ('"' || to_char(CURRENT_TIMESTAMP - INTERVAL '1 week 5 days', 'YYYY-MM-DD') || 'T' || to_char(CURRENT_TIMESTAMP - INTERVAL '1 week 5 days', 'HH24:MI:SS') || 'Z"')::jsonb -) -WHERE operation_number = '5'; - -- Add FLUX test data INSERT INTO logbook_reports ( @@ -286,4 +270,85 @@ INSERT INTO logbook_reports ( ((now() AT TIME ZONE 'UTC') - INTERVAL '1 month 10 minutes')::TIMESTAMP, NULL, 'ERS', false, NULL, NULL ) -; \ No newline at end of file +; +--WHEN log_type = 'FAR' THEN (SELECT MIN((haul->>'farDatetimeUtc')::TIMESTAMPTZ) AT TIME ZONE 'UTC' FROM jsonb_array_elements(value->'hauls') haul) +-- Set activity timestamps to operation_datetime_utc +UPDATE logbook_reports +SET value = jsonb_set( + value, + CASE + WHEN log_type = 'DEP' THEN '{departureDatetimeUtc}' + WHEN log_type = 'COE' THEN '{effortZoneEntryDatetimeUtc}' + WHEN log_type = 'CPS' THEN '{cpsDatetimeUtc}' + WHEN log_type = 'DIS' THEN '{discardDatetimeUtc}' + WHEN log_type = 'COX' THEN '{effortZoneExitDatetimeUtc}' + WHEN log_type = 'CRO' THEN '{effortZoneExitDatetimeUtc}' + WHEN log_type = 'EOF' THEN '{endOfFishingDatetimeUtc}' + WHEN log_type = 'PNO' THEN '{predictedArrivalDatetimeUtc}' + WHEN log_type = 'LAN' THEN '{landingDatetimeUtc}' + WHEN log_type = 'RTP' THEN '{returnDatetimeUtc}' + END::VARCHAR[], + ('"' || to_char(operation_datetime_utc + CASE WHEN log_type = 'PNO' THEN INTERVAL '4 hours' ELSE INTERVAL '0' END, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') || '"')::jsonb + ) +WHERE + transmission_format = 'ERS' + AND log_type IN ('DEP', 'COE', 'CPS', 'DIS', 'COX', 'CRO', 'EOF', 'PNO', 'LAN', 'RTP'); + + +UPDATE logbook_reports +SET value = jsonb_set( + value, + '{predictedLandingDatetimeUtc}' + , + ('"' || to_char(operation_datetime_utc + INTERVAL '4 hours', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') || '"')::jsonb + ) +WHERE + transmission_format = 'ERS' + AND log_type = 'PNO'; + + +WITH updated_hauls AS ( + SELECT + report_id, + jsonb_agg( + jsonb_set( + haul, + '{farDatetimeUtc}', + ('"' || to_char(operation_datetime_utc, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') || '"')::jsonb + ) + ) AS hauls + FROM logbook_reports, jsonb_array_elements(value->'hauls') haul + WHERE + transmission_format = 'ERS' + AND log_type = 'FAR' + GROUP BY report_id +) + +UPDATE logbook_reports lr +SET value = jsonb_set( + value, + '{hauls}', + hauls + ) +FROM updated_hauls uh +WHERE uh.report_id = lr.report_id; + +-- Add activity_datetime_utc values +UPDATE logbook_reports +SET activity_datetime_utc = CASE + WHEN log_type = 'DEP' THEN (value->>'departureDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'NOT-COE' THEN (value->>'effortZoneEntryDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'COE' THEN (value->>'effortZoneEntryDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'FAR' THEN (SELECT MIN((haul->>'farDatetimeUtc')::TIMESTAMPTZ) AT TIME ZONE 'UTC' FROM jsonb_array_elements(value->'hauls') haul) + WHEN log_type = 'CPS' THEN (value->>'cpsDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'DIS' THEN (value->>'discardDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'NOT-COX' THEN (value->>'effortZoneExitDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'COX' THEN (value->>'effortZoneExitDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'CRO' THEN (value->>'effortZoneExitDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'EOF' THEN (value->>'endOfFishingDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'PNO' THEN (value->>'predictedArrivalDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'LAN' THEN (value->>'landingDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + WHEN log_type = 'RTP' THEN (value->>'returnDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' + ELSE NULL + END +WHERE log_type IS NOT NULL; \ No newline at end of file diff --git a/datascience/tests/test_pipeline/test_flows/test_current_segments.py b/datascience/tests/test_pipeline/test_flows/test_current_segments.py index 51dfaadb6d..26bd845201 100644 --- a/datascience/tests/test_pipeline/test_flows/test_current_segments.py +++ b/datascience/tests/test_pipeline/test_flows/test_current_segments.py @@ -44,7 +44,7 @@ def current_segments() -> pd.DataFrame: ], "departure_datetime_utc": [ pd.NaT, - datetime.datetime(2018, 2, 27, 1, 5), + now - datetime.timedelta(days=2), now - datetime.timedelta(weeks=1, days=5), pd.NaT, pd.NaT, diff --git a/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py b/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py index a00c23d7ce..3d31a69069 100644 --- a/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py +++ b/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py @@ -432,22 +432,22 @@ def extracted_pnos() -> pd.DataFrame: ], "facade": ["NAMO", "SA", "NAMO", "SA", None, "SA", None, "NAMO", "SA"], "predicted_arrival_datetime_utc": [ - datetime(2020, 5, 6, 11, 41, 3, 340000), - datetime(2020, 5, 6, 11, 41, 3, 340000), - datetime(2020, 5, 6, 11, 41, 3, 340000), - datetime(2020, 5, 6, 11, 41, 3, 340000), - datetime(2020, 5, 6, 11, 41, 3, 340000), + now - relativedelta(months=1, hours=1) + relativedelta(hours=4), + now - relativedelta(months=1, minutes=25) + relativedelta(hours=4), + now - relativedelta(months=1, hours=2) + relativedelta(hours=4), + now - relativedelta(months=1, minutes=52) + relativedelta(hours=4), + now - relativedelta(months=1, minutes=32) + relativedelta(hours=4), datetime(2021, 5, 6, 7, 41, 3, 340000), datetime(2021, 5, 6, 7, 41, 3, 340000), datetime(2021, 5, 6, 7, 41, 3, 340000), datetime(2021, 5, 6, 7, 41, 3, 340000), ], "predicted_landing_datetime_utc": [ - datetime(2020, 5, 6, 16, 40, 0, 0), - None, - None, - None, - None, + now - relativedelta(months=1, hours=1) + relativedelta(hours=4), + now - relativedelta(months=1, minutes=25) + relativedelta(hours=4), + now - relativedelta(months=1, hours=2) + relativedelta(hours=4), + now - relativedelta(months=1, minutes=52) + relativedelta(hours=4), + now - relativedelta(months=1, minutes=32) + relativedelta(hours=4), datetime(2021, 5, 6, 11, 41, 3, 340000), datetime(2021, 5, 6, 11, 41, 3, 340000), datetime(2021, 5, 6, 11, 41, 3, 340000), @@ -1398,6 +1398,8 @@ def test_extract_pnos_to_generate(reset_test_data, extracted_pnos): "operation_datetime_utc", "report_datetime_utc", "last_control_datetime_utc", + "predicted_arrival_datetime_utc", + "predicted_landing_datetime_utc", ] pnos, generation_needed = extract_pnos_to_generate.run( diff --git a/datascience/tests/test_pipeline/test_flows/test_enrich_positions.py b/datascience/tests/test_pipeline/test_flows/test_enrich_positions.py index 69def68c8a..53c08a5505 100644 --- a/datascience/tests/test_pipeline/test_flows/test_enrich_positions.py +++ b/datascience/tests/test_pipeline/test_flows/test_enrich_positions.py @@ -477,7 +477,7 @@ def test_extract_enrich_load(reset_test_data): # The number of positions in the positions table should not change assert len(positions_after) == len(positions_before) - assert len(positions_after) == 29 + assert len(positions_after) == 31 # Positions outside of the selected Period should not be affected assert ( @@ -556,6 +556,8 @@ def test_extract_enrich_load(reset_test_data): timedelta(days=1, hours=3, minutes=30), None, ], + [13634203, "RV348407", 1.107, True, timedelta(), True], + [13634204, "RV348407", 1.107, True, timedelta(), True], [13634205, "RV348407", 1.107, False, timedelta(), False], [13637054, "RV348407", 0.355284, False, timedelta(hours=1), False], [13639642, "RV348407", 0.286178, False, timedelta(hours=2), True], @@ -729,10 +731,10 @@ def test_flow_can_compute_in_chunks(reset_test_data): flow.schedule = None state = flow.run( - start_hours_ago=48, + start_hours_ago=96, end_hours_ago=0, - minutes_per_chunk=30 * 60, - chunk_overlap_minutes=6 * 60, + minutes_per_chunk=60 * 60, + chunk_overlap_minutes=24 * 60, minimum_consecutive_positions=2, minimum_minutes_of_emission_at_sea=60, min_fishing_speed_threshold=0.1, @@ -746,10 +748,10 @@ def test_flow_can_compute_in_chunks(reset_test_data): flow.schedule = None state = flow.run( - start_hours_ago=48, + start_hours_ago=96, end_hours_ago=0, - minutes_per_chunk=48 * 60, - chunk_overlap_minutes=6 * 60, + minutes_per_chunk=96 * 60, + chunk_overlap_minutes=24 * 60, minimum_consecutive_positions=2, minimum_minutes_of_emission_at_sea=60, min_fishing_speed_threshold=0.1, diff --git a/datascience/tests/test_pipeline/test_flows/test_logbook.py b/datascience/tests/test_pipeline/test_flows/test_logbook.py index ae07caf153..dddcb9b85a 100644 --- a/datascience/tests/test_pipeline/test_flows/test_logbook.py +++ b/datascience/tests/test_pipeline/test_flows/test_logbook.py @@ -345,4 +345,5 @@ def test_flow(mock_move, reset_test_data): final_logbook_reports.is_test_message, "operation_number" ].values[0] ) == "FRA20200321502645" - assert final_logbook_reports.activity_datetime_utc.notnull().sum() == 14 + assert initial_logbook_reports.activity_datetime_utc.notnull().sum() == 30 + assert final_logbook_reports.activity_datetime_utc.notnull().sum() == 44 diff --git a/datascience/tests/test_pipeline/test_flows/test_missing_far_alerts.py b/datascience/tests/test_pipeline/test_flows/test_missing_far_alerts.py index 28b5aa9e60..a1c8fc6999 100644 --- a/datascience/tests/test_pipeline/test_flows/test_missing_far_alerts.py +++ b/datascience/tests/test_pipeline/test_flows/test_missing_far_alerts.py @@ -239,8 +239,8 @@ def test_extract_vessels_that_emitted_fars(reset_test_data): vessels_that_emitted_fars = extract_vessels_that_emitted_fars.run( declaration_min_datetime_utc=now - timedelta(days=2), declaration_max_datetime_utc=now - timedelta(days=1), - fishing_operation_min_datetime_utc=datetime(year=2018, month=7, day=21), - fishing_operation_max_datetime_utc=datetime(year=2018, month=7, day=22), + fishing_operation_min_datetime_utc=now - timedelta(days=2), + fishing_operation_max_datetime_utc=now - timedelta(days=1), ) assert vessels_that_emitted_fars == {"ABC000306959"} @@ -253,18 +253,18 @@ def test_extract_vessels_that_emitted_fars(reset_test_data): assert vessels_that_emitted_fars == set() vessels_that_emitted_fars = extract_vessels_that_emitted_fars.run( - declaration_min_datetime_utc=now - timedelta(days=5), - declaration_max_datetime_utc=now - timedelta(days=4), - fishing_operation_min_datetime_utc=datetime(year=2018, month=7, day=21), - fishing_operation_max_datetime_utc=datetime(year=2018, month=7, day=22), + declaration_min_datetime_utc=datetime(year=2015, month=7, day=21), + declaration_max_datetime_utc=datetime(year=2015, month=7, day=22), + fishing_operation_min_datetime_utc=now - timedelta(weeks=2), + fishing_operation_max_datetime_utc=now - timedelta(days=1), ) assert vessels_that_emitted_fars == set() vessels_that_emitted_fars = extract_vessels_that_emitted_fars.run( declaration_min_datetime_utc=now - timedelta(weeks=2), declaration_max_datetime_utc=now - timedelta(days=1), - fishing_operation_min_datetime_utc=datetime(year=2018, month=7, day=21), - fishing_operation_max_datetime_utc=datetime(year=2018, month=7, day=22), + fishing_operation_min_datetime_utc=now - timedelta(weeks=2), + fishing_operation_max_datetime_utc=now - timedelta(days=1), ) assert vessels_that_emitted_fars == {"ABC000306959", "ABC000542519"} diff --git a/datascience/tests/test_pipeline/test_flows/test_suspicions_of_under_declaration_alerts.py b/datascience/tests/test_pipeline/test_flows/test_suspicions_of_under_declaration_alerts.py new file mode 100644 index 0000000000..7dde0edf85 --- /dev/null +++ b/datascience/tests/test_pipeline/test_flows/test_suspicions_of_under_declaration_alerts.py @@ -0,0 +1,127 @@ +from datetime import datetime, timezone + +import pandas as pd +import pytest +from sqlalchemy import text + +from src.db_config import create_engine +from src.pipeline.flows.suspicions_of_under_declaration_alerts import ( + extract_suspicions_of_under_declaration, + flow, +) +from src.read_query import read_query +from tests.mocks import mock_check_flow_not_running + +flow.replace(flow.get_tasks("check_flow_not_running")[0], mock_check_flow_not_running) + + +@pytest.fixture +def expected_suspicions_of_under_declaration() -> pd.DataFrame: + return pd.DataFrame( + { + "cfr": ["ABC000306959"], + "external_immatriculation": ["RV348407"], + "ircs": ["LLUK"], + "vessel_id": [1], + "vessel_identifier": ["INTERNAL_REFERENCE_NUMBER"], + "vessel_name": ["ÉTABLIR IMPRESSION LORSQUE"], + "facade": ["SA"], + "dml": ["DML 29"], + "flag_state": ["FR"], + "risk_factor": [2.58], + "latitude": [49.606], + "longitude": [-0.736], + } + ) + + +@pytest.fixture +def reset_test_data_suspicions_of_under_declaration_alerts(reset_test_data): + e = create_engine(db="monitorfish_remote") + with e.begin() as con: + con.execute( + text( + """ + INSERT INTO last_positions ( + id, + cfr, + external_immatriculation, + ircs, + vessel_id, + vessel_identifier, + vessel_name, + impact_risk_factor, + probability_risk_factor, + detectability_risk_factor, + risk_factor, + is_at_port, + latitude, + longitude + ) VALUES + ( + 13639642, + 'ABC000306959', + 'RV348407', + 'LLUK', + 1, + 'INTERNAL_REFERENCE_NUMBER', + 'ÉTABLIR IMPRESSION LORSQUE', + 1.5, + 2.5, + 3.1, + 2.58, + false, + 49.606, + -0.736 + ) + """ + ) + ) + + +def test_extract_suspicions_of_under_declaration( + reset_test_data_suspicions_of_under_declaration_alerts, + expected_suspicions_of_under_declaration, +): + res = extract_suspicions_of_under_declaration.run() + pd.testing.assert_frame_equal(res, expected_suspicions_of_under_declaration) + + +def test_flow(reset_test_data_suspicions_of_under_declaration_alerts): + query = "SELECT * FROM pending_alerts WHERE alert_config_name = 'SUSPICION_OF_UNDER_DECLARATION'" + + initial_pending_alerts = read_query(query, db="monitorfish_remote") + + flow.schedule = None + state = flow.run() + assert state.is_successful() + + final_pending_alerts = read_query(query, db="monitorfish_remote") + + assert len(initial_pending_alerts) == 0 + assert len(final_pending_alerts) == 1 + + alert = final_pending_alerts.loc[0].to_dict() + alert.pop("id") + creation_date = alert.pop("creation_date") + + assert alert == { + "vessel_name": "ÉTABLIR IMPRESSION LORSQUE", + "internal_reference_number": "ABC000306959", + "external_reference_number": "RV348407", + "ircs": "LLUK", + "trip_number": None, + "value": { + "dml": "DML 29", + "type": "SUSPICION_OF_UNDER_DECLARATION", + "seaFront": "SA", + "riskFactor": 2.58, + }, + "vessel_identifier": "INTERNAL_REFERENCE_NUMBER", + "alert_config_name": "SUSPICION_OF_UNDER_DECLARATION", + "vessel_id": 1.0, + "latitude": 49.606, + "longitude": -0.736, + "flag_state": "FR", + } + assert abs((creation_date - datetime.now(timezone.utc)).total_seconds()) < 60 diff --git a/frontend/src/domain/entities/alerts/constants.ts b/frontend/src/domain/entities/alerts/constants.ts index f12719a388..b96792dd75 100644 --- a/frontend/src/domain/entities/alerts/constants.ts +++ b/frontend/src/domain/entities/alerts/constants.ts @@ -46,6 +46,11 @@ export const COMMON_ALERT_TYPE_OPTION: Record< nameWithAlertDetails: (percentOfTolerance, minimumWeightThreshold) => `Tolérance de ${percentOfTolerance}% non respectée, appliquée pour un poids minimum de ${minimumWeightThreshold}kg.` }, + SUSPICION_OF_UNDER_DECLARATION_ALERT: { + code: PendingAlertValueType.SUSPICION_OF_UNDER_DECLARATION_ALERT, + isOperationalAlert: true, + name: 'Suspicion de sous-déclaration' + }, THREE_MILES_TRAWLING_ALERT: { code: PendingAlertValueType.THREE_MILES_TRAWLING_ALERT, isOperationalAlert: true, diff --git a/frontend/src/domain/entities/alerts/types.ts b/frontend/src/domain/entities/alerts/types.ts index 95b6ed1d4e..84afd98ebd 100644 --- a/frontend/src/domain/entities/alerts/types.ts +++ b/frontend/src/domain/entities/alerts/types.ts @@ -9,6 +9,7 @@ export enum PendingAlertValueType { MISSING_DEP_ALERT = 'MISSING_DEP_ALERT', MISSING_FAR_48_HOURS_ALERT = 'MISSING_FAR_48_HOURS_ALERT', MISSING_FAR_ALERT = 'MISSING_FAR_ALERT', + SUSPICION_OF_UNDER_DECLARATION_ALERT = 'SUSPICION_OF_UNDER_DECLARATION_ALERT', THREE_MILES_TRAWLING_ALERT = 'THREE_MILES_TRAWLING_ALERT', TWELVE_MILES_FISHING_ALERT = 'TWELVE_MILES_FISHING_ALERT' }