From e8f3b10cb1bdeeccb2197b874ce5a06f0622f364 Mon Sep 17 00:00:00 2001 From: Vincent Date: Wed, 21 Aug 2024 14:29:44 +0200 Subject: [PATCH 1/3] Remove duplicate report_ids before PNO distribution --- .../src/pipeline/queries/monitorfish/pnos_to_generate.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql b/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql index c16a99fef9..2fbb531c85 100644 --- a/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql +++ b/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql @@ -17,7 +17,7 @@ acknowledged_messages AS ( AND value->>'returnStatus' = '000' ) -(SELECT +(SELECT DISTINCT ON (r.report_id) -- In rare cases the same PNO with the same data and the same report_id is sent multiple times in messages with different operation numbers r.id, r.operation_datetime_utc, r.report_id, @@ -78,7 +78,7 @@ WHERE (value->>'isInvalidated') IS NULL OR (value->>'isInvalidated')::BOOLEAN IS false ) -ORDER BY id) +ORDER BY report_id) UNION ALL From 6ecdc8b81fb5110070b5c7e73eb2c06a0bce3bbc Mon Sep 17 00:00:00 2001 From: Vincent Date: Wed, 21 Aug 2024 14:52:44 +0200 Subject: [PATCH 2/3] Fix pno list query --- .../interfaces/DBLogbookReportRepository.kt | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/infrastructure/database/repositories/interfaces/DBLogbookReportRepository.kt b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/infrastructure/database/repositories/interfaces/DBLogbookReportRepository.kt index 8e92415885..c2c8cccf9a 100644 --- a/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/infrastructure/database/repositories/interfaces/DBLogbookReportRepository.kt +++ b/backend/src/main/kotlin/fr/gouv/cnsp/monitorfish/infrastructure/database/repositories/interfaces/DBLogbookReportRepository.kt @@ -117,6 +117,19 @@ interface DBLogbookReportRepository : AND (:tripSegmentCodesAsSqlArrayString IS NULL OR trip_segment_codes && CAST(:tripSegmentCodesAsSqlArrayString AS TEXT[])) ), + acknowledged_report_ids AS ( + SELECT DISTINCT referenced_report_id + FROM logbook_reports lr + WHERE + -- This filter helps Timescale optimize the query since `operation_datetime_utc` is indexed + lr.operation_datetime_utc + BETWEEN CAST(:willArriveAfter AS TIMESTAMP) - INTERVAL '48 hours' + AND CAST(:willArriveBefore AS TIMESTAMP) + INTERVAL '48 hours' + + AND lr.operation_type = 'RET' + AND lr.value->>'returnStatus' = '000' + ), + del_pno_logbook_reports AS ( SELECT lr.*, @@ -134,19 +147,10 @@ interface DBLogbookReportRepository : AND CAST(:willArriveBefore AS TIMESTAMP) + INTERVAL '48 hours' AND lr.operation_type = 'DEL' - ), - - acknowledged_report_ids AS ( - SELECT DISTINCT referenced_report_id - FROM logbook_reports lr - WHERE - -- This filter helps Timescale optimize the query since `operation_datetime_utc` is indexed - lr.operation_datetime_utc - BETWEEN CAST(:willArriveAfter AS TIMESTAMP) - INTERVAL '48 hours' - AND CAST(:willArriveBefore AS TIMESTAMP) + INTERVAL '48 hours' - - AND lr.operation_type = 'RET' - AND lr.value->>'returnStatus' = '000' + AND ( + lr.operation_number IN (SELECT referenced_report_id FROM acknowledged_report_ids) + OR fdacplr.flag_state NOT IN ('FRA', 'GUF', 'VEN') -- flag_states for which we received RET messages + ) ) SELECT * @@ -159,9 +163,6 @@ interface DBLogbookReportRepository : SELECT * FROM del_pno_logbook_reports - WHERE - operation_number IN (SELECT referenced_report_id FROM acknowledged_report_ids) - OR flag_state NOT IN ('FRA', 'GUF', 'VEN') -- flag_states for which we received RET messages """, nativeQuery = true, ) From fb580e151433d718c679ba14cc46527e764a51ba Mon Sep 17 00:00:00 2001 From: Vincent Date: Wed, 21 Aug 2024 15:11:00 +0200 Subject: [PATCH 3/3] Fix ack message computation in pno distribution --- .../queries/monitorfish/pnos_to_generate.sql | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql b/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql index 2fbb531c85..cdd1e0e863 100644 --- a/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql +++ b/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql @@ -1,5 +1,7 @@ WITH deleted_messages AS ( - SELECT referenced_report_id + SELECT + operation_number, + referenced_report_id FROM logbook_reports WHERE operation_datetime_utc >= :start_datetime_utc - INTERVAL '4 hours' @@ -15,6 +17,13 @@ acknowledged_messages AS ( AND operation_datetime_utc < :end_datetime_utc + INTERVAL '8 hours' AND operation_type ='RET' AND value->>'returnStatus' = '000' +), + +acknowledged_deleted_messages AS ( + SELECT referenced_report_id + FROM deleted_messages + WHERE + operation_number IN (SELECT referenced_report_id FROM acknowledged_messages) ) (SELECT DISTINCT ON (r.report_id) -- In rare cases the same PNO with the same data and the same report_id is sent multiple times in messages with different operation numbers @@ -65,14 +74,17 @@ WHERE (value->>'isBeingSent')::BOOLEAN IS true OR report_id NOT IN (SELECT report_id FROM prior_notification_pdf_documents) ) - AND report_id NOT IN (SELECT referenced_report_id FROM deleted_messages) + AND NOT ( + report_id IN (SELECT referenced_report_id FROM acknowledged_deleted_messages) + OR ( + report_id IN (SELECT referenced_report_id FROM deleted_messages) + AND r.flag_state NOT IN ('FRA', 'GUF', 'VEN') + ) + ) AND ( - transmission_format = 'FLUX' + r.flag_state NOT IN ('FRA', 'GUF', 'VEN') -- Flag states for which we receive RET + OR report_id IN (SELECT referenced_report_id FROM acknowledged_messages) OR (value->>'isVerified')::BOOLEAN IS true - OR ( - transmission_format = 'ERS' - AND report_id IN (SELECT referenced_report_id FROM acknowledged_messages) - ) ) AND ( (value->>'isInvalidated') IS NULL