diff --git a/datascience/src/pipeline/flows/distribute_pnos.py b/datascience/src/pipeline/flows/distribute_pnos.py index d52cdb85c0..258534de84 100644 --- a/datascience/src/pipeline/flows/distribute_pnos.py +++ b/datascience/src/pipeline/flows/distribute_pnos.py @@ -701,8 +701,14 @@ def attribute_addressees( @task(checkpoint=False) def create_email(pno: RenderedPno, test_mode: bool) -> PnoToSend: - if pno.emails or (test_mode and PNO_TEST_EMAIL): - to = PNO_TEST_EMAIL if test_mode else pno.emails + if pno.emails: + if test_mode: + if PNO_TEST_EMAIL: + to = PNO_TEST_EMAIL + else: + return None + else: + to = pno.emails message = create_html_email( to=to, @@ -731,8 +737,14 @@ def create_email(pno: RenderedPno, test_mode: bool) -> PnoToSend: @task(checkpoint=False) def create_sms(pno: RenderedPno, test_mode: bool) -> PnoToSend: - if pno.phone_numbers or (test_mode and CNSP_SIP_DEPARTMENT_MOBILE_PHONE): - to = CNSP_SIP_DEPARTMENT_MOBILE_PHONE if test_mode else pno.phone_numbers + if pno.phone_numbers: + if test_mode: + if CNSP_SIP_DEPARTMENT_MOBILE_PHONE: + to = CNSP_SIP_DEPARTMENT_MOBILE_PHONE + else: + return None + else: + to = pno.phone_numbers return PnoToSend( pno=pno, @@ -803,6 +815,7 @@ def load_prior_notification_sent_messages( @task(checkpoint=False) def make_update_logbook_reports_statement( pnos_to_update: List[RenderedPno], + sent_messages: List[PriorNotificationSentMessage], start_datetime_utc: datetime, end_datetime_utc: datetime, ) -> Executable: @@ -812,6 +825,7 @@ def make_update_logbook_reports_statement( Args: pnos_to_update (List[RenderedPno]): PNOs to update + sent_messages (List[PriorNotificationSentMessage]): PNOs that were sent start_datetime_utc (datetime): start date end_datetime_utc (datetime): end date @@ -826,6 +840,16 @@ def make_update_logbook_reports_statement( pno.report_id for pno in pnos_to_update if pno.source == PnoSource.LOGBOOK ) + sent_pno_report_ids = tuple( + sorted( + set( + m.prior_notification_report_id + for m in sent_messages + if (m.success and m.prior_notification_source == PnoSource.LOGBOOK) + ) + ) + ) + logger = prefect.context.get("logger") if logbook_pno_report_ids: @@ -836,6 +860,29 @@ def make_update_logbook_reports_statement( ) ) + bind_params = [ + bindparam( + "logbook_pno_report_ids", value=logbook_pno_report_ids, expanding=True + ), + ] + + # Empty tuple is not expanded correctly by SQLAlchemy so this case must be + # handled separately + if sent_pno_report_ids: + is_sent_line = ( + " (CASE WHEN report_id IN :sent_pno_report_ids " + "THEN 'true' " + "ELSE 'false' END)::jsonb" + ) + bind_params.append( + bindparam( + "sent_pno_report_ids", value=sent_pno_report_ids, expanding=True + ) + ) + + else: + is_sent_line = " false::text::jsonb" + statement = text( "UPDATE public.logbook_reports " " SET value = jsonb_set(" @@ -845,16 +892,14 @@ def make_update_logbook_reports_statement( " false::text::jsonb" " ), " " '{isSent}', " - " true::text::jsonb" + f"{is_sent_line}" " ) " "WHERE " " operation_datetime_utc >= :start_datetime_utc " " AND operation_datetime_utc < :end_datetime_utc " " AND report_id IN :logbook_pno_report_ids" ).bindparams( - bindparam( - "logbook_pno_report_ids", value=logbook_pno_report_ids, expanding=True - ), + *bind_params, start_datetime_utc=start_datetime_utc, end_datetime_utc=end_datetime_utc, ) @@ -868,6 +913,7 @@ def make_update_logbook_reports_statement( @task(checkpoint=False) def make_manual_prior_notifications_statement( pnos_to_update: List[RenderedPno], + sent_messages: List[PriorNotificationSentMessage], ) -> Executable: """ Creates slqalchemy update statement to update `isBeingSent` and `isSent` fields of @@ -887,6 +933,16 @@ def make_manual_prior_notifications_statement( pno.report_id for pno in pnos_to_update if pno.source == PnoSource.MANUAL ) + sent_pno_report_ids = tuple( + sorted( + set( + m.prior_notification_report_id + for m in sent_messages + if (m.success and m.prior_notification_source == PnoSource.MANUAL) + ) + ) + ) + logger = prefect.context.get("logger") if manual_pno_report_ids: @@ -897,6 +953,29 @@ def make_manual_prior_notifications_statement( ) ) + bind_params = [ + bindparam( + "manual_pno_report_ids", value=manual_pno_report_ids, expanding=True + ), + ] + + # Empty tuple is not expanded correctly by SQLAlchemy so this case must be + # handled separately + if sent_pno_report_ids: + is_sent_line = ( + " (CASE WHEN report_id IN :sent_pno_report_ids " + "THEN 'true' " + "ELSE 'false' END)::jsonb" + ) + bind_params.append( + bindparam( + "sent_pno_report_ids", value=sent_pno_report_ids, expanding=True + ) + ) + + else: + is_sent_line = " false::text::jsonb" + statement = text( "UPDATE public.manual_prior_notifications " " SET value = jsonb_set(" @@ -906,14 +985,10 @@ def make_manual_prior_notifications_statement( " false::text::jsonb" " ), " " '{isSent}', " - " true::text::jsonb" + f"{is_sent_line}" " ) " "WHERE report_id IN :manual_pno_report_ids" - ).bindparams( - bindparam( - "manual_pno_report_ids", value=manual_pno_report_ids, expanding=True - ), - ) + ).bindparams(*bind_params) return statement @@ -1008,6 +1083,7 @@ def make_manual_prior_notifications_statement( update_logbook_reports_statement = ( make_update_logbook_reports_statement( pnos_to_update=pnos_to_distribute, + sent_messages=flatten(sent_messages), start_datetime_utc=start_datetime_utc, end_datetime_utc=end_datetime_utc, upstream_tasks=[loaded_prior_notification_sent_messages], @@ -1017,6 +1093,7 @@ def make_manual_prior_notifications_statement( update_manual_pnos_statement = ( make_manual_prior_notifications_statement( pnos_to_update=pnos_to_distribute, + sent_messages=flatten(sent_messages), upstream_tasks=[loaded_prior_notification_sent_messages], ) ) diff --git a/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py b/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py index ad72bb5540..de07855aaa 100644 --- a/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py +++ b/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py @@ -1424,6 +1424,54 @@ def messages_sent_by_sms() -> List[PriorNotificationSentMessage]: ] +@pytest.fixture +def some_more_sent_messages( + messages_sent_by_email, messages_sent_by_sms +) -> List[PriorNotificationSentMessage]: + return ( + messages_sent_by_email + + messages_sent_by_sms + + [ + PriorNotificationSentMessage( + prior_notification_report_id="Manual-Report-1", + prior_notification_source=PnoSource.MANUAL, + date_time_utc=datetime(2023, 6, 6, 16, 10), + communication_means=CommunicationMeans.SMS, + recipient_address_or_number="00000000000", + success=True, + error_message=None, + ), + PriorNotificationSentMessage( + prior_notification_report_id="Failed-logbook-report-123", + prior_notification_source=PnoSource.LOGBOOK, + date_time_utc=datetime(2023, 6, 6, 16, 10), + communication_means=CommunicationMeans.SMS, + recipient_address_or_number="00000000000", + success=False, + error_message=None, + ), + PriorNotificationSentMessage( + prior_notification_report_id="Other-logbook-report-123", + prior_notification_source=PnoSource.LOGBOOK, + date_time_utc=datetime(2023, 6, 6, 16, 10), + communication_means=CommunicationMeans.SMS, + recipient_address_or_number="00000000000", + success=True, + error_message=None, + ), + PriorNotificationSentMessage( + prior_notification_report_id="Manual-Report-failed", + prior_notification_source=PnoSource.MANUAL, + date_time_utc=datetime(2023, 6, 6, 16, 10), + communication_means=CommunicationMeans.SMS, + recipient_address_or_number="00000000000", + success=False, + error_message=None, + ), + ] + ) + + @pytest.fixture def loaded_sent_messages() -> pd.DataFrame: return pd.DataFrame( @@ -1953,6 +2001,11 @@ def test_create_email_with_no_email_addressees_returns_none( ) assert pno_to_send is None + pno_to_send = create_email.run( + pno_pdf_document_to_distribute_without_addressees_assigned, test_mode=True + ) + assert pno_to_send is None + @pytest.mark.parametrize("test_mode", [False, True]) def test_create_sms( @@ -1991,6 +2044,20 @@ def test_create_sms( assert pno_to_send.message.get_content() == "Message SMS préavis 123-abc\n" +def test_create_sms_with_no_phone_addressees_returns_none( + pno_pdf_document_to_distribute_without_addressees_assigned, +): + pno_to_send = create_sms.run( + pno_pdf_document_to_distribute_without_addressees_assigned, test_mode=False + ) + assert pno_to_send is None + + pno_to_send = create_sms.run( + pno_pdf_document_to_distribute_without_addressees_assigned, test_mode=True + ) + assert pno_to_send is None + + @patch( "src.pipeline.flows.distribute_pnos.datetime", mock_datetime_utcnow(datetime(2023, 6, 6, 16, 10, 0)), @@ -2033,10 +2100,39 @@ def test_load_prior_notification_sent_messages( def test_make_update_logbook_reports_statement( - logbook_rendered_pno, manual_rendered_pno + logbook_rendered_pno, manual_rendered_pno, some_more_sent_messages ): + # Test without sent messages + statement = make_update_logbook_reports_statement.run( + pnos_to_update=[logbook_rendered_pno, manual_rendered_pno], + sent_messages=[], + start_datetime_utc=datetime(2023, 6, 5, 12, 5, 6), + end_datetime_utc=datetime(2023, 7, 5, 15, 2, 38), + ) + assert isinstance(statement, sqlalchemy.TextClause) + compiled_statement = str(statement.compile(compile_kwargs={"literal_binds": True})) + + assert compiled_statement == ( + "UPDATE public.logbook_reports" + " SET value = jsonb_set(" + " jsonb_set(" + " value," + " '{isBeingSent}'," + " false::text::jsonb" + " )," + " '{isSent}'," + " false::text::jsonb" + " ) " + "WHERE" + " operation_datetime_utc >= '2023-06-05 12:05:06'" + " AND operation_datetime_utc < '2023-07-05 15:02:38'" + " AND report_id IN ('Report-1')" + ) + + # Test with sent messages statement = make_update_logbook_reports_statement.run( pnos_to_update=[logbook_rendered_pno, manual_rendered_pno], + sent_messages=some_more_sent_messages, start_datetime_utc=datetime(2023, 6, 5, 12, 5, 6), end_datetime_utc=datetime(2023, 7, 5, 15, 2, 38), ) @@ -2052,7 +2148,8 @@ def test_make_update_logbook_reports_statement( " false::text::jsonb" " )," " '{isSent}'," - " true::text::jsonb" + " (CASE WHEN report_id IN ('Other-logbook-report-123', 'Report-1') " + "THEN 'true' ELSE 'false' END)::jsonb" " ) " "WHERE" " operation_datetime_utc >= '2023-06-05 12:05:06'" @@ -2062,10 +2159,33 @@ def test_make_update_logbook_reports_statement( def test_make_update_manual_prior_notifications_statement( - logbook_rendered_pno, manual_rendered_pno + logbook_rendered_pno, manual_rendered_pno, some_more_sent_messages ): + # Test without sent messages + statement = make_manual_prior_notifications_statement.run( + pnos_to_update=[logbook_rendered_pno, manual_rendered_pno], sent_messages=[] + ) + assert isinstance(statement, sqlalchemy.TextClause) + compiled_statement = str(statement.compile(compile_kwargs={"literal_binds": True})) + + assert compiled_statement == ( + "UPDATE public.manual_prior_notifications" + " SET value = jsonb_set(" + " jsonb_set(" + " value," + " '{isBeingSent}'," + " false::text::jsonb" + " )," + " '{isSent}'," + " false::text::jsonb" + " ) " + "WHERE report_id IN ('Report-2')" + ) + + # Test with sent messages statement = make_manual_prior_notifications_statement.run( pnos_to_update=[logbook_rendered_pno, manual_rendered_pno], + sent_messages=some_more_sent_messages, ) assert isinstance(statement, sqlalchemy.TextClause) compiled_statement = str(statement.compile(compile_kwargs={"literal_binds": True})) @@ -2079,7 +2199,7 @@ def test_make_update_manual_prior_notifications_statement( " false::text::jsonb" " )," " '{isSent}'," - " true::text::jsonb" + " (CASE WHEN report_id IN ('Manual-Report-1') THEN 'true' ELSE 'false' END)::jsonb" " ) " "WHERE report_id IN ('Report-2')" )