diff --git a/backend/src/main/resources/db/migration/internal/V0.279__Update_facade_areas_subdivided_table.sql b/backend/src/main/resources/db/migration/internal/V0.279__Update_facade_areas_subdivided_table.sql new file mode 100644 index 0000000000..76ee4a25d2 --- /dev/null +++ b/backend/src/main/resources/db/migration/internal/V0.279__Update_facade_areas_subdivided_table.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.facade_areas_subdivided + ADD COLUMN email_address text; \ No newline at end of file diff --git a/datascience/src/pipeline/entities/pnos.py b/datascience/src/pipeline/entities/pnos.py index 7a3c75df6b..ee4d66a9ed 100644 --- a/datascience/src/pipeline/entities/pnos.py +++ b/datascience/src/pipeline/entities/pnos.py @@ -46,6 +46,7 @@ class PnoToRender: catch_onboard: List[dict] port_locode: str port_name: str + facade: str predicted_arrival_datetime_utc: datetime predicted_landing_datetime_utc: datetime trip_gears: List[dict] @@ -113,6 +114,7 @@ class PreRenderedPno: catch_onboard: pd.DataFrame port_locode: str port_name: str + facade: str predicted_arrival_datetime_utc: datetime predicted_landing_datetime_utc: datetime trip_gears: List[FishingGear] @@ -204,6 +206,7 @@ class RenderedPno: is_being_sent: bool trip_segments: list port_locode: str + facade: str source: PnoSource html_for_pdf: Optional[str] = None pdf_document: Optional[bytes] = None diff --git a/datascience/src/pipeline/flows/distribute_pnos.py b/datascience/src/pipeline/flows/distribute_pnos.py index 7574c13f41..6cdea2752f 100644 --- a/datascience/src/pipeline/flows/distribute_pnos.py +++ b/datascience/src/pipeline/flows/distribute_pnos.py @@ -112,6 +112,15 @@ def extract_pnos_to_generate( return (pnos, generation_needed) +@task(checkpoint=False) +def extract_facade_email_addresses() -> dict: + df = extract( + db_name="monitorfish_remote", + query_filepath="monitorfish/facade_email_addresses.sql", + ) + return df.set_index("facade").email_address.to_dict() + + @task(checkpoint=False) def to_pnos_to_render(pnos: pd.DataFrame) -> List[PnoToRender]: records = pnos.to_dict(orient="records") @@ -247,6 +256,7 @@ def format_sr_list(sr_list: list) -> str: catch_onboard=sum_by_species, port_locode=pno.port_locode, port_name=pno.port_name, + facade=pno.facade, predicted_arrival_datetime_utc=pno.predicted_arrival_datetime_utc, predicted_landing_datetime_utc=pno.predicted_landing_datetime_utc, trip_gears=trip_gears, @@ -474,6 +484,7 @@ def format_nullable_datetime(d: datetime, format: str = date_format): is_being_sent=pno.is_being_sent, trip_segments=pno.trip_segments, port_locode=pno.port_locode, + facade=pno.facade, source=pno.source, html_for_pdf=html_for_pdf, pdf_document=pdf, @@ -536,6 +547,7 @@ def attribute_addressees( units_targeting_vessels: pd.DataFrame, units_ports_and_segments_subscriptions: pd.DataFrame, control_units_contacts: pd.DataFrame, + facade_email_addresses: dict, ) -> RenderedPno: """ Returns a copy of the input `RenderedPno`'s with its `control_unit_ids`, @@ -560,6 +572,9 @@ def attribute_addressees( `unit_subscribed_segments` control_units_contacts (pd.DataFrame): DataFrame with columns `control_unit_id`, `emails` and `phone_numbers` + facade_email_addresses (dict): dictionnary with facade names as keys and FMC + facade email addresses as values. The email address of the corresponding + facade will be added to addressees in PNO emails. Returns: RenderedPno: copy of the input `pno_to_distribute` with addressees added @@ -622,6 +637,10 @@ def attribute_addressees( ) ) ) + if emails: + facade_email_address = facade_email_addresses.get(pno_to_distribute.facade) + if facade_email_address: + emails.append(facade_email_address) phone_numbers = sorted( set( @@ -995,11 +1014,14 @@ def make_manual_prior_notifications_statement( with case(distribution_needed, True): # Distribute PNOs + facade_email_addresses = extract_facade_email_addresses() + pnos_with_addressees = attribute_addressees.map( pnos_to_distribute, unmapped(units_targeting_vessels), unmapped(units_ports_and_segments_subscriptions), control_units_contacts=unmapped(control_units_contacts), + facade_email_addresses=unmapped(facade_email_addresses), ) email = create_email.map( diff --git a/datascience/src/pipeline/queries/cross/facade_areas.sql b/datascience/src/pipeline/queries/cross/facade_areas.sql index 7bc10a3e87..0320092838 100644 --- a/datascience/src/pipeline/queries/cross/facade_areas.sql +++ b/datascience/src/pipeline/queries/cross/facade_areas.sql @@ -1,4 +1,5 @@ SELECT facade_cnsp AS facade, + email_address, ST_SubDivide(geometry) AS geometry FROM prod.facade_areas \ No newline at end of file diff --git a/datascience/src/pipeline/queries/cross/local_execution/expand_facade_geometries.sql b/datascience/src/pipeline/queries/cross/local_execution/expand_facade_geometries.sql index fd30c08678..1cb837b0a9 100644 --- a/datascience/src/pipeline/queries/cross/local_execution/expand_facade_geometries.sql +++ b/datascience/src/pipeline/queries/cross/local_execution/expand_facade_geometries.sql @@ -21,6 +21,7 @@ facades_polygons_without_holes AS ( SELECT id, prod.facade_areas_unextended.facade, + prod.facade_areas_unextended.email_address, ST_Difference(ST_MakePolygon(ST_ExteriorRing((ST_Dump(geom)).geom)), other_facades_geom) AS geom FROM prod.facade_areas_unextended JOIN other_unexpanded_facades @@ -32,9 +33,10 @@ facades_multipolygons AS ( SELECT id, facade, + email_address, ST_Multi(ST_Union(geom)) AS geom FROM facades_polygons_without_holes - GROUP BY id, facade + GROUP BY id, facade, email_address ), all_facades AS ( @@ -45,6 +47,7 @@ all_facades AS ( expanded_facades AS ( SELECT facade, + email_address, ST_Difference( ST_Buffer(geom, 0.1), ST_Difference( @@ -69,6 +72,7 @@ expanded_facades_without_overlap AS ( SELECT ef.facade AS facade_cnsp, CASE WHEN ef.facade = 'Corse' THEN 'MED' ELSE ef.facade END AS facade_cacem, + email_address, ST_Difference( ef.expanded_geometry, other_expanded_facades_geom diff --git a/datascience/src/pipeline/queries/monitorfish/facade_email_addresses.sql b/datascience/src/pipeline/queries/monitorfish/facade_email_addresses.sql new file mode 100644 index 0000000000..fdbc9c86ce --- /dev/null +++ b/datascience/src/pipeline/queries/monitorfish/facade_email_addresses.sql @@ -0,0 +1,5 @@ +SELECT DISTINCT ON (facade) + facade, + email_address +FROM facade_areas_subdivided +ORDER BY facade \ No newline at end of file diff --git a/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql b/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql index cdd1e0e863..d921a46a27 100644 --- a/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql +++ b/datascience/src/pipeline/queries/monitorfish/pnos_to_generate.sql @@ -41,6 +41,7 @@ acknowledged_deleted_messages AS ( r.value->'catchOnboard' AS catch_onboard, r.value->>'port' AS port_locode, p.port_name, + p.facade, (r.value->>'predictedArrivalDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' AS predicted_arrival_datetime_utc, (r.value->>'predictedLandingDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' AS predicted_landing_datetime_utc, r.trip_gears, @@ -109,6 +110,7 @@ UNION ALL r.value->'catchOnboard' AS catch_onboard, r.value->>'port' AS port_locode, p.port_name, + p.facade, (r.value->>'predictedArrivalDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' AS predicted_arrival_datetime_utc, (r.value->>'predictedLandingDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC' AS predicted_landing_datetime_utc, (CASE WHEN jsonb_typeof(r.trip_gears) = 'array' THEN r.trip_gears ELSE '[]'::jsonb END) AS trip_gears, diff --git a/datascience/tests/test_data/remote_database/V666.29__Reset_test_ports.sql b/datascience/tests/test_data/remote_database/V666.29__Reset_test_ports.sql index 140869bee5..903354abba 100644 --- a/datascience/tests/test_data/remote_database/V666.29__Reset_test_ports.sql +++ b/datascience/tests/test_data/remote_database/V666.29__Reset_test_ports.sql @@ -2,13 +2,13 @@ DELETE FROM public.ports; INSERT INTO public.ports ( country_code_iso2, region, locode, port_name, is_active, facade) VALUES - ( 'FR', '56', 'FAKE', 'Fake Port initially active', true, NULL), + ( 'FR', '56', 'FAKE', 'Fake Port initially active', true, 'NAMO'), ( 'FR', '56', 'PNO_PORT', 'Fake Port with PNO', false, NULL), - ( 'FR', '56', 'LAN_PORT', 'Fake Port with LAN', false, NULL), - ( 'FR', '999', 'FRCQF', 'Somewhere over the rainbow', false, NULL), - ( 'FR', '999', 'FRBES', 'Somewhere over the hill', false, NULL), + ( 'FR', '56', 'LAN_PORT', 'Fake Port with LAN', false, 'NAMO'), + ( 'FR', '999', 'FRCQF', 'Somewhere over the rainbow', false, 'NAMO'), + ( 'FR', '999', 'FRBES', 'Somewhere over the hill', false, 'SA'), ( 'FR', '999', 'FRDPE', 'Somewhere over the clouds', false, NULL), - ( 'FR', '999', 'FRDKK', 'Somewhere over the swell', false, NULL), - ( 'FR', '999', 'FRLEH', 'Somewhere over the ocean', false, NULL), - ( 'FR', '999', 'FRZJZ', 'Somewhere over the top', false, NULL), + ( 'FR', '999', 'FRDKK', 'Somewhere over the swell', false, 'NAMO'), + ( 'FR', '999', 'FRLEH', 'Somewhere over the ocean', false, 'SA'), + ( 'FR', '999', 'FRZJZ', 'Somewhere over the top', false, 'SA'), ( 'FR', '999', 'GBPHD', 'Port with facade', false, 'MEMN'); \ No newline at end of file diff --git a/datascience/tests/test_data/remote_database/V666.9__Reset_test_facade_areas_subdivided.sql b/datascience/tests/test_data/remote_database/V666.9__Reset_test_facade_areas_subdivided.sql index 42dd6b6b7f..91f4710846 100644 --- a/datascience/tests/test_data/remote_database/V666.9__Reset_test_facade_areas_subdivided.sql +++ b/datascience/tests/test_data/remote_database/V666.9__Reset_test_facade_areas_subdivided.sql @@ -1,7 +1,7 @@ DELETE FROM facade_areas_subdivided; INSERT INTO facade_areas_subdivided ( - facade, geometry + facade, email_address, geometry ) VALUES -('NAMO', ST_Polygon('LINESTRING(10.0 45.0, -10.0 45.0, -10.0 0.0, 10.0 0.0, 10.0 45.0)'::geometry, 4326)), -('SA', ST_Polygon('LINESTRING(10.0 45.0, -10.0 45.0, -10.0 50.0, 10.0 50.0, 10.0 45.0)'::geometry, 4326)); +( 'NAMO', 'namo@email', ST_Polygon('LINESTRING(10.0 45.0, -10.0 45.0, -10.0 0.0, 10.0 0.0, 10.0 45.0)'::geometry, 4326)), +( 'SA', 'sa@email', ST_Polygon('LINESTRING(10.0 45.0, -10.0 45.0, -10.0 50.0, 10.0 50.0, 10.0 45.0)'::geometry, 4326)); diff --git a/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py b/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py index 2a3b27cb61..b330e80d93 100644 --- a/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py +++ b/datascience/tests/test_pipeline/test_flows/test_distribute_pnos.py @@ -32,6 +32,7 @@ attribute_addressees, create_email, create_sms, + extract_facade_email_addresses, extract_fishing_gear_names, extract_pnos_to_generate, extract_species_names, @@ -415,6 +416,7 @@ def extracted_pnos() -> pd.DataFrame: "Somewhere over the swell", "Somewhere over the ocean", ], + "facade": ["NAMO", "SA", "NAMO", "SA", None, "SA", None, "NAMO", "SA"], "predicted_arrival_datetime_utc": [ datetime(2020, 5, 6, 11, 41, 3, 340000), datetime(2020, 5, 6, 11, 41, 3, 340000), @@ -685,6 +687,7 @@ def pno_to_render_1() -> PnoToRender: ], port_locode="FRCQF", port_name="Somewhere over the rainbow", + facade="NAMO", predicted_arrival_datetime_utc=datetime(2020, 5, 6, 11, 41, 3, 340000), predicted_landing_datetime_utc=datetime(2020, 5, 6, 16, 40), trip_gears=[ @@ -778,6 +781,7 @@ def pre_rendered_pno_1(pre_rendered_pno_1_catch_onboard) -> PreRenderedPno: catch_onboard=pre_rendered_pno_1_catch_onboard, port_locode="FRCQF", port_name="Somewhere over the rainbow", + facade="NAMO", predicted_arrival_datetime_utc=datetime(2020, 5, 6, 11, 41, 3, 340000), predicted_landing_datetime_utc=datetime(2020, 5, 6, 16, 40), trip_gears=[ @@ -832,6 +836,7 @@ def pno_to_render_2() -> PnoToRender: catch_onboard=None, port_locode="FRZJZ", port_name="Somewhere over the top", + facade="SA", predicted_arrival_datetime_utc=datetime(2020, 5, 6, 11, 41, 3, 340000), predicted_landing_datetime_utc=pd.NaT, trip_gears=[], @@ -869,6 +874,7 @@ def pre_rendered_pno_2() -> PreRenderedPno: catch_onboard=None, port_locode="FRZJZ", port_name="Somewhere over the top", + facade="SA", predicted_arrival_datetime_utc=datetime(2020, 5, 6, 11, 41, 3, 340000), predicted_landing_datetime_utc=None, trip_gears=[], @@ -903,6 +909,7 @@ def pno_pdf_document_to_distribute_targeted_vessel_and_segments() -> RenderedPno FleetSegment("NWW02", "Autres"), ], port_locode="FRZJZ", + facade="NAMO", source=PnoSource.MANUAL, generation_datetime_utc=datetime(2023, 5, 6, 23, 52, 0), pdf_document=b"PDF Document", @@ -920,7 +927,7 @@ def pno_pdf_document_to_distribute_targeted_vessel_and_segments_assigned( pno_pdf_document_to_distribute_targeted_vessel_and_segments, control_unit_ids={1, 2, 3}, phone_numbers=["'00 11 22 33 44 55", "44 44 44 44 44"], - emails=["alternative@email", "some.email@control.unit.4"], + emails=["alternative@email", "some.email@control.unit.4", "namo@email"], ) @@ -935,6 +942,7 @@ def pno_pdf_document_to_distribute_receive_all_pnos_from_port() -> RenderedPno: is_being_sent=True, trip_segments=[], port_locode="FRDPE", + facade="Unknown facade", source=PnoSource.MANUAL, generation_datetime_utc=datetime(2023, 6, 6, 23, 50, 0), pdf_document=b"PDF Document", @@ -965,6 +973,7 @@ def pno_pdf_document_to_distribute_without_addressees() -> RenderedPno: is_being_sent=True, trip_segments=[], port_locode="FRDKK", + facade="SA", source=PnoSource.MANUAL, generation_datetime_utc=datetime(2023, 6, 6, 23, 50, 0), pdf_document=b"PDF Document", @@ -995,6 +1004,7 @@ def pno_pdf_document_to_distribute_verified() -> RenderedPno: is_being_sent=True, trip_segments=[], port_locode="FRLEH", + facade="NAMO", source=PnoSource.MANUAL, generation_datetime_utc=datetime(2023, 6, 6, 23, 50, 0), pdf_document=b"PDF Document", @@ -1009,7 +1019,7 @@ def pno_pdf_document_to_distribute_verified_assigned( return dataclasses.replace( pno_pdf_document_to_distribute_verified, control_unit_ids={2, 3}, - emails=["alternative@email", "some.email@control.unit.4"], + emails=["alternative@email", "some.email@control.unit.4", "namo@email"], phone_numbers=["'00 11 22 33 44 55", "44 44 44 44 44"], ) @@ -1055,6 +1065,7 @@ def logbook_rendered_pno(): is_being_sent=True, trip_segments=["Segment1", "Segment2"], port_locode="FRBOL", + facade="NAMO", source=PnoSource.LOGBOOK, html_for_pdf="Html for PDF", pdf_document=b"PDF document", @@ -1078,6 +1089,7 @@ def manual_rendered_pno(): is_being_sent=True, trip_segments=["Segment3"], port_locode="FRBOL", + facade="SA", source=PnoSource.MANUAL, html_for_pdf="Html for PDF manual", pdf_document=b"PDF document manual", @@ -1222,6 +1234,11 @@ def pnos_to_reset() -> List[RenderedPno]: return [RenderedPno()] +@pytest.fixture +def facade_email_addresses() -> dict: + return {"NAMO": "namo@email", "SA": "sa@email"} + + def test_extract_pnos_to_generate(reset_test_data, extracted_pnos): approximate_datetime_columns = [ "operation_datetime_utc", @@ -1256,6 +1273,11 @@ def test_extract_fishing_gear_names(reset_test_data, fishing_gear_names): assert res == fishing_gear_names +def test_extract_facade_email_addresses(reset_test_data, facade_email_addresses): + res = extract_facade_email_addresses.run() + assert res == facade_email_addresses + + def test_to_pnos_to_render(extracted_pnos): res = to_pnos_to_render.run(pnos=extracted_pnos) assert len(res) == 9 @@ -1481,6 +1503,7 @@ def test_load_pno_pdf_documents(reset_test_data): is_being_sent=is_being_sent, trip_segments=[], port_locode="FRBOL", + facade="NAMO", source=PnoSource.LOGBOOK, generation_datetime_utc=datetime(2020, 5, 6, 8, 52, 42), pdf_document=pdf, @@ -1557,12 +1580,14 @@ def test_attribute_addressees_uses_target_vessels_and_segments( pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ): res = attribute_addressees.run( pno_pdf_document_to_distribute_targeted_vessel_and_segments, pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ) assert res == pno_pdf_document_to_distribute_targeted_vessel_and_segments_assigned @@ -1573,12 +1598,14 @@ def test_attribute_addressees_uses_receive_all_pnos_from_port( pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ): res = attribute_addressees.run( pno_pdf_document_to_distribute_receive_all_pnos_from_port, pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ) assert res == pno_pdf_document_to_distribute_receive_all_pnos_from_port_assigned @@ -1589,12 +1616,14 @@ def test_attribute_addressees_returns_empty_addressees( pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ): res = attribute_addressees.run( pno_pdf_document_to_distribute_without_addressees, pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ) assert res == pno_pdf_document_to_distribute_without_addressees_assigned @@ -1605,12 +1634,14 @@ def test_attribute_addressees_when_is_verified( pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ): res = attribute_addressees.run( pno_pdf_document_to_distribute_verified, pno_units_targeting_vessels, pno_units_ports_and_segments_subscriptions, control_units_contacts, + facade_email_addresses, ) assert res == pno_pdf_document_to_distribute_verified_assigned @@ -1639,7 +1670,8 @@ def test_create_email( assert pno_to_send.message["To"] == "pno.test@email.fr" else: assert ( - pno_to_send.message["To"] == "alternative@email, some.email@control.unit.4" + pno_to_send.message["To"] + == "alternative@email, some.email@control.unit.4, namo@email" ) assert pno_to_send.message["From"] == "monitorfish@test.email" assert pno_to_send.message["Cc"] is None @@ -1972,7 +2004,7 @@ class UnexpectedFailureOfDeath(Exception): assert len(initial_pdf_documents) == 8 assert len(initial_sent_messages) == 0 assert len(final_pdf_documents) == 14 - assert len(final_sent_messages) == 12 + assert len(final_sent_messages) == 15 if is_integration: assert final_sent_messages.success.all() @@ -1995,7 +2027,7 @@ class UnexpectedFailureOfDeath(Exception): == CommunicationMeans.EMAIL.value ] ) - == 7 + == 10 ) assert final_sent_messages.loc[ (