Add proxy config to prefect flows

datascience/config.py

@@ -2,7 +2,17 @@
 
 ROOT_DIRECTORY = Path(__file__).parent
 LIBRARY_LOCATION = ROOT_DIRECTORY / "src"
+
+#
 ERS_FILES_LOCATION = Path("/opt2/monitorfish-data/ers")
+
+# Proxies for pipeline flows requiring Internet access
+PROXIES = {
+    "http": "http://172.27.229.197:8090",
+    "https": "http://172.27.229.197:8090",
+}
+
+# URLs to fetch data from
 PORTS_URL = (
     "https://www.data.gouv.fr/fr/datasets/r/60fe965d-5888-493b-9321-24bc3b1f84db"
 )

datascience/src/pipeline/flows/ports.py

@@ -9,7 +9,7 @@
 from dotenv import load_dotenv
 from prefect import Flow, Parameter, task
 
-from config import LIBRARY_LOCATION, PORTS_URL
+from config import LIBRARY_LOCATION, PORTS_URL, PROXIES
 from src.db_config import create_engine
 from src.pipeline.processing import combine_overlapping_columns
 from src.pipeline.utils import delete
@@ -411,8 +411,8 @@ def load_ports(ports):
 
 
 @task(checkpoint=False)
-def extract_datagouv_ports(ports_url: str = PORTS_URL):
-    r = requests.get(ports_url)
+def extract_datagouv_ports(ports_url: str = PORTS_URL, proxies: dict = None):
+    r = requests.get(ports_url, proxies=proxies)
     f = io.StringIO(r.text)
 
     dtype = {
@@ -467,5 +467,5 @@ def load_ports_to_monitorfish(ports):
 
 
 with Flow("Extract ports from data.gouv.fr and load to Monitorfish") as flow:
-    ports = extract_datagouv_ports()
+    ports = extract_datagouv_ports(ports_url=PORTS_URL, proxies=PROXIES)
     load_ports_to_monitorfish(ports)