fix: Removed parallel logic that could lead to memory issues (#397)

fix: Removed parallel logic that could lead to memory issues when running on the cloud or locally

ra2ce/network/hazard/hazard_intersect/hazard_intersect_builder_for_gpkg.py

@@ -30,9 +30,6 @@
 from ra2ce.network.hazard.hazard_intersect.hazard_intersect_builder_base import (
     HazardIntersectBuilderBase,
 )
-from ra2ce.network.hazard.hazard_intersect.hazard_intersect_parallel_run import (
-    get_hazard_parallel_process,
-)
 
 
 @dataclass
@@ -90,10 +87,8 @@ def networkx_overlay(hazard_shp_file: Path, race_name: str):
                         race_name + "_" + self.hazard_aggregate_wl[:2]
                     ] = 0
 
-        # Run in parallel to boost performance.
-        self._overlay_in_parallel(networkx_overlay)
-        # for i, _ra2ce_name in self.ra2ce_names:
-        #     networkx_overlay(self.hazard_shp_files[i], _ra2ce_name)
+        self._overlay_hazard_files(networkx_overlay)
+
         return hazard_overlay
 
     def _from_geodataframe(self, hazard_overlay: GeoDataFrame) -> GeoDataFrame:
@@ -127,22 +122,13 @@ def geodataframe_overlay(hazard_shp_file: Path, ra2ce_name: str):
                 inplace=True,
             )
 
-        # Run in parallel to boost performance.
-        self._overlay_in_parallel(geodataframe_overlay)
-        # for i, _ra2ce_name in self.ra2ce_names:
-        #     geodataframe_overlay(self.hazard_shp_files[i], _ra2ce_name)
+        self._overlay_hazard_files(geodataframe_overlay)
 
         if hazard_overlay.crs != gdf_crs_original:
             hazard_overlay = hazard_overlay.to_crs(gdf_crs_original)
 
         return hazard_overlay
 
-    def _overlay_in_parallel(self, overlay_func: Callable):
-        # Run in parallel to boost performance.
-        get_hazard_parallel_process(
-            overlay_func,
-            lambda delayed_func: (
-                delayed_func(self.hazard_gpkg_files[i], _ra2ce_name)
-                for i, _ra2ce_name in enumerate(self.ra2ce_names)
-            ),
-        )
+    def _overlay_hazard_files(self, overlay_func: Callable[[str, str], None]):
+        for i, _ra2ce_name in enumerate(self.ra2ce_names):
+            overlay_func(self.hazard_gpkg_files[i], _ra2ce_name)

ra2ce/network/hazard/hazard_intersect/hazard_intersect_builder_for_tif.py

@@ -37,9 +37,7 @@
 from ra2ce.network.hazard.hazard_intersect.hazard_intersect_builder_base import (
     HazardIntersectBuilderBase,
 )
-from ra2ce.network.hazard.hazard_intersect.hazard_intersect_parallel_run import (
-    get_hazard_parallel_process,
-)
+
 from ra2ce.network.networks_utils import (
     fraction_flooded,
     get_graph_edges_extent,
@@ -162,8 +160,7 @@ def overlay_network_x(hazard_tif_file: Path, hazard_name: str, ra2ce_name: str):
                 },
             )
 
-        # Run in parallel to boost performance.
-        self._overlay_in_parallel(overlay_network_x)
+        self._overlay_hazard_files(overlay_network_x)
         return hazard_overlay
 
     def _from_geodataframe(self, hazard_overlay: GeoDataFrame):
@@ -231,16 +228,9 @@ def _get_attributes(gen_flood_stat: list[dict]) -> tuple:
                 lambda x, _hz_str=_hazard_files_str: fraction_flooded(x, _hz_str)
             )
 
-        # Run in parallel to boost performance.
-        self._overlay_in_parallel(overlay_geodataframe)
+        self._overlay_hazard_files(overlay_geodataframe)
         return hazard_overlay
 
-    def _overlay_in_parallel(self, overlay_func: Callable):
-        # Run in parallel to boost performance.
-        get_hazard_parallel_process(
-            overlay_func,
-            lambda delayed_func: (
-                delayed_func(self.hazard_tif_files[i], hn, rn)
-                for i, (hn, rn) in enumerate(self._combined_names)
-            ),
-        )
+    def _overlay_hazard_files(self, overlay_func: Callable[[str, str, str], None]):
+        for i, (hn, rn) in enumerate(self._combined_names):
+            overlay_func(self.hazard_tif_files[i], hn, rn)