refacto: dataframe loop with itertuples to spot memory leak (#238)

dataforgoodfr · Sep 13, 2024 · 39ab7ec · 39ab7ec · github-actions · Sep 13, 2024
1 parent 4e8d60d
commit 39ab7ec
Showing 1 changed file with 6 additions and 5 deletions.
diff --git a/quotaclimat/data_processing/mediatree/api_import.py b/quotaclimat/data_processing/mediatree/api_import.py
@@ -103,16 +103,17 @@ async def get_and_save_api_data(exit_event):
                     try:
                         programs_for_this_day = get_programs_for_this_day(day, channel, df_programs)
 
-                        for index, program in programs_for_this_day.iterrows():
-                            start_epoch = program['start']
-                            end_epoch = program['end']
-                            channel_program = str(program['program_name'])
-                            channel_program_type = str(program['program_type'])
+                        for program in programs_for_this_day.itertuples(index=False):
+                            start_epoch = program.start
+                            end_epoch = program.end
+                            channel_program = str(program.program_name)
+                            channel_program_type = str(program.program_type)
                             logging.info(f"Querying API for {channel} - {channel_program} - {channel_program_type} - {start_epoch} - {end_epoch}")
                             df = extract_api_sub(token, channel, type_sub, start_epoch,end_epoch, channel_program,channel_program_type) 
                             if(df is not None):
                                 logging.debug(f"Memory df {df.memory_usage()}")
                                 save_to_pg(df, keywords_table, conn)
+                                del df
                             else:
                                 logging.info("Nothing to save to Postgresql")
                         gc.collect()