Skip to content

Commit

Permalink
added columns for counters without hrfp
Browse files Browse the repository at this point in the history
  • Loading branch information
@RDiPiazza
RDiPiazza committed Nov 24, 2024
1 parent af0ae0d commit 81f14ac
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 0 deletions.
11 changes: 11 additions & 0 deletions postgres/schemas/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,17 @@ class Keywords(Base):
number_of_keywords_climat = Column(Integer) # sum of all climatique counters without duplicate (like number_of_keywords)
number_of_keywords_biodiversite = Column(Integer) # sum of all biodiversite counters without duplicate
number_of_keywords_ressources = Column(Integer) # sum of all ressources counters without duplicate
number_of_changement_climatique_constat_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_changement_climatique_constat integer;
number_of_changement_climatique_causes_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_changement_climatique_causes_directes integer;
number_of_changement_climatique_consequences_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_changement_climatique_consequences integer;
number_of_attenuation_climatique_solutions_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_attenuation_climatique_solutions_directes integer;
number_of_adaptation_climatique_solutions_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_adaptation_climatique_solutions_directes integer;
number_of_ressources_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_ressources_naturelles_concepts_generaux integer;
number_of_ressources_solutions_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_ressources_solutions integer;
number_of_biodiversite_concepts_generaux_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_concepts_generaux integer;
number_of_biodiversite_causes_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_causes_directes integer;
number_of_biodiversite_consequences_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_consequences integer;
number_of_biodiversite_solutions_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_solutions_directes integer;

class Channel_Metadata(Base):
__tablename__ = channel_metadata_table
Expand Down
34 changes: 34 additions & 0 deletions quotaclimat/data_processing/mediatree/detect_keywords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,18 @@ def get_themes_keywords_duration(plaintext: str, subtitle_duration: List[str], s
number_of_biodiversite_consequences = count_keywords_duration_overlap(filtered_keywords_with_timestamp, start,theme=["biodiversite_consequences"])
number_of_biodiversite_solutions = count_keywords_duration_overlap(filtered_keywords_with_timestamp, start,theme=["biodiversite_solutions"])

number_of_changement_climatique_constat_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["changement_climatique_constat"])
number_of_changement_climatique_causes_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["changement_climatique_causes"])
number_of_changement_climatique_consequences_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["changement_climatique_consequences"])
number_of_attenuation_climatique_solutions_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["attenuation_climatique_solutions"])
number_of_adaptation_climatique_solutions_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["adaptation_climatique_solutions"])
number_of_ressources_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["ressources"])
number_of_ressources_solutions_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["ressources_solutions"])
number_of_biodiversite_concepts_generaux_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["biodiversite_concepts_generaux"])
number_of_biodiversite_causes_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["biodiversite_causes"])
number_of_biodiversite_consequences_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["biodiversite_consequences"])
number_of_biodiversite_solutions_no_hrfp = count_keywords_duration_overlap(keywords_with_timestamp, start,theme=["biodiversite_solutions"])

return [
theme
,keywords_with_timestamp
Expand All @@ -211,6 +223,17 @@ def get_themes_keywords_duration(plaintext: str, subtitle_duration: List[str], s
,number_of_keywords_climat
,number_of_keywords_biodiversite
,number_of_keywords_ressources
,number_of_changement_climatique_constat_no_hrfp
,number_of_changement_climatique_causes_no_hrfp
,number_of_changement_climatique_consequences_no_hrfp
,number_of_attenuation_climatique_solutions_no_hrfp
,number_of_adaptation_climatique_solutions_no_hrfp
,number_of_ressources_no_hrfp
,number_of_ressources_solutions_no_hrfp
,number_of_biodiversite_concepts_generaux_no_hrfp
,number_of_biodiversite_causes_no_hrfp
,number_of_biodiversite_consequences_no_hrfp
,number_of_biodiversite_solutions_no_hrfp
]
else:
return [None] * number_of_elements_in_array
Expand Down Expand Up @@ -274,6 +297,17 @@ def filter_and_tag_by_theme(df: pd.DataFrame) -> pd.DataFrame :
,"number_of_keywords_climat"
,"number_of_keywords_biodiversite"
,"number_of_keywords_ressources"
,"number_of_changement_climatique_constat_no_hrfp"
,"number_of_changement_climatique_causes_no_hrfp"
,"number_of_changement_climatique_consequences_no_hrfp"
,"number_of_attenuation_climatique_solutions_no_hrfp"
,"number_of_adaptation_climatique_solutions_no_hrfp"
,"number_of_ressources_no_hrfp"
,"number_of_ressources_solutions_no_hrfp"
,"number_of_biodiversite_concepts_generaux_no_hrfp"
,"number_of_biodiversite_causes_no_hrfp"
,"number_of_biodiversite_consequences_no_hrfp"
,"number_of_biodiversite_solutions_no_hrfp"
]
] = df[['plaintext','srt', 'start']]\
.swifter.apply(\
Expand Down
11 changes: 11 additions & 0 deletions quotaclimat/data_processing/mediatree/update_pg_keywords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,17 @@ def update_keywords(session: Session, batch_size: int = 50000, start_date : str
,number_of_biodiversite_causes_directes
,number_of_biodiversite_consequences
,number_of_biodiversite_solutions_directes
,number_of_changement_climatique_constat_no_hrfp
,number_of_changement_climatique_causes_directes_no_hrfp
,number_of_changement_climatique_consequences_no_hrfp
,number_of_attenuation_climatique_solutions_directes_no_hrfp
,number_of_adaptation_climatique_solutions_directes_no_hrfp
,number_of_ressources_no_hrfp
,number_of_ressources_solutions_no_hrfp
,number_of_biodiversite_concepts_generaux_no_hrfp
,number_of_biodiversite_causes_directes_no_hrfp
,number_of_biodiversite_consequences_no_hrfp
,number_of_biodiversite_solutions_directes_no_hrfp
,channel_title=channel_title
,number_of_keywords_climat=new_number_of_keywords_climat
,number_of_keywords_biodiversite=new_number_of_keywords_biodiversite
Expand Down

0 comments on commit 81f14ac

Please sign in to comment.