review: keywords category (industrie/general) (#231)

dataforgoodfr · Sep 6, 2024 · 3341d0a · 3341d0a · github-actions · Sep 6, 2024
1 parent f1ebcee
commit 3341d0a
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 19 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 secrets/pwd_api.txt
 secrets/username_api.txt
+documents-experts/
 cc-bio.json
 *.xlsx
 coverage_re

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,6 +30,7 @@ tenacity = "^8.2.3"
 sentry-sdk = "^2.13.0"
 modin = {extras = ["ray"], version = "^0.31.0"}
 numpy = "1.26.4"
+openpyxl = "^3.1.5"
 [build-system]
 requires = ["poetry-core>=1.1"]
 build-backend = "poetry.core.masonry.api"

diff --git a/quotaclimat/data_processing/mediatree/api_import.py b/quotaclimat/data_processing/mediatree/api_import.py
@@ -230,7 +230,7 @@ def parse_reponse_subtitle(response_sub, channel = None, channel_program = "", c
         logging.getLogger("modin.logging.default").setLevel(logging.WARNING)
         if(total_results > 0):
             logging.info(f"{total_results} 'total_results' field")
-            new_df : pd.DataFrame = json_normalize(response_sub.get('data'))
+            new_df : pd.DataFrame = json_normalize(response_sub.get('data')) # TODO UserWarning: json_normalize is not currently supported by PandasOnRay, defaulting to pandas implementation.
             logging.debug("Schema from API before formatting :\n%s", new_df.dtypes)
             pd.set_option('display.max_columns', None)
             logging.debug("head:  :\n%s", new_df.head())

diff --git a/quotaclimat/data_processing/mediatree/keyword/keyword.py b/quotaclimat/data_processing/mediatree/keyword/keyword.py
@@ -1048,19 +1048,19 @@
         },
         {
             "keyword": "polluer",
-            "category": "general"
+            "category": "General"
         },
         {
             "keyword": "pollution",
-            "category": "general"
+            "category": "General"
         },
         {
             "keyword": "pollution de l’air",
             "category": "General"
         },
         {
             "keyword": "rejets industriels",
-            "category": "industrie"
+            "category": "Industrie"
         },
         {
             "keyword": "site minier",
@@ -2986,7 +2986,7 @@
         },
         {
             "keyword": "relocaliser la production",
-            "category": "industrie"
+            "category": "Industrie"
         },
         {
             "keyword": "rénovation",
@@ -3016,10 +3016,6 @@
             "keyword": "restreindre",
             "category": "General"
         },
-        {
-            "keyword": "restreindre",
-            "category": "general"
-        },
         {
             "keyword": "restriction",
             "category": "General"
@@ -3416,7 +3412,7 @@
         },
         {
             "keyword": "usine",
-            "category": "industrie"
+            "category": "Industrie"
         },
         {
             "keyword": "vache",
@@ -4652,10 +4648,6 @@
             "keyword": "norme",
             "category": "General"
         },
-        {
-            "keyword": "norme",
-            "category": "General"
-        },
         {
             "keyword": "parc",
             "category": "Ecosystème"
@@ -4720,10 +4712,6 @@
             "keyword": "restreindre",
             "category": "General"
         },
-        {
-            "keyword": "restreindre",
-            "category": "general"
-        },
         {
             "keyword": "restriction",
             "category": "General"
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	43	7	84%	36–38, 56–58, 63
insert_existing_data_example.py	19	3	84%	25–27
postgres/schemas
models.py	147	10	93%	121–128, 140–141, 199–200, 214–215
quotaclimat/data_ingestion
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	55	37	33%	21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	211	132	37%	44–48, 53–69, 73–76, 82, 85–126, 132–147, 151–152, 165–177, 181–187, 200–212, 215–219, 225, 261–262, 265–301, 304–306
channel_program.py	157	56	64%	28–30, 41–43, 60–61, 64–66, 93, 105, 114, 154–195
config.py	15	2	87%	7, 16
detect_keywords.py	214	12	94%	169–172, 216, 266–273
update_pg_keywords.py	54	39	28%	14–100, 125–129, 152–178, 184
utils.py	69	22	68%	27–51, 54, 63, 84–85
quotaclimat/utils
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	24	11	54%	22–24, 28–37
sentry.py	11	2	82%	22–23
TOTAL	1244	367	70%