feat/multiple-update-from-ci (#262)

dataforgoodfr · Oct 10, 2024 · acada53 · acada53 · github-actions · Oct 10, 2024
1 parent e11ef9a
commit acada53
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 3 deletions.
diff --git a/.github/workflows/scaleway-start-import-job-update.yml b/.github/workflows/scaleway-start-import-job-update.yml
@@ -8,7 +8,27 @@ jobs:
   start-job-image:
     strategy:
       matrix:
-        start_date: ["2023-04-01", "2023-05-01","2023-06-01","2023-07-01"]
+        dates: [ 
+          {start_date: "2023-04-01", end_date: "2023-05-01"}
+          ,{start_date: "2023-05-01", end_date: "2023-06-01"}
+          ,{start_date: "2023-06-01", end_date: "2023-07-01"}
+          ,{start_date: "2023-07-01", end_date: "2023-08-01"}
+          ,{start_date: "2023-08-01", end_date: "2023-09-01"}
+          ,{start_date: "2023-09-01", end_date: "2023-10-01"}
+          ,{start_date: "2023-10-01", end_date: "2023-11-01"}
+          ,{start_date: "2023-11-01", end_date: "2023-12-01"}
+          ,{start_date: "2023-12-01", end_date: "2024-01-01"}
+          ,{start_date: "2024-01-01", end_date: "2024-02-01"}
+          ,{start_date: "2024-02-01", end_date: "2024-03-01"}
+          ,{start_date: "2024-03-01", end_date: "2024-04-01"}
+          ,{start_date: "2024-04-01", end_date: "2024-05-01"}
+          ,{start_date: "2024-05-01", end_date: "2024-06-01"}
+          ,{start_date: "2024-06-01", end_date: "2024-07-01"}
+          ,{start_date: "2024-07-01", end_date: "2024-08-01"}
+          ,{start_date: "2024-08-01", end_date: "2024-09-01"}
+          ,{start_date: "2024-09-01", end_date: "2024-10-01"}
+          ,{start_date: "2024-10-01", end_date: "2024-11-01"}
+        ]
     runs-on: ubuntu-latest
     steps:
     - name: start import job to reapply logic to all elements start_date matrix
@@ -19,4 +39,6 @@ jobs:
         SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
         SCW_ZONE: ${{ secrets.SCW_ZONE }}
       with:
-        args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true environment-variables.START_DATE_UPDATE=${{ matrix.start_date }}
+        args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true \
+         environment-variables.START_DATE_UPDATE=${{ matrix.dates.start_date }} \
+         environment-variables.END_DATE=${{ matrix.dates.end_date }}
diff --git a/README.md b/README.md
@@ -380,6 +380,8 @@ poetry run python3 transform_program.py
 ```
 The SQL queries are based on this file that generate the Program Metadata table.
 
+Program data will not be updated to avoid lock concurrent issues when using `UPDATE=true` for keywords logic. Note: The default case will update them.
+
 **With the docker-entrypoint.sh this command is done automatically, so for production uses, you will not have to run this command.**
 
 ## Production monitoring

diff --git a/postgres/schemas/models.py b/postgres/schemas/models.py
@@ -140,7 +140,10 @@ def create_tables():
 
         Base.metadata.create_all(engine, checkfirst=True)
         update_channel_metadata(engine)
-        update_program_metadata(engine)
+        if(os.environ.get("UPDATE") != "true"):
+            update_program_metadata(engine)
+        else:
+            logging.warning("No program update as UPDATE=true as it can create lock issues")
         logging.info("Table creation done, if not already done.")
     except (Exception) as error:
         logging.error(error)
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	43	7	84%	36–38, 56–58, 63
insert_existing_data_example.py	19	3	84%	25–27
postgres/schemas
models.py	157	11	93%	126–133, 146, 148–149, 214–215, 229–230
quotaclimat/data_ingestion
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	55	37	33%	21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	211	133	37%	44–48, 53–74, 78–81, 87, 90–132, 138–153, 158, 171–183, 187–193, 206–218, 221–225, 231, 266–267, 270–301, 304–306
channel_program.py	162	57	65%	21–23, 34–36, 53–54, 57–59, 98–99, 108, 124, 175–216
config.py	15	2	87%	7, 16
detect_keywords.py	222	9	96%	221, 279–286, 322
update_pg_keywords.py	67	49	27%	15–108, 132, 135, 142–157, 180–206, 213
utils.py	79	25	68%	29–53, 56, 65, 86–87, 117–120
quotaclimat/utils
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	24	11	54%	22–24, 28–37
sentry.py	11	2	82%	22–23
TOTAL	1291	380	71%