Feat/ Program grid can change (#253)

* wip: program grid evolution

* fix: test

* wip test

* chores: update some programs

* chores: alembic

* chores: default values for columns

* alembic: new columns for program metadata

* fix: use timezone for comparing start date

* chores: update programs evolution

* fix: tesdt

* fix: right column type for date

* fix: default time for empty date

README.md

@@ -339,10 +339,10 @@ Using [Alembic](https://alembic.sqlalchemy.org/en/latest/autogenerate.html) Auto
 # If changes have already been applied (on your feature vranch) and you have to recreate your alembic file by doing :
 # 1. change to your main branch 
 git  switch main
-# 2. start test container and run "pytest -vv -k api" to rebuild the state of the DB (or drop table the table you want)
+# 2. start test container (docker compose up testconsole -d / docker compose exec testconsole bash) and run "pytest -vv -k api" to rebuild the state of the DB (or drop table the table you want) - just let it run a few seconds.
 # 3. rechange to your WIP branch 
 git switch -
-# 4. connect to the test container : docker compose up test -d / docker compose exec test bash
+# 4. connect to the test container : docker compose up testconsole -d / docker compose exec testconsole bash
 # 5. reapply the latest saved state : 
 poetry run alembic stamp head
 # 6. Save the new columns
@@ -372,6 +372,8 @@ poetry run python3 quotaclimat/transform_excel_to_json.py
 ## Program Metadata table
 The media perimeter is defined here : "quotaclimat/data_processing/mediatree/channel_program.json"
 
+To evolve the media perimeter, we use `program_grid_start` and `program_grid_end` columns to version all evolutions.
+
 To calculate the right total duration for each channel, after updating "quotaclimat/data_processing/mediatree/channel_program.json" you need to execute this command to update `postgres/program_metadata.json` 
 ```
 poetry run python3 transform_program.py

alembic/versions/30abfd828007_program_metadata.py

@@ -0,0 +1,32 @@
+"""program metadata
+
+Revision ID: 30abfd828007
+Revises: 43103d5b49c9
+Create Date: 2024-10-03 14:18:09.874225
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '30abfd828007'
+down_revision: Union[str, None] = '43103d5b49c9'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('program_metadata', sa.Column('program_grid_start', sa.DateTime(), nullable=True))
+    op.add_column('program_metadata', sa.Column('program_grid_end', sa.DateTime(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('program_metadata', 'program_grid_end')
+    op.drop_column('program_metadata', 'program_grid_start')
+    # ### end Alembic commands ###

alembic/versions/43103d5b49c9_program_add_start_end_date_for_grid_.py

@@ -0,0 +1,30 @@
+"""program: add start/end date for grid evolution
+
+Revision ID: 43103d5b49c9
+Revises: af956a85658f
+Create Date: 2024-10-02 13:18:56.251135
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '43103d5b49c9'
+down_revision: Union[str, None] = 'af956a85658f'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

docker-compose.yml

@@ -152,8 +152,8 @@ services:
       #START_DATE_UPDATE: "2024-02-01" # to batch update PG from a date
       #END_DATE: "2024-02-29" # optional - otherwise end of the month
       BATCH_SIZE: 100 # number of records to update in one batch
-      # START_DATE: 1717227223 # to test batch import 
-      CHANNEL : france-info # to reimport only one channel
+     # START_DATE: 1727610071 # to test batch import 
+      CHANNEL : fr3-idf # to reimport only one channel
       MEDIATREE_USER : /run/secrets/username_api
       MEDIATREE_PASSWORD:  /run/secrets/pwd_api
       MEDIATREE_AUTH_URL: https://keywords.mediatree.fr/api/auth/token/

docker-entrypoint.sh

@@ -6,7 +6,7 @@ poetry run alembic upgrade head
 
 
 echo "update program metadata file"
-python transform_program.py
+poetry run python3 transform_program.py
 if [[ $? -eq 0 ]]; then
     echo "Command succeeded"
 else