[55] update content of country exportsh and its deployment for turnke…

…y curator portal (#60)

* Update country exporter

* Update export/aggregate container creation process

* Remove covid related outdated code

* Update full export and transform

* Update tests

* Update env for test

* Update aggregate-ci.yml

* Update aggregate-ci.yml

* Remove SGTF field and update format of all_data file

* verification in query

* Lint issue resolve

.github/workflows/aggregate-ci.yml

@@ -5,19 +5,24 @@ on:
     branches: [main]
     paths:
     - '.github/workflows/aggregate-ci.yml'
-    - 'data-serving/scripts/aggregate-covid19/aggregate/**'
+    - 'data-serving/scripts/aggregate-data/aggregate/**'
   pull_request:
     paths:
     - '.github/workflows/aggregate-ci.yml'
-    - 'data-serving/scripts/aggregate-covid19/aggregate/**'
+    - 'data-serving/scripts/aggregate-data/aggregate/**'
   workflow_dispatch:
 
 jobs:
   ci:
     runs-on: ubuntu-20.04
     defaults:
       run:
-        working-directory: data-serving/scripts/aggregate-covid19/aggregate
+        working-directory: data-serving/scripts/aggregate-data/aggregate
+    env:
+      S3_AGGREGATE_BUCKET: 'mock-aggregate-bucket'
+      S3_MAP_DATA_BUCKET: 'mock-map-data-bucket'
+      CONN: 'mock-conn'
+      DATABASE_NAME: 'mock-database-name'
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.10

.github/workflows/aggregate-deploy.yml

@@ -5,8 +5,8 @@ on:
     branches: [main]
     paths:
     - '.github/workflows/aggregate-deploy.yml'
-    - 'data-serving/scripts/aggregate-covid19/**'
-    - '!data-serving/scripts/aggregate-covid19/README.md'
+    - 'data-serving/scripts/aggregate-data/**'
+    - '!data-serving/scripts/aggregate-data/README.md'
   workflow_dispatch:
 
 jobs:
@@ -26,10 +26,10 @@ jobs:
       uses: aws-actions/amazon-ecr-login@v1
 
     - name: Build, tag, and push image to Amazon ECR (latest)
-      working-directory: data-serving/scripts/aggregate-covid19/aggregate
+      working-directory: data-serving/scripts/aggregate-data/aggregate
       env:
         REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-        REPO: gdh-map-aggregator
+        REPO: gdh-tcp-map-aggregator
         IMAGE_TAG: ${{ github.sha }}
       run: |
         docker build -t $REGISTRY/$REPO:$IMAGE_TAG -t $REGISTRY/$REPO .

.github/workflows/export-deploy.yml

@@ -29,8 +29,8 @@ jobs:
       working-directory: data-serving/scripts/export-data
       env:
         REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-        REPO_COUNTRY: gdh-country-exporter
-        REPO_FULL: gdh-full-exporter
+        REPO_COUNTRY: gdh-tcp-country-exporter
+        REPO_FULL: gdh-tcp-full-exporter
         IMAGE_TAG: ${{ github.sha }}
       run: |
         docker build -t $REGISTRY/$REPO_COUNTRY:$IMAGE_TAG -t $REGISTRY/$REPO_COUNTRY .