From 273c2caba529589a6ba1ae405e894e5cce27b933 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 24 Apr 2024 15:17:05 -0700 Subject: [PATCH 01/29] TODO added to change to master branch in YML file --- .github/workflows/image_build_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 9fd15dfa0..3db336372 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -4,6 +4,7 @@ name: docker image # Controls when the action will run. Triggers the workflow on push or pull request # events but only for the master branch +# TODO: Change to master branch once changes are final. on: push: branches: [ image-push-merge ] From 2fdb46994e9fa5af64701c7d31794760f7c4c0e8 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 24 Apr 2024 15:44:33 -0700 Subject: [PATCH 02/29] Adding image-push-merge branch to automated CI/CD tests --- .github/workflows/test-with-docker.yml | 4 ++-- .github/workflows/test-with-manual-install.yml | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-with-docker.yml b/.github/workflows/test-with-docker.yml index e38499544..f5ae3e239 100644 --- a/.github/workflows/test-with-docker.yml +++ b/.github/workflows/test-with-docker.yml @@ -6,9 +6,9 @@ name: test-with-docker # events but only for the master branch on: push: - branches: [ master ] + branches: [ image-push-merge ] pull_request: - branches: [ master ] + branches: [ image-push-merge ] schedule: # * is a special character in YAML so you have to quote this string - cron: '5 4 * * 0' diff --git a/.github/workflows/test-with-manual-install.yml b/.github/workflows/test-with-manual-install.yml index 4a81eb000..62863e8e1 100644 --- a/.github/workflows/test-with-manual-install.yml +++ b/.github/workflows/test-with-manual-install.yml @@ -7,12 +7,14 @@ name: ubuntu-only-test-with-manual-install on: push: branches: - - master - - gis-based-mode-detection + # - master + # - gis-based-mode-detection + - image-push-merge pull_request: branches: - - master - - gis-based-mode-detection + # - master + # - gis-based-mode-detection + - image-push-merge schedule: # * is a special character in YAML so you have to quote this string - cron: '5 4 * * 0' From 91abdeaf2eaff57917434c417209a8c31f35b1f1 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 02:52:53 -0700 Subject: [PATCH 03/29] Upload artifact test - 1 Removed test workflow execution. Added upload artifact. --- .github/workflows/image_build_push.yml | 12 ++++++++++++ .github/workflows/test-with-docker.yml | 4 ++-- .github/workflows/test-with-manual-install.yml | 12 ++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 3db336372..8690196a6 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -47,3 +47,15 @@ jobs: - name: push docker image run: | docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + + - name: Create a text file + run: | + echo "hello world" > tag_file.txt + echo "Created tag text file" + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: docker-image-tag + path: tag_file.txt + overwrite: true diff --git a/.github/workflows/test-with-docker.yml b/.github/workflows/test-with-docker.yml index f5ae3e239..108dd62fa 100644 --- a/.github/workflows/test-with-docker.yml +++ b/.github/workflows/test-with-docker.yml @@ -6,9 +6,9 @@ name: test-with-docker # events but only for the master branch on: push: - branches: [ image-push-merge ] + branches: [ master ] pull_request: - branches: [ image-push-merge ] + branches: [ master ] schedule: # * is a special character in YAML so you have to quote this string - cron: '5 4 * * 0' diff --git a/.github/workflows/test-with-manual-install.yml b/.github/workflows/test-with-manual-install.yml index 62863e8e1..7212b9ebf 100644 --- a/.github/workflows/test-with-manual-install.yml +++ b/.github/workflows/test-with-manual-install.yml @@ -7,14 +7,14 @@ name: ubuntu-only-test-with-manual-install on: push: branches: - # - master - # - gis-based-mode-detection - - image-push-merge + - master + - gis-based-mode-detection + # - image-push-merge pull_request: branches: - # - master - # - gis-based-mode-detection - - image-push-merge + - master + - gis-based-mode-detection + # - image-push-merge schedule: # * is a special character in YAML so you have to quote this string - cron: '5 4 * * 0' From bbdaaaede9c819be371776879b8699d5ceec3cb0 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 02:59:42 -0700 Subject: [PATCH 04/29] Upload artifact test - 2 Commented image build and push for testing purposes. Fixed indentation. Learnt that YAML do not permit using tabs as indentation; spaces allowed. --- .github/workflows/image_build_push.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 8690196a6..73d06d4d6 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -39,23 +39,23 @@ jobs: run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} # Runs a set of commands using the runners shell - - name: build docker image - run: | - docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . - docker images + # - name: build docker image + # run: | + # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + # docker images - - name: push docker image - run: | - docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + # - name: push docker image + # run: | + # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - name: Create a text file run: | echo "hello world" > tag_file.txt - echo "Created tag text file" + echo "Created tag text file" - name: Upload Artifact uses: actions/upload-artifact@v4 - with: + with: name: docker-image-tag - path: tag_file.txt - overwrite: true + path: tag_file.txt + overwrite: true From 5d0ca0237d1dc5925452de40e432cbd16a5ffb57 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 10:50:46 -0700 Subject: [PATCH 05/29] Added temporary test file Temporary test file while working on automating tags modification. Checking to see if new artifacts generated even if overwrite is set to true. --- .docker/test.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .docker/test.txt diff --git a/.docker/test.txt b/.docker/test.txt new file mode 100644 index 000000000..9310e7312 --- /dev/null +++ b/.docker/test.txt @@ -0,0 +1,2 @@ +Temporary test file while working on automating tags modification. +Checking to see if new artifacts generated even if overwrite is set to true. \ No newline at end of file From f5aae4760062cc357a7f2e806f009028525d253b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 16:19:53 -0700 Subject: [PATCH 06/29] Upload artifact test - 3 Storing actual required tag with date timestamp. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 73d06d4d6..3cd0f61a9 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -50,7 +50,7 @@ jobs: - name: Create a text file run: | - echo "hello world" > tag_file.txt + echo ${{ steps.date.outputs.date }} > tag_file.txt echo "Created tag text file" - name: Upload Artifact From 051ef66b6f19a435c2a24f65137b4515ad7e3ba3 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 18:36:53 -0700 Subject: [PATCH 07/29] Repository dispatch send - 1 Checked out from image-push-merge branch which has artifact method working. - Changed trigger branch on push to tags-dispatch - Dispatching repository event from server so that join page repo's workflow will be triggered. - Once this dispatch method works, might not even need artifacts since repository dispatch has a client_payload option to pass data. - This is helpful since only 500 artifacts are allowed in any repo: https://github.com/actions/upload-artifact#:~:text=has%20a%20limit-,of%20500%20artifacts.,-For%20assistance%20with --- .github/workflows/image_build_push.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 3cd0f61a9..9009a94c8 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -7,7 +7,7 @@ name: docker image # TODO: Change to master branch once changes are final. on: push: - branches: [ image-push-merge ] + branches: [ tags-dispatch ] # Env variable @@ -59,3 +59,13 @@ jobs: name: docker-image-tag path: tag_file.txt overwrite: true + + - name: Trigger workflow in join-page, admin-dash, public-dash + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GH_PAT_TAG }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/MukuFlash03/nrel-openpath-join-page/dispatches \ + -d '{"event_type":"tags-dispatch","client_payload":{"docker-image-tag":${{ steps.date.outputs.date }}}}' \ No newline at end of file From 6c448656d1c52c48c15f1a2f0d28fe4fe9ee2e49 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 21:16:22 -0700 Subject: [PATCH 08/29] Workflow dispatch send - 1 Trying with workflow dispatch. --- .github/workflows/image_build_push.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 9009a94c8..4bbd9aef6 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -62,10 +62,10 @@ jobs: - name: Trigger workflow in join-page, admin-dash, public-dash run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GH_PAT_TAG }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/MukuFlash03/nrel-openpath-join-page/dispatches \ - -d '{"event_type":"tags-dispatch","client_payload":{"docker-image-tag":${{ steps.date.outputs.date }}}}' \ No newline at end of file + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/MukuFlash03/nrel-openpath-join-page/actions/workflows/90148778/dispatches \ + -d '{"ref":"tags-dispatch","inputs":{"name":"Mona the Octocat","home":"San Francisco, CA"}}' \ No newline at end of file From 4569dfb00f656da06851b7c8befb0f092ef15c3a Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 21:18:49 -0700 Subject: [PATCH 09/29] Workflow dispatch send - 2 Removing failed inputs --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 4bbd9aef6..c9cdab548 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -68,4 +68,4 @@ jobs: -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/MukuFlash03/nrel-openpath-join-page/actions/workflows/90148778/dispatches \ - -d '{"ref":"tags-dispatch","inputs":{"name":"Mona the Octocat","home":"San Francisco, CA"}}' \ No newline at end of file + -d '{"ref":"tags-dispatch"}' \ No newline at end of file From b562c2c155a1d958326cf0de5bab3171eaccb6f6 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 21:21:30 -0700 Subject: [PATCH 10/29] Workflow dispatch send - 3 Testing triggering workflow dispatch again. --- .docker/test.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.docker/test.txt b/.docker/test.txt index 9310e7312..9a745cd3d 100644 --- a/.docker/test.txt +++ b/.docker/test.txt @@ -1,2 +1,8 @@ Temporary test file while working on automating tags modification. -Checking to see if new artifacts generated even if overwrite is set to true. \ No newline at end of file +Checking to see if new artifacts generated even if overwrite is set to true. + +----- + +Testing again to see if workflow dispatch works + +------ \ No newline at end of file From 25109b31bacf3f20bf9dd6a0a1e893cf99c4898b Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 21:39:29 -0700 Subject: [PATCH 11/29] Workflow dispatch send - 3 Changed branch name to new branch so it picks workflow events in join from this branch. --- .docker/test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.docker/test.txt b/.docker/test.txt index 9a745cd3d..63a1bf80c 100644 --- a/.docker/test.txt +++ b/.docker/test.txt @@ -5,4 +5,5 @@ Checking to see if new artifacts generated even if overwrite is set to true. Testing again to see if workflow dispatch works +Yes, working with updated branch as well. ------ \ No newline at end of file From 709f3cf480face7f886a18612ac7735bc4b37a1e Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 22:48:48 -0700 Subject: [PATCH 12/29] Workflow dispatch send - 4 Testing if working with updated token scopes to have only actions: write --- .docker/test.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.docker/test.txt b/.docker/test.txt index 63a1bf80c..351c280d9 100644 --- a/.docker/test.txt +++ b/.docker/test.txt @@ -6,4 +6,8 @@ Checking to see if new artifacts generated even if overwrite is set to true. Testing again to see if workflow dispatch works Yes, working with updated branch as well. ------- \ No newline at end of file +------ + +Testing if working with updated token scopes to have only actions: write + +----- \ No newline at end of file From 4b4218529faabe2c9f7809331eaaf5f57e1975eb Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 23:20:53 -0700 Subject: [PATCH 13/29] Workflow dispatch send - 5 Testing sending docker image tags as input values in the POST request. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index c9cdab548..8cd707682 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -68,4 +68,4 @@ jobs: -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/MukuFlash03/nrel-openpath-join-page/actions/workflows/90148778/dispatches \ - -d '{"ref":"tags-dispatch"}' \ No newline at end of file + -d '{"ref":"tags-dispatch", "inputs": {"docker_image_tag" : "test_tag"}}' \ No newline at end of file From 12d09ae06e9a9c5e44954a9718a33a038bde15aa Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 23:23:39 -0700 Subject: [PATCH 14/29] Workflow dispatch send - 6 Sending actual date timestamp which is used as a suffix to the docker image tag. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 8cd707682..30bb7add0 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -68,4 +68,4 @@ jobs: -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/MukuFlash03/nrel-openpath-join-page/actions/workflows/90148778/dispatches \ - -d '{"ref":"tags-dispatch", "inputs": {"docker_image_tag" : "test_tag"}}' \ No newline at end of file + -d '{"ref":"tags-dispatch", "inputs": {"docker_image_tag" : "${{ steps.date.outputs.date }}"}}' \ No newline at end of file From 94c129bb3b2f1b827661c64093b7803918c18ccd Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 25 Apr 2024 23:52:48 -0700 Subject: [PATCH 15/29] Workflow dispatch send - 7 Removed code related to artifact method for sending docker image tags. Workflow dispatch uses POST request via which input data values can be sent to the repository in which the workflow is to be triggered. --- .github/workflows/image_build_push.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 30bb7add0..1c6f19a84 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -48,18 +48,6 @@ jobs: # run: | # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} - - name: Create a text file - run: | - echo ${{ steps.date.outputs.date }} > tag_file.txt - echo "Created tag text file" - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: docker-image-tag - path: tag_file.txt - overwrite: true - - name: Trigger workflow in join-page, admin-dash, public-dash run: | curl -L \ From 0dd1245205e6901b0d0ce1d302d56e3db41ffbac Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 00:43:23 -0700 Subject: [PATCH 16/29] Matrix build send - 1 Copied over YAML file from e-mission-server repo after successfully implementing transmission of docker image tags from e-mission-server to join with artifacts and workflow dispatch. Added another dispatch job to trigger workflow via matrix strategy. --- .github/workflows/image_build_push.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 1c6f19a84..5ee206b30 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -7,7 +7,7 @@ name: docker image # TODO: Change to master branch once changes are final. on: push: - branches: [ tags-dispatch ] + branches: [ tags-matrix ] # Env variable @@ -48,6 +48,15 @@ jobs: # run: | # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + dispatch: + runs-on: ubuntu-latest + strategy: + matrix: + repo: ['MukuFlash03/nrel-openpath-join-page', 'MukuFlash03/op-admin-dashboard', 'MukuFlash03/em-public-dashboard'] + + steps: + - uses: actions/checkout@v2 + - name: Trigger workflow in join-page, admin-dash, public-dash run: | curl -L \ @@ -55,5 +64,5 @@ jobs: -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/MukuFlash03/nrel-openpath-join-page/actions/workflows/90148778/dispatches \ + https://api.github.com/repos/${{ matrix.repo }}/actions/workflows/image_build_push.yml/dispatches \ -d '{"ref":"tags-dispatch", "inputs": {"docker_image_tag" : "${{ steps.date.outputs.date }}"}}' \ No newline at end of file From 5434914ac17ddef894920be56d5e5b42d02fc4b2 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 00:50:33 -0700 Subject: [PATCH 17/29] Matrix build send - 2 Updated branch name which was set for the workflow dispatch without matrix build. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 5ee206b30..2cbb56fb8 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -65,4 +65,4 @@ jobs: -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/${{ matrix.repo }}/actions/workflows/image_build_push.yml/dispatches \ - -d '{"ref":"tags-dispatch", "inputs": {"docker_image_tag" : "${{ steps.date.outputs.date }}"}}' \ No newline at end of file + -d '{"ref":"tags-matrix", "inputs": {"docker_image_tag" : "${{ steps.date.outputs.date }}"}}' \ No newline at end of file From 0fd3e9db732fd17ce50f7cb8811fa98333bffb60 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 00:58:06 -0700 Subject: [PATCH 18/29] Matrix build send - 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jobs completed successfully but nothing happens in the other three target repos. I observed in real-time that the matrix jobs - "dispatch”, start before or at the same time the main build job and get over really quickly. Hence, the error makes sense as the docker image tag is not generated yet: { "message": "Required input 'docker_image_tag' not provided", "documentation_url": "https://docs.github.com/rest/actions/workflows#create-a-workflow-dispatch-event" } Hence, the timeout-minutes parameter makes sense to use as was mentioned in the blog post: https://www.amaysim.technology/blog/using-github-actions-to-trigger-actions-across-repos 5 minutes seems too much, I’ll give it a minute for now. Alternatively, timeout not needed; can use "needs" similar to how fetch_run_id job was run first before trying to download artifact. --- .github/workflows/image_build_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 2cbb56fb8..e250f0967 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -49,6 +49,7 @@ jobs: # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} dispatch: + needs: build runs-on: ubuntu-latest strategy: matrix: From ab399ea70dd8e215a35564e980bafc2d056417a2 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 01:02:45 -0700 Subject: [PATCH 19/29] Matrix build send - 3 Since the date is being generated in a different jobs, must access it using needs keyword. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index e250f0967..74d5d240d 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -66,4 +66,4 @@ jobs: -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/${{ matrix.repo }}/actions/workflows/image_build_push.yml/dispatches \ - -d '{"ref":"tags-matrix", "inputs": {"docker_image_tag" : "${{ steps.date.outputs.date }}"}}' \ No newline at end of file + -d '{"ref":"tags-matrix", "inputs": {"docker_image_tag" : "${{ needs.build.outputs.date }}"}}' \ No newline at end of file From 91c0c1f86dbc4048782a953eb01c07fd8e2fecf5 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 01:13:34 -0700 Subject: [PATCH 20/29] Matrix build send - 4 Properly added date by setting it as an output value of the 1st build job. Then storing it as an environment variable in the 2nd job by accessing it via the needs keyword. Finally using it in the curl POST request by referencing the environment variable. --- .github/workflows/image_build_push.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 74d5d240d..f3ce7c1c5 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -22,6 +22,9 @@ jobs: # The type of runner that the job will run on runs-on: ubuntu-latest + outputs: + date: ${{ steps.date.outputs.date }} + # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it @@ -51,6 +54,10 @@ jobs: dispatch: needs: build runs-on: ubuntu-latest + + env: + DOCKER_IMAGE_TAG: ${{needs.build.outputs.date}} + strategy: matrix: repo: ['MukuFlash03/nrel-openpath-join-page', 'MukuFlash03/op-admin-dashboard', 'MukuFlash03/em-public-dashboard'] @@ -66,4 +73,4 @@ jobs: -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/${{ matrix.repo }}/actions/workflows/image_build_push.yml/dispatches \ - -d '{"ref":"tags-matrix", "inputs": {"docker_image_tag" : "${{ needs.build.outputs.date }}"}}' \ No newline at end of file + -d '{"ref":"tags-matrix", "inputs": {"docker_image_tag" : "${{ env.DOCKER_IMAGE_TAG }}"}}' \ No newline at end of file From acecf3e0974094b17dc32dcf4d0ad0d07d16d7ac Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 26 Apr 2024 01:18:00 -0700 Subject: [PATCH 21/29] Matrix build send - 5 Editing sample file as a sanity check to see if matrix build works. --- .docker/test.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.docker/test.txt b/.docker/test.txt index 351c280d9..c640a8df8 100644 --- a/.docker/test.txt +++ b/.docker/test.txt @@ -10,4 +10,8 @@ Yes, working with updated branch as well. Testing if working with updated token scopes to have only actions: write +----- + +Testing to see if matrix build dispatch strategy works again. + ----- \ No newline at end of file From e778b3f3cf050eec33ef439a1b6763b6aaf533e0 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 29 Apr 2024 20:35:39 -0700 Subject: [PATCH 22/29] Fix for "url" KeyError observed in public-dash redesign testing The bug was triggered by a .gitignore-d conf/ directory in public-dash which had a db.conf file. This was being loaded when docker-compose.dev.yml was used to test the dev version of public-dash. This was occurring since there was a nested dictionary in the db.conf.sample and db.conf files while I had initially only stored the nested keys (url, result_limit). Since it was still reading from the file, it stored the nested dictionary format with timeseries as the parent key followed by (url, result_limit) as children. Hence, fixing it by adding the same nested dictionary structure in the emission/core/config.py and emission/core/get_database.py --- emission/core/config.py | 14 +++++++++----- emission/core/get_database.py | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/emission/core/config.py b/emission/core/config.py index 8dbbabe9d..3f4c28b3e 100644 --- a/emission/core/config.py +++ b/emission/core/config.py @@ -4,15 +4,19 @@ def get_config_data_from_env(): config_data_env = { - "url": os.getenv('DB_HOST', "localhost"), - "result_limit": os.getenv('DB_TS_RESULT_LIMIT', 250000) + "timeseries": { + "url": os.getenv('DB_HOST', "localhost"), + "result_limit": os.getenv('DB_TS_RESULT_LIMIT', 250000) + } } return config_data_env def check_unset_env_vars(): config_data_env = { - "url": os.getenv('DB_HOST'), - "result_limit": os.getenv('DB_TS_RESULT_LIMIT') + "timeseries": { + "url": os.getenv('DB_HOST'), + "result_limit": os.getenv('DB_TS_RESULT_LIMIT') + } } return not any(config_data_env.values()) @@ -26,7 +30,7 @@ def get_config_data(): # if check_unset_env_vars(): # print("All DB environment variables are set to None") ret_val = get_config_data_from_env() - if ret_val["url"] == "localhost": + if ret_val["timeseries"]["url"] == "localhost": print("storage not configured, falling back to sample, default configuration") return ret_val diff --git a/emission/core/get_database.py b/emission/core/get_database.py index 005abe3ad..c8d370fcd 100644 --- a/emission/core/get_database.py +++ b/emission/core/get_database.py @@ -12,8 +12,8 @@ import emission.core.config as ecc -url = ecc.get_config()["url"] -result_limit = ecc.get_config()["result_limit"] +url = ecc.get_config()["timeseries"]["url"] +result_limit = ecc.get_config()["timeseries"]["result_limit"] try: parsed=pymongo.uri_parser.parse_uri(url) From 776f0b9a50552e58a2dbdf519c323460021a6120 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 17:50:13 -0700 Subject: [PATCH 23/29] Artifact + Matrix - 1 Added build and push to have latest image available in docker hub that is generated with updated YAML file. Removed join repo from workflow dispatch. Added artifact so that this can be used in case of push trigger event in admin-dash and public-dash repos with the help of the python script fetch_runID.py in those repos, to get the latest run_ID for a specific branch with the artifact. Workflow dispatch would still use the input parameters which contains the latest timestamp tag. --- .github/workflows/image_build_push.yml | 35 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index f3ce7c1c5..c89f55a18 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -7,8 +7,7 @@ name: docker image # TODO: Change to master branch once changes are final. on: push: - branches: [ tags-matrix ] - + branches: [ tags-combo-approach ] # Env variable env: @@ -42,28 +41,40 @@ jobs: run: echo running in repo ${GITHUB_REPOSITORY#*/} branch ${GITHUB_REF##*/} on ${{ steps.date.outputs.date }} # Runs a set of commands using the runners shell - # - name: build docker image - # run: | - # docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . - # docker images + - name: build docker image + run: | + docker build -t $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} . + docker images - # - name: push docker image - # run: | - # docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + - name: push docker image + run: | + docker push $DOCKER_USER/${GITHUB_REPOSITORY#*/}:${GITHUB_REF##*/}_${{ steps.date.outputs.date }} + + - name: Create a text file + run: | + echo ${{ steps.date.outputs.date }} > tag_file.txt + echo "Created tag text file" + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: docker-image-tag + path: tag_file.txt + overwrite: true dispatch: needs: build runs-on: ubuntu-latest env: - DOCKER_IMAGE_TAG: ${{needs.build.outputs.date}} + DOCKER_IMAGE_TAG: ${{ needs.build.outputs.date }} strategy: matrix: - repo: ['MukuFlash03/nrel-openpath-join-page', 'MukuFlash03/op-admin-dashboard', 'MukuFlash03/em-public-dashboard'] + repo: ['MukuFlash03/op-admin-dashboard', 'MukuFlash03/em-public-dashboard'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Trigger workflow in join-page, admin-dash, public-dash run: | From 17ac3cc6e5a88624755fe3d0af8074ea19d64911 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 18:05:57 -0700 Subject: [PATCH 24/29] Artifact + Matrix - 2 Changed branch name to tags-combo-approach; resulted in a failed workflow in admin-dash, public-dash since branch was still set to tags-matrix. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index c89f55a18..08a0a3006 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -84,4 +84,4 @@ jobs: -H "Authorization: Bearer ${{ secrets.GH_FG_PAT_TAGS }}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/${{ matrix.repo }}/actions/workflows/image_build_push.yml/dispatches \ - -d '{"ref":"tags-matrix", "inputs": {"docker_image_tag" : "${{ env.DOCKER_IMAGE_TAG }}"}}' \ No newline at end of file + -d '{"ref":"tags-combo-approach", "inputs": {"docker_image_tag" : "${{ env.DOCKER_IMAGE_TAG }}"}}' \ No newline at end of file From 706f74c63683742e3b892a83eacf63e0b8154b12 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Wed, 1 May 2024 18:32:21 -0700 Subject: [PATCH 25/29] Artifact + Matrix - 3 Changing step name to exclude join-page; included earlier for testing purposes but now focussing on the dashboard repos which actually use the server as the base image and hence need the workflow to be triggered. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 08a0a3006..c47147a2c 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -76,7 +76,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Trigger workflow in join-page, admin-dash, public-dash + - name: Trigger workflow in admin-dash, public-dash run: | curl -L \ -X POST \ From d724fd1d293f7dd01a683ba5a1da4ed0601967ff Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 2 May 2024 00:33:58 -0700 Subject: [PATCH 26/29] Revert "Adding image-push-merge branch to automated CI/CD tests" This reverts commit 2fdb46994e9fa5af64701c7d31794760f7c4c0e8. --- .github/workflows/test-with-docker.yml | 2 +- .github/workflows/test-with-manual-install.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-with-docker.yml b/.github/workflows/test-with-docker.yml index 108dd62fa..e38499544 100644 --- a/.github/workflows/test-with-docker.yml +++ b/.github/workflows/test-with-docker.yml @@ -6,7 +6,7 @@ name: test-with-docker # events but only for the master branch on: push: - branches: [ master ] + branches: [ master ] pull_request: branches: [ master ] schedule: diff --git a/.github/workflows/test-with-manual-install.yml b/.github/workflows/test-with-manual-install.yml index 7212b9ebf..4a81eb000 100644 --- a/.github/workflows/test-with-manual-install.yml +++ b/.github/workflows/test-with-manual-install.yml @@ -9,12 +9,10 @@ on: branches: - master - gis-based-mode-detection - # - image-push-merge pull_request: branches: - master - gis-based-mode-detection - # - image-push-merge schedule: # * is a special character in YAML so you have to quote this string - cron: '5 4 * * 0' From e6a2d7911de69dcb9f18b468b23b34eff697770a Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Thu, 2 May 2024 00:34:30 -0700 Subject: [PATCH 27/29] Revert "TODO added to change to master branch in YML file" This reverts commit 273c2caba529589a6ba1ae405e894e5cce27b933. --- .github/workflows/image_build_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index c47147a2c..e3a8e828e 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -4,7 +4,6 @@ name: docker image # Controls when the action will run. Triggers the workflow on push or pull request # events but only for the master branch -# TODO: Change to master branch once changes are final. on: push: branches: [ tags-combo-approach ] From f033f0618de488cbf92d03a948d237e1855b3d55 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 03:14:16 -0700 Subject: [PATCH 28/29] Added TODOs in github actions workflow YAML file Reminder for things to change as per master branch of e-mission-server once changes are finalized. --- .github/workflows/image_build_push.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index e3a8e828e..3618e0804 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -6,6 +6,7 @@ name: docker image # events but only for the master branch on: push: + # TODO: Change branch to master branches: [ tags-combo-approach ] # Env variable @@ -70,12 +71,15 @@ jobs: strategy: matrix: + # TODO: Change user / organization to emission instead of MukuFlash03 repo: ['MukuFlash03/op-admin-dashboard', 'MukuFlash03/em-public-dashboard'] steps: - uses: actions/checkout@v4 - name: Trigger workflow in admin-dash, public-dash + # TODO: Create Fine-grained token with "Actions: write" permissions + # TODO: Change branch name under "ref" in the curl command run: | curl -L \ -X POST \ From 1207d791150866c84bd8c7fafc737326a4052374 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Fri, 3 May 2024 13:41:49 -0700 Subject: [PATCH 29/29] Artifact + Matrix - 4 Adding extra words to TODO just to trigger a workflow dispatch run and check if .env commit action works successfully for this trigger type too. --- .github/workflows/image_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image_build_push.yml b/.github/workflows/image_build_push.yml index 3618e0804..b098c7cde 100644 --- a/.github/workflows/image_build_push.yml +++ b/.github/workflows/image_build_push.yml @@ -6,7 +6,7 @@ name: docker image # events but only for the master branch on: push: - # TODO: Change branch to master + # TODO: Change branch to master branch of e-mission-server branches: [ tags-combo-approach ] # Env variable