GHA - Only publish docker builds on main branch (#200)

* GHA - Only publish docker builds on main branch * GHA - Return develop_tag input * GHA - push_image input required false * GHA - remove input last_commit_tag * GHA - remove input last_commit_tag 2 * GHA - Fix if statement * GHA - Debug event name * GHA - Fix event name check * GHA - Fix tags env var * GHA - Fix load input * GHA - Separate steps for pushing images because images can't be loaded with multiple platforms * GHA - Add profile for pulling non-local images * GHA - Clean env vars and correct inputs * GHA - Remove requirement of removed job * GHA - Fix push_image type * GHA - Remove profile from local image containers * GHA - Store docker image artifact and download in e2e test job * GHA - remove custom driver input * GHA - Fix end of line * GHA - Set file location * Test file location Signed-off-by: Bradley Bown <[email protected]> * Testing image publishing * Fix load and push error * Commit tag from current branch, fix tags env var * Test fetch depth * Test * Revert commit tag changes * Testing removal of docker login step before push * TEST - Removal of cache * Only login to docker on main * Testing pull image without docker login * Testing pull image without docker login - 2 * Test force build * Test no changes * Docker secrets required false * Remove requirements for dockerhub credentials for pulls * Rename arfifacts and compose profile * Fix testing.yml secrets * Replace usage of inputs with env vars * Remove unnecessary inputs * Reinstate dockerhub login for e2e tests * Reinstate dockerhub login for coordinator-testing.yml * Reinstate dockerhub login for build steps * Fix secrets for build-and-publish.yml * Non concurrent manual-docker-build-and-e2e-tests * Optional docker secrets, check if they're set before running login step * Set to env vars * feat: add support for transaction-exclusion-api in new github workflow * feat: remove new line pipeline testing * feat: add echo for github.ref pipeline testing --------- Signed-off-by: Bradley Bown <[email protected]> Co-authored-by: jonesho <[email protected]> Co-authored-by: jonesho <[email protected]>
Consensys · Oct 24, 2024 · 80e12cd · 80e12cd
1 parent e71ec80
commit 80e12cd
Show file tree

Hide file tree

Showing 22 changed files with 343 additions and 331 deletions.
diff --git a/.github/actions/check-image-tags-exist/action.yml b/.github/actions/check-image-tags-exist/action.yml
@@ -5,48 +5,24 @@ inputs:
   last_commit_tag:
     description: 'The tag of the last commit image'
     required: true
-  common_ancestor_tag:
-    description: 'The tag of the common ancestor commit image'
-    required: true
   image_name:
     description: 'The name of the image to check'
     required: true
-  docker_username:
-    description: 'The username to login to Docker Hub'
-    required: true
-  docker_password:
-    description: 'The password to login to Docker Hub'
-    required: true
 outputs:
   last_commit_tag_exists:
     description: 'Whether the last commit image tag exists'
     value: ${{ steps.last_commit_image_exists.outputs.last_commit_tag_exists }}
-  common_ancestor_commit_tag_exists:
-    description: 'Whether the common ancestor commit image tag exists'
-    value: ${{ steps.ancestor_commit_image_exists.outputs.common_ancestor_commit_tag_exists }}
 
 runs:
   using: 'composite'
   steps:
-    - name: Login to Docker Hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ inputs.docker_username }}
-        password: ${{ inputs.docker_password }}
     - name: Check last commit image tag exists
       id: last_commit_image_exists
       shell: bash
       continue-on-error: true
       run: |
         echo last_commit_tag_exists=$(docker pull ${{ inputs.image_name }}:${{ inputs.last_commit_tag }} > /dev/null ; echo $?) >> $GITHUB_OUTPUT
-    - name: Check ancestor commit image tag exists
-      shell: bash
-      id: ancestor_commit_image_exists
-      continue-on-error: true
-      run: |
-        echo common_ancestor_commit_tag_exists=$(docker pull ${{ inputs.image_name }}:${{ inputs.common_ancestor_tag }} > /dev/null ; echo $?) >> $GITHUB_OUTPUT
     - name: Show outputs
       shell: bash
       run: |
         echo "last_commit_tag_exists: ${{ steps.last_commit_image_exists.outputs.last_commit_tag_exists }}"
-        echo "common_ancestor_commit_tag_exists: ${{ steps.ancestor_commit_image_exists.outputs.common_ancestor_commit_tag_exists }}"
diff --git a/.github/actions/image-tag-and-push/action.yml b/.github/actions/image-tag-and-push/action.yml
@@ -8,40 +8,32 @@ inputs:
   last_commit_tag:
     description: 'The tag of the last commit image'
     required: true
-  common_ancestor_tag:
-    description: 'The tag of the common ancestor commit image'
-    required: true
   develop_tag:
     description: 'The tag of the develop image'
     required: true
-  untested_tag_suffix:
-    description: 'The suffix to add to untested images'
-    required: true
   image_name:
     description: 'The name of the image to tag and push'
     required: true
   last_commit_tag_exists:
     description: 'Whether the last commit image tag exists'
     required: true
-  common_ancestor_commit_tag_exists:
-    description: 'Whether the common ancestor commit image tag exists'
-    required: true
   docker_username:
     description: 'The username to login to Docker Hub'
-    required: true
+    required: false
   docker_password:
     description: 'The password to login to Docker Hub'
-    required: true
+    required: false
 
 outputs:
-  image_tagged: 
+  image_tagged:
     description: 'Whether the image was tagged'
     value: ${{ steps.set-output.outputs.image_tagged }}
 
 runs:
   using: 'composite'
   steps:
     - name: Login to Docker Hub
+      if: ${{ github.ref == 'refs/heads/main' && inputs.last_commit_tag_exists == '0' }}
       uses: docker/login-action@v2
       with:
         username: ${{ inputs.docker_username }}
@@ -50,29 +42,15 @@ runs:
       shell: bash
       run: |
         echo IMAGE_TAGGED=false >> $GITHUB_ENV
-    - name: Tag Docker image with last commit tag with the commit hash plus w/o "untested" suffix
-      shell: bash
-      if: ${{ inputs.last_commit_tag != '0000000' && inputs.last_commit_tag_exists == '0' }}
-      run: |
-        docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }}
-        docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }}
-        echo IMAGE_TAGGED=true >> $GITHUB_ENV
-    - name: Tag Docker image with common ancestor commit tag with the commit hash plus w/o "untested" suffix
-      shell: bash
-      if: ${{ inputs.last_commit_tag == '0000000' && inputs.common_ancestor_commit_tag_exists == '0' }}
-      run: |
-        docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }} ${{ inputs.image_name }}:${{ inputs.common_ancestor_tag }}
-        docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} ${{ inputs.image_name }}:${{ inputs.common_ancestor_tag }}
-        echo IMAGE_TAGGED=true >> $GITHUB_ENV
     - name: Tag Docker image with develop if on main branch
       shell: bash
       if: ${{ github.ref == 'refs/heads/main' && inputs.last_commit_tag_exists == '0' }}
       run: |
         docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.develop_tag }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }}
+        docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }}
         echo IMAGE_TAGGED=true >> $GITHUB_ENV
-    
     - name: Set output from environment variable
       shell: bash
       id: set-output
       run: |
-        echo "image_tagged=$IMAGE_TAGGED" >> $GITHUB_OUTPUT
+        echo "image_tagged=$IMAGE_TAGGED" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/all-tools.yml b/.github/workflows/all-tools.yml
@@ -50,10 +50,7 @@ jobs:
         uses: ./.github/actions/check-image-tags-exist
         with:
           last_commit_tag: ${{ needs.store_image_name_and_tags.outputs.last_commit_tag }}
-          common_ancestor_tag: ${{ needs.store_image_name_and_tags.outputs.common_ancestor_tag }}
           image_name: consensys/linea-alltools
-          docker_username: ${{ secrets.DOCKERHUB_USERNAME }}
-          docker_password: ${{ secrets.DOCKERHUB_TOKEN }}
 
   all-tools-tag-only:
     runs-on: [self-hosted, ubuntu-20.04, X64, small]
@@ -71,12 +68,9 @@ jobs:
         with:
           commit_tag: ${{ needs.store_image_name_and_tags.outputs.commit_tag }}
           last_commit_tag: ${{ needs.store_image_name_and_tags.outputs.last_commit_tag }}
-          common_ancestor_tag: ${{ needs.store_image_name_and_tags.outputs.common_ancestor_tag }}
           develop_tag: ${{ needs.store_image_name_and_tags.outputs.develop_tag }}
-          untested_tag_suffix: ${{ needs.store_image_name_and_tags.outputs.untested_tag_suffix }}
           image_name: consensys/linea-alltools
           last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists }}
-          common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists }}
           docker_username: ${{ secrets.DOCKERHUB_USERNAME }}
           docker_password: ${{ secrets.DOCKERHUB_TOKEN }}
 

diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml
@@ -6,18 +6,13 @@ on:
       commit_tag:
         required: true
         type: string
-      last_commit_tag:
-        required: true
-        type: string
-      common_ancestor_tag:
-        required: true
-        type: string
       develop_tag:
         required: true
         type: string
-      untested_tag_suffix:
-        required: true
-        type: string
+      push_image:
+        required: false
+        type: boolean
+        default: false
       coordinator_changed:
         required: true
         type: string
@@ -50,67 +45,57 @@ on:
         type: string
     secrets:
       DOCKERHUB_USERNAME:
-        required: true
+        required: false
       DOCKERHUB_TOKEN:
-        required: true
+        required: false
 
 jobs:
   coordinator:
     uses: ./.github/workflows/coordinator-build-and-publish.yml
     if: ${{ always() && (inputs.coordinator_changed == 'true' || inputs.coordinator_image_tagged != 'true') }}
     with:
       commit_tag: ${{ inputs.commit_tag }}
-      last_commit_tag: ${{ inputs.last_commit_tag }}
-      common_ancestor_tag: ${{ inputs.common_ancestor_tag }}
       develop_tag: ${{ inputs.develop_tag }}
-      untested_tag_suffix: ${{ inputs.untested_tag_suffix }}
       image_name: consensys/linea-coordinator
+      push_image: ${{ inputs.push_image }}
     secrets: inherit
 
   prover:
     uses: ./.github/workflows/prover-build-and-publish.yml
     if: ${{ always() && (inputs.prover_changed == 'true' || inputs.prover_image_tagged != 'true') }}
     with:
       commit_tag: ${{ inputs.commit_tag }}
-      last_commit_tag: ${{ inputs.last_commit_tag }}
-      common_ancestor_tag: ${{ inputs.common_ancestor_tag }}
       develop_tag: ${{ inputs.develop_tag }}
-      untested_tag_suffix: ${{ inputs.untested_tag_suffix }}
       image_name: consensys/linea-prover
+      push_image: ${{ inputs.push_image }}
     secrets: inherit
 
   postman:
     uses: ./.github/workflows/postman-build-and-publish.yml
     if: ${{ always() && (inputs.postman_changed == 'true' || inputs.postman_image_tagged != 'true') }}
     with:
       commit_tag: ${{ inputs.commit_tag }}
-      last_commit_tag: ${{ inputs.last_commit_tag }}
-      common_ancestor_tag: ${{ inputs.common_ancestor_tag }}
       develop_tag: ${{ inputs.develop_tag }}
-      untested_tag_suffix: ${{ inputs.untested_tag_suffix }}
       image_name: consensys/linea-postman
+      push_image: ${{ inputs.push_image }}
     secrets: inherit
 
   traces-api-facade:
     uses: ./.github/workflows/traces-api-facade-build-and-publish.yml
     if: ${{ always() && (inputs.traces_api_facade_changed == 'true' || inputs.traces_api_facade_image_tagged != 'true') }}
     with:
       commit_tag: ${{ inputs.commit_tag }}
-      last_commit_tag: ${{ inputs.last_commit_tag }}
-      common_ancestor_tag: ${{ inputs.common_ancestor_tag }}
       develop_tag: ${{ inputs.develop_tag }}
-      untested_tag_suffix: ${{ inputs.untested_tag_suffix }}
       image_name: consensys/linea-traces-api-facade
+      push_image: ${{ inputs.push_image }}
     secrets: inherit
 
   transaction_exclusion_api:
     uses: ./.github/workflows/transaction-exclusion-api-build-and-publish.yml
     if: ${{ always() && (inputs.transaction_exclusion_api_changed == 'true' || inputs.transaction_exclusion_api_image_tagged != 'true') }}
     with:
       commit_tag: ${{ inputs.commit_tag }}
-      last_commit_tag: ${{ inputs.last_commit_tag }}
-      common_ancestor_tag: ${{ inputs.common_ancestor_tag }}
       develop_tag: ${{ inputs.develop_tag }}
-      untested_tag_suffix: ${{ inputs.untested_tag_suffix }}
       image_name: consensys/linea-transaction-exclusion-api
+      push_image: ${{ inputs.push_image }}
     secrets: inherit
diff --git a/.github/workflows/coordinator-build-and-publish.yml b/.github/workflows/coordinator-build-and-publish.yml
@@ -6,26 +6,28 @@ on:
       commit_tag:
         required: true
         type: string
-      last_commit_tag:
-        required: true
-        type: string
-      common_ancestor_tag:
-        required: true
-        type: string
       develop_tag:
         required: true
         type: string
-      untested_tag_suffix:
-        required: true
-        type: string
       image_name:
         required: true
         type: string
+      push_image:
+        required: false
+        type: boolean
+        default: false
     secrets:
       DOCKERHUB_USERNAME:
         required: true
       DOCKERHUB_TOKEN:
         required: true
+  workflow_dispatch:
+    inputs:
+      push_image:
+        description: 'Toggle whether to push image to docker registry'
+        required: false
+        type: boolean
+        default: true
 
 concurrency:
   group: coordinator-build-and-publish-${{ github.workflow }}-${{ github.ref }}
@@ -38,10 +40,16 @@ jobs:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
       DEVELOP_TAG: ${{ inputs.develop_tag }}
-      UNTESTED_TAG_SUFFIX: ${{ inputs.untested_tag_suffix }}
       IMAGE_NAME: ${{ inputs.image_name }}
-      GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
+      PUSH_IMAGE: ${{ inputs.push_image }}
+      TAGS: ${{ inputs.image_name }}:${{ inputs.commit_tag }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
     steps:
+      - name: Set develop tag if main branch
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: |
+          echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV
       - name: Checkout
         uses: actions/checkout@v4
       - uses: actions/setup-java@v4
@@ -54,29 +62,53 @@ jobs:
         run: |
           ./gradlew coordinator:app:distZip --no-daemon
       - name: Login to Docker Hub
+        if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
+      - name: Set up Docker Buildx - local
         id: buildx
         uses: docker/setup-buildx-action@v3
       - name: Docker meta
         id: coordinator
         uses: docker/metadata-action@v3
         with:
           images: ${{ env.IMAGE_NAME }}
+      - name: Build for testing
+        uses: docker/build-push-action@v6
+        if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }}
+        with:
+          context: .
+          build-contexts: zip=./coordinator/app/build/distributions/
+          file: ./coordinator/Dockerfile
+          platforms: linux/amd64
+          load: true
+          push: false
+          tags: ${{ env.TAGS }}
+      - name: Save Docker image as artifact
+        if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }}
+        run: |
+          docker save ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }} | gzip > linea-coordinator-docker-image.tar.gz
+        shell: bash
+      - name: Upload Docker image artifact
+        if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: linea-coordinator
+          path: linea-coordinator-docker-image.tar.gz
       - name: Build & push
         uses: docker/build-push-action@v6
+        if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }}
         with:
           context: .
           build-contexts: zip=./coordinator/app/build/distributions/
           file: ./coordinator/Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: |
-            ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }}-${{ env.UNTESTED_TAG_SUFFIX }}
+          tags: ${{ env.TAGS }}
           cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache
           cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max
+
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
@@ -8,9 +8,9 @@ on:
         type: string
     secrets:
       DOCKERHUB_USERNAME:
-        required: true
+        required: false
       DOCKERHUB_TOKEN:
-        required: true
+        required: false
 
 concurrency:
   group: coordinator-testing-${{ github.workflow }}-${{ github.ref }}
@@ -24,6 +24,8 @@ jobs:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
       GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
     runs-on: [self-hosted, ubuntu-22.04, X64, medium]
     name: Coordinator tests
     steps:
@@ -52,6 +54,7 @@ jobs:
         run: |
           ./gradlew -V coordinator:app:buildNeeded
       - name: Login to Docker Hub
+        if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}