Artifacts mono

.codespellrc

@@ -0,0 +1,6 @@
+[codespell]
+skip = .git,*.pdf,*.svg,go.sum,go.mod,*requirements.txt,gen
+# some strings with unicodes, constructs like [o]utput
+ignore-regex = ".*\\0[0-9][0-9].*"|json:"[^"]*"|\b[a-z]*\[[a-z]\][a-z]*\b
+# some ad-hoc variable names etc
+ignore-words-list = te,nd,querys,ser

.github/workflows/bump-tags.yml

@@ -0,0 +1,57 @@
+name: Bump git tags
+
+on:
+  workflow_call:
+    outputs:
+      next_version:
+        description: "Tag name"
+        value: ${{ jobs.bump_version.outputs.next_version }}
+    secrets:
+      FLYTE_BOT_PAT:
+        required: true
+jobs:
+  bump_version:
+    name: Bump Version
+    runs-on: ubuntu-latest
+    outputs:
+      next_version: ${{ steps.get_semver.outputs.next_version }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: '0'
+      # Bump patch version
+      - uses: rickstaa/action-get-semver@v1
+        id: get_semver
+        with:
+          bump_level: "patch"
+      - name: Print current and next versions
+        run: |
+          echo "Current version: ${{ steps.get_semver.outputs.current_version  }}"
+          echo "Next version: ${{ steps.get_semver.outputs.next_version }}"
+      # Generate all tags for all components
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/${{ steps.get_semver.outputs.next_version }}`,
+              sha: context.sha
+            })
+            const components = [
+              "datacatalog",
+              "flyteadmin",
+              "flytecopilot",
+              "flyteplugins",
+              "flytepropeller",
+              "flytestdlib",
+            ];
+            for (const c of components) {
+              github.rest.git.createRef({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                ref: `refs/tags/${c}/${{ steps.get_semver.outputs.next_version }}`,
+                sha: context.sha
+              })
+            }

.github/workflows/checks.yml

@@ -0,0 +1,179 @@
+name: Components Checks
+
+on:
+  pull_request:
+    paths:
+      - 'datacatalog/**'
+      - 'flyteadmin/**'
+      - 'flytecopilot/**'
+      - 'flyteidl/**'
+      - 'flyteplugins/**'
+      - 'flytepropeller/**'
+      - 'flytestdlib/**'
+  push:
+    branches:
+      - master
+    paths:
+      - 'datacatalog/**'
+      - 'flyteadmin/**'
+      - 'flytecopilot/**'
+      - 'flyteidl/**'
+      - 'flyteplugins/**'
+      - 'flytepropeller/**'
+      - 'flytestdlib/**'
+env:
+  GO_VERSION: "1.19"
+  PRIORITIES: "P0"
+jobs:
+  unpack-envvars:
+    runs-on: ubuntu-latest
+    outputs:
+      go-version: ${{ steps.step.outputs.go-version }}
+    steps:
+      - id: step
+        run: |
+          echo "go-version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT
+
+  lint:
+    name: Lint
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          # TODO(monorepo): Enable lint flytecopilot
+          # - flytecopilot
+          - flyteidl
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    uses: ./.github/workflows/lint.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+  unit-tests:
+    name: Unit Tests
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          # TODO(monorepo): Enable flyteidl unit tests
+          # - flyteidl
+          - flyteplugins
+          - flytepropeller
+          - flytestdlib
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+  docker-build:
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+    name: Docker Build Images
+    uses: ./.github/workflows/component_docker_build.yml
+    with:
+       component: ${{ matrix.component }}
+
+  # TODO(monorepo): these tests are broken. They never test an actual change.
+  # endtoend:
+  #   name: End2End Test
+  #   needs: [ docker-build ]
+  #   uses: ./.github/workflows/end2end.yml
+  #   with:
+  #     # Reusing the output of the matrix is ok as they are essentially writing the same value (i.e. the directory artifacts are written + run id)
+  #     cache_key: ${{ needs.docker-build.outputs.cache_key }}
+  #     priorities: "P0"
+
+  integration:
+    name: Integration Test
+    needs:
+      - docker-build
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - flyteadmin
+    uses: ./.github/workflows/integration.yml
+    with:
+      component: ${{ matrix.component }}
+      cache_key: ${{ needs.docker-build.outputs.cache_key }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+
+  generate:
+    name: Check Go Generate
+    needs:
+      - unpack-envvars
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+    uses: ./.github/workflows/go_generate.yml
+    with:
+      component: ${{ matrix.component }}
+      go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+
+  bump-tags:
+    name: Bump git tags
+    # TODO(monorepo): skip this if author is flyte-bot?
+    if: ${{ github.event_name != 'pull_request' }}
+    needs:
+      - integration
+      - lint
+      - unit-tests
+      - generate
+    uses: ./.github/workflows/bump-tags.yml
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+
+  # TODO(monorepo): we are not going to release any binary
+  # goreleaser:
+  #   name: Goreleaser
+  #   needs: [ bump_version ] # Only to ensure it can successfully build
+  #   uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master
+  #   with:
+  #     go-version: "1.19"
+  #   secrets:
+  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+
+  push_docker_image:
+    name: Build & Push Image
+    needs: [ bump-tags ]
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - datacatalog
+          - flyteadmin
+          - flytecopilot
+          - flytepropeller
+          - flytescheduler
+    uses: ./.github/workflows/publish.yml
+    with:
+      version: ${{ needs.bump-tags.outputs.next_version }}
+      component: ${{ matrix.component }}
+      dockerfile: Dockerfile.${{ matrix.component }}
+      push: true
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}

.github/workflows/codespell.yml

@@ -0,0 +1,22 @@
+---
+name: Codespell
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Codespell
+        uses: codespell-project/actions-codespell@v2

.github/workflows/component_docker_build.yml

@@ -0,0 +1,49 @@
+name: Build Docker Image
+
+on:
+  workflow_call:
+    inputs:
+      component:
+        required: true
+        type: string
+    outputs:
+      cache_key:
+        description: "Docker Cache key"
+        value: ${{ jobs.build_docker.outputs.cache_key }}
+jobs:
+  build_docker:
+    runs-on: ubuntu-latest
+    outputs:
+      cache_key: ${{ steps.cache_key.outputs.cache_key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/tmp/docker-images-${{ inputs.component }}
+          key: /tmp/docker-images-${{ github.run_id }}-${{ inputs.component }}
+          restore-keys: |
+            /tmp/docker-images-
+      - name: Set cache key output
+        id: cache_key
+        run: |
+          echo ::set-output name=cache_key::/tmp/docker-images-${{ github.run_id }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        env:
+          # We are unable to leverage docker buildx here without changing the
+          # caching mechanism significantly. See here for the caching options
+          # available for docker buildx: https://docs.docker.com/engine/reference/commandline/buildx_build/#cache-from
+          # For now at least enable DOCKER_BUILDKIT for faster builds. Eventually we
+          # should rewrite this pipeline to use docker buildx with cache type=gha.
+          DOCKER_BUILDKIT: "1"
+        run: |
+          docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:builder --target builder --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} .
+          docker build -t ${{ github.repository_owner }}/${{ inputs.component }}:latest --cache-from=${{ github.repository_owner }}/${{ inputs.component }}:builder --file Dockerfile.${{ inputs.component }} .
+
+      - name: Tag and cache docker image
+        run: |
+          mkdir -p /tmp/tmp/docker-images-${{ inputs.component }} && docker save ${{ github.repository_owner }}/${{ inputs.component }}:builder -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-builder-${{ inputs.component }}.tar && docker save ${{ github.repository_owner }}/${{ inputs.component }}:latest -o /tmp/tmp/docker-images-${{ inputs.component }}/snapshot-${{ inputs.component }}.tar

.github/workflows/create_release.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Tag image to release version
         run: |
           for release in latest ${{ needs.bump-version.outputs.version }}; do
-            docker buildx imagetools create --tag "ghcr.io/flyteorg/flyte-binary-release:${release}" "ghcr.io/flyteorg/flyte-binary:sha-${{ github.sha }}"
+            docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/flyte-binary-release:${release}" "ghcr.io/${{ github.repository_owner }}/flyte-binary:sha-${{ github.sha }}"
           done
 
   publish-flyte-component-image:
@@ -93,7 +93,7 @@ jobs:
       - name: Tag Image to release version
         run: |
           for release in latest ${{ needs.bump-version.outputs.version }}; do
-            docker buildx imagetools create --tag "ghcr.io/flyteorg/${{ matrix.component }}-release:${release}" "ghcr.io/flyteorg/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
+            docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}-release:${release}" "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
           done
 
   helm-release:
@@ -116,7 +116,7 @@ jobs:
         run: |
           git config user.name "${{ github.actor }}"
           git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
-      - name: Preprare Flyte Helm Release
+      - name: Prepare Flyte Helm Release
         env:
           VERSION: ${{ needs.bump-version.outputs.version }}
           REPOSITORY: "https://flyteorg.github.io/flyte"
@@ -138,7 +138,7 @@ jobs:
         with:
           fetch-depth: "0"
 
-      - name: Preprare Flyte Release
+      - name: Prepare Flyte Release
         env:
           VERSION: ${{ needs.bump-version.outputs.version }}
         run: |