Merge branch 'master' into patch-1

Signed-off-by: Eduardo Apolinario <[email protected]>
flyteorg · Dec 26, 2024 · 0023f53 · 0023f53
2 parents 268c1a4 + 7aeaa26
commit 0023f53
Show file tree

Hide file tree

Showing 855 changed files with 23,623 additions and 14,207 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -9,6 +9,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
 env:
   GO_VERSION: "1.22"
   PRIORITIES: "P0"
@@ -32,8 +33,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
-          # TODO(monorepo): Enable lint flytecopilot
-          # - flytecopilot
+          - flytecopilot
           - flytectl
           - flyteidl
           - flyteplugins

diff --git a/.github/workflows/flytectl-install.yml b/.github/workflows/flytectl-install.yml
@@ -8,6 +8,7 @@ on:
   pull_request:
     paths:
       - flytectl/**
+      - 'release-v**'
   push:
     branches:
       - master

diff --git a/.github/workflows/flyteidl-buf-publish.yml b/.github/workflows/flyteidl-buf-publish.yml
@@ -6,6 +6,7 @@ on:
       - artifacts-shell-2
       - artifacts
       - master
+      - 'release-v**'
     paths:
       - 'flyteidl/**'
 jobs:

diff --git a/.github/workflows/flyteidl-checks.yml b/.github/workflows/flyteidl-checks.yml
@@ -9,6 +9,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
 env:
   GO_VERSION: "1.22"
 jobs:

diff --git a/.github/workflows/helm-charts.yaml b/.github/workflows/helm-charts.yaml
@@ -10,6 +10,7 @@ on:
     branches:
       - master
       - rc/*
+      - 'release-v**'
   workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -59,7 +59,7 @@ jobs:
       - name: Before Build
         run: ${{ inputs.before-build }}
       - name: Build and Push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ inputs.dockerfile }}

diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
@@ -53,7 +53,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64

diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -10,6 +10,7 @@ on:
     branches:
       - master
       - rc/*
+      - 'release-v**'
   workflow_dispatch:
 
 jobs:
@@ -72,7 +73,7 @@ jobs:
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64
@@ -83,7 +84,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64
@@ -107,7 +108,7 @@ jobs:
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push Image
         if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -143,7 +144,7 @@ jobs:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Build sandbox image for functional tests
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           load: true
@@ -158,6 +159,8 @@ jobs:
         with:
           python-version: "3.12"
       - uses: unionai/[email protected]
+        with:
+          version: '0.9.2'
       - name: Setup sandbox
         run: |
           mkdir -p ~/.flyte/sandbox
@@ -196,6 +199,9 @@ jobs:
                   --version ${{ env.FLYTESNACKS_VERSION }} \
                   flytesnacks/$line;
           done < flytesnacks/flyte_tests.txt
+      - name: Install Pytest
+        run: |
+          pip install pytest
       - name: End2End
         run: |
           make end2end_execute
@@ -239,7 +245,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push multi-arch image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,6 +8,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
   pull_request:
 jobs:
   compile:

diff --git a/.github/workflows/validate-helm-charts.yaml b/.github/workflows/validate-helm-charts.yaml
@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - 'release-v**'
     paths:
       - "charts/**"
       - "deployment/**"

diff --git a/.gitignore b/.gitignore
@@ -38,3 +38,4 @@ docs/examples
 docs/_src
 docs/_projects
 docs/tests
+empty-config.yaml
diff --git a/CHANGELOG/CHANGELOG-v1.14.0.md b/CHANGELOG/CHANGELOG-v1.14.0.md
diff --git a/CHANGELOG/CHANGELOG-v1.14.1.md b/CHANGELOG/CHANGELOG-v1.14.1.md
@@ -0,0 +1,19 @@
+# Flyte 1.14.1 Release Notes
+
+* Update flytestdlib and affected tools (copilot) for missing config.
+
+## What's Changed
+* docs: Refactor merge sort code example to use literalinclude by @davidlin20dev in https://github.com/flyteorg/flyte/pull/6091
+* [DOCS] Using ImageSpec in ContainerTask by @machichima in https://github.com/flyteorg/flyte/pull/6095
+* Eager doc updates by @wild-endeavor in https://github.com/flyteorg/flyte/pull/6099
+* Revert "fix: return the config file not found error" by @eapolinario in https://github.com/flyteorg/flyte/pull/6100
+* Remove notes on deprecated Batch size by @wild-endeavor in https://github.com/flyteorg/flyte/pull/6102
+* Upstream: Add labels to published execution events by @katrogan in https://github.com/flyteorg/flyte/pull/6104
+* Fix: Make distributed error aggregation opt-in by @fg91 in https://github.com/flyteorg/flyte/pull/6103
+* Add default labels and annotations to Ray worker pods too. by @katrogan in https://github.com/flyteorg/flyte/pull/6107
+* Fix: Remove the default search dialog if it exists (on CMD + K) by @chmod77 in https://github.com/flyteorg/flyte/pull/6106
+
+## New Contributors
+* @chmod77 made their first contribution in https://github.com/flyteorg/flyte/pull/6106
+
+**Full Changelog**: https://github.com/flyteorg/flyte/compare/v1.14.0...v1.14.1
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1,3 +1,3 @@
 # Automatically request docs team for docs PR review
-/docs/              @neverett @ppiegaze
+/docs/              @ppiegaze
 /docs/deployment/   @davidmirror-ops
diff --git a/Dockerfile b/Dockerfile
@@ -5,8 +5,8 @@ FROM ghcr.io/flyteorg/flyteconsole:${FLYTECONSOLE_VERSION} AS flyteconsole
 FROM --platform=${BUILDPLATFORM} golang:1.22-bookworm AS flytebuilder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 WORKDIR /flyteorg/build
 
@@ -29,10 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/r
 FROM debian:bookworm-slim
 
 ARG FLYTE_VERSION
-ENV FLYTE_VERSION "${FLYTE_VERSION}"
+ENV FLYTE_VERSION="${FLYTE_VERSION}"
 
-ENV DEBCONF_NONINTERACTIVE_SEEN true
-ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+ENV DEBIAN_FRONTEND=noninteractive
 
 # Install core packages
 RUN apt-get update && apt-get install --no-install-recommends --yes \

diff --git a/Dockerfile.datacatalog b/Dockerfile.datacatalog
@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 

diff --git a/Dockerfile.flyteadmin b/Dockerfile.flyteadmin
@@ -3,11 +3,11 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -41,7 +41,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin
 

diff --git a/Dockerfile.flytecopilot b/Dockerfile.flytecopilot
@@ -3,12 +3,12 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -32,7 +32,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/lyft/flyteplugins
+LABEL org.opencontainers.image.source=https://github.com/lyft/flyteplugins
 
 COPY --from=builder /artifacts /bin
 

diff --git a/Dockerfile.flytepropeller b/Dockerfile.flytepropeller
@@ -4,12 +4,12 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -33,7 +33,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flytepropeller
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytepropeller
 
 COPY --from=builder /artifacts /bin
 

diff --git a/Dockerfile.flytescheduler b/Dockerfile.flytescheduler
@@ -4,12 +4,12 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.22-alpine3.18 AS builder
 
 
 ARG TARGETARCH
-ENV GOARCH "${TARGETARCH}"
-ENV GOOS linux
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
 
 RUN apk add git openssh-client make curl
 
@@ -36,7 +36,7 @@ ENV PATH="/artifacts:${PATH}"
 
 # This will eventually move to centurylink/ca-certs:latest for minimum possible image size
 FROM alpine:3.18
-LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flyteadmin
 
 COPY --from=builder /artifacts /bin
 

diff --git a/Makefile b/Makefile
@@ -6,10 +6,10 @@ define PIP_COMPILE
 pip-compile $(1) --upgrade --verbose --resolver=backtracking --annotation-style=line
 endef
 
-GIT_VERSION := $(shell git describe --always --tags)
+GIT_VERSION := $(shell git describe --tags --long --match "v*" --first-parent)
 GIT_HASH := $(shell git rev-parse --short HEAD)
 TIMESTAMP := $(shell date '+%Y-%m-%d')
-PACKAGE ?=github.com/flyteorg/flytestdlib
+PACKAGE ?=github.com/flyteorg/flyte/flytestdlib
 LD_FLAGS="-s -w -X $(PACKAGE)/version.Version=$(GIT_VERSION) -X $(PACKAGE)/version.Build=$(GIT_HASH) -X $(PACKAGE)/version.BuildTime=$(TIMESTAMP)"
 TMP_BUILD_DIR := .tmp_build
 
@@ -135,6 +135,7 @@ go-tidy:
 	make -C flyteplugins go-tidy
 	make -C flytestdlib go-tidy
 	make -C flytecopilot go-tidy
+	make -C flytectl go-tidy
 
 .PHONY: lint-helm-charts
 lint-helm-charts:

diff --git a/README.md b/README.md
diff --git a/boilerplate/flyte/end2end/Makefile b/boilerplate/flyte/end2end/Makefile
@@ -7,8 +7,12 @@
 end2end_execute: export FLYTESNACKS_PRIORITIES ?= P0
 end2end_execute: export FLYTESNACKS_VERSION ?= $(shell curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name)
 end2end_execute:
-	./boilerplate/flyte/end2end/end2end.sh ./boilerplate/flyte/end2end/functional-test-config.yaml --return_non_zero_on_failure
-
+	pytest ./boilerplate/flyte/end2end/test_run.py \
+		--flytesnacks_release_tag=$(FLYTESNACKS_VERSION) \
+		--priorities=$(FLYTESNACKS_PRIORITIES) \
+		--config_file=./boilerplate/flyte/end2end/functional-test-config.yaml \
+		--return_non_zero_on_failure
+
 .PHONY: k8s_integration_execute
 k8s_integration_execute:
 	echo "pass"
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,6 +9,7 @@ on: @@
       push:
         branches:
           - master
+          - 'release-v**'
     env:
       GO_VERSION: "1.22"
     jobs:
@@ Expand Down @@