Merge remote-tracking branch 'origin' into union/upstream

Signed-off-by: Eduardo Apolinario <[email protected]>

.github/workflows/single-binary.yml

@@ -175,7 +175,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install uv
-          uv pip install --system flytekit flytekitplugins-deck-standard "numpy<2.0.0" pyarrow
+          uv pip install --system flytekit flytekitplugins-deck-standard "numpy<2.0.0" pyarrow pandas
           uv pip freeze
       - name: Checkout flytesnacks
         uses: actions/checkout@v4
@@ -188,7 +188,7 @@ jobs:
           flytekit_version=$(pip show flytekit | grep -i version | awk '{ print $2 }')
           while read -r line;
           do
-              pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
+              pyflyte -vv --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \

.github/workflows/tests.yml

@@ -69,6 +69,9 @@ jobs:
       - name: Build the documentation
         working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
+        env:
+          # this is a fake key for algolia docsearch to make docs build process pass
+          DOCSEARCH_API_KEY: fake_docsearch_api_key
         run: |
           conda activate monodocs-env
           make docs

.gitignore

@@ -30,12 +30,11 @@ vendor/
 /docker/sandbox-bundled/images/tar
 **/bin/
 docs/_tags/
-docs/flytectl
-docs/protos
-docs/flytekit
+docs/api/flytectl
+docs/api/flytekit
+docs/api/flyteidl
 docs/flytesnacks
 docs/examples
 docs/_src
 docs/_projects
-docs/api
 docs/tests

CHANGELOG/CHANGELOG-v1.13.2.md

@@ -0,0 +1,53 @@
+# Flyte 1.13.2 Release Notes
+
+## What's Changed
+* Enable echo plugin by default by @pingsutw in https://github.com/flyteorg/flyte/pull/5679
+* Do not emit execution id label by default in single binary by @eapolinario in https://github.com/flyteorg/flyte/pull/5704
+* Using new offloaded metadata literal message for literal offloading by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5705
+* Improve error message for MismatchingTypes by @pingsutw in https://github.com/flyteorg/flyte/pull/5639
+* [Docs] Echo Task by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5707
+* Improve execution name readability by @wayner0628 in https://github.com/flyteorg/flyte/pull/5637
+* Configure imagePullPolicy to be Always pull on flyte sandbox environment by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5709
+* Set IsDefault to False for echo plugin by @pingsutw in https://github.com/flyteorg/flyte/pull/5713
+* Move default execution name generation to flyteadmin by @wayner0628 in https://github.com/flyteorg/flyte/pull/5714
+* Update helm/docs per changes in supported task discovery by @Sovietaced in https://github.com/flyteorg/flyte/pull/5694
+* [flyteagent] Add Logging for Agent Supported Task Types by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5718
+* extend pod customization to include init containers by @samhita-alla in https://github.com/flyteorg/flyte/pull/5685
+* Update "Try Serverless" language in Quickstart guide by @neverett in https://github.com/flyteorg/flyte/pull/5698
+* Refactor flyteadmin to pass proto structs as pointers by @Sovietaced in https://github.com/flyteorg/flyte/pull/5717
+* fix: Use deterministic execution names in scheduler by @pingsutw in https://github.com/flyteorg/flyte/pull/5724
+* [flyteagent] Enable `agent-service` in every configuration  by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5730
+* [Docs][flyteagent] Remove `agent-service` config in flyte agent documentation by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5731
+* Fix flytectl returning the oldest workflow when using --latest flag by @RRap0so in https://github.com/flyteorg/flyte/pull/5716
+* Remove explicit go toolchain versions by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5723
+* Add listing api to stow storage by @bgedik in https://github.com/flyteorg/flyte/pull/5741
+* Use latest upload/download-artifact action version by @Sovietaced in https://github.com/flyteorg/flyte/pull/5743
+* Introduced SMTP notification by @robert-ulbrich-mercedes-benz in https://github.com/flyteorg/flyte/pull/5535
+* Added literal offloading for array node map tasks by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5697
+* pytorch object.inv moved by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5755
+* [RFC] Binary IDL With MessagePack Bytes by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5742
+* Rename literal offloading flags and enable offloading in local config by @eapolinario in https://github.com/flyteorg/flyte/pull/5754
+* Add semver check for dev and beta version by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5757
+* Add comments for semver regex by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5758
+* Enable caching in flyte-core helmchart by @cpaulik in https://github.com/flyteorg/flyte/pull/5739
+* [Flyte][1][IDL] Binary IDL With MessagePack by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5751
+* [Flyte][2][Literal Type For Scalar] Binary IDL With MessagePack by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5761
+* Backoff on etcd errors by @EngHabu in https://github.com/flyteorg/flyte/pull/5710
+* [Docs][flyteagent] Update Databricks Agent Setup to V2 by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5766
+* [Docs][flyteagent] Improve Agent Secret Setup by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5765
+* [RFC] Offloaded Raw Literals by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5103
+* Use pluggable clock for auto refresh cache and make unit tests fast by @Sovietaced in https://github.com/flyteorg/flyte/pull/5767
+* [Docs] Flyte Deck example by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5777
+* Update ImageSpec documentation by @pingsutw in https://github.com/flyteorg/flyte/pull/5748
+* Revert "Improve execution name readability" by @pingsutw in https://github.com/flyteorg/flyte/pull/5740
+* [Docs] Flyte Deck example V2 by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5781
+* [flytepropeller][flyteadmin] Compiler unknown literal type error handling by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5651
+* [Temporary fix] Pin flytekit version for single binary builds by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5798
+* Fix propeller crash when inferring literal type for an offloaded literal by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5771
+
+## New Contributors
+* @wayner0628 made their first contribution in https://github.com/flyteorg/flyte/pull/5637
+* @robert-ulbrich-mercedes-benz made their first contribution in https://github.com/flyteorg/flyte/pull/5535
+* @cpaulik made their first contribution in https://github.com/flyteorg/flyte/pull/5739
+
+**Full Changelog**: https://github.com/flyteorg/flyte/compare/v1.13.1...v1.13.2

CODEOWNERS

@@ -1,3 +1,3 @@
 # Automatically request docs team for docs PR review
-/docs/          @neverett @ppiegaze
-
+/docs/              @neverett @ppiegaze
+/docs/deployment/   @davidmirror-ops

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 # Contributing to Flyte
 
-For information related to contributing to Flyte, please check out the [Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute.html) section of the documentation.
+For information related to contributing to Flyte, please check out the [Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute/index.html) section of the documentation.

charts/flyte-binary/README.md

@@ -41,7 +41,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.13.1"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.13.2"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |

charts/flyte-binary/values.yaml

@@ -159,7 +159,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot  # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.13.1  # FLYTECOPILOT_TAG
+      tag: v1.13.2  # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent:

charts/flyte-core/README.md

@@ -95,8 +95,8 @@ helm install gateway bitnami/contour -n flyte
 | configmap.clusters.clusterConfigs | list | `[]` |  |
 | configmap.clusters.labelClusterMap | object | `{}` |  |
 | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI |
-| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
-| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
+| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
+| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
 | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
 | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). |
 | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
@@ -130,7 +130,7 @@ helm install gateway bitnami/contour -n flyte
 | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment |
-| datacatalog.image.tag | string | `"v1.13.1"` | Docker image tag |
+| datacatalog.image.tag | string | `"v1.13.2"` | Docker image tag |
 | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment |
 | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods |
 | datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables |
@@ -166,7 +166,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command |
 | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment |
-| flyteadmin.image.tag | string | `"v1.13.1"` |  |
+| flyteadmin.image.tag | string | `"v1.13.2"` |  |
 | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create |
 | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment |
 | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods |
@@ -237,7 +237,7 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment |
-| flytepropeller.image.tag | string | `"v1.13.1"` |  |
+| flytepropeller.image.tag | string | `"v1.13.2"` |  |
 | flytepropeller.manager | bool | `false` |  |
 | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment |
 | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods |
@@ -269,7 +269,7 @@ helm install gateway bitnami/contour -n flyte
 | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files |
 | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment |
-| flytescheduler.image.tag | string | `"v1.13.1"` | Docker image tag |
+| flytescheduler.image.tag | string | `"v1.13.2"` | Docker image tag |
 | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment |
 | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods |
 | flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables |

charts/flyte-core/values.yaml

@@ -16,7 +16,7 @@ flyteadmin:
   image:
     # -- Docker image for Flyteadmin deployment
     repository: cr.flyte.org/flyteorg/flyteadmin  # FLYTEADMIN_IMAGE
-    tag: v1.13.1  # FLYTEADMIN_TAG
+    tag: v1.13.2  # FLYTEADMIN_TAG
     pullPolicy: IfNotPresent
   # -- Additional flyteadmin container environment variables
   #
@@ -144,7 +144,7 @@ flytescheduler:
     # -- Docker image for Flytescheduler deployment
     repository: cr.flyte.org/flyteorg/flytescheduler  # FLYTESCHEDULER_IMAGE
     # -- Docker image tag
-    tag: v1.13.1  # FLYTESCHEDULER_TAG
+    tag: v1.13.2  # FLYTESCHEDULER_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytescheduler deployment
@@ -210,7 +210,7 @@ datacatalog:
     # -- Docker image for Datacatalog deployment
     repository: cr.flyte.org/flyteorg/datacatalog  # DATACATALOG_IMAGE
     # -- Docker image tag
-    tag: v1.13.1  # DATACATALOG_TAG
+    tag: v1.13.2  # DATACATALOG_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Datacatalog deployment
@@ -307,7 +307,7 @@ flytepropeller:
   image:
     # -- Docker image for Flytepropeller deployment
     repository: cr.flyte.org/flyteorg/flytepropeller  # FLYTEPROPELLER_IMAGE
-    tag: v1.13.1  # FLYTEPROPELLER_TAG
+    tag: v1.13.2  # FLYTEPROPELLER_TAG
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytepropeller deployment
   resources:
@@ -801,7 +801,7 @@ configmap:
         # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig)
         co-pilot:
           name: flyte-copilot-
-          image: cr.flyte.org/flyteorg/flytecopilot:v1.13.1  # FLYTECOPILOT_IMAGE
+          image: cr.flyte.org/flyteorg/flytecopilot:v1.13.2  # FLYTECOPILOT_IMAGE
           start-timeout: 30s
 
   # -- Core propeller configuration