Enable literal offloading in flyte-binary and flyte-core (#6087)

* Enable literal offloading in flyte-binary and flyte-core Signed-off-by: Eduardo Apolinario <[email protected]> * Expose flag in flyte-binary values file Signed-off-by: Eduardo Apolinario <[email protected]> --------- Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
flyteorg · Dec 5, 2024 · 7fcbdf9 · 7fcbdf9
1 parent de12c82
commit 7fcbdf9
Show file tree

Hide file tree

Showing 16 changed files with 52 additions and 24 deletions.
diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
@@ -64,6 +64,7 @@ Chart for basic single Flyte executable deployment
 | configuration.logging.plugins.stackdriver.enabled | bool | `false` |  |
 | configuration.logging.plugins.stackdriver.templateUri | string | `""` |  |
 | configuration.propeller.createCRDs | bool | `true` |  |
+| configuration.propeller.literalOffloadingConfigEnabled | bool | `true` |  |
 | configuration.storage.metadataContainer | string | `"my-organization-flyte-container"` |  |
 | configuration.storage.provider | string | `"s3"` |  |
 | configuration.storage.providerConfig.azure.account | string | `"storage-account-name"` |  |

diff --git a/charts/flyte-binary/templates/configmap.yaml b/charts/flyte-binary/templates/configmap.yaml
@@ -41,6 +41,10 @@ data:
       level: {{ default 1 .Values.configuration.logging.level }}
     propeller:
       create-flyteworkflow-crd: {{ .Values.configuration.propeller.createCRDs }}
+      {{- if .Values.configuration.propeller.literalOffloadingConfigEnabled }}
+      literal-offloading-config:
+        enabled: true
+      {{- end}}
     webhook:
       certDir: /var/run/flyte/certs
       localCert: true

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -180,6 +180,9 @@ configuration:
   propeller:
     # createCRDs If true, Propeller will install CRDs at runtime, if false, CRDs will be installed during helm install
     createCRDs: true
+    # enableOffloading If true, big literals are offloaded to blob store
+    literalOffloadingConfigEnabled: true
+
   # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte
   # If set, no Flyte configuration will be generated by this chart
   externalConfigMap: ""

diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md
@@ -112,9 +112,9 @@ helm install gateway bitnami/contour -n flyte
 | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI |
 | configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
 | configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.13.2","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
-| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
+| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","literal-offloading-config":{"enabled":true},"max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
 | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). |
-| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
+| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","literal-offloading-config":{"enabled":true},"max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
 | configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"heartbeat-grace-period-multiplier":3,"max-reservation-heartbeat":"30s","metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config |
 | configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. |
 | configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service","echo"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) |

diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml
@@ -851,6 +851,8 @@ configmap:
           type: bucket
           rate: 10
           capacity: 100
+      literal-offloading-config:
+        enabled: true
     webhook:
       certDir: /etc/webhook/certs
       serviceName: flyte-pod-webhook

diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml
@@ -461,6 +461,8 @@ data:
         renew-deadline: 10s
         retry-period: 2s
       limit-namespace: all
+      literal-offloading-config:
+        enabled: true
       max-workflow-retries: 50
       metadata-prefix: metadata/propeller
       metrics-prefix: flyte
@@ -1278,7 +1280,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
+        configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -1362,7 +1364,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.2
       annotations:
-        configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
+        configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml
@@ -119,6 +119,8 @@ data:
         renew-deadline: 10s
         retry-period: 2s
       limit-namespace: all
+      literal-offloading-config:
+        enabled: true
       max-workflow-retries: 50
       metadata-prefix: metadata/propeller
       metrics-prefix: flyte
@@ -428,7 +430,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
+        configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -512,7 +514,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.2
       annotations:
-        configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
+        configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml
@@ -492,6 +492,8 @@ data:
         renew-deadline: 10s
         retry-period: 2s
       limit-namespace: all
+      literal-offloading-config:
+        enabled: true
       max-workflow-retries: 50
       metadata-prefix: metadata/propeller
       metrics-prefix: flyte
@@ -1408,7 +1410,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
+        configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -1492,7 +1494,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.2
       annotations:
-        configChecksum: "6572aa999f8e6842b4dba120e12e6ccb8cdfa506373de2a267b62a63146ccde"
+        configChecksum: "d1a06e03d8567840a12eb893aee369cfe8b658cc36c3a307684b00811a35803"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml
@@ -119,6 +119,8 @@ data:
         renew-deadline: 10s
         retry-period: 2s
       limit-namespace: all
+      literal-offloading-config:
+        enabled: true
       max-workflow-retries: 50
       metadata-prefix: metadata/propeller
       metrics-prefix: flyte
@@ -436,7 +438,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
+        configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -519,7 +521,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.2
       annotations:
-        configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
+        configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml
@@ -505,6 +505,8 @@ data:
         renew-deadline: 10s
         retry-period: 2s
       limit-namespace: all
+      literal-offloading-config:
+        enabled: true
       max-workflow-retries: 50
       metadata-prefix: metadata/propeller
       metrics-prefix: flyte
@@ -1431,7 +1433,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
+        configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -1514,7 +1516,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.2
       annotations:
-        configChecksum: "8562f7f608d4936e13f6ad70c18c7c095068e742243e7f380f89694d2182110"
+        configChecksum: "6ea7ca6e738c35f382aaf08e3256e6c40fbdff3d9b87e0e7fdd680ff6f26393"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml
@@ -79,6 +79,8 @@ data:
       level: 1
     propeller:
       create-flyteworkflow-crd: true
+      literal-offloading-config:
+        enabled: true
     webhook:
       certDir: /var/run/flyte/certs
       localCert: true
@@ -363,7 +365,7 @@ spec:
         app.kubernetes.io/instance: flyte
         app.kubernetes.io/component: flyte-binary
       annotations:
-        checksum/configuration: 886440a42b3eeec802cfe60d37885f69e35ffd83e53e625b3c877da5e8c7eb38
+        checksum/configuration: 4032606d756c88462f340dbaaa8d2f78434fb4aaf63d07c77e8b49430b6a6368
         checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
         checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
     spec:

diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml
@@ -612,6 +612,8 @@ data:
         renew-deadline: 10s
         retry-period: 2s
       limit-namespace: all
+      literal-offloading-config:
+        enabled: true
       max-workflow-retries: 30
       metadata-prefix: metadata/propeller
       metrics-prefix: flyte
@@ -7182,7 +7184,7 @@ spec:
   template:
     metadata:
       annotations:
-        configChecksum: "7ab9aee83ad8109354235eee7f46c3f091d7c70cd55157a195f4997d247f933"
+        configChecksum: "3f4f81a5040b0a69e9a9755baabda29ab7a3c36275036b9ca5a17b3ec910c31"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
       labels: 
@@ -7258,7 +7260,7 @@ spec:
         app.kubernetes.io/name: flyte-pod-webhook
         app.kubernetes.io/version: v1.13.2
       annotations:
-        configChecksum: "7ab9aee83ad8109354235eee7f46c3f091d7c70cd55157a195f4997d247f933"
+        configChecksum: "3f4f81a5040b0a69e9a9755baabda29ab7a3c36275036b9ca5a17b3ec910c31"
         prometheus.io/path: "/metrics"
         prometheus.io/port: "10254"
     spec:

diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml
@@ -431,6 +431,8 @@ data:
       level: 5
     propeller:
       create-flyteworkflow-crd: true
+      literal-offloading-config:
+        enabled: true
     webhook:
       certDir: /var/run/flyte/certs
       localCert: true
@@ -821,7 +823,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: ZncyUmNBaWdYWlJmTUVMVA==
+  haSharedSecret: dWJKZFZBWWtWSGM4ajFZUw==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1252,7 +1254,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
-        checksum/configuration: 73e77e790b0ce72a7f3f7a81e4ca3f279f33c1aaea5e5172f61959f3960b0d7b
+        checksum/configuration: 4554674e2366a0ec10044a41caab60124bf69e1988b7dd3d0550d89808622e3a
         checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
       labels:
         app.kubernetes.io/component: flyte-binary
@@ -1418,7 +1420,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: f2f7afffa66c57f22b3c423e1ec672e533bededf3848df42ae23e805da175146
+        checksum/secret: fb07b2b50b56c5acaf388e9b1cc802fc9dc8ed8bcf4b1ffab1423e85f7abc78b
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml
@@ -420,6 +420,8 @@ data:
       level: 5
     propeller:
       create-flyteworkflow-crd: true
+      literal-offloading-config:
+        enabled: true
     webhook:
       certDir: /var/run/flyte/certs
       localCert: true
@@ -803,7 +805,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: VWxGQXNKU1RyMk5haUpZWA==
+  haSharedSecret: NTlsaHl4YVdGVmJjcXlkdA==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1201,7 +1203,7 @@ spec:
     metadata:
       annotations:
         checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035
-        checksum/configuration: 956d7a4502a3f977583a7b1bf4da2e985d64c09c94f357798ce289efa25facd7
+        checksum/configuration: dc08b44088d4fd052cba8a322a4f66a369d96a504bf69d4029fd5180479e65a7
         checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914
       labels:
         app.kubernetes.io/component: flyte-binary
@@ -1367,7 +1369,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 421896ac3426efc3fb1d427ce98c9c6d631de7dcc4599a53726fb37eb59e6aad
+        checksum/secret: c904d62f5ffb3182576fa85ba6dafc845e40663bff6da7e579c5503b0029e8ea
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml
@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: QXRvTWlFM1NYcE5NRWlidg==
+  haSharedSecret: bkpOR0xac3FOaWJZMXFTTw==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -934,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: ba6d3d6226ec4daefb7d441c51c19ea2363808c06739832a58c342a92df951e8
+        checksum/secret: 9994f07a562e31de28569ef54dfdd7d6443f2dc0d2d43847d828f84c9d8cafc4
       labels:
         app: docker-registry
         release: flyte-sandbox

diff --git a/flytepropeller/pkg/controller/config/config.go b/flytepropeller/pkg/controller/config/config.go
@@ -128,7 +128,7 @@ var (
 		LiteralOffloadingConfig: LiteralOffloadingConfig{
 			Enabled: false, // Default keep this disabled and we will followup when flytekit is released with the offloaded changes.
 			SupportedSDKVersions: map[string]string{ // The key is the SDK name (matches the supported SDK in core.RuntimeMetadata_RuntimeType)  and the value is the minimum supported version
-				"FLYTE_SDK": "1.13.5", // Expected release number with flytekit support from this PR https://github.com/flyteorg/flytekit/pull/2685
+				"FLYTE_SDK": "1.13.14", // Expected release number with flytekit support from this PR https://github.com/flyteorg/flytekit/pull/2685
 			},
 			MinSizeInMBForOffloading: 10,   // 10 MB is the default size for offloading
 			MaxSizeInMBForOffloading: 1000, // 1 GB is the default size before failing fast.