Attempt to fix oom issues (#2428)

opensource-observer · Oct 30, 2024 · 4d4ff64 · 4d4ff64
1 parent e1d8d3b
commit 4d4ff64
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 4 deletions.
diff --git a/ops/k8s-apps/base/dagster/dagster.yaml b/ops/k8s-apps/base/dagster/dagster.yaml
@@ -60,10 +60,10 @@ spec:
                 resources:
                   limits:
                     cpu: 500m
-                    memory: 1024Mi
+                    memory: 2048Mi
                   requests:
                     cpu: 500m
-                    memory: 768Mi
+                    memory: 1024Mi
       dagsterWebserver:
         enableReadOnly: true
         env:

diff --git a/warehouse/oso_dagster/assets/clickhouse_dbt_marts.py b/warehouse/oso_dagster/assets/clickhouse_dbt_marts.py
@@ -93,11 +93,11 @@ def clickhouse_assets_from_manifests_map(
                                         "resources": {
                                             "requests": {
                                                 "cpu": "1000m",
-                                                "memory": "1024Mi",
+                                                "memory": "1536Mi",
                                             },
                                             "limits": {
                                                 "cpu": "1000m",
-                                                "memory": "1024Mi",
+                                                "memory": "1536Mi",
                                             },
                                         },
                                     },

diff --git a/warehouse/oso_dagster/factories/alerts.py b/warehouse/oso_dagster/factories/alerts.py
@@ -27,6 +27,36 @@ def failure_job():
     )
     def failure_sensor(context: RunFailureSensorContext):
         yield RunRequest(
+            tags={
+                "dagster-k8s/config": {
+                    "merge_behavior": "SHALLOW",
+                    "container_config": {
+                        "resources": {
+                            "requests": {
+                                "cpu": "500m",
+                                "memory": "768Mi",
+                            },
+                            "limits": {
+                                "cpu": "500m",
+                                "memory": "1536Mi",
+                            },
+                        },
+                    },
+                    "pod_spec_config": {
+                        "node_selector": {
+                            "pool_type": "persistent",
+                        },
+                        "tolerations": [
+                            {
+                                "key": "pool_type",
+                                "operator": "Equal",
+                                "value": "persistent",
+                                "effect": "NoSchedule",
+                            }
+                        ],
+                    },
+                },
+            },
             run_key=context.dagster_run.run_id,
             run_config=RunConfig(
                 ops={