From 67199f676a88a2f486909b63b254abcf24821931 Mon Sep 17 00:00:00 2001 From: Jake Ferriero Date: Fri, 20 Dec 2024 12:36:55 -0800 Subject: [PATCH 1/2] skip setproctitle in `task_runner` on Mac OS On some newer versions of Mac OS setproctitle can cause segfault https://github.com/benoitc/gunicorn/issues/3021 --- .../src/airflow/sdk/execution_time/task_runner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/task_sdk/src/airflow/sdk/execution_time/task_runner.py b/task_sdk/src/airflow/sdk/execution_time/task_runner.py index 48dd3ecbfcd67..51230d0e7130f 100644 --- a/task_sdk/src/airflow/sdk/execution_time/task_runner.py +++ b/task_sdk/src/airflow/sdk/execution_time/task_runner.py @@ -214,9 +214,14 @@ def startup() -> tuple[RuntimeTaskInstance, Logger]: msg = SUPERVISOR_COMMS.get_message() if isinstance(msg, StartupDetails): - from setproctitle import setproctitle - - setproctitle(f"airflow worker -- {msg.ti.id}") + # setproctitle causes issue on Mac OS: https://github.com/benoitc/gunicorn/issues/3021 + os_type = sys.platform + if os_type == "darwin": + log.info("Mac OS detected, skipping setproctitle") + else: + from setproctitle import setproctitle + + setproctitle(f"airflow worker -- {msg.ti.id}") log = structlog.get_logger(logger_name="task") # TODO: set the "magic loop" context vars for parsing From 3abfa1a4a92e1ac91e71a6cea88c3d712565278a Mon Sep 17 00:00:00 2001 From: Jake Ferriero Date: Fri, 20 Dec 2024 13:51:22 -0800 Subject: [PATCH 2/2] also skip in dag processing manager --- airflow/dag_processing/manager.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/airflow/dag_processing/manager.py b/airflow/dag_processing/manager.py index 0e9f8b4bc37b0..e48154dec4e00 100644 --- a/airflow/dag_processing/manager.py +++ b/airflow/dag_processing/manager.py @@ -39,7 +39,6 @@ from typing import TYPE_CHECKING, Any, NamedTuple import attrs -from setproctitle import setproctitle from sqlalchemy import delete, select, update from tabulate import tabulate from uuid6 import uuid7 @@ -181,7 +180,15 @@ def _run_processor_manager( # to iterate the child processes set_new_process_group() - setproctitle("airflow scheduler -- DagFileProcessorManager") + + # setproctitle causes issue on Mac OS: https://github.com/benoitc/gunicorn/issues/3021 + os_type = sys.platform + if os_type == "darwin": + log.info("Mac OS detected, skipping setproctitle") + else: + from setproctitle import setproctitle + setproctitle("airflow scheduler -- DagFileProcessorManager") + reload_configuration_for_dag_processing() processor_manager = DagFileProcessorManager( dag_directory=dag_directory,