From 6dfee41a9dd8213151879b37a94deb8157ad5094 Mon Sep 17 00:00:00 2001 From: Daniel Suveges Date: Mon, 20 May 2024 11:59:38 +0100 Subject: [PATCH] fix(spark helpers): sorter function updated to avoid crash (#613) --- src/gentropy/common/spark_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gentropy/common/spark_helpers.py b/src/gentropy/common/spark_helpers.py index bd22d91c9..913e67a67 100644 --- a/src/gentropy/common/spark_helpers.py +++ b/src/gentropy/common/spark_helpers.py @@ -1,4 +1,5 @@ """Common utilities in Spark that can be used across the project.""" + from __future__ import annotations import re @@ -363,6 +364,7 @@ def order_array_of_structs_by_field(column_name: str, field_name: str) -> Column array_sort( {column_name}, (left, right) -> case + when left.{field_name} is null and right.{field_name} is null then 0 when left.{field_name} is null then 1 when right.{field_name} is null then -1 when left.{field_name} < right.{field_name} then 1