feat(sentrykube): introduce additional possibilities to override config

libsentrykube/kube.py

@@ -21,6 +21,7 @@
     get_service_values,
     get_service_value_overrides,
     get_tools_managed_service_value_overrides,
+    get_hierarchical_value_overrides,
 )
 from libsentrykube.utils import (
     deep_merge_dict,
@@ -111,6 +112,13 @@ def _consolidate_variables(
        overrides are managed manually and they can contain comments. Tools cannot
        preserve comments.
     4. overridden by the cluster file. Which is likely going to be replaced by 2 and 3.
+    5. overridden by creating a hierarchical structure. Adding an intermediate directory
+       in 'region_override' with a '_values.yaml' file allows to have a common config
+       between a set of regions. This is useful if regions in a service are different, but
+       subset of them are similar.
+       It works like point 1 and 2 but with an additional layer in between
+    6. overridden by a common '_values.yaml' within a region folder that has a shared config
+       between all clusters in the region. Values in point 2 will still override those values
 
     TODO: write the minimum components of a yaml parser to remove step 3 and
           patch the regional override preserving comments.
@@ -131,6 +139,11 @@ def _consolidate_variables(
     )
     deep_merge_dict(service_values, managed_values)
 
+    hierarchical_values = get_hierarchical_value_overrides(
+        service_name, customer_name, cluster_name, external
+    )
+    deep_merge_dict(service_values, hierarchical_values)
+
     # Service data overrides from clusters/
     customer_values, _ = get_service_data(
         customer_name,

libsentrykube/service.py

@@ -7,7 +7,7 @@
 from collections import OrderedDict
 from libsentrykube.config import Config
 from libsentrykube.customer import load_customer_data
-from libsentrykube.utils import workspace_root
+from libsentrykube.utils import workspace_root, deep_merge_dict
 
 _services = OrderedDict()
 
@@ -100,9 +100,21 @@ def get_service_value_overrides(
     external: bool = False,
 ) -> dict:
     """
-    For the given service, return the values specified in the corresponding _values.yaml.
+    Loads service configuration with regional and cluster-specific overrides.
 
-    If "external=True" is specified, treat the service name as the full service path.
+    This function implements a two-level configuration system where common regional settings
+    can be shared across all clusters, with cluster-specific overrides on top.
+
+    Directory Structure:
+        region_overrides/
+        └── {region_name}/
+            ├── _values.yaml           # Common settings for all clusters in this region
+            ├── cluster1.yaml          # Cluster-specific overrides
+            └── cluster2.yaml          # Cluster-specific overrides
+
+    Override Precedence (highest to lowest):
+    1. {cluster_name}.yaml            # Cluster-specific settings
+    2. _values.yaml                   # Common regional settings
     """
     try:
         service_override_file = (
@@ -112,9 +124,109 @@ def get_service_value_overrides(
 
         with open(service_override_file, "rb") as f:
             values = yaml.safe_load(f)
+
+        # Try to load the config only after {cluster_name}.yaml was successfully loaded because the common override
+        # only makes sense if there is a cluster config
+        common_override_values = get_common_regional_override(
+            service_name, region_name, external
+        )
+
+        if common_override_values:
+            deep_merge_dict(common_override_values, values)
+            return common_override_values
+
+        return values
     except FileNotFoundError:
-        values = {}
-    return values
+        return {}
+
+
+def get_common_regional_override(
+    service_name: str, region_name: str, external: bool = False
+) -> dict:
+    """
+    Helper function to load common regional configuration values.
+
+    Looks for a '_values.yaml' file in the region's override directory that contains
+    settings shared across all clusters in that region.
+    """
+    try:
+        common_service_override_file = (
+            get_service_value_override_path(service_name, region_name, external)
+            / "_values.yaml"
+        )
+
+        with open(common_service_override_file, "rb") as f:
+            common_override_values = yaml.safe_load(f)
+
+        return common_override_values
+    except FileNotFoundError:
+        return {}
+
+
+def get_hierarchical_value_overrides(
+    service_name: str,
+    region_name: str,
+    cluster_name: str = "default",
+    external: bool = False,
+) -> dict:
+    """
+    Enables hierarchical configuration overrides with shared base values.
+
+    This function extends the standard region_overrides system by adding support for
+    shared base configurations. This helps reduce duplication across region-specific
+    configurations.
+
+    Directory Structure:
+        region_overrides/
+        └── common_shared_config/      # Arbitrary name for the shared config group
+            ├── _values.yaml           # Base values for this group
+            └── {region_name}/         # Region-specific overrides
+                └── {cluster_name}.yaml # Cluster-specific overrides
+
+    Override Precedence (highest to lowest):
+    1. region_name/cluster_name.yaml
+    2. common_shared_config/_values.yaml
+    3. Top-level configuration
+    """
+    if external:
+        service_regions_path = workspace_root() / service_name
+    else:
+        service_regions_path = get_service_path(service_name)
+
+    service_regions_path = service_regions_path / "region_overrides"
+
+    if not service_regions_path.exists():
+        return {}
+
+    for override_group in service_regions_path.iterdir():
+        if not override_group.is_dir():
+            continue
+
+        try:
+            service_override_file = (
+                get_service_value_override_path(
+                    service_name, override_group.name, external
+                )
+                / "_values.yaml"
+            )
+
+            with open(service_override_file, "rb") as f:
+                base_values = yaml.safe_load(f)
+        except FileNotFoundError:
+            base_values = {}
+
+        region_path = f"{override_group.name}/{region_name}"
+        region_values = get_service_value_overrides(
+            service_name, region_path, cluster_name, external
+        )
+
+        if not region_values:
+            continue
+
+        deep_merge_dict(base_values, region_values)
+        return base_values
+
+    return {}
 
 
 def get_tools_managed_service_value_overrides(

libsentrykube/tests/conftest.py

@@ -79,6 +79,12 @@ def set_workspaceroot() -> Iterator[None]:
     },
 }
 
+COMMON_SHARED_CONFIG = {
+    "config": {"foo": "123", "baz": "test", "settings": {"abc": 10, "def": "test"}}
+}
+
+CLUSTER_OVERRIDE_CONFIG = {"config": {"foo": "not-foo", "settings": {"abc": 20}}}
+
 
 @pytest.fixture
 def config_structure() -> Generator[str, None, None]:
@@ -114,6 +120,78 @@ def config_structure() -> Generator[str, None, None]:
         yield temp_dir
 
 
+@pytest.fixture
+def hierarchical_override_structure() -> Generator[str, None, None]:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        k8s = Path(temp_dir) / "k8s"
+        os.makedirs(k8s / "clusters")
+        clusters = k8s / "clusters"
+        with open(clusters / "cluster1.yaml", "w") as f:
+            f.write(safe_dump(CLUSTER_1))
+        with open(clusters / "cluster2.yaml", "w") as f:
+            f.write(safe_dump(CLUSTER_2))
+
+        service_dir = k8s / "services" / "my_service"
+        region_overrides = service_dir / "region_overrides"
+        os.makedirs(region_overrides)
+
+        os.makedirs(k8s / "services" / "another_service")
+        common_config = region_overrides / "common_shared_config"
+        os.makedirs(common_config)
+
+        with open(common_config / "_values.yaml", "w") as f:
+            f.write(safe_dump(COMMON_SHARED_CONFIG))
+
+        region_dir = common_config / "customer1"
+        os.makedirs(region_dir)
+
+        with open(region_dir / "cluster1.yaml", "w") as f:
+            f.write(safe_dump(CLUSTER_OVERRIDE_CONFIG))
+
+        os.makedirs(Path(temp_dir) / "cli_config")
+        with open(Path(temp_dir) / "cli_config" / "configuration.yaml", "w") as f:
+            f.write(safe_dump(CONFIGURATION))
+
+        os.environ["SENTRY_KUBE_CONFIG_FILE"] = str(
+            Path(temp_dir) / "cli_config" / "configuration.yaml"
+        )
+
+        yield temp_dir
+
+
+@pytest.fixture
+def regional_cluster_specific_override_structure() -> Generator[str, None, None]:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        k8s = Path(temp_dir) / "k8s"
+        os.makedirs(k8s / "clusters")
+        clusters = k8s / "clusters"
+        with open(clusters / "cluster1.yaml", "w") as f:
+            f.write(safe_dump(CLUSTER_1))
+        with open(clusters / "cluster2.yaml", "w") as f:
+            f.write(safe_dump(CLUSTER_2))
+
+        service_dir = k8s / "services" / "my_service"
+        region_overrides = service_dir / "region_overrides"
+        os.makedirs(region_overrides)
+
+        os.makedirs(k8s / "services" / "another_service")
+
+        region_dir = region_overrides / "customer1"
+        os.makedirs(region_dir)
+
+        with open(region_dir / "_values.yaml", "w") as f:
+            f.write(safe_dump(COMMON_SHARED_CONFIG))
+
+        with open(region_dir / "cluster1.yaml", "w") as f:
+            f.write(safe_dump(CLUSTER_OVERRIDE_CONFIG))
+
+        os.makedirs(Path(temp_dir) / "cli_config")
+        with open(Path(temp_dir) / "cli_config" / "configuration.yaml", "w") as f:
+            f.write(safe_dump(CONFIGURATION))
+
+        yield temp_dir
+
+
 @pytest.fixture
 def initialized_config_structure(config_structure: str) -> Generator[str, None, None]:
     directory = config_structure

libsentrykube/tests/test_service.py

@@ -1,6 +1,7 @@
 from libsentrykube.context import init_cluster_context
 import os
 from libsentrykube.service import (
+    get_hierarchical_value_overrides,
     get_service_data,
     get_service_values,
     get_service_value_overrides,
@@ -63,6 +64,10 @@
     }
 }
 
+expected_hierarchical_and_regional_cluster_values = {
+    "config": {"foo": "not-foo", "baz": "test", "settings": {"abc": 20, "def": "test"}}
+}
+
 
 def test_get_service_values_not_external():
     region = "saas"
@@ -186,3 +191,35 @@ def test_write_managed_file(config_structure) -> None:
     assert path.is_file()
 
     set_workspace_root_start(start_workspace_root)
+
+
+def test_get_hierarchical_value_overrides(hierarchical_override_structure: str) -> None:
+    set_workspace_root_start(hierarchical_override_structure)
+    os.environ["SENTRY_KUBE_CONFIG_FILE"] = str(
+        workspace_root() / "cli_config/configuration.yaml"
+    )
+    init_cluster_context("customer1", "cluster1")
+
+    returned = get_hierarchical_value_overrides(
+        service_name="my_service", region_name="customer1", cluster_name="cluster1"
+    )
+
+    assert returned == expected_hierarchical_and_regional_cluster_values
+
+
+def test_regional_cluster_value_overrides(
+    regional_cluster_specific_override_structure: str,
+) -> None:
+    set_workspace_root_start(regional_cluster_specific_override_structure)
+    os.environ["SENTRY_KUBE_CONFIG_FILE"] = str(
+        workspace_root() / "cli_config/configuration.yaml"
+    )
+    init_cluster_context("customer1", "cluster1")
+
+    returned = get_service_value_overrides(
+        service_name="my_service",
+        region_name="customer1",
+        cluster_name="cluster1",
+    )
+
+    assert returned == expected_hierarchical_and_regional_cluster_values