Imperative "kubectl scale deploy busybox --replicas=15" command does NOT prevent scaling as per replicalimit OPA GK Policy

[hikkart]

Hello All,

I was trying OPA GK library replicalimits sample.
https://github.com/open-policy-agent/gatekeeper-library/tree/master/library/general/replicalimits

I kept max_replicas": 4 in the constraint.yaml.
Declarative "kubectl apply -f example_allowed.yaml" approach throws proper error if it exceeds the max replica limit.

Error from server ([replica-limits] The provided number of replicas is not allowed for deployment: allowed-deployment. Allowed ranges: {"ranges": [{"max_replicas": 4, "min_replicas": 1}]}): error when applying patch:
{"metadata":{"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"name\":\"allowed-deployment\",\"namespace\":\"default\"},\"spec\":{\"replicas\":5,\"selector\":{\"matchLabels\":{\"app\":\"nginx\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"nginx\"}},\"spec\":{\"containers\":[{\"image\":\"nginx:1.14.2\",\"name\":\"nginx\",\"ports\":[{\"containerPort\":80}]}]}}}}\n"}},"spec":{"replicas":5}}
to:

Imperative "kubectl scale deploy busybox --replicas=5" command does NOT prevent scaling as per replicalimit OPA GK Policy

[kk@Ik1 replica-limit]$ k scale deployment allowed-deployment --replicas=5
deployment.apps/allowed-deployment scaled
[kk@Ik1 replica-limit]$ k get po
NAME                                                        READY   STATUS    RESTARTS   AGE
allowed-deployment-66b6c48dd5-bq2ln                         1/1     Running   0          27s
allowed-deployment-66b6c48dd5-dzcsn                         1/1     Running   0          45m
allowed-deployment-66b6c48dd5-m96zm                         1/1     Running   0          27s
allowed-deployment-66b6c48dd5-s6j82                         1/1     Running   0          45m
allowed-deployment-66b6c48dd5-sl6kc                         1/1     Running   0          45m

How to prevent imperative scale command using OPA GK Policy? Am I missing something here

Thanks in advance.

[anderseknert]

Hi @hikkart! A scale request is updating the replicaset associated with the deployment. You'll want to add a rule that checks UPDATE requests on kind: ReplicaSet and compare the request.object.spec.replicas against request.oldObject.spec.replicas. An example AdmissionReview object changing the number of replicas from 3 to 1 would look something like the one below.

 {
    "apiVersion": "admission.k8s.io/v1",
    "kind": "AdmissionReview",
    "request": {
      "dryRun": false,
      "kind": {
        "group": "apps",
        "kind": "ReplicaSet",
        "version": "v1"
      },
      "name": "allowed-deployment-66b6c48dd5",
      "namespace": "default",
      "object": {
        "apiVersion": "apps/v1",
        "kind": "ReplicaSet",
        "metadata": {
          "annotations": {
            "deployment.kubernetes.io/desired-replicas": "1",
            "deployment.kubernetes.io/max-replicas": "2",
            "deployment.kubernetes.io/revision": "1"
          },
          "creationTimestamp": "2021-09-23T07:41:37Z",
          "generation": 2,
          "labels": {
            "app": "nginx",
            "pod-template-hash": "66b6c48dd5"
          },
          "name": "allowed-deployment-66b6c48dd5",
          "namespace": "default",
          "ownerReferences": [
            {
              "apiVersion": "apps/v1",
              "blockOwnerDeletion": true,
              "controller": true,
              "kind": "Deployment",
              "name": "allowed-deployment",
              "uid": "fc14c113-271d-45e3-b131-877912243bcb"
            }
          ],
          "resourceVersion": "2353",
          "uid": "da561a4a-320c-491c-9a09-5e8796bb0349"
        },
        "spec": {
          "replicas": 1,
          "selector": {
            "matchLabels": {
              "app": "nginx",
              "pod-template-hash": "66b6c48dd5"
            }
          },
          "template": {
            "metadata": {
              "creationTimestamp": null,
              "labels": {
                "app": "nginx",
                "pod-template-hash": "66b6c48dd5"
              }
            },
            "spec": {
              "containers": [
                {
                  "image": "nginx:1.14.2",
                  "imagePullPolicy": "IfNotPresent",
                  "name": "nginx",
                  "ports": [
                    {
                      "containerPort": 80,
                      "protocol": "TCP"
                    }
                  ],
                  "resources": {},
                  "terminationMessagePath": "/dev/termination-log",
                  "terminationMessagePolicy": "File"
                }
              ],
              "dnsPolicy": "ClusterFirst",
              "restartPolicy": "Always",
              "schedulerName": "default-scheduler",
              "securityContext": {},
              "terminationGracePeriodSeconds": 30
            }
          }
        },
        "status": {
          "availableReplicas": 3,
          "fullyLabeledReplicas": 3,
          "observedGeneration": 1,
          "readyReplicas": 3,
          "replicas": 3
        }
      },
      "oldObject": {
        "apiVersion": "apps/v1",
        "kind": "ReplicaSet",
        "metadata": {
          "annotations": {
            "deployment.kubernetes.io/desired-replicas": "3",
            "deployment.kubernetes.io/max-replicas": "4",
            "deployment.kubernetes.io/revision": "1"
          },
          "creationTimestamp": "2021-09-23T07:41:37Z",
          "generation": 1,
          "labels": {
            "app": "nginx",
            "pod-template-hash": "66b6c48dd5"
          },
          "name": "allowed-deployment-66b6c48dd5",
          "namespace": "default",
          "ownerReferences": [
            {
              "apiVersion": "apps/v1",
              "blockOwnerDeletion": true,
              "controller": true,
              "kind": "Deployment",
              "name": "allowed-deployment",
              "uid": "fc14c113-271d-45e3-b131-877912243bcb"
            }
          ],
          "resourceVersion": "2353",
          "uid": "da561a4a-320c-491c-9a09-5e8796bb0349"
        },
        "spec": {
          "replicas": 3,
          "selector": {
            "matchLabels": {
              "app": "nginx",
              "pod-template-hash": "66b6c48dd5"
            }
          },
          "template": {
            "metadata": {
              "creationTimestamp": null,
              "labels": {
                "app": "nginx",
                "pod-template-hash": "66b6c48dd5"
              }
            },
            "spec": {
              "containers": [
                {
                  "image": "nginx:1.14.2",
                  "imagePullPolicy": "IfNotPresent",
                  "name": "nginx",
                  "ports": [
                    {
                      "containerPort": 80,
                      "protocol": "TCP"
                    }
                  ],
                  "resources": {},
                  "terminationMessagePath": "/dev/termination-log",
                  "terminationMessagePolicy": "File"
                }
              ],
              "dnsPolicy": "ClusterFirst",
              "restartPolicy": "Always",
              "schedulerName": "default-scheduler",
              "securityContext": {},
              "terminationGracePeriodSeconds": 30
            }
          }
        },
        "status": {
          "availableReplicas": 3,
          "fullyLabeledReplicas": 3,
          "observedGeneration": 1,
          "readyReplicas": 3,
          "replicas": 3
        }
      },
      "operation": "UPDATE",
      "options": {
        "apiVersion": "meta.k8s.io/v1",
        "kind": "UpdateOptions"
      },
      "requestKind": {
        "group": "apps",
        "kind": "ReplicaSet",
        "version": "v1"
      },
      "requestResource": {
        "group": "apps",
        "resource": "replicasets",
        "version": "v1"
      },
      "resource": {
        "group": "apps",
        "resource": "replicasets",
        "version": "v1"
      },
      "uid": "8c07b0a8-2e43-4d1c-b53a-37ee6d2685c7",
      "userInfo": {
        "groups": [
          "system:serviceaccounts",
          "system:serviceaccounts:kube-system",
          "system:authenticated"
        ],
        "uid": "0edbf1ad-8659-4efe-882e-0893955c092a",
        "username": "system:serviceaccount:kube-system:deployment-controller"
      }
    }
  }

See the docs on debugging if you'd like to review the objects as they are passed from the Kubernetes API server.

[hikkart]

Thank you @anderseknert . I will check it out with replicaset.