From e51f7df8995b3b3f48d89c9442034405a8ecac7c Mon Sep 17 00:00:00 2001 From: Daniel Lee Date: Fri, 16 Dec 2016 15:04:22 +0100 Subject: [PATCH] adds dynamic node element to all pod metrics --- README.md | 27 +++++++++++----------- kubestate/deployment.go | 4 ++++ kubestate/deployment_test.go | 3 +++ kubestate/kubestate.go | 20 ++++++++++++++++ kubestate/pod.go | 20 +++++++++------- kubestate/pod_test.go | 44 ++++++++++++++++++------------------ 6 files changed, 75 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index de7486d..89cb3e6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# snap collector plugin - kube state +# snap collector plugin - kubestate This plugin collects metrics from Kubernetes about the state of pods, nodes and deployments. @@ -100,22 +100,22 @@ This plugin has the ability to gather the following metrics: Namespace | Description (optional) ----------|----------------------- -/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/condition/ready | specifies if the pod is ready to serve requests -/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/condition/scheduled | status of the scheduling process for the pod -/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Pending | This includes time before being bound to a node, as well as time spent pulling images onto the host. -/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Running | The pod has been bound to a node and all of the containers have been started. -/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Succeeded | All containers in the pod have voluntarily terminated with a container exit code of 0, and the system is not going to restart any of these containers. -/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Failed | All containers in the pod have terminated, and at least one container has terminated in a failure. -/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Unknown | For some reason the state of the pod could not be obtained, typically due to an error in communicating with the host of the pod. +/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/condition/ready | specifies if the pod is ready to serve requests +/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/condition/scheduled | status of the scheduling process for the pod +/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Pending | This includes time before being bound to a node, as well as time spent pulling images onto the host. +/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Running | The pod has been bound to a node and all of the containers have been started. +/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Succeeded | All containers in the pod have voluntarily terminated with a container exit code of 0, and the system is not going to restart any of these containers. +/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Failed | All containers in the pod have terminated, and at least one container has terminated in a failure. +/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Unknown | For some reason the state of the pod could not be obtained, typically due to an error in communicating with the host of the pod. /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/limits/cpu/cores | The limit on cpu cores to be used by a container. /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/limits/memory/bytes | The limit on memory to be used by a container in bytes. /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/requested/cpu/cores | The number of requested cpu cores by a container. /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/requested/memory/bytes | The number of requested memory bytes by a container. -/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/ready | specifies whether the container has passed its readiness probe -/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/restarts | number of times the container has been restarted -/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/running | value 1 if container is running else value 0 -/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/terminated | value 1 if container is terminated else value 0 -/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/waiting | value 1 if container is waiting else value 0 +/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/ready | specifies whether the container has passed its readiness probe +/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/restarts | number of times the container has been restarted +/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/running | value 1 if container is running else value 0 +/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/terminated | value 1 if container is terminated else value 0 +/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/waiting | value 1 if container is waiting else value 0 #### Nodes @@ -139,6 +139,7 @@ Namespace | Description (optional) /grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/status/availablereplicas | Total number of available pods (ready for at least minReadySeconds) targeted by this deployment. /grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/status/unavailablereplicas | Total number of unavailable pods targeted by this deployment. /grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/spec/desiredreplicas | Number of desired pods. +/grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/status/deploynotfinished | If desired and observed generation are not the same, then either an ongoing deploy or a failed deploy. ### Examples diff --git a/kubestate/deployment.go b/kubestate/deployment.go index 314fb1b..a01c2e5 100644 --- a/kubestate/deployment.go +++ b/kubestate/deployment.go @@ -51,6 +51,10 @@ func (*deploymentCollector) Collect(mts []plugin.Metric, deployment v1beta1.Depl } else if ns[5] == "spec" && ns[6] == "paused" { metric := createDeploymentMetric(mt, ns, deployment, boolInt(deployment.Spec.Paused)) metrics = append(metrics, metric) + } else if ns[5] == "status" && ns[6] == "deploynotfinished" { + notFinished := deployment.Generation-deployment.Status.ObservedGeneration > 0 + metric := createDeploymentMetric(mt, ns, deployment, boolInt(notFinished)) + metrics = append(metrics, metric) } } diff --git a/kubestate/deployment_test.go b/kubestate/deployment_test.go index 86bc06c..eb56123 100644 --- a/kubestate/deployment_test.go +++ b/kubestate/deployment_test.go @@ -75,6 +75,7 @@ var deploymentCases = []struct { "grafanalabs.kubestate.deployment.default.BeingDeployed.status.updatedreplicas 2", "grafanalabs.kubestate.deployment.default.BeingDeployed.spec.desiredreplicas 16", "grafanalabs.kubestate.deployment.default.BeingDeployed.spec.paused 0", + "grafanalabs.kubestate.deployment.default.BeingDeployed.status.deploynotfinished 1", }, }, { @@ -88,6 +89,7 @@ var deploymentCases = []struct { "grafanalabs.kubestate.deployment.default.NoDesiredReplicas.status.unavailablereplicas 0", "grafanalabs.kubestate.deployment.default.NoDesiredReplicas.status.updatedreplicas 0", "grafanalabs.kubestate.deployment.default.NoDesiredReplicas.spec.paused 0", + "grafanalabs.kubestate.deployment.default.NoDesiredReplicas.status.deploynotfinished 0", }, }, { @@ -102,6 +104,7 @@ var deploymentCases = []struct { "grafanalabs.kubestate.deployment.default.PausedDeploy.status.updatedreplicas 2", "grafanalabs.kubestate.deployment.default.PausedDeploy.spec.desiredreplicas 16", "grafanalabs.kubestate.deployment.default.PausedDeploy.spec.paused 1", + "grafanalabs.kubestate.deployment.default.PausedDeploy.status.deploynotfinished 1", }, }, { diff --git a/kubestate/kubestate.go b/kubestate/kubestate.go index 283de01..5bd57c5 100644 --- a/kubestate/kubestate.go +++ b/kubestate/kubestate.go @@ -114,6 +114,7 @@ func getPodMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddStaticElements("status", "phase", "Pending"), Version: 1, @@ -122,6 +123,7 @@ func getPodMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddStaticElements("status", "phase", "Running"), Version: 1, @@ -130,6 +132,7 @@ func getPodMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddStaticElements("status", "phase", "Succeeded"), Version: 1, @@ -138,6 +141,7 @@ func getPodMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddStaticElements("status", "phase", "Failed"), Version: 1, @@ -146,6 +150,7 @@ func getPodMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddStaticElements("status", "phase", "Unknown"), Version: 1, @@ -154,6 +159,7 @@ func getPodMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddStaticElement("status"). AddStaticElements("condition", "ready"), @@ -163,6 +169,7 @@ func getPodMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddStaticElement("status"). AddStaticElements("condition", "scheduled"), @@ -178,6 +185,7 @@ func getPodContainerMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddDynamicElement("container", "container name"). AddStaticElements("status", "restarts"), @@ -187,6 +195,7 @@ func getPodContainerMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddDynamicElement("container", "container name"). AddStaticElements("status", "ready"), @@ -196,6 +205,7 @@ func getPodContainerMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddDynamicElement("container", "container name"). AddStaticElements("status", "waiting"), @@ -205,6 +215,7 @@ func getPodContainerMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddDynamicElement("container", "container name"). AddStaticElements("status", "running"), @@ -214,6 +225,7 @@ func getPodContainerMetricTypes() []plugin.Metric { mts = append(mts, plugin.Metric{ Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container"). AddDynamicElement("namespace", "kubernetes namespace"). + AddDynamicElement("node", "node name"). AddDynamicElement("pod", "pod name"). AddDynamicElement("container", "container name"). AddStaticElements("status", "terminated"), @@ -393,6 +405,14 @@ func getDeploymentMetricTypes() []plugin.Metric { Version: 1, }) + mts = append(mts, plugin.Metric{ + Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "deployment"). + AddDynamicElement("namespace", "Kubernetes namespace"). + AddDynamicElement("deployment", "deployment name"). + AddStaticElements("status", "deploynotfinished"), + Version: 1, + }) + return mts } diff --git a/kubestate/pod.go b/kubestate/pod.go index 32ed9a3..7e48c13 100644 --- a/kubestate/pod.go +++ b/kubestate/pod.go @@ -14,12 +14,13 @@ const ( minPodNamespaceSize = 8 metricTypeNsPart = 2 namespaceNsPart = metricTypeNsPart + 1 - podNameNsPart = metricTypeNsPart + 2 - podStatusNsPart = metricTypeNsPart + 3 - podStatusTypeNsPart = metricTypeNsPart + 4 - podStatusValueNsPart = metricTypeNsPart + 5 - containerStatusNsPart = metricTypeNsPart + 4 - containerStatusValueNsPart = metricTypeNsPart + 5 + nodeNsPart = metricTypeNsPart + 2 + podNameNsPart = metricTypeNsPart + 3 + podStatusNsPart = metricTypeNsPart + 4 + podStatusTypeNsPart = metricTypeNsPart + 5 + podStatusValueNsPart = metricTypeNsPart + 6 + containerStatusNsPart = metricTypeNsPart + 5 + containerStatusValueNsPart = metricTypeNsPart + 6 containerResourceNsPart = metricTypeNsPart + 5 containerResourceValueNsPart = metricTypeNsPart + 6 ) @@ -40,6 +41,7 @@ func (*podCollector) Collect(mts []plugin.Metric, pod v1.Pod) ([]plugin.Metric, if ns[metricTypeNsPart] == "pod" && ns[podStatusNsPart] == "status" { if ns[podStatusTypeNsPart] == "phase" { ns[namespaceNsPart] = pod.Namespace + ns[nodeNsPart] = slugify(pod.Spec.NodeName) ns[podNameNsPart] = pod.Name mt.Namespace = plugin.NewNamespace(ns...) @@ -54,6 +56,7 @@ func (*podCollector) Collect(mts []plugin.Metric, pod v1.Pod) ([]plugin.Metric, metrics = append(metrics, mt) } else if ns[podStatusTypeNsPart] == "condition" { ns[namespaceNsPart] = pod.Namespace + ns[nodeNsPart] = slugify(pod.Spec.NodeName) ns[podNameNsPart] = pod.Name mt.Namespace = plugin.NewNamespace(ns...) @@ -140,8 +143,9 @@ func (*podCollector) Collect(mts []plugin.Metric, pod v1.Pod) ([]plugin.Metric, func createContainerStatusMetric(mt plugin.Metric, ns []string, pod v1.Pod, cs v1.ContainerStatus, value interface{}) plugin.Metric { ns[namespaceNsPart] = pod.Namespace - ns[namespaceNsPart+1] = pod.Name - ns[namespaceNsPart+2] = cs.Name + ns[nodeNsPart] = slugify(pod.Spec.NodeName) + ns[nodeNsPart+1] = pod.Name + ns[nodeNsPart+2] = cs.Name mt.Namespace = plugin.NewNamespace(ns...) mt.Data = value diff --git a/kubestate/pod_test.go b/kubestate/pod_test.go index a04ed41..0ef85c9 100644 --- a/kubestate/pod_test.go +++ b/kubestate/pod_test.go @@ -132,24 +132,24 @@ var cases = []struct { pod: mockPods[0], metrics: getPodMetricTypes(), expected: []string{ - "grafanalabs.kubestate.pod.default.pod1.status.phase.Pending 0", - "grafanalabs.kubestate.pod.default.pod1.status.phase.Running 1", - "grafanalabs.kubestate.pod.default.pod1.status.phase.Succeeded 0", - "grafanalabs.kubestate.pod.default.pod1.status.phase.Failed 0", - "grafanalabs.kubestate.pod.default.pod1.status.phase.Unknown 0", - "grafanalabs.kubestate.pod.default.pod1.status.condition.ready 1", - "grafanalabs.kubestate.pod.default.pod1.status.condition.scheduled 1", + "grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Pending 0", + "grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Running 1", + "grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Succeeded 0", + "grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Failed 0", + "grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Unknown 0", + "grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.condition.ready 1", + "grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.condition.scheduled 1", }, }, { pod: mockPods[0], metrics: getPodContainerMetricTypes(), expected: []string{ - "grafanalabs.kubestate.container.default.pod1.container1.status.restarts 3", - "grafanalabs.kubestate.container.default.pod1.container1.status.ready 1", - "grafanalabs.kubestate.container.default.pod1.container1.status.waiting 0", - "grafanalabs.kubestate.container.default.pod1.container1.status.running 1", - "grafanalabs.kubestate.container.default.pod1.container1.status.terminated 0", + "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.restarts 3", + "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.ready 1", + "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.waiting 0", + "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.running 1", + "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.terminated 0", "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.requested.cpu.cores 0.1", "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.requested.memory.bytes 1e+08", "grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.limits.cpu.cores 0.2", @@ -160,16 +160,16 @@ var cases = []struct { pod: mockPods[1], metrics: getPodContainerMetricTypes(), expected: []string{ - "grafanalabs.kubestate.container.kube-system.pod2.container1.status.restarts 3", - "grafanalabs.kubestate.container.kube-system.pod2.container2.status.restarts 5", - "grafanalabs.kubestate.container.kube-system.pod2.container1.status.ready 1", - "grafanalabs.kubestate.container.kube-system.pod2.container2.status.ready 0", - "grafanalabs.kubestate.container.kube-system.pod2.container1.status.waiting 1", - "grafanalabs.kubestate.container.kube-system.pod2.container2.status.waiting 0", - "grafanalabs.kubestate.container.kube-system.pod2.container1.status.running 0", - "grafanalabs.kubestate.container.kube-system.pod2.container2.status.running 0", - "grafanalabs.kubestate.container.kube-system.pod2.container1.status.terminated 0", - "grafanalabs.kubestate.container.kube-system.pod2.container2.status.terminated 1", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.restarts 3", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.restarts 5", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.ready 1", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.ready 0", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.waiting 1", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.waiting 0", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.running 0", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.running 0", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.terminated 0", + "grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.terminated 1", "grafanalabs.kubestate.container.kube-system.node1.pod2.container1.limits.cpu.cores 0.2", "grafanalabs.kubestate.container.kube-system.node1.pod2.container2.limits.cpu.cores 0.2", "grafanalabs.kubestate.container.kube-system.node1.pod2.container1.limits.memory.bytes 2e+08",