adds dynamic node element to all pod metrics

README.md

@@ -1,4 +1,4 @@
-# snap collector plugin - kube state
+# snap collector plugin - kubestate
 
 This plugin collects metrics from Kubernetes about the state of pods, nodes and deployments.
 
@@ -100,22 +100,22 @@ This plugin has the ability to gather the following metrics:
 
 Namespace | Description (optional)
 ----------|-----------------------
-/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/condition/ready | specifies if the pod is ready to serve requests
-/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/condition/scheduled | status of the scheduling process for the pod
-/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Pending | This includes time before being bound to a node, as well as time spent pulling images onto the host.
-/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Running | The pod has been bound to a node and all of the containers have been started.
-/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Succeeded | All containers in the pod have voluntarily terminated with a container exit code of 0, and the system is not going to restart any of these containers.
-/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Failed |  All containers in the pod have terminated, and at least one container has terminated in a failure.
-/grafanalabs/kubestate/pod/[NAMESPACE]/[POD]/status/phase/Unknown | For some reason the state of the pod could not be obtained, typically due to an error in communicating with the host of the pod.
+/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/condition/ready | specifies if the pod is ready to serve requests
+/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/condition/scheduled | status of the scheduling process for the pod
+/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Pending | This includes time before being bound to a node, as well as time spent pulling images onto the host.
+/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Running | The pod has been bound to a node and all of the containers have been started.
+/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Succeeded | All containers in the pod have voluntarily terminated with a container exit code of 0, and the system is not going to restart any of these containers.
+/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Failed |  All containers in the pod have terminated, and at least one container has terminated in a failure.
+/grafanalabs/kubestate/pod/[NODE]/[NAMESPACE]/[POD]/status/phase/Unknown | For some reason the state of the pod could not be obtained, typically due to an error in communicating with the host of the pod.
 /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/limits/cpu/cores | The limit on cpu cores to be used by a container.
 /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/limits/memory/bytes | The limit on memory to be used by a container in bytes.
 /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/requested/cpu/cores | The number of requested cpu cores by a container.
 /grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/requested/memory/bytes | The number of requested memory bytes by a container.
-/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/ready | specifies whether the container has passed its readiness probe
-/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/restarts | number of times the container has been restarted
-/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/running | value 1 if container is running else value 0
-/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/terminated | value 1 if container is terminated else value 0
-/grafanalabs/kubestate/container/[NAMESPACE]/[POD]/[CONTAINER]/status/waiting | value 1 if container is waiting else value 0
+/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/ready | specifies whether the container has passed its readiness probe
+/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/restarts | number of times the container has been restarted
+/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/running | value 1 if container is running else value 0
+/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/terminated | value 1 if container is terminated else value 0
+/grafanalabs/kubestate/container/[NAMESPACE]/[NODE]/[POD]/[CONTAINER]/status/waiting | value 1 if container is waiting else value 0
 
 #### Nodes
 
@@ -139,6 +139,7 @@ Namespace | Description (optional)
 /grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/status/availablereplicas | Total number of available pods (ready for at least minReadySeconds) targeted by this deployment.
 /grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/status/unavailablereplicas | Total number of unavailable pods targeted by this deployment.
 /grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/spec/desiredreplicas | Number of desired pods.
+/grafanalabs/kubestate/deployment/[NAMESPACE]/[DEPLOYMENT]/status/deploynotfinished | If desired and observed generation are not the same, then either an ongoing deploy or a failed deploy.
 
 ### Examples
 

kubestate/deployment.go

@@ -51,6 +51,10 @@ func (*deploymentCollector) Collect(mts []plugin.Metric, deployment v1beta1.Depl
 		} else if ns[5] == "spec" && ns[6] == "paused" {
 			metric := createDeploymentMetric(mt, ns, deployment, boolInt(deployment.Spec.Paused))
 			metrics = append(metrics, metric)
+		} else if ns[5] == "status" && ns[6] == "deploynotfinished" {
+			notFinished := deployment.Generation-deployment.Status.ObservedGeneration > 0
+			metric := createDeploymentMetric(mt, ns, deployment, boolInt(notFinished))
+			metrics = append(metrics, metric)
 		}
 	}
 

kubestate/deployment_test.go

@@ -75,6 +75,7 @@ var deploymentCases = []struct {
 			"grafanalabs.kubestate.deployment.default.BeingDeployed.status.updatedreplicas 2",
 			"grafanalabs.kubestate.deployment.default.BeingDeployed.spec.desiredreplicas 16",
 			"grafanalabs.kubestate.deployment.default.BeingDeployed.spec.paused 0",
+			"grafanalabs.kubestate.deployment.default.BeingDeployed.status.deploynotfinished 1",
 		},
 	},
 	{
@@ -88,6 +89,7 @@ var deploymentCases = []struct {
 			"grafanalabs.kubestate.deployment.default.NoDesiredReplicas.status.unavailablereplicas 0",
 			"grafanalabs.kubestate.deployment.default.NoDesiredReplicas.status.updatedreplicas 0",
 			"grafanalabs.kubestate.deployment.default.NoDesiredReplicas.spec.paused 0",
+			"grafanalabs.kubestate.deployment.default.NoDesiredReplicas.status.deploynotfinished 0",
 		},
 	},
 	{
@@ -102,6 +104,7 @@ var deploymentCases = []struct {
 			"grafanalabs.kubestate.deployment.default.PausedDeploy.status.updatedreplicas 2",
 			"grafanalabs.kubestate.deployment.default.PausedDeploy.spec.desiredreplicas 16",
 			"grafanalabs.kubestate.deployment.default.PausedDeploy.spec.paused 1",
+			"grafanalabs.kubestate.deployment.default.PausedDeploy.status.deploynotfinished 1",
 		},
 	},
 	{

kubestate/kubestate.go

@@ -114,6 +114,7 @@ func getPodMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddStaticElements("status", "phase", "Pending"),
 		Version: 1,
@@ -122,6 +123,7 @@ func getPodMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddStaticElements("status", "phase", "Running"),
 		Version: 1,
@@ -130,6 +132,7 @@ func getPodMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddStaticElements("status", "phase", "Succeeded"),
 		Version: 1,
@@ -138,6 +141,7 @@ func getPodMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddStaticElements("status", "phase", "Failed"),
 		Version: 1,
@@ -146,6 +150,7 @@ func getPodMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddStaticElements("status", "phase", "Unknown"),
 		Version: 1,
@@ -154,6 +159,7 @@ func getPodMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddStaticElement("status").
 			AddStaticElements("condition", "ready"),
@@ -163,6 +169,7 @@ func getPodMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "pod").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddStaticElement("status").
 			AddStaticElements("condition", "scheduled"),
@@ -178,6 +185,7 @@ func getPodContainerMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddDynamicElement("container", "container name").
 			AddStaticElements("status", "restarts"),
@@ -187,6 +195,7 @@ func getPodContainerMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddDynamicElement("container", "container name").
 			AddStaticElements("status", "ready"),
@@ -196,6 +205,7 @@ func getPodContainerMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddDynamicElement("container", "container name").
 			AddStaticElements("status", "waiting"),
@@ -205,6 +215,7 @@ func getPodContainerMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddDynamicElement("container", "container name").
 			AddStaticElements("status", "running"),
@@ -214,6 +225,7 @@ func getPodContainerMetricTypes() []plugin.Metric {
 	mts = append(mts, plugin.Metric{
 		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "container").
 			AddDynamicElement("namespace", "kubernetes namespace").
+			AddDynamicElement("node", "node name").
 			AddDynamicElement("pod", "pod name").
 			AddDynamicElement("container", "container name").
 			AddStaticElements("status", "terminated"),
@@ -393,6 +405,14 @@ func getDeploymentMetricTypes() []plugin.Metric {
 		Version: 1,
 	})
 
+	mts = append(mts, plugin.Metric{
+		Namespace: plugin.NewNamespace("grafanalabs", "kubestate", "deployment").
+			AddDynamicElement("namespace", "Kubernetes namespace").
+			AddDynamicElement("deployment", "deployment name").
+			AddStaticElements("status", "deploynotfinished"),
+		Version: 1,
+	})
+
 	return mts
 }
 

kubestate/pod.go

@@ -14,12 +14,13 @@ const (
 	minPodNamespaceSize          = 8
 	metricTypeNsPart             = 2
 	namespaceNsPart              = metricTypeNsPart + 1
-	podNameNsPart                = metricTypeNsPart + 2
-	podStatusNsPart              = metricTypeNsPart + 3
-	podStatusTypeNsPart          = metricTypeNsPart + 4
-	podStatusValueNsPart         = metricTypeNsPart + 5
-	containerStatusNsPart        = metricTypeNsPart + 4
-	containerStatusValueNsPart   = metricTypeNsPart + 5
+	nodeNsPart                   = metricTypeNsPart + 2
+	podNameNsPart                = metricTypeNsPart + 3
+	podStatusNsPart              = metricTypeNsPart + 4
+	podStatusTypeNsPart          = metricTypeNsPart + 5
+	podStatusValueNsPart         = metricTypeNsPart + 6
+	containerStatusNsPart        = metricTypeNsPart + 5
+	containerStatusValueNsPart   = metricTypeNsPart + 6
 	containerResourceNsPart      = metricTypeNsPart + 5
 	containerResourceValueNsPart = metricTypeNsPart + 6
 )
@@ -40,6 +41,7 @@ func (*podCollector) Collect(mts []plugin.Metric, pod v1.Pod) ([]plugin.Metric,
 		if ns[metricTypeNsPart] == "pod" && ns[podStatusNsPart] == "status" {
 			if ns[podStatusTypeNsPart] == "phase" {
 				ns[namespaceNsPart] = pod.Namespace
+				ns[nodeNsPart] = slugify(pod.Spec.NodeName)
 				ns[podNameNsPart] = pod.Name
 
 				mt.Namespace = plugin.NewNamespace(ns...)
@@ -54,6 +56,7 @@ func (*podCollector) Collect(mts []plugin.Metric, pod v1.Pod) ([]plugin.Metric,
 				metrics = append(metrics, mt)
 			} else if ns[podStatusTypeNsPart] == "condition" {
 				ns[namespaceNsPart] = pod.Namespace
+				ns[nodeNsPart] = slugify(pod.Spec.NodeName)
 				ns[podNameNsPart] = pod.Name
 				mt.Namespace = plugin.NewNamespace(ns...)
 
@@ -140,8 +143,9 @@ func (*podCollector) Collect(mts []plugin.Metric, pod v1.Pod) ([]plugin.Metric,
 
 func createContainerStatusMetric(mt plugin.Metric, ns []string, pod v1.Pod, cs v1.ContainerStatus, value interface{}) plugin.Metric {
 	ns[namespaceNsPart] = pod.Namespace
-	ns[namespaceNsPart+1] = pod.Name
-	ns[namespaceNsPart+2] = cs.Name
+	ns[nodeNsPart] = slugify(pod.Spec.NodeName)
+	ns[nodeNsPart+1] = pod.Name
+	ns[nodeNsPart+2] = cs.Name
 	mt.Namespace = plugin.NewNamespace(ns...)
 
 	mt.Data = value

kubestate/pod_test.go

@@ -132,24 +132,24 @@ var cases = []struct {
 		pod:     mockPods[0],
 		metrics: getPodMetricTypes(),
 		expected: []string{
-			"grafanalabs.kubestate.pod.default.pod1.status.phase.Pending 0",
-			"grafanalabs.kubestate.pod.default.pod1.status.phase.Running 1",
-			"grafanalabs.kubestate.pod.default.pod1.status.phase.Succeeded 0",
-			"grafanalabs.kubestate.pod.default.pod1.status.phase.Failed 0",
-			"grafanalabs.kubestate.pod.default.pod1.status.phase.Unknown 0",
-			"grafanalabs.kubestate.pod.default.pod1.status.condition.ready 1",
-			"grafanalabs.kubestate.pod.default.pod1.status.condition.scheduled 1",
+			"grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Pending 0",
+			"grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Running 1",
+			"grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Succeeded 0",
+			"grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Failed 0",
+			"grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.phase.Unknown 0",
+			"grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.condition.ready 1",
+			"grafanalabs.kubestate.pod.default.127_0_0_1.pod1.status.condition.scheduled 1",
 		},
 	},
 	{
 		pod:     mockPods[0],
 		metrics: getPodContainerMetricTypes(),
 		expected: []string{
-			"grafanalabs.kubestate.container.default.pod1.container1.status.restarts 3",
-			"grafanalabs.kubestate.container.default.pod1.container1.status.ready 1",
-			"grafanalabs.kubestate.container.default.pod1.container1.status.waiting 0",
-			"grafanalabs.kubestate.container.default.pod1.container1.status.running 1",
-			"grafanalabs.kubestate.container.default.pod1.container1.status.terminated 0",
+			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.restarts 3",
+			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.ready 1",
+			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.waiting 0",
+			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.running 1",
+			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.status.terminated 0",
 			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.requested.cpu.cores 0.1",
 			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.requested.memory.bytes 1e+08",
 			"grafanalabs.kubestate.container.default.127_0_0_1.pod1.container1.limits.cpu.cores 0.2",
@@ -160,16 +160,16 @@ var cases = []struct {
 		pod:     mockPods[1],
 		metrics: getPodContainerMetricTypes(),
 		expected: []string{
-			"grafanalabs.kubestate.container.kube-system.pod2.container1.status.restarts 3",
-			"grafanalabs.kubestate.container.kube-system.pod2.container2.status.restarts 5",
-			"grafanalabs.kubestate.container.kube-system.pod2.container1.status.ready 1",
-			"grafanalabs.kubestate.container.kube-system.pod2.container2.status.ready 0",
-			"grafanalabs.kubestate.container.kube-system.pod2.container1.status.waiting 1",
-			"grafanalabs.kubestate.container.kube-system.pod2.container2.status.waiting 0",
-			"grafanalabs.kubestate.container.kube-system.pod2.container1.status.running 0",
-			"grafanalabs.kubestate.container.kube-system.pod2.container2.status.running 0",
-			"grafanalabs.kubestate.container.kube-system.pod2.container1.status.terminated 0",
-			"grafanalabs.kubestate.container.kube-system.pod2.container2.status.terminated 1",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.restarts 3",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.restarts 5",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.ready 1",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.ready 0",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.waiting 1",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.waiting 0",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.running 0",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.running 0",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container1.status.terminated 0",
+			"grafanalabs.kubestate.container.kube-system.node1.pod2.container2.status.terminated 1",
 			"grafanalabs.kubestate.container.kube-system.node1.pod2.container1.limits.cpu.cores 0.2",
 			"grafanalabs.kubestate.container.kube-system.node1.pod2.container2.limits.cpu.cores 0.2",
 			"grafanalabs.kubestate.container.kube-system.node1.pod2.container1.limits.memory.bytes 2e+08",