From 4f5503ba920289deb1dc60dfb38e068cbc16fc0a Mon Sep 17 00:00:00 2001 From: Fabian Ruff Date: Mon, 3 Jun 2019 16:36:23 +0200 Subject: [PATCH] use container ID instead of pod UID when getting info from docker This PR also simplifies the Makefile and Dockerfile and makes use of a multi stage build process. --- .dockerignore | 1 + Dockerfile | 16 +++++++++++++- Makefile | 23 +------------------ cache/main.go | 25 +++++++++++++++++++++ main.go | 61 ++++++++++++++------------------------------------- main_test.go | 22 +++++++++++++------ 6 files changed, 74 insertions(+), 74 deletions(-) create mode 100644 .dockerignore create mode 100644 cache/main.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..48b8bf90 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +vendor/ diff --git a/Dockerfile b/Dockerfile index 5371b065..71d1a815 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,22 @@ +FROM golang:1.11 + +WORKDIR /go/src/github.com/sapcc/kuberntes-oomkill-exporter +ENV GO111MODULE=on \ + CGOENABLED=0 +ADD go.mod go.sum ./ +RUN go mod download +ADD cache/main.go . +RUN go build -v -o /dev/null +ADD . . +RUN go build -v -o /kubernetes-oomkill-exporter +RUN go test -v +RUN go vet + FROM alpine:3.8 LABEL maintainer="jan.knipper@sap.com" RUN apk --no-cache add ca-certificates -COPY kubernetes-oomkill-exporter /kubernetes-oomkill-exporter +COPY /kubernetes-oomkill-exporter /kubernetes-oomkill-exporter ENTRYPOINT ["/kubernetes-oomkill-exporter"] CMD ["-logtostderr"] diff --git a/Makefile b/Makefile index a2cd8512..48195872 100644 --- a/Makefile +++ b/Makefile @@ -1,40 +1,19 @@ TARGET = kubernetes-oomkill-exporter GOTARGET = github.com/sapcc/$(TARGET) -BUILDMNT = /go/src/$(GOTARGET) REGISTRY ?= sapcc -VERSION ?= 0.2.0 +VERSION ?= 0.3.0 IMAGE = $(REGISTRY)/$(BIN) -BUILD_IMAGE ?= golang:1.11-alpine3.8 DOCKER ?= docker -DIR := ${CURDIR} - -ifneq ($(VERBOSE),) -VERBOSE_FLAG = -v -endif -TESTARGS ?= $(VERBOSE_FLAG) -timeout 60s -TEST_PKGS ?= $(GOTARGET)/... -TEST = CGO_ENABLED=0 go test $(TEST_PKGS) $(TESTARGS) -VET_PKGS ?= $(GOTARGET)/... -VET = CGO_ENABLED=0 go vet $(VET_PKGS) - -DOCKER_BUILD ?= $(DOCKER) run --rm -v $(DIR):$(BUILDMNT) -w $(BUILDMNT) $(BUILD_IMAGE) /bin/sh -c all: container container: - $(DOCKER_BUILD) 'go build' $(DOCKER) build -t $(REGISTRY)/$(TARGET):latest -t $(REGISTRY)/$(TARGET):$(VERSION) . push: $(DOCKER) push $(REGISTRY)/$(TARGET):latest $(DOCKER) push $(REGISTRY)/$(TARGET):$(VERSION) -test: - $(DOCKER_BUILD) '$(TEST)' - -vet: - $(DOCKER_BUILD) '$(VET)' - .PHONY: all local container push clean: diff --git a/cache/main.go b/cache/main.go new file mode 100644 index 00000000..d41b75bb --- /dev/null +++ b/cache/main.go @@ -0,0 +1,25 @@ +//This file is only used to speedup the docker build +//We use this to download and compile the go module dependencies before adding our own source code. +//See Dockerfile for more details + +package main + +import ( + _ "flag" + _ "net/http" + _ "regexp" + _ "strings" + + _ "docker.io/go-docker" + _ "docker.io/go-docker/api/types" + _ "github.com/golang/glog" + _ "github.com/prometheus/client_golang/prometheus" + _ "github.com/prometheus/client_golang/prometheus/promhttp" + _ "golang.org/x/net/context" + _ "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/kmsg" + _ "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types" +) + +func main() { + +} diff --git a/main.go b/main.go index b27e20e9..c9f2b2c5 100644 --- a/main.go +++ b/main.go @@ -2,14 +2,12 @@ package main import ( "flag" - "fmt" "net/http" "regexp" "strings" docker_client "docker.io/go-docker" docker_types "docker.io/go-docker/api/types" - docker_filters "docker.io/go-docker/api/types/filters" "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -18,10 +16,8 @@ import ( "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types" ) -const ( - OOMMatchExpression = ".*killed as a result of limit of.*" - PodExtractExpression = "^.+/pod(\\w+\\-\\w+\\-\\w+\\-\\w+\\-\\w+)/.+$" - PodUIDLabel = "io.kubernetes.pod.uid" +var ( + kmesgRE = regexp.MustCompile("/pod(\\w+\\-\\w+\\-\\w+\\-\\w+\\-\\w+)/([a-f0-9]+) killed as a result of limit of /kubepods") ) var ( @@ -74,66 +70,43 @@ func main() { } for log := range logCh { - podUID := getPodUIDFromLog(log.Message) - if podUID != "" { - container, err := getContainerFromPod(podUID, dockerClient) - + podUID, containerID := getContainerIDFromLog(log.Message) + if containerID != "" { + container, err := getContainer(containerID, dockerClient) if err != nil { - glog.Warningf("Could not get container for pod UID %s: %v", podUID, err) + glog.Warningf("Could not get container %s for pod %s: %v", containerID, podUID, err) } else { - prometheusCount(container) + prometheusCount(container.Config.Labels) } } } } -func getPodUIDFromLog(log string) string { - match, err := regexp.MatchString(OOMMatchExpression, log) - if err != nil { - return "" - } - - var ret []string - if match { - re := regexp.MustCompile(PodExtractExpression) - ret = re.FindStringSubmatch(log) - if len(ret) == 2 { - return ret[1] - } +func getContainerIDFromLog(log string) (string, string) { + if matches := kmesgRE.FindStringSubmatch(log); matches != nil { + return matches[1], matches[2] } - return "" + return "", "" } -func getContainerFromPod(podUID string, cli *docker_client.Client) (docker_types.Container, error) { - filters := docker_filters.NewArgs() - filters.Add("label", fmt.Sprintf("%s=%s", PodUIDLabel, podUID)) - filters.Add("label", fmt.Sprintf("%s=%s", "io.kubernetes.docker.type", "container")) - - listOpts := docker_types.ContainerListOptions{ - Filters: filters, - } - - containers, err := cli.ContainerList(context.Background(), listOpts) +func getContainer(containerID string, cli *docker_client.Client) (docker_types.ContainerJSON, error) { + container, err := cli.ContainerInspect(context.Background(), containerID) if err != nil { - return docker_types.Container{}, err - } - - if len(containers) < 1 { - return docker_types.Container{}, fmt.Errorf("There should be at least one container with UID %s", podUID) + return docker_types.ContainerJSON{}, err } + return container, nil - return containers[0], nil } -func prometheusCount(container docker_types.Container) { +func prometheusCount(containerLabels map[string]string) { var counter prometheus.Counter var err error var labels map[string]string labels = make(map[string]string) for key, label := range prometheusContainerLabels { - labels[label] = container.Labels[key] + labels[label] = containerLabels[key] } glog.V(5).Infof("Labels: %v\n", labels) diff --git a/main_test.go b/main_test.go index ec21fb0f..bb3d324d 100644 --- a/main_test.go +++ b/main_test.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "reflect" "strings" "testing" @@ -11,17 +10,22 @@ import ( ) func TestGetPodUIDFromLog(t *testing.T) { - klog, podUIDs := getTestData() - var extractedUIDs []string + klog, podUIDs, containerIDs := getTestData() + + var extractedContainerIDs []string + var extractedPodUIDs []string for _, msg := range klog { parsedMsg, err := parseMessage(msg) require.NoError(t, err, "There should be no error while parsing kernel log") - uid := getPodUIDFromLog(parsedMsg.Message) - extractedUIDs = append(extractedUIDs, uid) + uid, cid := getContainerIDFromLog(parsedMsg.Message) + fmt.Println(uid) + extractedContainerIDs = append(extractedContainerIDs, cid) + extractedPodUIDs = append(extractedPodUIDs, uid) } - require.True(t, reflect.DeepEqual(podUIDs, extractedUIDs), "Extracted UIDs do not match the expected result") + require.Equal(t, containerIDs, extractedContainerIDs, "Extracted container ids do not match the expected result") + require.Equal(t, podUIDs, extractedPodUIDs, "Extracted container ids do not match the expected result") } func parseMessage(input string) (kmsgparser.Message, error) { @@ -44,7 +48,7 @@ func parseMessage(input string) (kmsgparser.Message, error) { }, nil } -func getTestData() ([]string, []string) { +func getTestData() ([]string, []string, []string) { return []string{ "6,22743,6115623303887,-;Task in /kubepods/burstable/pode501ca8a-ec23-11e8-b17a-0a586444015a/f24766bce80e0ce4f0ca2887da2be9d0d250448d7ef503d9f85bf5e549c757d5 killed as a result of limit of /kubepods/burstable/pode501ca8a-ec23-11e8-b17a-0a586444015a", "6,23800,6780904484233,-;Task in /kubepods/burstable/pod0c4e2576-ef09-11e8-b17a-0a586444015a/9df959ad4292532c5d551226063bd840b906cbf118983fffefa0e3ab90331dc2 killed as a result of limit of /kubepods/burstable/pod0c4e2576-ef09-11e8-b17a-0a586444015a/9df959ad4292532c5d551226063bd840b906cbf118983fffefa0e3ab90331dc2", @@ -52,5 +56,9 @@ func getTestData() ([]string, []string) { []string{ "e501ca8a-ec23-11e8-b17a-0a586444015a", "0c4e2576-ef09-11e8-b17a-0a586444015a", + }, + []string{ + "f24766bce80e0ce4f0ca2887da2be9d0d250448d7ef503d9f85bf5e549c757d5", + "9df959ad4292532c5d551226063bd840b906cbf118983fffefa0e3ab90331dc2", } }