feat: update config (#56)

* - merge {docker,cri}-runtime.conf to container-runtime.conf - enable multiline in tail input * get runtime_parser in entrypoint.sh * update local conf file and testdata * build: use bitnami/fluent-bit as base image * fix typo in fluent-bit.conf include * remove useless firstline parser for old multiline in tail input * set default container_runtime to docker
erda-project · Feb 29, 2024 · de6266f · de6266f
1 parent 485d879
commit de6266f
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 206 deletions.
diff --git a/build.sh b/build.sh
@@ -14,27 +14,49 @@ echo "version=${v}"
 
 image="${DOCKER_REGISTRY}/erda-fluent-bit:${v}"
 
-echo "image=${image}"
-
-docker login -u "${DOCKER_REGISTRY_USERNAME}" -p "${DOCKER_REGISTRY_PASSWORD}" "${DOCKER_REGISTRY}"
-#docker buildx create --use
-#docker buildx build --platform linux/amd64,linux/arm64 -t "${image}" \
-#docker buildx build --platform linux/${ARCH} -t "${image}" \
-#    --label "branch=$(git rev-parse --abbrev-ref HEAD)" \
-#    --label "commit=$(git rev-parse HEAD)" \
-#    --label "build-time=$(date '+%Y-%m-%d %T%z')" \
-#    --push \
-#    -f dockerfiles/Dockerfile .
-
-buildctl --addr tcp://buildkitd.default.svc.cluster.local:1234 \
+echo "image: ${image}"
+
+platforms=${PLATFORMS:-"linux/amd64,linux/arm64"}
+echo "platforms: ${platforms}"
+
+function local_build_func() {
+  docker login -u "${DOCKER_REGISTRY_USERNAME}" -p "${DOCKER_REGISTRY_PASSWORD}" "${DOCKER_REGISTRY}"
+  docker buildx create --use
+  docker buildx build --platform ${platforms} -t "${image}" \
+    --label "branch=$(git rev-parse --abbrev-ref HEAD)" \
+    --label "commit=$(git rev-parse HEAD)" \
+    --label "build-time=$(date '+%Y-%m-%d %T%z')" \
+    --push \
+    -f dockerfiles/Dockerfile .
+}
+
+function k8s_build_func() {
+  buildctl --addr tcp://buildkitd.default.svc.cluster.local:1234 \
     --tlscacert=/.buildkit/ca.pem \
     --tlscert=/.buildkit/cert.pem \
     --tlskey=/.buildkit/key.pem \
     build \
     --frontend dockerfile.v0 \
-    --opt platform=${PLATFORMS} \
+    --opt platform=${platforms} \
     --local context=/.pipeline/container/context/fluent-bit \
     --local dockerfile=/.pipeline/container/context/fluent-bit/dockerfiles \
     --output type=image,name=${image},push=true
-
-echo "image=${image}" >> $METAFILE
+  echo "image=${image}" >> $METAFILE
+}
+
+# switch by local or k8s
+local_build=${LOCAL_BUILD:-"false"}
+case $local_build in
+  "true")
+    echo "local build"
+    local_build_func
+    ;;
+  "false")
+    echo "k8s build"
+    k8s_build_func
+    ;;
+  *)
+    echo "unknown build"
+    exit 1
+    ;;
+esac
diff --git a/conf/ds/cri-runtime.conf → conf/ds/container-runtime.conf b/conf/ds/cri-runtime.conf → conf/ds/container-runtime.conf
@@ -13,12 +13,19 @@
     DB.locking           true
     DB.journal_mode      WAL
     Mem_Buf_Limit        300MB
+    Read_from_Head       false
 
-    Parser               cri
+    Parser               ${RUNTIME_PARSER}
+    multiline.parser     docker, cri
 
 [FILTER]
     Name                   multiline
     match                  kube.*
-    multiline.parser       java
+    multiline.parser       java, go, python
     multiline.key_content  log
     buffer                 off
+
+[FILTER]
+    Name   modify
+    match  *
+    add    container_runtime ${DICE_CONTAINER_RUNTIME}
diff --git a/conf/ds/docker-runtime.conf b/conf/ds/docker-runtime.conf
diff --git a/conf/ds/fluent-bit.conf b/conf/ds/fluent-bit.conf
@@ -9,16 +9,7 @@
     http_listen  0.0.0.0
     http_port    2020
 
-@include ${INCLUDE_RUNTIME_CONF}
-
-# [FILTER]
-#     Name         throttle
-#     Match        *
-#     Rate         ${FLUENTBIT_THROTTLE_RATE}
-#     Window       5
-#     Interval     1s
-#     Print_Status ${FLUENTBIT_THROTTLE_PRINT_STATUS}
-#     Retain       ${FLUENTBIT_THROTTLE_RETAIN}
+@include container-runtime.conf
 
 [FILTER]
     name            parser
@@ -28,19 +19,6 @@
     Reserve_Data    True
     Parser          truncate-log
 
-# may crash and lost log when a high load (issue: https://github.com/fluent/fluent-bit/issues/4940)
-# if enable filesystem storage may occupy huge disk space when a high load (issue: https://github.com/fluent/fluent-bit/issues/3718)
-# In fact, the emitter seems don't handle the back pressure correctly.
-# So I only have to increase emitter_mem_buf_limit and decrease flush_ms, hope to decrease the possibility of log lost.
-# TODO. wait for address
-# [FILTER]
-#     Name                   multiline
-#     match                  kube.*
-#     multiline.key_content  log
-#     multiline.parser       java
-#     flush_ms               500
-#     emitter_mem_buf_limit  ${FLUENTBIT_FILTER_MULTILINE_MEM_BUF_LIMIT}
-
 [FILTER]
     Name                kubernetes
     Match               kube.*

diff --git a/conf/ds/fluent-bit.conf.local b/conf/ds/fluent-bit.conf.local
@@ -2,18 +2,18 @@
     flush        0.5
     daemon       Off
     grace        5
-    log_level    info
+    log_level    ${LOG_LEVEL}
     parsers_file parsers.conf
     http_server  On
     http_listen  0.0.0.0
     http_port    2020
 
 [INPUT]
     name                 tail
-    path                 ./testdata/k8s/cri.log
-    DB                   ./flb_k8s.db
+    # set in .envrc
+    path                 ./testdata/k8s/${RUNTIME_PARSER}.log
+    # DB                   ./flb_k8s.db
     Tag                  kube.*
-    Read_From_Head       true
     Buffer_Chunk_Size    1MB
     Buffer_Max_Size      10MB
     Skip_Long_Lines      On
@@ -23,16 +23,23 @@
     DB.locking           true
     DB.journal_mode      WAL
     Mem_Buf_Limit        300MB
+    Read_from_Head       true
 
-    Parser               cri
+    Parser               ${RUNTIME_PARSER}
+    Multiline.parser     cri, docker
 
 [FILTER]
     Name                   multiline
     match                  kube.*
-    multiline.parser       java
+    multiline.parser       java, go, python
     multiline.key_content  log
     buffer                 off
 
+[FILTER]
+    Name   modify
+    match  *
+    add    container_runtime ${DICE_CONTAINER_RUNTIME}
+
 [FILTER]
     name modify
     match *
@@ -53,17 +60,15 @@
     match *
     remove_wildcard __pri_
 
+[FILTER]
+    name lua
+    match *
+    protected_mode true
+    time_as_table false
+    script time.lua
+    call ensure_time
+
 [OUTPUT]
-    name                http
+    name                stdout
     Match               kube.*
-    host                ${COLLECTOR_HOST}
-    port                ${COLLECTOR_PORT}
-    uri                 /collect/logs-all
-    compress            gzip
-    format              json
-    http_user           ${COLLECTOR_AUTH_USERNAME}
-    http_passwd         ${COLLECTOR_AUTH_PASSWORD}
-    json_date_key       false
-    json_date_format    iso8601
-    Retry_Limit         100
-    log_response_payload false
+    Format json
diff --git a/conf/ds/parsers.conf b/conf/ds/parsers.conf
@@ -16,24 +16,10 @@
     Time_Format         %Y-%m-%dT%H:%M:%S.%LZ
     Time_Keep           On
 
-[PARSER]
-    Name                docker_firstline
-    Format              regex
-    Regex               (?<log>(?<="log":")\d{4}[\/-]\d{1,2}[\/-]\d{1,2}[ T]\d{2}:\d{2}:\d{2}.*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
-    Time_Key            time
-    Time_Format         %Y-%m-%dT%H:%M:%S.%LZ
-
 [PARSER]
     Name        cri
     Format      regex
     Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<__pri_logtag>[^ ]*)\s?(?<log>[\s\S]*)$
     Time_Key    time
     Time_Format %Y-%m-%dT%H:%M:%S.%L%z
     Time_Keep   On
-
-[PARSER]
-    Name                cri_firstline
-    Format              regex
-    Regex               ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<__pri_logtag>[^ ]*) (?<log>(\d{4}[\/-]\d{1,2}[\/-]\d{1,2}[ T]\d{2}:\d{2}:\d{2}.*?).*)$
-    Time_Key            time
-    Time_Format         %Y-%m-%dT%H:%M:%S.%L%z
diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile
@@ -1,89 +1,20 @@
+# build erda.so
 FROM registry.erda.cloud/erda-x/golang:1.17 as gobuilder
 
 COPY . /build
 WORKDIR /build
 
 RUN make build
 
-FROM registry.erda.cloud/erda-x/debian:10 as builder
+# bitnami/fluent-bit (gnu sed checked)
+FROM bitnami/fluent-bit:2.2.2 AS fluentbit
+USER root
 
-# Fluent Bit version
-ENV FLB_MAJOR 1
-ENV FLB_MINOR 9
-ENV FLB_PATCH 10
-ENV FLB_VERSION 1.9.10
+RUN ln -s /opt/bitnami/fluent-bit /fluent-bit
+WORKDIR /fluent-bit
+RUN mkdir -p {etc,bin}
 
-#ARG FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.tar.gz
-ARG FLB_TARBALL=https://erda-project.oss-cn-hangzhou.aliyuncs.com/fluent-bit/fluent-bit-$FLB_VERSION.tar.gz
-ENV FLB_SOURCE $FLB_TARBALL
-RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/
+COPY conf/ entrypoint.sh etc/
+COPY --from=gobuilder /build/out_erda.so bin/
 
-ENV DEBIAN_FRONTEND noninteractive
-
-COPY filter_throttle_patch /tmp/filter_throttle_patch
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    build-essential \
-    curl \
-    ca-certificates \
-    cmake \
-    make \
-    tar \
-    libssl-dev \
-    libsasl2-dev \
-    pkg-config \
-    libsystemd-dev \
-    zlib1g-dev \
-    libpq-dev \
-    postgresql-server-dev-all \
-    flex \
-    bison \
-    && curl -L -o "/tmp/fluent-bit.tar.gz" ${FLB_SOURCE} \
-    && cd tmp/ && mkdir fluent-bit \
-    && tar zxfv fluent-bit.tar.gz -C ./fluent-bit --strip-components=1 \
-    && cd fluent-bit/build/ \
-    && rm -rf /tmp/fluent-bit/build/* \
-    && if [ "$FLB_VERSION" = "1.9.3" ] ; then rm -rf /tmp/fluent-bit/plugins/filter_throttle/throttle.c;  rm -rf /tmp/fluent-bit/plugins/filter_throttle/throttle.h; cp /tmp/filter_throttle_patch/v1.9.3/* /tmp/fluent-bit/plugins/filter_throttle/; fi
-
-WORKDIR /tmp/fluent-bit/build/
-RUN cmake -DFLB_RELEASE=On \
-          -DFLB_TRACE=Off \
-          -DFLB_JEMALLOC=On \
-          -DFLB_TLS=On \
-          -DFLB_SHARED_LIB=Off \
-          -DFLB_EXAMPLES=Off \
-          -DFLB_HTTP_SERVER=On \
-          -DFLB_IN_SYSTEMD=On \
-          -DFLB_OUT_KAFKA=On \
-          -DFLB_CONFIG_YAML=Off \
-          -DFLB_OUT_PGSQL=On .. \
-          -DFLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
-
-RUN make -j $(getconf _NPROCESSORS_ONLN)
-RUN install bin/fluent-bit /fluent-bit/bin/
-
-# Configuration files
-#COPY conf/fluent-bit.conf \
-#     conf/parsers.conf \
-#     conf/parsers_ambassador.conf \
-#     conf/parsers_java.conf \
-#     conf/parsers_extra.conf \
-#     conf/parsers_openstack.conf \
-#     conf/parsers_cinder.conf \
-#     conf/plugins.conf \
-#     /fluent-bit/etc/
-
-COPY conf/ /fluent-bit/etc/
-COPY entrypoint.sh /fluent-bit/etc/
-
-COPY --from=gobuilder /build/out_erda.so /fluent-bit/bin/
-
-FROM registry.erda.cloud/erda-x/debian:10
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    bash gdb valgrind build-essential vim curl sqlite3 ca-certificates libpq5 sed
-
-COPY --from=builder /fluent-bit /fluent-bit
-
-CMD ["/fluent-bit/etc/entrypoint.sh"]
+ENTRYPOINT ["/fluent-bit/etc/entrypoint.sh"]
diff --git a/dockerfiles/Dockerfile_no_throttle b/dockerfiles/Dockerfile_no_throttle