From e40109f69c41b142123a94ad3202b60101fa1b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geyslan=20Greg=C3=B3rio?= Date: Mon, 20 May 2024 21:14:04 -0300 Subject: [PATCH 1/3] fix(build): static build might require zstd lib libelf might be built with zstd support in some environments, so for static builds (mainly) it is necessary to ensure that the zstd library is available. https://sourceware.org/git/?p=elfutils.git;a=commit;h=ed688a59b4d4f5ccf6ef15244e5a9139f71769a3 --- .github/actions/build-dependencies/action.yaml | 2 +- Makefile | 11 +++++++++-- Vagrantfile | 2 +- builder/Dockerfile.alpine-tracee-container | 3 ++- builder/Dockerfile.alpine-tracee-make | 1 + builder/Dockerfile.ubuntu-tracee-make | 1 + docs/contributing/building/building.md | 5 +++-- 7 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/actions/build-dependencies/action.yaml b/.github/actions/build-dependencies/action.yaml index e81417094e3f..2a94d2807af4 100644 --- a/.github/actions/build-dependencies/action.yaml +++ b/.github/actions/build-dependencies/action.yaml @@ -8,7 +8,7 @@ runs: run: | sudo apt-get update sudo apt-get install -y bsdutils build-essential pkgconf - sudo apt-get install -y zlib1g-dev libelf-dev + sudo apt-get install -y zlib1g-dev libelf-dev libzstd-dev sudo apt-get install -y software-properties-common shell: bash - name: Install Golang diff --git a/Makefile b/Makefile index 57c54011a252..03885d15c958 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ CMD_CONTROLLER_GEN ?= controller-gen LIB_ELF ?= libelf LIB_ZLIB ?= zlib +LIB_ZSTD ?= libzstd define pkg_config $(CMD_PKGCONFIG) --libs $(1) @@ -172,6 +173,7 @@ env: @echo --------------------------------------- @echo "LIB_ELF $(LIB_ELF)" @echo "LIB_ZLIB $(LIB_ZLIB)" + @echo "LIB_ZSTD $(LIB_ZSTD)" @echo --------------------------------------- @echo "VERSION $(VERSION)" @echo "LAST_GIT_TAG $(LAST_GIT_TAG)" @@ -316,7 +318,7 @@ $(OUTPUT_DIR)/btfhub: # LIBBPF_CFLAGS = "-fPIC" -LIBBPF_LDLAGS = +LIBBPF_LDFLAGS = LIBBPF_SRC = ./3rdparty/libbpf/src $(OUTPUT_DIR)/libbpf/libbpf.a: \ @@ -392,7 +394,10 @@ TRACEE_SRC_DIRS = ./cmd/ ./pkg/ ./signatures/ TRACEE_SRC = $(shell find $(TRACEE_SRC_DIRS) -type f -name '*.go' ! -name '*_test.go') CUSTOM_CGO_CFLAGS = "-I$(abspath $(OUTPUT_DIR)/libbpf)" -CUSTOM_CGO_LDFLAGS = "$(shell $(call pkg_config, $(LIB_ELF))) $(shell $(call pkg_config, $(LIB_ZLIB))) $(abspath $(OUTPUT_DIR)/libbpf/libbpf.a)" +CUSTOM_CGO_LDFLAGS = "$(shell $(call pkg_config, $(LIB_ELF))) \ + $(shell $(call pkg_config, $(LIB_ZLIB))) \ + $(shell $(call pkg_config, $(LIB_ZSTD))) \ + $(abspath $(OUTPUT_DIR)/libbpf/libbpf.a)" GO_ENV_EBPF = GO_ENV_EBPF += GOOS=linux @@ -439,6 +444,7 @@ $(OUTPUT_DIR)/tracee: \ | .checkver_$(CMD_GO) \ .checklib_$(LIB_ELF) \ .checklib_$(LIB_ZLIB) \ + .checklib_$(LIB_ZSTD) \ btfhub \ signatures # @@ -472,6 +478,7 @@ $(OUTPUT_DIR)/tracee-ebpf: \ | .checkver_$(CMD_GO) \ .checklib_$(LIB_ELF) \ .checklib_$(LIB_ZLIB) \ + .checklib_$(LIB_ZSTD) \ btfhub # $(MAKE) $(OUTPUT_DIR)/btfhub diff --git a/Vagrantfile b/Vagrantfile index 58951af8bd72..e7848c5bf8b8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -95,7 +95,7 @@ Vagrant.configure("2") do |config| ln -s "$path" "${path%-*}" done - apt-get install --yes zlib1g-dev libelf-dev + apt-get install --yes zlib1g-dev libelf-dev libzstd-dev apt-get install --yes protobuf-compiler apt-get install --yes linux-tools-"$(uname -r)" || apt-get install --yes linux-tools-generic diff --git a/builder/Dockerfile.alpine-tracee-container b/builder/Dockerfile.alpine-tracee-container index b214e808e5ad..26303d65847d 100644 --- a/builder/Dockerfile.alpine-tracee-container +++ b/builder/Dockerfile.alpine-tracee-container @@ -18,7 +18,7 @@ USER root RUN apk --no-cache update && \ apk --no-cache add coreutils && \ apk --no-cache add sudo curl && \ - apk --no-cache add libelf zlib && \ + apk --no-cache add libelf zlib zstd && \ apk --no-cache add libc6-compat # install OPA @@ -47,6 +47,7 @@ RUN apk --no-cache update && \ apk --no-cache add elfutils-dev && \ apk --no-cache add libelf-static && \ apk --no-cache add zlib-static && \ + apk --no-cache add zstd-static && \ rm -f /usr/bin/cc && \ rm -f /usr/bin/clang && \ rm -f /usr/bin/clang++ && \ diff --git a/builder/Dockerfile.alpine-tracee-make b/builder/Dockerfile.alpine-tracee-make index fa20e6458fcf..bb5acaf21955 100644 --- a/builder/Dockerfile.alpine-tracee-make +++ b/builder/Dockerfile.alpine-tracee-make @@ -19,6 +19,7 @@ RUN apk --no-cache update && \ apk --no-cache add elfutils-dev && \ apk --no-cache add libelf-static && \ apk --no-cache add zlib-static && \ + apk --no-cache add zstd-static && \ rm -f /usr/bin/cc && \ rm -f /usr/bin/clang && \ rm -f /usr/bin/clang++ && \ diff --git a/builder/Dockerfile.ubuntu-tracee-make b/builder/Dockerfile.ubuntu-tracee-make index 0030365b4b7b..7fa57a5797bd 100644 --- a/builder/Dockerfile.ubuntu-tracee-make +++ b/builder/Dockerfile.ubuntu-tracee-make @@ -18,6 +18,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y linux-headers-generic && \ apt-get install -y libelf-dev && \ apt-get install -y zlib1g-dev && \ + apt-get install -y libzstd-dev && \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 130 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 --slave /usr/bin/llc llc /usr/bin/llc-12 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-12 --slave /usr/bin/clangd clangd /usr/bin/clangd-12 && \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 140 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-14 --slave /usr/bin/llc llc /usr/bin/llc-14 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-14 --slave /usr/bin/clangd clangd /usr/bin/clangd-14 diff --git a/docs/contributing/building/building.md b/docs/contributing/building/building.md index 420bf25465a8..e107a66c2954 100644 --- a/docs/contributing/building/building.md +++ b/docs/contributing/building/building.md @@ -14,10 +14,11 @@ 1. **clang** && **llvm** (12, 13 or 14) 1. **golang** (1.21) - 1. **libelf** and **libelf-dev** + 1. **libelf** and **libelf-dev** (or elfutils-libelf and elfutils-libelf-devel) - 1. **zlib1g** and **zlib1g-dev** + 1. **zlib1g** and **zlib1g-dev** (or zlib and zlib-devel) + 1. **libzstd-dev** for static build (libelf linkage) 1. **clang-format-12** (specific version) for `fix-fmt` > You might take a look at the following files to understand how to have a From 656df9ac86d8b792a23fb72cfc166623fa8570e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geyslan=20Greg=C3=B3rio?= Date: Tue, 21 May 2024 15:21:54 -0300 Subject: [PATCH 2/3] chore: install required bpftool from btfhub --- builder/Dockerfile.alpine-tracee-container | 7 +------ builder/Dockerfile.alpine-tracee-make | 7 +++++++ builder/Dockerfile.ubuntu-tracee-make | 7 +++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/builder/Dockerfile.alpine-tracee-container b/builder/Dockerfile.alpine-tracee-container index 26303d65847d..f0dc975edc9d 100644 --- a/builder/Dockerfile.alpine-tracee-container +++ b/builder/Dockerfile.alpine-tracee-container @@ -86,12 +86,7 @@ RUN apk --no-cache update && \ RUN cd /tmp && \ git clone https://github.com/aquasecurity/btfhub.git && \ cd ./btfhub && \ - git submodule update --init --recursive 3rdparty/bpftool && \ - cd ./3rdparty/bpftool && \ - make -C src clean && \ - make -C src all && \ - cp ./src/bpftool /usr/sbin/bpftool && \ - make -C src clean + ./3rdparty/bpftool.sh # # tracee-make diff --git a/builder/Dockerfile.alpine-tracee-make b/builder/Dockerfile.alpine-tracee-make index bb5acaf21955..f3688c43db38 100644 --- a/builder/Dockerfile.alpine-tracee-make +++ b/builder/Dockerfile.alpine-tracee-make @@ -53,6 +53,13 @@ RUN apk --no-cache update && \ ln -s /usr/lib/llvm14/bin/llvm-readelf /usr/bin/llvm-readelf && \ ln -s /usr/lib/llvm14/bin/opt /usr/bin/opt +# install bpftool from btfhub + +RUN cd /tmp && \ + git clone https://github.com/aquasecurity/btfhub.git && \ + cd ./btfhub && \ + ./3rdparty/bpftool.sh + # install OPA RUN altarch=$(uname -m | sed 's:x86_64:amd64:g' | sed 's:aarch64:arm64:g') && \ diff --git a/builder/Dockerfile.ubuntu-tracee-make b/builder/Dockerfile.ubuntu-tracee-make index 7fa57a5797bd..19c80f8edbaa 100644 --- a/builder/Dockerfile.ubuntu-tracee-make +++ b/builder/Dockerfile.ubuntu-tracee-make @@ -22,6 +22,13 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 130 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-12 --slave /usr/bin/llc llc /usr/bin/llc-12 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-12 --slave /usr/bin/clangd clangd /usr/bin/clangd-12 && \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 140 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-14 --slave /usr/bin/llc llc /usr/bin/llc-14 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-14 --slave /usr/bin/clangd clangd /usr/bin/clangd-14 +# install bpftool from btfhub + +RUN cd /tmp && \ + git clone https://github.com/aquasecurity/btfhub.git && \ + cd ./btfhub && \ + ./3rdparty/bpftool.sh + # install OPA RUN altarch=$(uname -m | sed 's:x86_64:amd64:g' | sed 's:aarch64:arm64:g') && \ From ac9bdb162c877ee3c69aab946ee10df6ab373f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geyslan=20Greg=C3=B3rio?= Date: Tue, 21 May 2024 15:48:05 -0300 Subject: [PATCH 3/3] chore: temp fix for install libzstd on AMIs AMIs have to be updated and installations like this commented out. --- tests/e2e-install-deps.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/e2e-install-deps.sh b/tests/e2e-install-deps.sh index c361bd0a20ca..c68fce138c0e 100755 --- a/tests/e2e-install-deps.sh +++ b/tests/e2e-install-deps.sh @@ -264,6 +264,22 @@ remove_golang_os_packages() { apt-get --purge autoremove -y } +install_libzstd_os_packages() { + case $ID in + "ubuntu") + wait_for_apt_locks + apt-get install -y libzstd-dev + ;; + "almalinux") + yum install -y libzstd-devel + ;; + *) + echo "Unsupported OS: $ID" + exit 1 + ;; + esac +} + # Main logic. # Note: I left commented out the commands that would (re)install clang-14. This @@ -328,3 +344,6 @@ if [[ $ID == "almalinux" ]]; then remove_golang_usr_bin_files install_golang_from_github fi + +# for static builds libelf might require libzstd +install_libzstd_os_packages