Revamp Testing Infrastructure and run Multi-Kernel Tests in CI

.github/workflows/formatting-build.yml

@@ -0,0 +1,47 @@
+name: Formatting and Build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  formatting-and-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Test code formatting
+        run: make test-format
+      - name: Cache hash
+        run: echo "GITHUB_CACHE_HASH=$(md5sum Makefile docker/Dockerfile.builder | md5sum | awk '{print $1}')" >> $GITHUB_ENV
+      - name: Setup docker cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/docker
+          key: ${{ env.GITHUB_CACHE_HASH }}-v0
+          restore-keys: |
+            ${{ env.GITHUB_CACHE_HASH }}-v0
+      - name: Configure Docker
+        run: |
+          sudo systemctl stop docker
+          sudo rm -rf /var/lib/docker
+          sudo mkdir -p /var/lib/docker
+          sudo mkdir -p ~/.cache/docker
+          sudo chown -fR root:root ~/.cache/docker
+          sudo mount --rbind ~/.cache/docker /var/lib/docker
+          sudo systemctl start docker
+      - name: Test build
+        run: make build
+      - name: Test for source differences post-build
+        run: git diff --exit-code
+      - name: Test container image build
+        run: make container
+      - name: Fix permissions (last step)
+        run: |
+          docker system prune -f
+          sudo systemctl stop docker
+          sudo umount /var/lib/docker
+          sudo chown -R $USER:$USER ~/.cache/docker
+          sudo rm -rf ~/.cache/docker/volumes/backingFsBlockDev
+          sudo find ~/.cache/docker -name work | sudo xargs chmod -R 700

.github/workflows/multikernel-tester.yml

@@ -0,0 +1,83 @@
+name: Multi Kernel Testing
+on: [ pull_request ]
+
+jobs:
+  multikernel_tester:
+    strategy:
+      # If a failure occurs, run the other arches/distros to the end. It's useful to see if it
+      # occurs on other kernels as well
+      fail-fast: false
+      matrix:
+        kernel_flavor: [ mainline ]
+        arch: [ x86_64, aarch64 ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    - name: Build eBPF probes and userspace components
+      # Enable BPF printk to dump useful information if tests fail
+      run: make build ARCH=${{ matrix.arch }} EXTRA_CMAKE_FLAGS="-DENABLE_BPF_PRINTK=TRUE"
+    - name: Auth GCP
+      uses: 'google-github-actions/auth@v0'
+      with:
+        credentials_json: '${{ secrets.ACTIONS_GCP_JSON_CREDENTIALS }}'
+    - name: 'Setup gcloud'
+      uses: 'google-github-actions/setup-gcloud@v0'
+    - name: Create kernel images directory
+      run: |
+        sudo mkdir -p /kernel-images
+        sudo chown -fR $USER:$USER /kernel-images
+    # We get the hash of every kernel image and then hash the result. This results in a re-fetch
+    # if anything in the GCS directory changes at all
+    - name: Calculate kernel image cache key
+      run: |
+        echo "GCS_HASH=$(gsutil hash -h $HASH_PATH | md5sum | awk '{ print $1 }')" >> $GITHUB_ENV
+      env:
+        HASH_PATH: gs://ebpf-ci-kernel-images/${{ matrix.kernel_flavor }}/${{ matrix.arch }}/images/*
+    - name: Attempt to get kernel images from cache
+      id: cache
+      uses: actions/cache@v3
+      with:
+        path: /kernel-images
+        key: ${{ matrix.kernel_flavor }}-${{ matrix.arch }}-${{ env.GCS_HASH }}-kernel-images
+    - name: Download kernel images from GCS (if uncached)
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: gsutil -m rsync -r gs://ebpf-ci-kernel-images/${{ matrix.kernel_flavor}}/${{ matrix.arch }}/images /kernel-images/
+    - name: Install packages needed for testing
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends \
+        gcc-aarch64-linux-gnu \
+        libc6-dev-arm64-cross \
+        parallel \
+        qemu-system-x86 \
+        qemu-system-arm
+      env:
+        DEBIAN_FRONTEND: noninteractive
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: '1.17'
+    - name: Install Bluebox
+      run: |
+        go install github.com/florianl/bluebox@b8590fb1850f56df6e6d7786931fcabdc1e9173d
+        echo "$HOME/go" >> $GITHUB_PATH
+    - name: Run tests
+      working-directory: testing
+      run: |
+        ./run_tests.sh \
+        -a ${{ matrix.arch }} \
+        -d /kernel-images \
+        -e ../artifacts-${{ matrix.arch }}/non-GPL/EventsTrace/EventsTrace
+    - name: Archive test summary
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: run-summary-${{ matrix.kernel_flavor }}-${{ matrix.arch }}.txt
+        path: testing/bpf-check-summary.txt
+    - name: Archive individual test results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: results-${{ matrix.kernel_flavor }}-${{ matrix.arch }}
+        path: testing/results

GPL/EventProbe/CMakeLists.txt

@@ -7,7 +7,7 @@
 
 # BPF program
 
-if (CMAKE_BUILD_TYPE STREQUAL Debug)
+if (CMAKE_BUILD_TYPE STREQUAL Debug OR ENABLE_BPF_PRINTK)
     set(BPF_DEBUG_TRACE 1)
 else()
     set(BPF_DEBUG_TRACE 0)

GPL/EventProbe/PathResolver.h

@@ -238,7 +238,21 @@ static void ebpf_resolve_kernfs_node_to_string(char *buf, struct kernfs_node *kn
 
 static void ebpf_resolve_pids_ss_cgroup_path_to_string(char *buf, const struct task_struct *task)
 {
-    struct kernfs_node *kn = BPF_CORE_READ(task, cgroups, subsys[pids_cgrp_id], cgroup, kn);
+    /*
+     * Since pids_cgrp_id is an enum value, we need to get it at runtime as it
+     * can change kernel-to-kernel depending on the kconfig or possibly not be
+     * enabled at all.
+     */
+    int cgrp_id;
+    if (bpf_core_enum_value_exists(enum cgroup_subsys_id, pids_cgrp_id)) {
+        cgrp_id = bpf_core_enum_value(enum cgroup_subsys_id, pids_cgrp_id);
+    } else {
+        /* Pids cgroup is not enabled on this kernel */
+        buf[0] = '\0';
+        return;
+    }
+
+    struct kernfs_node *kn = BPF_CORE_READ(task, cgroups, subsys[cgrp_id], cgroup, kn);
     ebpf_resolve_kernfs_node_to_string(buf, kn);
 }
 

Makefile

@@ -1,39 +1,68 @@
 ARCH ?= $(shell arch)
 BUILD_DIR ?= artifacts-${ARCH}
-SUDO ?= 
-PWD ?= $(shell pwd)
-CONTAINER_RUNTIME ?= docker
+PWD = $(shell pwd)
 DOCKER_IMG_UBUNTU_VERSION ?= jammy
-BUILDER_PULL_TAG ?= us-docker.pkg.dev/elastic-security-dev/ebpf-public/builder:20220620-0715
+BUILDER_PULL_TAG ?= us-docker.pkg.dev/elastic-security-dev/ebpf-public/builder:20220621-0034
 BUILDER_TAG ?= us-docker.pkg.dev/elastic-security-dev/ebpf-public/builder:${USER}-latest
-C_INCLUDE_PATH ?=
-DOCKER_CACHE ?=
+CMAKE_FLAGS = -DARCH=${ARCH} -DBUILD_STATIC_EVENTSTRACE=True -DUSE_BUILTIN_VMLINUX=True -B${BUILD_DIR} -S${PWD}
 
+.PHONY = build build-debug build-local clean container fix-permissions format test-format
 
-.PHONY = build build-local clean container fix-permissions format test-format
+# Kludge to get around a missing header. If we don't do this, we'll get the following error when
+# building:
+#
+# In file included from /home/vagrant/ebpf/contrib/libbpf/include/uapi/linux/bpf.h:11:
+# In file included from /home/vagrant/ebpf/contrib/libbpf/include/linux/types.h:8:
+# In file included from /usr/lib/llvm-14/lib/clang/14.0.0/include/stdint.h:52:
+# /usr/include/stdint.h:26:10: fatal error: 'bits/libc-header-start.h' file not found
+# include <bits/libc-header-start.h>
+#        ^~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# The HostIsolation probes include linux/bpf.h (copied into the libbpf repo) which includes
+# linux/types.h (also copied into the libbpf repo) which includes stdint.h. The clang stdint.h
+# includes bits/libc-header-start.h which is not in our include path. The correct one to use
+# depends on which arch we're compiling for.
+ifeq ($(ARCH),x86_64)
+	export C_INCLUDE_PATH = /usr/include/x86_64-linux-gnu
+else
+	export C_INCLUDE_PATH = /usr/aarch64-linux-gnu/include
+endif
 
-container:
-	${CONTAINER_RUNTIME} build ${DOCKER_CACHE} -t ${BUILDER_TAG} --build-arg PULL_TAG=${DOCKER_IMG_UBUNTU_VERSION} -f docker/Dockerfile.builder .
-
-build-local:
-	mkdir -p ${BUILD_DIR}
-	C_INCLUDE_PATH=${C_INCLUDE_PATH} cmake -DUSE_BUILTIN_VMLINUX=True -B${BUILD_DIR} -S${PWD}
-	C_INCLUDE_PATH=${C_INCLUDE_PATH} make -C${BUILD_DIR}
+export CC=${ARCH}-linux-gnu-gcc
+export CXX=${ARCH}-linux-gnu-g++
+export AR=${ARCH}-linux-gnu-ar
+export LD=${ARCH}-linux-gnu-ld
 
 build:
-	docker run --rm -v${PWD}:${PWD} -w${PWD} ${BUILDER_PULL_TAG} 
+	docker run --rm -v${PWD}:${PWD} -w${PWD} ${BUILDER_PULL_TAG} \
+		/usr/bin/env make _internal-build ARCH=${ARCH} EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS}
+	sudo chown -fR ${USER}:${USER} ${BUILD_DIR}
 	@echo "\n++ Build Successful at `date` ++\n"
 
-fix-permissions:
+# Convenience target to pass -DCMAKE_BUILD_TYPE=Debug and -DCMAKE_C_FLAGS="-g -O0"
+build-debug:
+	docker run --rm -v${PWD}:${PWD} -w${PWD} ${BUILDER_PULL_TAG} \
+		/usr/bin/env make _internal-build-debug ARCH=${ARCH} EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS}
 	sudo chown -fR ${USER}:${USER} ${BUILD_DIR}
+	@echo "\n++ Build Successful at `date` ++\n"
+
+_internal-build-debug: CMAKE_FLAGS = -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-g -O0" ${CMAKE_COMMON_FLAGS}
+_internal-build-debug: _internal-build
+_internal-build:
+	mkdir -p ${BUILD_DIR}/
+	cmake ${EXTRA_CMAKE_FLAGS} ${CMAKE_FLAGS}
+	make -C${BUILD_DIR} -j$(shell nproc)
+
+container:
+	docker build -t ${BUILDER_TAG} -f docker/Dockerfile.builder .
 
 format:
-	find . \( -path ./contrib -o -path ./artifacts* \) -prune \
-	-o -name "*.c" -o -name "*.cpp" -o -name "*.h" -print | xargs /usr/bin/env clang-format -i
+	find non-GPL/ GPL/ testing/test_bins/ -name "*.c" -o -name "*.h" -o -name "*.cpp" | \
+		xargs /usr/bin/env clang-format -i
 
 test-format:
-	find . \( -path ./contrib -o -path ./artifacts* \) -prune \
-	-o -name "*.c" -o -name "*.cpp" -o -name "*.h" -print | xargs /usr/bin/env clang-format --dry-run -Werror
+	find non-GPL/ GPL/ testing/test_bins/ -name "*.c" -o -name "*.h" -o -name "*.cpp" | \
+		xargs /usr/bin/env clang-format -i --dry-run -Werror
 
 clean:
-	${SUDO} rm -rf artifacts-*
+	sudo rm -rf artifacts-*

docker/Dockerfile.builder

@@ -1,29 +1,45 @@
-ARG PULL_TAG=jammy
+FROM ubuntu:jammy
 
-FROM ubuntu:${PULL_TAG}
+ENV BPFTOOL_VERSION=5.15.0-33
+ENV DEBIAN_FRONTEND=noninteractive
 
-ARG BPFTOOL_VERSION=5.15.0-33
-ENV BPFTOOL_VERSION=${BPFTOOL_VERSION}
+# https://askubuntu.com/a/1323570
+#
+# We need the aarch64 libz package. We need to add ports.ubuntu.com (the
+# default repos don't have aarch64 packages) and then run dpkg
+# --add-architecture arm64 to be able to retrieve and install it.
+#
+# Furthermore, we need to mark the repos in /etc/apt/sources.list (usually
+# archive.ubuntu.org) as explicitly being amd64 only. Otherwise apt will look
+# for aarch64 packages there after we run dpkg --add-architecture arm64. This
+# will cause failures as Ubuntu doesn't store aarch64 packages at
+# archive.ubuntu.org.
+RUN echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted" >> /etc/apt/sources.list.d/arm-sources.list \
+    && echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted" >> /etc/apt/sources.list.d/arm-sources.list \
+    && echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy universe" >> /etc/apt/sources.list.d/arm-sources.list \
+    && echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates universe" >> /etc/apt/sources.list.d/arm-sources.list \
+    && echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy multiverse" >> /etc/apt/sources.list.d/arm-sources.list \
+    && echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates multiverse" >> /etc/apt/sources.list.d/arm-sources.list \
+    && echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-backports main restricted universe multiverse" >> /etc/apt/sources.list.d/arm-sources.list \
+    && dpkg --add-architecture arm64 \
+    && sed -i 's/deb/deb [arch=amd64]/g' /etc/apt/sources.list \
+    && apt-get update
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
         llvm \
         clang \
+        gcc-aarch64-linux-gnu \
+        g++-aarch64-linux-gnu \
+        libz-dev:arm64 \
         libz-dev \
         m4 \
         cmake \
         make \
         build-essential \
         groff-base \
         bmake \
-        linux-tools-generic linux-tools-${BPFTOOL_VERSION}-generic \
-    && apt-get autoremove -y --purge && apt-get autoclean && apt-get clean \
-    && sed -i -e 's/-soname /-soname=/g' /usr/share/mk/lib.mk \
-    && update-alternatives --install /usr/local/sbin/bpftool bpftool /usr/lib/linux-tools-${BPFTOOL_VERSION}/bpftool 1
-
-CMD [ \
-    "/usr/bin/env", \
-    "C_INCLUDE_PATH=/usr/include/`arch`-linux-gnu", \
-    "make", \
-    "build-local" \
-]
+        linux-tools-generic linux-tools-${BPFTOOL_VERSION}-generic && \
+    apt-get autoremove -y --purge && apt-get autoclean && apt-get clean && \
+    sed -i -e 's/-soname /-soname=/g' /usr/share/mk/lib.mk && \
+    update-alternatives --install /usr/local/sbin/bpftool bpftool /usr/lib/linux-tools-${BPFTOOL_VERSION}/bpftool 1

non-GPL/EventsTrace/EventsTrace.c

@@ -68,13 +68,15 @@ static const struct argp_option opts[] = {
      "Whether or not to consider network connection attempted events", 1},
     {"net-conn-closed", EBPF_EVENT_NETWORK_CONNECTION_CLOSED, NULL, false,
      "Whether or not to consider network connection closed events", 1},
+    {"features-autodetect", 'd', NULL, false, "Autodetect features based on running kernel", 2},
     {"set-bpf-tramp", EBPF_FEATURE_BPF_TRAMP, NULL, false, "Set feature supported: bpf trampoline",
      2},
     {},
 };
 
-uint64_t g_events_env   = 0;
-uint64_t g_features_env = 0;
+uint64_t g_events_env          = 0;
+uint64_t g_features_env        = 0;
+uint64_t g_features_autodetect = 0;
 
 bool g_print_initialized = 0;
 bool g_unbuffer_stdout   = 0;
@@ -95,6 +97,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
     case 'a':
         g_events_env = UINT64_MAX;
         break;
+    case 'd':
+        g_features_autodetect = 1;
+        break;
     case EBPF_EVENT_FILE_DELETE:
     case EBPF_EVENT_FILE_CREATE:
     case EBPF_EVENT_FILE_RENAME:
@@ -659,10 +664,10 @@ int main(int argc, char **argv)
     if (g_libbpf_verbose)
         ebpf_set_verbose_logging();
 
-    struct ebpf_event_ctx_opts opts = {
-        .events   = g_events_env,
-        .features = g_features_env,
-    };
+    struct ebpf_event_ctx_opts opts = {.events              = g_events_env,
+                                       .features            = g_features_env,
+                                       .features_autodetect = g_features_autodetect};
+
     err = ebpf_event_ctx__new(&ctx, event_ctx_callback, opts);
 
     if (err < 0) {