diff --git a/.github/actions/build-dependencies/action.yaml b/.github/actions/build-dependencies/action.yaml index 312cdba9..f8a6496b 100644 --- a/.github/actions/build-dependencies/action.yaml +++ b/.github/actions/build-dependencies/action.yaml @@ -2,75 +2,16 @@ name: Build Dependencies description: | Install build dependencies to test and compile btfhub artifacts inputs: - # set default versions in a single place (as default) - go-version: - description: go version + install-deps-opt: + description: 'Install dependencies script option (pr, cron)' required: true - default: "1.19.5" + default: 'pr' runs: using: composite steps: - name: Install ubuntu packages run: | - export DEBIAN_FRONTEND=noninteractive - sudo dpkg --purge unattended-upgrades - sudo apt-get update - sudo apt-get install -y bsdutils build-essential pkgconf - sudo apt-get install -y zlib1g-dev libelf-dev - sudo apt-get install -y software-properties-common - sudo apt-get install -y devscripts ubuntu-dev-tools - shell: bash - - name: Install golang - run: | - rm -rf /usr/local/go - curl -L -o /tmp/go.tar.xz https://go.dev/dl/go${{ inputs.go-version }}.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/go.tar.xz - sudo update-alternatives --remove-all go || true - sudo update-alternatives --remove-all gofmt || true - sudo update-alternatives --install /usr/bin/go go /usr/local/go/bin/go 1 - sudo update-alternatives --install /usr/bin/gofmt gofmt /usr/local/go/bin/gofmt 1 - shell: bash - - name: Install llvm - run: | - export DEBIAN_FRONTEND=noninteractive - cd /tmp - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 14 - sudo apt-get install -y clang-14 clangd-14 clang-tools-14 clang-format-14 - sudo apt-get install -y llvm-14 llvm-14-runtime llvm-14-tools - sudo update-alternatives --remove-all cc || true - sudo update-alternatives --remove-all clang || true - sudo update-alternatives --remove-all clang++ || true - sudo update-alternatives --remove-all llc || true - sudo update-alternatives --remove-all lld || true - sudo update-alternatives --remove-all clangd || true - sudo update-alternatives --remove-all clang-format || true - sudo update-alternatives --remove-all llvm-strip || true - sudo update-alternatives --remove-all llvm-config || true - sudo update-alternatives --remove-all ld.lld || true - sudo update-alternatives --remove-all llvm-ar || true - sudo update-alternatives --remove-all llvm-nm || true - sudo update-alternatives --remove-all llvm-objcopy || true - sudo update-alternatives --remove-all llvm-objdump || true - sudo update-alternatives --remove-all llvm-readelf || true - sudo update-alternatives --remove-all opt || true - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 140 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-14 \ - --slave /usr/bin/clangd clangd /usr/bin/clangd-14 \ - --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-14 \ - --slave /usr/bin/lld lld /usr/bin/lld-14 \ - --slave /usr/bin/llc llc /usr/bin/llc-14 \ - --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-14 \ - --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 \ - --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-14 \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-14 \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-14 \ - --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-14 \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-14 \ - --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-14 \ - --slave /usr/bin/opt opt /usr/bin/opt-14 \ - --slave /usr/bin/cc cc /usr/bin/clang-14 + sudo ./tests/install-deps.sh ${{ inputs.install-deps-opt }} shell: bash - name: Install pahole run: | @@ -80,13 +21,3 @@ runs: run: | ./3rdparty/bpftool.sh shell: bash - - name: Install staticchecker - run: | - GOROOT=/usr/local/go GOPATH=$HOME/go go install honnef.co/go/tools/cmd/staticcheck@latest - sudo cp $HOME/go/bin/staticcheck /usr/bin/ - shell: bash - - name: Install goimports-reviser - run: | - GOROOT=/usr/local/go GOPATH=$HOME/go go install github.com/incu6us/goimports-reviser/v3@latest - sudo cp $HOME/go/bin/goimports-reviser /usr/bin/ - shell: bash diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index c26b5788..d71a4deb 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -24,59 +24,16 @@ jobs: swapon /swapfile shell: bash # - - name: Install needed ubuntu packages - run: | - export DEBIAN_FRONTEND=noninteractive - dpkg --purge unattended-upgrades - apt-get update - apt-get install -y bsdutils build-essential pkgconf - apt-get install -y zlib1g-dev libelf-dev - apt-get install -y software-properties-common - apt-get install -y devscripts ubuntu-dev-tools - shell: bash - # - - name: Update alternatives for LLVM - run: | - export DEBIAN_FRONTEND=noninteractive - update-alternatives --remove-all cc || true - update-alternatives --remove-all clang || true - update-alternatives --remove-all clang++ || true - update-alternatives --remove-all llc || true - update-alternatives --remove-all lld || true - update-alternatives --remove-all clangd || true - update-alternatives --remove-all clang-format || true - update-alternatives --remove-all llvm-strip || true - update-alternatives --remove-all llvm-config || true - update-alternatives --remove-all ld.lld || true - update-alternatives --remove-all llvm-ar || true - update-alternatives --remove-all llvm-nm || true - update-alternatives --remove-all llvm-objcopy || true - update-alternatives --remove-all llvm-objdump || true - update-alternatives --remove-all llvm-readelf || true - update-alternatives --remove-all opt || true - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 140 \ - --slave /usr/bin/clang++ clang++ /usr/bin/clang++-14 \ - --slave /usr/bin/clangd clangd /usr/bin/clangd-14 \ - --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-14 \ - --slave /usr/bin/lld lld /usr/bin/lld-14 \ - --slave /usr/bin/llc llc /usr/bin/llc-14 \ - --slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-14 \ - --slave /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 \ - --slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-14 \ - --slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-14 \ - --slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-14 \ - --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-14 \ - --slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-14 \ - --slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-14 \ - --slave /usr/bin/opt opt /usr/bin/opt-14 \ - --slave /usr/bin/cc cc /usr/bin/clang-14 - shell: bash - # - name: Check out BTFHub uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: repository: aquasecurity/btfhub - path: ./btfhub + # path: ./btfhub + # + - name: "Prepare Image (Fix AMI)" + uses: ./.github/actions/build-dependencies + with: + install-deps-opt: cron # - name: Checkout BTFHub Archive uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -86,18 +43,18 @@ jobs: persist-credentials: false fetch-depth: 1 path: ./btfhub-archive - # - - name: Install pahole - run: | - cd btfhub - ./3rdparty/pahole.sh - shell: bash - # - - name: Install bpftool - run: | - cd btfhub - ./3rdparty/bpftool.sh - shell: bash + # # + # - name: Install pahole + # run: | + # cd btfhub + # ./3rdparty/pahole.sh + # shell: bash + # # + # - name: Install bpftool + # run: | + # cd btfhub + # ./3rdparty/bpftool.sh + # shell: bash # - name: Bring current BTFHub Archive run: | diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 95fc28cd..d393d6cb 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -30,12 +30,29 @@ jobs: name: Verify and Analyze Code runs-on: ubuntu-latest steps: + # - name: Checkout Code uses: actions/checkout@v2 with: submodules: true + # - name: Install Dependencies uses: ./.github/actions/build-dependencies + with: + install-deps-opt: pr + # + - name: Install staticchecker + run: | + GOROOT=/usr/local/go GOPATH=$HOME/go go install honnef.co/go/tools/cmd/staticcheck@latest + sudo cp $HOME/go/bin/staticcheck /usr/bin/ + shell: bash + # + - name: Install goimports-reviser + run: | + GOROOT=/usr/local/go GOPATH=$HOME/go go install github.com/incu6us/goimports-reviser/v3@latest + sudo cp $HOME/go/bin/goimports-reviser /usr/bin/ + shell: bash + # - name: Lint run: | if test -z "$(gofmt -l .)"; then @@ -45,9 +62,11 @@ jobs: gofmt -s -d . exit 1 fi + # - name: Check Golang Vet run: | make check-vet + # - name: Check with StaticCheck run: | make check-staticcheck @@ -60,12 +79,17 @@ jobs: - verify-analyze-code runs-on: ubuntu-latest steps: + # - name: Checkout Code uses: actions/checkout@v2 with: submodules: true + # - name: Install Dependencies uses: ./.github/actions/build-dependencies + with: + install-deps-opt: pr + # - name: Run Unit Tests run: | make test-unit diff --git a/tests/install-deps.sh b/tests/install-deps.sh new file mode 100755 index 00000000..80dac6b5 --- /dev/null +++ b/tests/install-deps.sh @@ -0,0 +1,331 @@ +#!/bin/bash + +set -e +set -x # for debugging + +# This script installs the dependencies for compiling btfhub and running the +# tests. +# +# If AMIs need to be updated, you can provision an AMI and run this script on +# it. It will install dependencies and update the OS. You may then create a new +# AMI from the provisioned one (and stop using this script until new changes +# are required). + +wait_for_apt_locks() { + local lock="/var/lib/dpkg/lock" + local lock_frontend="/var/lib/dpkg/lock-frontend" + local lock_lists="/var/lib/apt/lists/lock" + local lock_archives="/var/cache/apt/archives/lock" + + local timeout=20 + local elapsed=0 + local wait_interval=2 + + echo "Checking for unattended-upgrades..." + while pgrep -f unattended-upgrades > /dev/null; do + if (( elapsed >= timeout )); then + echo "Timed out waiting for unattended-upgrades to finish. Attempting to kill..." + pkill -SIGQUIT -f unattended-upgrades || true + pkill -SIGKILL -f unattended-upgrades || true + break + fi + + echo "unattended-upgrades is still running. Waiting..." + sleep $wait_interval + ((elapsed += wait_interval)) + done + + timeout=5 # reduce timeout for apt locks + elapsed=0 # reset timer + + while : ; do + if ! fuser $lock >/dev/null 2>&1 && + ! fuser $lock_frontend >/dev/null 2>&1 && + ! fuser $lock_lists >/dev/null 2>&1 && + ! fuser $lock_archives >/dev/null 2>&1; then + echo "All apt locks are free." + break + fi + + if (( elapsed >= timeout )); then + echo "Timed out waiting for apt locks to be released. Attempting to kill locking processes." + fuser -k -SIGQUIT $lock >/dev/null 2>&1 || true + fuser -k -SIGQUIT $lock_frontend >/dev/null 2>&1 || true + fuser -k -SIGQUIT $lock_lists >/dev/null 2>&1 || true + fuser -k -SIGQUIT $lock_archives >/dev/null 2>&1 || true + + # Give some time for processes to terminate gracefully + sleep 2 + + fuser -k -SIGKILL $lock >/dev/null 2>&1 || true + fuser -k -SIGKILL $lock_frontend >/dev/null 2>&1 || true + fuser -k -SIGKILL $lock_lists >/dev/null 2>&1 || true + fuser -k -SIGKILL $lock_archives >/dev/null 2>&1 || true + + # Delete lock files if they still exist + rm -f $lock $lock_frontend $lock_lists $lock_archives + + echo "Forced removal of processes locking apt. System may be in an inconsistent state." + break + fi + + echo "Waiting for other software managers to finish..." + sleep $wait_interval + ((elapsed += wait_interval)) + done +} + +disable_unattended_upgrades() { + # This is a pain point. Make sure to always disable anything touching the + # dpkg database, otherwise it will fail with locking errors. + systemctl stop unattended-upgrades || true + systemctl disable --now unattended-upgrades || true + + wait_for_apt_locks + apt-get remove -y --purge unattended-upgrades || true + apt-get remove -y --purge ubuntu-advantage-tools || true +} + +remove_alternatives() { + tools=("$@") + + for tool in "${tools[@]}"; do + update-alternatives --remove-all "$tool" || true + done +} + +# +# LLVM +# + +remove_llvm_alternatives() { + tools=( + clang + clang++ + clangd + clang-format + llc + lld + llvm-strip + llvm-config + ld.lld + llvm-ar + llvm-nm + llvm-objcopy + llvm-objdump + llvm-readelf + opt + cc + ) + + remove_alternatives "${tools[@]}" +} + +update_llvm_alternatives() { + remove_llvm_alternatives + + # Get the major version + local version + version=$(echo "${1}" | cut -d. -f1) + update-alternatives \ + --install /usr/bin/clang clang /usr/local/clang/bin/clang-"${version}" 0 \ + --slave /usr/bin/clang++ clang++ /usr/local/clang/bin/clang++-"${version}" \ + --slave /usr/bin/clangd clangd /usr/local/clang/bin/clangd-"${version}" \ + --slave /usr/bin/clang-format clang-format /usr/local/clang/bin/clang-format-"${version}" \ + --slave /usr/bin/llc llc /usr/local/clang/bin/llc-"${version}" \ + --slave /usr/bin/lld lld /usr/local/clang/bin/lld-"${version}" \ + --slave /usr/bin/llvm-strip llvm-strip /usr/local/clang/bin/llvm-strip-"${version}" \ + --slave /usr/bin/llvm-config llvm-config /usr/local/clang/bin/llvm-config-"${version}" \ + --slave /usr/bin/ld.lld ld.lld /usr/local/clang/bin/ld.lld-"${version}" \ + --slave /usr/bin/llvm-ar llvm-ar /usr/local/clang/bin/llvm-ar-"${version}" \ + --slave /usr/bin/llvm-nm llvm-nm /usr/local/clang/bin/llvm-nm-"${version}" \ + --slave /usr/bin/llvm-objcopy llvm-objcopy /usr/local/clang/bin/llvm-objcopy-"${version}" \ + --slave /usr/bin/llvm-objdump llvm-objdump /usr/local/clang/bin/llvm-objdump-"${version}" \ + --slave /usr/bin/llvm-readelf llvm-readelf /usr/local/clang/bin/llvm-readelf-"${version}" \ + --slave /usr/bin/opt opt /usr/local/clang/bin/opt-"${version}" \ + --slave /usr/bin/cc cc /usr/local/clang/bin/clang-"${version}" +} + +remove_llvm_os_packages() { + wait_for_apt_locks + apt-get remove -y clang-12 clangd-12 lld-12 llvm-12 || true + apt-get remove -y clang-13 clangd-13 lld-13 llvm-13 || true + apt-get remove -y clang-14 clangd-14 lld-14 llvm-14 || true + apt-get --purge autoremove -y +} + +remove_llvm_usr_bin_files() { + rm -f /usr/bin/clang* + rm -f /usr/bin/clang++* + rm -f /usr/bin/clangd* + rm -f /usr/bin/clang-format* + + rm -f /usr/bin/lld* + rm -f /usr/bin/llc* + rm -f /usr/bin/llvm-strip* + rm -f /usr/bin/llvm-config* + rm -f /usr/bin/ld.lld* + rm -f /usr/bin/llvm-ar* + rm -f /usr/bin/llvm-nm* + rm -f /usr/bin/llvm-objcopy* + rm -f /usr/bin/llvm-objdump* + rm -f /usr/bin/llvm-readelf* + rm -f /usr/bin/opt + rm -f /usr/bin/cc +} + +link_llvm_usr_local_clang() { + ln -s /usr/local/clang/bin/clang /usr/bin/clang + ln -s /usr/local/clang/bin/clang++ /usr/bin/clang++ + ln -s /usr/local/clang/bin/clangd /usr/bin/clangd + ln -s /usr/local/clang/bin/clang-format /usr/bin/clang-format + ln -s /usr/local/clang/bin/lld /usr/bin/lld + ln -s /usr/local/clang/bin/llc /usr/bin/llc + ln -s /usr/local/clang/bin/llvm-strip /usr/bin/llvm-strip + ln -s /usr/local/clang/bin/llvm-config /usr/bin/llvm-config + ln -s /usr/local/clang/bin/ld.lld /usr/bin/ld.lld + ln -s /usr/local/clang/bin/llvm-ar /usr/bin/llvm-ar + ln -s /usr/local/clang/bin/llvm-nm /usr/bin/llvm-nm + ln -s /usr/local/clang/bin/llvm-objcopy /usr/bin/llvm-objcopy + ln -s /usr/local/clang/bin/llvm-objdump /usr/bin/llvm-objdump + ln -s /usr/local/clang/bin/llvm-readelf /usr/bin/llvm-readelf + ln -s /usr/local/clang/bin/opt /usr/bin/opt + ln -s /usr/local/clang/bin/clang /usr/bin/cc +} + +install_clang_from_github() { + local version=$1 + + remove_llvm_os_packages + remove_llvm_usr_bin_files + + LLVM_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${version}/" + + if [[ $ARCH == x86_64 ]]; then + LLVM_URL=$LLVM_URL"clang+llvm-${version}-x86_64-linux-gnu-rhel-8.4.tar.xz" + else + LLVM_URL=$LLVM_URL"clang+llvm-${version}-aarch64-linux-gnu.tar.xz" + fi + + LLVM_FILE=$(basename "$LLVM_URL") + LLVM_DIR="${LLVM_FILE%.tar.xz}" + + # Download + rm -f "/tmp/$LLVM_FILE" + curl -L -o "/tmp/$LLVM_FILE" "$LLVM_URL" + + # Install + cd /usr/local + rm -rf ./clang + tar xfJ /tmp/"$LLVM_FILE" + mv "$LLVM_DIR" ./clang + cd - + + link_llvm_usr_local_clang +} + +# +# Golang +# + +remove_golang_alternatives() { + tools=( + go + gofmt + ) + + remove_alternatives "${tools[@]}" +} + +update_golang_alternatives() { + remove_golang_alternatives + + update-alternatives \ + --install /usr/bin/go go /usr/local/go/bin/go 0 \ + --slave /usr/bin/gofmt gofmt /usr/local/go/bin/gofmt +} + +remove_golang_os_packages() { + wait_for_apt_locks + apt-get remove -y golang golang-go + apt-get --purge autoremove -y +} + +remove_golang_usr_bin_files() { + rm -f /usr/bin/go + rm -f /usr/bin/gofmt +} + +link_golang_usr_local_go() { + ln -s /usr/local/go/bin/go /usr/bin/go + ln -s /usr/local/go/bin/gofmt /usr/bin/gofmt +} + +install_golang_from_github() { + remove_golang_alternatives + remove_golang_os_packages + remove_golang_usr_bin_files + + local version=$1 + if [[ $ARCH == x86_64 ]]; then + GO_URL="https://go.dev/dl/go$version.linux-amd64.tar.gz" + else + GO_URL="https://go.dev/dl/go$version.linux-arm64.tar.gz" + fi + + GO_FILE=$(basename "$GO_URL") + + # Download + rm -f "/tmp/$GO_FILE" + curl -L -o "/tmp/$GO_FILE" "$GO_URL" + + # Install + cd /usr/local + rm -rf ./go + tar xfz /tmp/"$GO_FILE" + cd - + + link_golang_usr_local_go +} + +# +# Main +# + +FROM="$1" + +# pr: when called from the pr action +# cron: when called from the cron job action +if [ "$FROM" != "pr" ] && [ "$FROM" != "cron" ]; then + echo "Usage: $0 [pr|cron]" + exit 1 +fi + +# Common dependencies + +export DEBIAN_FRONTEND=noninteractive +disable_unattended_upgrades + +wait_for_apt_locks +apt-get update + +wait_for_apt_locks +apt-get install -y \ + bsdutils build-essential pkgconf \ + zlib1g-dev libelf-dev \ + software-properties-common \ + devscripts ubuntu-dev-tools + +# Install dependencies based on the origin + +GO_VERSION="1.22.2" +LLVM_VERSION="14.0.6" + +if [ "$FROM" == "pr" ]; then + # pr dependencies + install_golang_from_github "$GO_VERSION" + install_clang_from_github "$LLVM_VERSION" +elif [ "$FROM" == "cron" ]; then + # cron dependencies + update_llvm_alternatives "$LLVM_VERSION" +fi