[workflow] Download bpf objects for veristat from S3 bucket

Use bpf objects from S3 bucker for Veristat validation and comparison

Signed-off-by: Nikolay Yurin <[email protected]>

.github/actions/veristat_baseline_compare/action.yml

@@ -0,0 +1,58 @@
+name: 'run-veristat'
+description: 'Run veristat benchmark'
+inputs:
+  arch_and_tool:
+    description: 'arch and build tool string'
+    required: true
+  veristat_output:
+    description: 'veristat output file'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    # veristat.csv is produced by run-qemu run_veristat action
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.arch_and_tool }}-baseline-${{ inputs.veristat_output }}
+        if-no-files-found: error
+        path: '${{ github.workspace }}/${{ inputs.veristat_output }}'
+
+    # For pull request:
+    # - get baseline log from cache
+    # - compare it to current run
+    - if: ${{ github.event_name == 'pull_request' }}
+      uses: actions/cache/restore@v3
+      with:
+        key: ${{ env.CACHE_RESULT_KEY }}
+        restore-keys: ${{ env.CACHE_RESULT_KEY }}
+        path: '${{ env.CACHE_RESULT_KEY }}.csv'
+      env:
+        CACHE_RESULT_KEY: ${{ inputs.arch_and_tool }}-baseline-${{ inputs.veristat_output }}
+
+    - if: ${{ github.event_name == 'pull_request' }}
+      name: Show veristat comparison
+      shell: bash
+      run: |
+        cd ${{ github.workspace }}
+        if [[ ! -f ${{ env.CACHE_RESULT_KEY }}.csv ]]; then
+          echo "No ${{ env.CACHE_RESULT_KEY }}.csv available"
+          echo "# No ${{ env.CACHE_RESULT_KEY }}.csv available" >> $GITHUB_STEP_SUMMARY
+          exit
+        fi
+        selftests/bpf/veristat \
+          --output-format csv \
+          --emit file,prog,verdict,states \
+          --compare ${{ env.CACHE_RESULT_KEY }}.csv veristat.csv > compare.csv
+        python3 ./.github/scripts/veristat-compare.py compare.csv
+      env:
+        CACHE_RESULT_KEY: ${{ inputs.arch_and_tool }}-baseline-${{ inputs.veristat_output }}
+
+    # For push: just put baseline log to cache
+    - if: ${{ github.event_name == 'push' }}
+      uses: actions/cache/save@v3
+      with:
+        key: ${{ env.CACHE_RESULT_KEY }}.csv
+        path: '${{ github.workspace }}/${{ env.CACHE_RESULT_KEY }}.csv'
+      env:
+        CACHE_RESULT_KEY: ${{ inputs.arch_and_tool }}-baseline-${{ inputs.veristat_output }}
+

.github/scripts/bpf-objects-rootfs.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+eval "$(guestfish --listen)"
+
+guestfish --verbose --remote \
+    add /tmp/root.img label:img : \
+    launch : \
+    mount /dev/disk/guestfs/img / : \
+    mkdir-p /bpf_objects : \
+    copy-in /tmp/bpf_objects /bpf_objects
+
+guestfish --remote exit

.github/workflows/test.yml

@@ -70,10 +70,10 @@ jobs:
 
           matrix = [
             {"kernel": "LATEST", "runs_on": [], "arch": Arch.x86_64.value, "toolchain": "gcc", "llvm-version": "16"},
-            {"kernel": "LATEST", "runs_on": [], "arch": Arch.x86_64.value, "toolchain": "llvm", "llvm-version": "16"},
-            {"kernel": "LATEST", "runs_on": [], "arch": Arch.aarch64.value, "toolchain": "gcc", "llvm-version": "16"},
+            # {"kernel": "LATEST", "runs_on": [], "arch": Arch.x86_64.value, "toolchain": "llvm", "llvm-version": "16"},
+            # {"kernel": "LATEST", "runs_on": [], "arch": Arch.aarch64.value, "toolchain": "gcc", "llvm-version": "16"},
             #{"kernel": "LATEST", "runs_on": [], "arch": Arch.aarch64.value, "toolchain": "llvm", "llvm-version": "16"},
-            {"kernel": "LATEST", "runs_on": [], "arch": Arch.s390x.value, "toolchain": "gcc", "llvm-version": "16", "parallel_tests": False},
+            # {"kernel": "LATEST", "runs_on": [], "arch": Arch.s390x.value, "toolchain": "gcc", "llvm-version": "16", "parallel_tests": False},
           ]
           self_hosted_repos = [
             "kernel-patches/bpf",
@@ -377,19 +377,22 @@ jobs:
       REPO_ROOT: ${{ github.workspace }}
       REPO_PATH: ""
       KBUILD_OUTPUT: kbuild-output/
+    permissions:
+      id-token: write
+      contents: read
     steps:
       - name: Setup environment variables
         run: |
-            echo arch_and_tool=${{ env.veristat_arch }}-${{ env.veristat_toolchain }} > \
+            echo ARCH_AND_TOOL=${{ env.veristat_arch }}-${{ env.veristat_toolchain }} > \
               ${GITHUB_ENV}
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
-          name: vmlinux-${{ env.arch_and_tool }}
+          name: vmlinux-${{ env.ARCH_AND_TOOL }}
           path: .
       - name: Untar artifacts
         # zstd is installed by default in the runner images.
-        run: zstd -d -T0  vmlinux-${{ env.arch_and_tool }}.tar.zst --stdout | tar -xf -
+        run: zstd -d -T0  vmlinux-${{ env.ARCH_AND_TOOL }}.tar.zst --stdout | tar -xf -
 
       - name: Prepare rootfs
         uses: libbpf/ci/prepare-rootfs@main
@@ -400,61 +403,48 @@ jobs:
           kernel-root: '.'
           kbuild-output: ${{ env.KBUILD_OUTPUT }}
           image-output: '/tmp/root.img'
-          test: run_veristat
+          test: run_veristat_kernel
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          role-session-name: github-action-bpf-ci
+
+      - name: Download BPF objects
+        run: |
+          set -eux
+          if [ -n "$AWS_ROLE_ARN" ]; then
+            mkdir /tmp/bpf_objects
+            aws s3 sync s3://veristat-bpf-binaries /tmp/bpf_objects
+          fi
+        env:
+          AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
+
+      - name: Add BPF objects to rootfs
+        run: |
+          bash .github/scripts/bpf-objects-rootfs.sh
 
       - name: Run veristat
         uses: libbpf/ci/run-qemu@main
-        timeout-minutes: 10
         with:
           arch: x86_64
           img: '/tmp/root.img'
           vmlinuz: '${{ github.workspace }}/vmlinuz'
           kernel-root: '.'
           max-cpu: 8
-          kernel-test: run_veristat
+          kernel-test: run_veristat_meta
           output-dir: '${{ github.workspace }}'
 
-      # veristat.csv is produced by run-qemu run_veristat action
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.arch_and_tool }}-veristat-log
-          if-no-files-found: error
-          path: '${{ github.workspace }}/veristat.csv'
-
-      # For pull request:
-      # - get baseline log from cache
-      # - compare it to current run
-      - if: ${{ github.event_name == 'pull_request' }}
-        uses: actions/cache/restore@v3
+      - name: Compare and save veristat.kernel.csv
+        uses: ./.github/actions/veristat_baseline_compare
         with:
-          key: ${{ env.arch_and_tool }}-veristat-baseline
-          restore-keys: |
-            ${{ env.arch_and_tool }}-veristat-baseline-
-          path: '${{ github.workspace }}/veristat-baseline.csv'
-
-      - if: ${{ github.event_name == 'pull_request' }}
-        name: Show veristat comparison
-        run: |
-          cd ${{ github.workspace }}
-          if [[ ! -f veristat-baseline.csv ]]; then
-            echo "No veristat-baseline.csv available"
-            echo "# No veristat-baseline.csv available" >> $GITHUB_STEP_SUMMARY
-            exit
-          fi
-          selftests/bpf/veristat \
-            --output-format csv \
-            --emit file,prog,verdict,states \
-            --compare veristat-baseline.csv veristat.csv > compare.csv
-          python3 ./.github/scripts/veristat-compare.py compare.csv
-
-      # For push: just put baseline log to cache
-      - if: ${{ github.event_name == 'push' }}
-        run: |
-          mv '${{ github.workspace }}/veristat.csv' \
-             '${{ github.workspace }}/veristat-baseline.csv'
+          arch_and_tool: ${{ env.ARCH_AND_TOOL}}
+          veristat_output: veristat.kernel.csv
 
-      - if: ${{ github.event_name == 'push' }}
-        uses: actions/cache/save@v3
+      - name: Compare and save veristat.meta.csv
+        uses: ./.github/actions/veristat_baseline_compare
         with:
-          key: ${{ env.arch_and_tool }}-veristat-baseline-${{ github.run_id }}
-          path: '${{ github.workspace }}/veristat-baseline.csv'
+          arch_and_tool: ${{ env.ARCH_AND_TOOL}}
+          veristat_output: veristat.meta.csv

ci/vmtest/configs/run_veristat.kernel.cfg

@@ -0,0 +1,3 @@
+VERISTAT_OBJECTS_DIR="/${PROJECT_NAME}/selftests/bpf"
+VERISTAT_OBJECTS_GLOB=$(awk '/^#/ { next; } { print $0 ".bpf.o"; }' "${BPF_SELFTESTS_DIR}/veristat.cfg")
+VERISTAT_OUTPUT="veristat.kernel.csv"

ci/vmtest/configs/run_veristat.meta.cfg

@@ -0,0 +1,3 @@
+VERISTAT_OBJECTS_DIR="/bpf_objects"
+VERISTAT_OBJECTS_GLOB="*.o"
+VERISTAT_OUTPUT="veristat.meta.csv"

ci/vmtest/run_selftests.sh

@@ -17,7 +17,8 @@ ARCH=$(uname -m)
 STATUS_FILE=/exitstatus
 OUTPUT_DIR=/command_output
 
-declare -a TEST_NAMES=()
+BPF_SELFTESTS_DIR="/${PROJECT_NAME}/selftests/bpf"
+VMTEST_CONFIGS_PATH="/${PROJECT_NAME}/vmtest/configs"
 
 read_lists() {
 	(for path in "$@"; do
@@ -27,6 +28,21 @@ read_lists() {
 	done) | cut -d'#' -f1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '\n' ','
 }
 
+DENYLIST=$(read_lists \
+	"$BPF_SELFTESTS_DIR/DENYLIST" \
+	"$BPF_SELFTESTS_DIR/DENYLIST.${ARCH}" \
+	"$VMTEST_CONFIGS_PATH/DENYLIST" \
+	"$VMTEST_CONFIGS_PATH/DENYLIST.${ARCH}" \
+)
+ALLOWLIST=$(read_lists \
+	"$BPF_SELFTESTS_DIR/ALLOWLIST" \
+	"$BPF_SELFTESTS_DIR/ALLOWLIST.${ARCH}" \
+	"$VMTEST_CONFIGS_PATH/ALLOWLIST" \
+	"$VMTEST_CONFIGS_PATH/ALLOWLIST.${ARCH}" \
+)
+
+declare -a TEST_NAMES=()
+
 read_test_names() {
     foldable start read_test_names "Reading test names from boot parameters and command line arguments"
     # Check if test names were passed as boot parameter.
@@ -99,8 +115,8 @@ test_verifier() {
   foldable end test_verifier
 }
 
-run_veristat() {
-  foldable start run_veristat "Running veristat"
+run_veristat_helper() {
+  local mode="${1}"
 
   # Make veristat commands visible in the log
   if [ -o xtrace ]; then
@@ -110,17 +126,38 @@ run_veristat() {
       set -x
   fi
 
-  globs=$(awk '/^#/ { next; } { print $0 ".bpf.o"; }' ./veristat.cfg)
-  mkdir -p ${OUTPUT_DIR}
-  ./veristat -o csv -q -e file,prog,verdict,states ${globs} > ${OUTPUT_DIR}/veristat.csv
-  echo "run_veristat:$?" >> ${STATUS_FILE}
+  (
+    # shellcheck source=ci/vmtest/configs/run_veristat.default.cfg
+    # shellcheck source=ci/vmtest/configs/run_veristat.meta.cfg
+    source "${VMTEST_CONFIGS_PATH}/run_veristat.${mode}.cfg"
+    mkdir -p ${OUTPUT_DIR}
+    pushd "${VERISTAT_OBJECTS_DIR}"
+
+    ls ${VERISTAT_OBJECTS_DIR}
+    "${BPF_SELFTESTS_DIR}/veristat" -o csv -q -e file,prog,verdict,states ${VERISTAT_OBJECTS_GLOB} > \
+      "${OUTPUT_DIR}/${VERISTAT_OUTPUT}"
+
+    echo "run_veristat_${mode}:$?" >> ${STATUS_FILE}
+    popd
+  )
 
   # Hide commands again
   if [ -z "$xtrace_was_on" ]; then
       set +x
   fi
 
-  foldable end run_veristat
+}
+
+run_veristat_kernel() {
+  foldable start run_veristat_kernel "Running veristat.kernel"
+  run_veristat_helper "kernel"
+  foldable end run_veristat_kernel
+}
+
+run_veristat_meta() {
+  foldable start run_veristat_meta "Running veristat.meta"
+  run_veristat_helper "meta"
+  foldable end run_veristat_meta
 }
 
 foldable end vm_init
@@ -131,21 +168,6 @@ zcat /proc/config.gz
 
 foldable end kernel_config
 
-configs_path=${PROJECT_NAME}/selftests/bpf
-local_configs_path=${PROJECT_NAME}/vmtest/configs
-DENYLIST=$(read_lists \
-	"$configs_path/DENYLIST" \
-	"$configs_path/DENYLIST.${ARCH}" \
-	"$local_configs_path/DENYLIST" \
-	"$local_configs_path/DENYLIST.${ARCH}" \
-)
-ALLOWLIST=$(read_lists \
-	"$configs_path/ALLOWLIST" \
-	"$configs_path/ALLOWLIST.${ARCH}" \
-	"$local_configs_path/ALLOWLIST" \
-	"$local_configs_path/ALLOWLIST.${ARCH}" \
-)
-
 echo "DENYLIST: ${DENYLIST}"
 echo "ALLOWLIST: ${ALLOWLIST}"