Add job to update amazon 2 in cron workflow

Co-authored-by: Geyslan Gregório <[email protected]>

.github/workflows/cron.yml

@@ -4,6 +4,116 @@ on:
     - cron: "0 1 * * *"
   workflow_dispatch: {}
 jobs:
+  amazon-update:
+    name: Update Amazon 2 BTF Archive
+    runs-on: ubuntu-latest
+    container:
+      image: amazonlinux:2023
+    steps:
+      - name: Install needed amazon packages
+        run: |
+          yum install -y yum-utils tar gzip xz golang clang make cmake git libdwarf-devel elfutils-libelf-devel elfutils-devel rsync
+        shell: bash
+      #
+      - name: Setup Amazon Debuginfo Repositories
+        run: |
+          # disable default debuginfo repositories
+          yum-config-manager -y --disable amazonlinux-debuginfo
+
+          # add Amazon Linux 2 debuginfo repositories
+          append_repo() {
+            local arch=$1
+            local repo_name="amzn2-core-debuginfo-$arch"
+            
+            echo "[$repo_name]
+          name=Amazon Linux 2 core repository - debuginfo packages $arch
+          mirrorlist=http://amazonlinux.default.amazonaws.com/2/core/latest/debuginfo/$arch/mirror.list
+          enabled=1
+          "
+          }
+
+          cat << EOF >> /etc/yum.repos.d/amazonlinux.repo
+          $(append_repo "x86_64")
+          $(append_repo "aarch64")
+          EOF
+        shell: bash
+      #
+      - name: Checkout BTFHub
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          repository: aquasecurity/btfhub
+          submodules: recursive
+          path: ./btfhub
+      #
+      - name: Build and install pahole
+        run: |
+          cd btfhub/3rdparty/dwarves
+          mkdir build
+          cd build
+          cmake -D__LIB=lib -DDWARF_INCLUDE_DIR=/usr/include ..
+          make install
+          echo "/usr/local/lib" >> /etc/ld.so.conf.d/pahole.conf
+          ldconfig
+        shell: bash
+      #
+      - name: Build and install bpftool
+        run: |
+          cd btfhub/3rdparty/bpftool
+          make -C src clean
+          CC=clang make -C src all
+          cp ./src/bpftool /usr/sbin/bpftool
+          make -C src clean
+        shell: bash
+      #
+      - name: Compile BTFHub Tool
+        run: |
+          cd btfhub
+          make
+        shell: bash
+      #
+      - name: Checkout BTFHub Archive
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          repository: aquasecurity/btfhub-archive
+          token: ${{ secrets.GEYSLAN_BTFHUB_PAT }}
+          persist-credentials: false
+          fetch-depth: 1
+          path: ./btfhub-archive
+          sparse-checkout: |
+            amzn
+      #
+      - name: Bring current BTFHub Archive
+        run: |
+          cd btfhub
+          make bring
+        shell: bash
+      #
+      - name: Fetch and Generate new BTFs (AMAZON 2)
+        run: |
+          cd btfhub
+          ./btfhub -workers 6 -d amzn -r 2
+      #
+      - name: Take new BTFs to BTFHub Archive
+        run: |
+          cd btfhub
+          make take
+      #
+      - name: Check Status
+        run: |
+          cd btfhub-archive
+          git status
+      #
+      - name: Commit and Push to BTFHub Archive
+        uses: actions-js/push@5a7cbd780d82c0c937b5977586e641b2fd94acc5 # v1.5
+        with:
+          directory: ./btfhub-archive
+          author_email: '[email protected]'
+          author_name: 'Geyslan Gregório'
+          github_token: ${{ secrets.GEYSLAN_BTFHUB_PAT }}
+          message: 'Update BTFHUB Archive from BTFHUB'
+          repository: aquasecurity/btfhub-archive
+          branch: main
+
   build:
     name: Update BTF Archive
     env:
@@ -24,7 +134,7 @@ jobs:
           swapon /swapfile
         shell: bash
       #
-      - name: Check out BTFHub
+      - name: Checkout BTFHub
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           repository: aquasecurity/btfhub
@@ -44,6 +154,13 @@ jobs:
           persist-credentials: false
           fetch-depth: 1
           path: ./btfhub-archive
+          sparse-checkout: |
+            centos
+            debian
+            fedora
+            ol
+            ubuntu
+      #
       - name: Bring current BTFHub Archive
         run: |
           cd btfhub
@@ -56,22 +173,17 @@ jobs:
           make
         shell: bash
       #
-      - name: Fetch and Generate new BTFs (UBUNTU)
+      - name: Fetch and Generate new BTFs (CENTOS)
         run: |
           cd btfhub
-          ./btfhub -workers 6 -d ubuntu
+          ./btfhub -workers 6 -d centos
       # debian stretch seems to be gone, updates for buster and bullseye only
       - name: Fetch and Generate new BTFs (DEBIAN)
         run: |
           cd btfhub
           ./btfhub -workers 6 -d debian -r buster
           ./btfhub -workers 6 -d debian -r bullseye
       #
-      - name: Fetch and Generate new BTFs (CENTOS)
-        run: |
-          cd btfhub
-          ./btfhub -workers 6 -d centos
-      #
       - name: Fetch and Generate new BTFs (FEDORA)
         run: |
           cd btfhub
@@ -82,6 +194,11 @@ jobs:
           cd btfhub
           ./btfhub -workers 6 -d ol
       #
+      - name: Fetch and Generate new BTFs (UBUNTU)
+        run: |
+          cd btfhub
+          ./btfhub -workers 6 -d ubuntu
+      #
       - name: Take new BTFs to BTFHub Archive
         run: |
           cd btfhub

pkg/repo/amazon.go

@@ -1,11 +1,18 @@
 package repo
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"log"
+	"os/exec"
 	"sort"
+	"strings"
+
+	"golang.org/x/exp/maps"
 
 	"github.com/aquasecurity/btfhub/pkg/job"
 	"github.com/aquasecurity/btfhub/pkg/kernel"
@@ -34,11 +41,12 @@ func (d *AmazonRepo) GetKernelPackages(
 	force bool,
 	jobChan chan<- job.Job,
 ) error {
-	searchOut, err := yumSearch(ctx, "kernel-debuginfo")
+	altArch := d.archs[arch]
+	searchOut, err := repoquery(ctx, "kernel-debuginfo", altArch)
 	if err != nil {
 		return err
 	}
-	pkgs, err := parseYumPackages(searchOut, kernel.NewKernelVersion(""))
+	pkgs, err := parseRepoqueryPackages(searchOut, kernel.NewKernelVersion(""))
 	if err != nil {
 		return fmt.Errorf("parse package listing: %s", err)
 	}
@@ -57,3 +65,55 @@ func (d *AmazonRepo) GetKernelPackages(
 
 	return nil
 }
+
+func repoquery(ctx context.Context, pkg string, arch string) (*bytes.Buffer, error) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	binary, args := utils.SudoCMD("repoquery", "--archlist="+arch, "--show-duplicates", pkg)
+	cmd := exec.CommandContext(ctx, binary, args...)
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("repoquery search %s: %s\n%s", pkg, err, stderr.String())
+	}
+	return stdout, nil
+}
+
+func parseRepoqueryPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package, error) {
+	pkgs := map[string]pkg.Package{}
+	bio := bufio.NewScanner(rdr)
+	for bio.Scan() {
+		line := bio.Text()
+		if !strings.HasPrefix(line, "kernel-debuginfo-") {
+			continue
+		}
+		if strings.HasPrefix(line, "kernel-debuginfo-common-") {
+			continue
+		}
+		_, version, found := strings.Cut(line, ":")
+		if !found {
+			continue
+		}
+		filename := version
+		lastdot := strings.LastIndex(filename, ".")
+		if lastdot == -1 {
+			continue
+		}
+		p := &pkg.RHELPackage{
+			Name:          fmt.Sprintf("kernel-debuginfo-%s", version),
+			NameOfFile:    filename,
+			KernelVersion: kernel.NewKernelVersion(filename[:lastdot]),
+			Architecture:  filename[lastdot+1:],
+		}
+		if !minVersion.IsZero() && p.Version().Less(minVersion) {
+			continue
+		}
+		if _, ok := pkgs[p.Name]; !ok {
+			pkgs[p.Name] = p
+		}
+	}
+	if err := bio.Err(); err != nil {
+		return nil, err
+	}
+	return maps.Values(pkgs), nil
+}