Add job to update amazon 2 in cron workflow

.github/workflows/cron.yml

@@ -4,6 +4,82 @@ on:
     - cron: "0 1 * * *"
   workflow_dispatch: {}
 jobs:
+  amazon-update:
+    name: Update Amazon 2 BTF Archive
+    runs-on: ubuntu-latest
+    container:
+      image: amazonlinux:2
+    steps:
+      - name: Install needed amazon packages
+        run: |
+          yum install -y yum-utils tar gzip xz clang make cmake git libdwarf-devel elfutils-libelf-devel elfutils-devel
+          yum-config-manager -y --enable amzn2-core-debuginfo
+          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
+          yum install -y gh
+          echo "[amzn2-core-debuginfo-alt]" >> /etc/yum.repos.d/amzn2-core.repo
+          awk '/^\[/{found=""} /^\[amzn2-core-debuginfo]$/{found=1;next} found && NF' /etc/yum.repos.d/amzn2-core.repo | sed 's/\$basearch/aarch64/g' >> /etc/yum.repos.d/amzn2-core.repo
+        shell: bash
+
+      - name: Authenticate
+        run: |
+          gh auth login --with-token <<<'${{ secrets.PAT_RAFAEL }}'
+
+      - name: Check out BTFHub
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+          token: ${{ secrets.PAT_RAFAEL }}
+
+      - run: git config --system --add safe.directory $(pwd)
+        shell: bash
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+
+      - name: build pahole
+        run: |
+          cd ./3rdparty/dwarves
+          mkdir build
+          cd build
+          cmake -D__LIB=lib -DDWARF_INCLUDE_DIR=/usr/include ..
+          make install
+          echo "/usr/local/lib" >> /etc/ld.so.conf.d/pahole.conf
+          ldconfig
+        shell: bash
+
+      - name: Compile BTFHub Tool
+        run: |
+          make
+        shell: bash
+
+      - name: Checkout BTFHub Archive
+        uses: actions/checkout@v3
+        with:
+          repository: aquasecurity/btfhub-archive
+          token: ${{ secrets.PAT_RAFAEL }}
+          path: archive
+          sparse-checkout: |
+            amzn
+
+      - name: Fetch and Generate new BTFs (AMAZON 2)
+        run: |
+          ./btfhub -workers 6 -d amzn -r 2
+
+      - name: Check Status
+        run: |
+          cd archive
+          git status
+
+      - name: Commit and Push to BTFHub Archive
+        run: |
+          cd archive
+          git config --local user.name "Rafael David Tinoco"
+          git config --local user.email "[email protected]"
+          git add -A
+          git diff-index --quiet HEAD || git commit -m "Update BTFHUB Archive from BTFHUB"
+          git push
+
   build:
     name: Update BTF Archive
     env:
@@ -72,88 +148,68 @@ jobs:
             --slave /usr/bin/opt opt /usr/bin/opt-14 \
             --slave /usr/bin/cc cc /usr/bin/clang-14
         shell: bash
-     #
+    #
       - name: Authenticate
         run: |
           gh auth login --with-token <<<'${{ secrets.PAT_RAFAEL }}'
-      #
+    #
       - name: Check out BTFHub
-        uses: actions/checkout@v3
-        with:
-          repository: aquasecurity/btfhub
-          path: ./btfhub
-          token: ${{ secrets.PAT_RAFAEL }}
-      # Authenticate
-      - name: Checkout BTFHub Archive
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-          repository: aquasecurity/btfhub-archive
-          path: ./btfhub-archive
           token: ${{ secrets.PAT_RAFAEL }}
-      #
+    #
       - name: Install pahole
         run: |
-          cd btfhub 
           ./3rdparty/pahole.sh
         shell: bash
-      #
-      - name: Install bpftool
-        run: |
-          cd btfhub
-          ./3rdparty/bpftool.sh
-        shell: bash
-      #
-      - name: Bring current BTFHub Archive
-        run: |
-          cd btfhub 
-          make bring
-        shell: bash
-      #
+    #
       - name: Compile BTFHub Tool
         run: |
-          cd btfhub 
           make
         shell: bash
-      #
+    #
+      - name: Checkout BTFHub Archive
+        uses: actions/checkout@v4
+        with:
+          repository: aquasecurity/btfhub-archive
+          path: archive
+          sparse-checkout: |
+            centos
+            debian
+            fedora
+            ol
+            ubuntu            
+
+    #
       - name: Fetch and Generate new BTFs (UBUNTU)
         run: |
-          cd btfhub
           ./btfhub -workers 6 -d ubuntu
-      # debian stretch seems to be gone, updates for buster and bullseye only
+    # debian stretch seems to be gone, updates for buster and bullseye only
       - name: Fetch and Generate new BTFs (DEBIAN)
         run: |
-          cd btfhub
           ./btfhub -workers 6 -d debian -r buster
           ./btfhub -workers 6 -d debian -r bullseye
-      #
+    #
       - name: Fetch and Generate new BTFs (CENTOS)
         run: |
-          cd btfhub
           ./btfhub -workers 6 -d centos
-      #
+    #
       - name: Fetch and Generate new BTFs (FEDORA)
         run: |
-          cd btfhub
           ./btfhub -workers 6 -d fedora
-      #
+    #
       - name: Fetch and Generate new BTFs (ORACLE)
         run: |
-          cd btfhub
           ./btfhub -workers 6 -d ol
-      #
-      - name: Take new BTFs to BTFHub Archive
-        run: |
-          cd btfhub
-          make take
-      #
+    #
       - name: Check Status
         run: |
-          cd btfhub-archive
+          cd archive
           git status
-      #
+    #
       - name: Commit and Push to BTFHub Archive
         run: |
-          cd btfhub-archive
+          cd archive
           git config --local user.name "Rafael David Tinoco"
           git config --local user.email "[email protected]"
           git add -A

.gitignore

@@ -1,7 +1,8 @@
+# binary
 btfhub
-# arvhices
-archive/*
-custom-archive/*
+# archives
+archive/
 # makefile leftovers
 .check*
-# binary
+# JetBrains
+.idea/

pkg/pkg/utils.go

@@ -31,9 +31,8 @@ func yumDownload(ctx context.Context, pkg string, destdir string) error {
 
 	destDirParam := fmt.Sprintf("--downloaddir=%s", destdir)
 
-	cmd := exec.CommandContext(ctx,
-		"sudo", "yum", "install", "-y", "--downloadonly", destDirParam, pkg,
-	)
+	binary, args := utils.SudoCMD("yum", "install", "-y", "--downloadonly", destDirParam, pkg)
+	cmd := exec.CommandContext(ctx, binary, args...)
 
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = stderr

pkg/repo/amazon.go

@@ -1,11 +1,18 @@
 package repo
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"log"
+	"os/exec"
 	"sort"
+	"strings"
+
+	"golang.org/x/exp/maps"
 
 	"github.com/aquasecurity/btfhub/pkg/job"
 	"github.com/aquasecurity/btfhub/pkg/kernel"
@@ -34,11 +41,12 @@ func (d *AmazonRepo) GetKernelPackages(
 	force bool,
 	jobChan chan<- job.Job,
 ) error {
-	searchOut, err := yumSearch(ctx, "kernel-debuginfo")
+	altArch := d.archs[arch]
+	searchOut, err := repoquery(ctx, "kernel-debuginfo", altArch)
 	if err != nil {
 		return err
 	}
-	pkgs, err := parseYumPackages(searchOut, kernel.NewKernelVersion(""))
+	pkgs, err := parseRepoqueryPackages(searchOut, kernel.NewKernelVersion(""))
 	if err != nil {
 		return fmt.Errorf("parse package listing: %s", err)
 	}
@@ -57,3 +65,55 @@ func (d *AmazonRepo) GetKernelPackages(
 
 	return nil
 }
+
+func repoquery(ctx context.Context, pkg string, arch string) (*bytes.Buffer, error) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	binary, args := utils.SudoCMD("repoquery", "--archlist="+arch, "--show-duplicates", pkg)
+	cmd := exec.CommandContext(ctx, binary, args...)
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("repoquery search %s: %s\n%s", pkg, err, stderr.String())
+	}
+	return stdout, nil
+}
+
+func parseRepoqueryPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package, error) {
+	pkgs := map[string]pkg.Package{}
+	bio := bufio.NewScanner(rdr)
+	for bio.Scan() {
+		line := bio.Text()
+		if !strings.HasPrefix(line, "kernel-debuginfo-") {
+			continue
+		}
+		if strings.HasPrefix(line, "kernel-debuginfo-common-") {
+			continue
+		}
+		_, version, found := strings.Cut(line, ":")
+		if !found {
+			continue
+		}
+		filename := version
+		lastdot := strings.LastIndex(filename, ".")
+		if lastdot == -1 {
+			continue
+		}
+		p := &pkg.RHELPackage{
+			Name:          fmt.Sprintf("kernel-debuginfo-%s", version),
+			NameOfFile:    filename,
+			KernelVersion: kernel.NewKernelVersion(filename[:lastdot]),
+			Architecture:  filename[lastdot+1:],
+		}
+		if !minVersion.IsZero() && p.Version().Less(minVersion) {
+			continue
+		}
+		if _, ok := pkgs[p.Name]; !ok {
+			pkgs[p.Name] = p
+		}
+	}
+	if err := bio.Err(); err != nil {
+		return nil, err
+	}
+	return maps.Values(pkgs), nil
+}

pkg/repo/rhel.go

@@ -45,7 +45,8 @@ func (d *RHELRepo) GetKernelPackages(
 ) error {
 	altArch := d.archs[arch]
 	rver := d.releaseVersions[release+":"+altArch]
-	if err := utils.RunCMD(ctx, "", "sudo", "subscription-manager", "release", fmt.Sprintf("--set=%s", rver)); err != nil {
+	binary, args := utils.SudoCMD("subscription-manager", "release", fmt.Sprintf("--set=%s", rver))
+	if err := utils.RunCMD(ctx, "", binary, args...); err != nil {
 		return err
 	}
 

pkg/repo/utils.go

@@ -12,11 +12,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/aquasecurity/btfhub/pkg/job"
 	"github.com/aquasecurity/btfhub/pkg/kernel"
 	"github.com/aquasecurity/btfhub/pkg/pkg"
 	"github.com/aquasecurity/btfhub/pkg/utils"
-	"golang.org/x/exp/maps"
 )
 
 func parseYumPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package, error) {
@@ -62,7 +63,8 @@ func parseYumPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package,
 func yumSearch(ctx context.Context, pkg string) (*bytes.Buffer, error) {
 	stdout := &bytes.Buffer{}
 	stderr := &bytes.Buffer{}
-	cmd := exec.CommandContext(ctx, "sudo", "yum", "search", "--showduplicates", pkg)
+	binary, args := utils.SudoCMD("yum", "search", "--showduplicates", pkg)
+	cmd := exec.CommandContext(ctx, binary, args...)
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	if err := cmd.Run(); err != nil {

pkg/utils/btf.go

@@ -34,3 +34,11 @@ func RunCMD(ctx context.Context, cwd string, binary string, args ...string) erro
 
 	return nil
 }
+
+func SudoCMD(binary string, args ...string) (string, []string) {
+	_, err := exec.LookPath("sudo")
+	if err == nil {
+		return "sudo", append([]string{binary}, args...)
+	}
+	return binary, args
+}