Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add job to update amazon 2 in cron workflow #104

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 103 additions & 47 deletions .github/workflows/cron.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,82 @@ on:
- cron: "0 1 * * *"
workflow_dispatch: {}
jobs:
amazon-update:
name: Update Amazon 2 BTF Archive
runs-on: ubuntu-latest
container:
image: amazonlinux:2
steps:
- name: Install needed amazon packages
run: |
yum install -y yum-utils tar gzip xz clang make cmake git libdwarf-devel elfutils-libelf-devel elfutils-devel
yum-config-manager -y --enable amzn2-core-debuginfo
yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
yum install -y gh
echo "[amzn2-core-debuginfo-alt]" >> /etc/yum.repos.d/amzn2-core.repo
awk '/^\[/{found=""} /^\[amzn2-core-debuginfo]$/{found=1;next} found && NF' /etc/yum.repos.d/amzn2-core.repo | sed 's/\$basearch/aarch64/g' >> /etc/yum.repos.d/amzn2-core.repo
shell: bash

- name: Authenticate
run: |
gh auth login --with-token <<<'${{ secrets.PAT_RAFAEL }}'

- name: Check out BTFHub
uses: actions/checkout@v3
with:
submodules: 'recursive'
token: ${{ secrets.PAT_RAFAEL }}

- run: git config --system --add safe.directory $(pwd)
shell: bash

- uses: actions/setup-go@v4
with:
go-version: '1.20'

- name: build pahole
run: |
cd ./3rdparty/dwarves
mkdir build
cd build
cmake -D__LIB=lib -DDWARF_INCLUDE_DIR=/usr/include ..
make install
echo "/usr/local/lib" >> /etc/ld.so.conf.d/pahole.conf
ldconfig
shell: bash

- name: Compile BTFHub Tool
run: |
make
shell: bash

- name: Checkout BTFHub Archive
uses: actions/checkout@v3
with:
repository: aquasecurity/btfhub-archive
token: ${{ secrets.PAT_RAFAEL }}
path: archive
sparse-checkout: |
amzn

- name: Fetch and Generate new BTFs (AMAZON 2)
run: |
./btfhub -workers 6 -d amzn -r 2

- name: Check Status
run: |
cd archive
git status

- name: Commit and Push to BTFHub Archive
run: |
cd archive
git config --local user.name "Rafael David Tinoco"
git config --local user.email "[email protected]"
git add -A
git diff-index --quiet HEAD || git commit -m "Update BTFHUB Archive from BTFHUB"
git push

build:
name: Update BTF Archive
env:
Expand Down Expand Up @@ -72,88 +148,68 @@ jobs:
--slave /usr/bin/opt opt /usr/bin/opt-14 \
--slave /usr/bin/cc cc /usr/bin/clang-14
shell: bash
#
#
- name: Authenticate
run: |
gh auth login --with-token <<<'${{ secrets.PAT_RAFAEL }}'
#
#
- name: Check out BTFHub
uses: actions/checkout@v3
with:
repository: aquasecurity/btfhub
path: ./btfhub
token: ${{ secrets.PAT_RAFAEL }}
# Authenticate
- name: Checkout BTFHub Archive
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
repository: aquasecurity/btfhub-archive
path: ./btfhub-archive
token: ${{ secrets.PAT_RAFAEL }}
#
#
- name: Install pahole
run: |
cd btfhub
./3rdparty/pahole.sh
shell: bash
#
- name: Install bpftool
run: |
cd btfhub
./3rdparty/bpftool.sh
shell: bash
#
- name: Bring current BTFHub Archive
run: |
cd btfhub
make bring
shell: bash
#
#
- name: Compile BTFHub Tool
run: |
cd btfhub
make
shell: bash
#
#
- name: Checkout BTFHub Archive
uses: actions/checkout@v4
with:
repository: aquasecurity/btfhub-archive
path: archive
sparse-checkout: |
centos
debian
fedora
ol
ubuntu

#
- name: Fetch and Generate new BTFs (UBUNTU)
run: |
cd btfhub
./btfhub -workers 6 -d ubuntu
# debian stretch seems to be gone, updates for buster and bullseye only
# debian stretch seems to be gone, updates for buster and bullseye only
- name: Fetch and Generate new BTFs (DEBIAN)
run: |
cd btfhub
./btfhub -workers 6 -d debian -r buster
./btfhub -workers 6 -d debian -r bullseye
#
#
- name: Fetch and Generate new BTFs (CENTOS)
run: |
cd btfhub
./btfhub -workers 6 -d centos
#
#
- name: Fetch and Generate new BTFs (FEDORA)
run: |
cd btfhub
./btfhub -workers 6 -d fedora
#
#
- name: Fetch and Generate new BTFs (ORACLE)
run: |
cd btfhub
./btfhub -workers 6 -d ol
#
- name: Take new BTFs to BTFHub Archive
run: |
cd btfhub
make take
#
#
- name: Check Status
run: |
cd btfhub-archive
cd archive
git status
#
#
- name: Commit and Push to BTFHub Archive
run: |
cd btfhub-archive
cd archive
git config --local user.name "Rafael David Tinoco"
git config --local user.email "[email protected]"
git add -A
Expand Down
5 changes: 2 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# binary
btfhub
# arvhices
archive/*
custom-archive/*
# archives
archive/
# makefile leftovers
.check*
# JetBrains
Expand Down
1 change: 0 additions & 1 deletion archive/.gitignore

This file was deleted.

1 change: 0 additions & 1 deletion custom-archive/.gitignore

This file was deleted.

2 changes: 1 addition & 1 deletion pkg/pkg/rhel.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func (pkg *RHELPackage) Download(ctx context.Context, dir string, force bool) (s
return rpmpath, nil
}

err := yumDownload(ctx, pkg.Name, dir)
err := yumDownload(ctx, pkg.Name, pkg.Architecture, dir)
if err != nil {
os.Remove(rpmpath)
return "", fmt.Errorf("rpm download: %s", err)
Expand Down
9 changes: 2 additions & 7 deletions pkg/pkg/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,10 @@ func TarballBTF(ctx context.Context, btf string, out string) error {
// RHEL packages
//

func yumDownload(ctx context.Context, pkg string, destdir string) error {

func yumDownload(ctx context.Context, pkg string, arch string, destdir string) error {
stderr := &bytes.Buffer{}

destDirParam := fmt.Sprintf("--downloaddir=%s", destdir)

binary, args := utils.SudoCMD("yum", "install", "-y", "--downloadonly", destDirParam, pkg)
binary, args := utils.SudoCMD("yumdownloader", "--archlist="+arch, "--destdir="+destdir, pkg)
cmd := exec.CommandContext(ctx, binary, args...)

cmd.Stdout = os.Stdout
cmd.Stderr = stderr

Expand Down
64 changes: 62 additions & 2 deletions pkg/repo/amazon.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
package repo

import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"log"
"os/exec"
"sort"
"strings"

"golang.org/x/exp/maps"

"github.com/aquasecurity/btfhub/pkg/job"
"github.com/aquasecurity/btfhub/pkg/kernel"
Expand Down Expand Up @@ -34,11 +41,12 @@ func (d *AmazonRepo) GetKernelPackages(
force bool,
jobChan chan<- job.Job,
) error {
searchOut, err := yumSearch(ctx, "kernel-debuginfo")
altArch := d.archs[arch]
searchOut, err := repoquery(ctx, "kernel-debuginfo", altArch)
if err != nil {
return err
}
pkgs, err := parseYumPackages(searchOut, kernel.NewKernelVersion(""))
pkgs, err := parseRepoqueryPackages(searchOut, kernel.NewKernelVersion(""))
if err != nil {
return fmt.Errorf("parse package listing: %s", err)
}
Expand All @@ -57,3 +65,55 @@ func (d *AmazonRepo) GetKernelPackages(

return nil
}

func repoquery(ctx context.Context, pkg string, arch string) (*bytes.Buffer, error) {
stdout := &bytes.Buffer{}
stderr := &bytes.Buffer{}
binary, args := utils.SudoCMD("repoquery", "--archlist="+arch, "--show-duplicates", pkg)
cmd := exec.CommandContext(ctx, binary, args...)
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("repoquery search %s: %s\n%s", pkg, err, stderr.String())
}
return stdout, nil
}

func parseRepoqueryPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package, error) {
pkgs := map[string]pkg.Package{}
bio := bufio.NewScanner(rdr)
for bio.Scan() {
line := bio.Text()
if !strings.HasPrefix(line, "kernel-debuginfo-") {
continue
}
if strings.HasPrefix(line, "kernel-debuginfo-common-") {
continue
}
_, version, found := strings.Cut(line, ":")
if !found {
continue
}
filename := version
lastdot := strings.LastIndex(filename, ".")
if lastdot == -1 {
continue
}
p := &pkg.RHELPackage{
Name: fmt.Sprintf("kernel-debuginfo-%s", version),
NameOfFile: filename,
KernelVersion: kernel.NewKernelVersion(filename[:lastdot]),
Architecture: filename[lastdot+1:],
}
if !minVersion.IsZero() && p.Version().Less(minVersion) {
continue
}
if _, ok := pkgs[p.Name]; !ok {
pkgs[p.Name] = p
}
}
if err := bio.Err(); err != nil {
return nil, err
}
return maps.Values(pkgs), nil
}
Loading