diff --git a/.gitignore b/.gitignore index d888caa2..8b813b6e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ +# binary btfhub # arvhices archive/* custom-archive/* # makefile leftovers .check* -# binary +# JetBrains +.idea/ diff --git a/cmd/btfhub/main.go b/cmd/btfhub/main.go index ac225f95..eb9d8fdf 100644 --- a/cmd/btfhub/main.go +++ b/cmd/btfhub/main.go @@ -11,9 +11,10 @@ import ( "path/filepath" "runtime" + "golang.org/x/sync/errgroup" + "github.com/aquasecurity/btfhub/pkg/job" "github.com/aquasecurity/btfhub/pkg/repo" - "golang.org/x/sync/errgroup" ) var distroReleases = map[string][]string{ @@ -24,6 +25,7 @@ var distroReleases = map[string][]string{ "ol": {"7", "8"}, "rhel": {"7", "8"}, "amzn": {"1", "2"}, + "sles": {"12.3", "12.5", "15.1", "15.2", "15.3", "15.4"}, } type repoFunc func() repo.Repository @@ -36,6 +38,7 @@ var repoCreators = map[string]repoFunc{ "ol": repo.NewOracleRepo, "rhel": repo.NewRHELRepo, "amzn": repo.NewAmazonRepo, + "sles": repo.NewSUSERepo, } var distro, release, arch string @@ -43,8 +46,8 @@ var numWorkers int var force bool func init() { - flag.StringVar(&distro, "distro", "", "distribution to update (ubuntu,debian,centos,fedora,ol,rhel,amazon)") - flag.StringVar(&distro, "d", "", "distribution to update (ubuntu,debian,centos,fedora,ol,rhel,amazon)") + flag.StringVar(&distro, "distro", "", "distribution to update (ubuntu,debian,centos,fedora,ol,rhel,amazon,sles)") + flag.StringVar(&distro, "d", "", "distribution to update (ubuntu,debian,centos,fedora,ol,rhel,amazon,sles)") flag.StringVar(&release, "release", "", "distribution release to update, requires specifying distribution") flag.StringVar(&release, "r", "", "distribution release to update, requires specifying distribution") flag.StringVar(&arch, "arch", "", "architecture to update (x86_64,arm64)") diff --git a/pkg/pkg/suse.go b/pkg/pkg/suse.go new file mode 100644 index 00000000..13fab0f5 --- /dev/null +++ b/pkg/pkg/suse.go @@ -0,0 +1,59 @@ +package pkg + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/aquasecurity/btfhub/pkg/kernel" + "github.com/aquasecurity/btfhub/pkg/utils" +) + +type SUSEPackage struct { + Name string + NameOfFile string + Architecture string + KernelVersion kernel.Version + Repo string + Flavor string + Downloaddir string +} + +func (pkg *SUSEPackage) Filename() string { + return pkg.NameOfFile +} + +func (pkg *SUSEPackage) Version() kernel.Version { + return pkg.KernelVersion +} + +func (pkg *SUSEPackage) String() string { + return fmt.Sprintf("%s-%s.%s", pkg.Name, pkg.KernelVersion.String(), pkg.Architecture) +} + +func (pkg *SUSEPackage) ExtractKernel(ctx context.Context, pkgpath string, vmlinuxPath string) error { + // vmlinux at: /usr/lib/debug/boot/vmlinux--.debug + return utils.ExtractVmlinuxFromRPM(ctx, pkgpath, vmlinuxPath) +} + +func (pkg *SUSEPackage) Download(ctx context.Context, _ string, force bool) (string, error) { + localFile := fmt.Sprintf("%s-%s.%s.rpm", pkg.Name, pkg.KernelVersion.String(), pkg.Architecture) + rpmpath := filepath.Join(pkg.Downloaddir, localFile) + if !force && utils.Exists(rpmpath) { + return rpmpath, nil + } + + if err := zypperDownload(ctx, fmt.Sprintf("%s=%s", pkg.Name, pkg.KernelVersion.String())); err != nil { + os.Remove(rpmpath) + return "", fmt.Errorf("zypper download: %s", err) + } + + return rpmpath, nil +} + +func zypperDownload(ctx context.Context, pkg string) error { + stdout, err := utils.RunZypperCMD(ctx, "-q", "install", "-y", "--no-recommends", "--download-only", pkg) + _, _ = fmt.Fprint(os.Stdout, stdout.String()) + return err +} diff --git a/pkg/pkg/ubuntu.go b/pkg/pkg/ubuntu.go index 0be4363f..36747241 100644 --- a/pkg/pkg/ubuntu.go +++ b/pkg/pkg/ubuntu.go @@ -12,9 +12,10 @@ import ( "path/filepath" "strings" + "pault.ag/go/debian/deb" + "github.com/aquasecurity/btfhub/pkg/kernel" "github.com/aquasecurity/btfhub/pkg/utils" - "pault.ag/go/debian/deb" ) // UbuntuPackage represents a package in Ubuntu @@ -116,6 +117,7 @@ func (pkg *UbuntuPackage) ExtractKernel(ctx context.Context, pkgPath string, vml return fmt.Errorf("create vmlinux file: %s", err) } counter := &utils.ProgressCounter{ + Ctx: ctx, Op: "Extract", Name: hdr.Name, Size: uint64(hdr.Size), diff --git a/pkg/pkg/utils.go b/pkg/pkg/utils.go index 6760300c..0f3eb566 100644 --- a/pkg/pkg/utils.go +++ b/pkg/pkg/utils.go @@ -31,9 +31,8 @@ func yumDownload(ctx context.Context, pkg string, destdir string) error { destDirParam := fmt.Sprintf("--downloaddir=%s", destdir) - cmd := exec.CommandContext(ctx, - "sudo", "yum", "install", "-y", "--downloadonly", destDirParam, pkg, - ) + binary, args := utils.SudoCMD("yum", "install", "-y", "--downloadonly", destDirParam, pkg) + cmd := exec.CommandContext(ctx, binary, args...) cmd.Stdout = os.Stdout cmd.Stderr = stderr diff --git a/pkg/repo/centos.go b/pkg/repo/centos.go index 1e0b1ee4..e8b32f6a 100644 --- a/pkg/repo/centos.go +++ b/pkg/repo/centos.go @@ -51,7 +51,7 @@ func (d *CentosRepo) GetKernelPackages( repoURL := fmt.Sprintf(d.repos[release], altArch) - links, err := utils.GetLinks(repoURL) + links, err := utils.GetLinks(ctx, repoURL) if err != nil { return fmt.Errorf("ERROR: list packages: %s", err) } diff --git a/pkg/repo/fedora.go b/pkg/repo/fedora.go index 54a0135d..bb443101 100644 --- a/pkg/repo/fedora.go +++ b/pkg/repo/fedora.go @@ -90,7 +90,7 @@ func (d *FedoraRepo) GetKernelPackages( // Pick all the links from multiple repositories for _, repo := range repos { - rlinks, err := utils.GetLinks(repo) + rlinks, err := utils.GetLinks(ctx, repo) if err != nil { log.Printf("ERROR: list packages: %s\n", err) continue diff --git a/pkg/repo/oracle.go b/pkg/repo/oracle.go index 38f189b1..e3139fac 100644 --- a/pkg/repo/oracle.go +++ b/pkg/repo/oracle.go @@ -51,7 +51,7 @@ func (d *oracleRepo) GetKernelPackages( repoURL := d.repos[release] - links, err := utils.GetLinks(repoURL) + links, err := utils.GetLinks(ctx, repoURL) if err != nil { return fmt.Errorf("ERROR: list packages: %s", err) } diff --git a/pkg/repo/rhel.go b/pkg/repo/rhel.go index 9b6833df..098af1fd 100644 --- a/pkg/repo/rhel.go +++ b/pkg/repo/rhel.go @@ -45,7 +45,8 @@ func (d *RHELRepo) GetKernelPackages( ) error { altArch := d.archs[arch] rver := d.releaseVersions[release+":"+altArch] - if err := utils.RunCMD(ctx, "", "sudo", "subscription-manager", "release", fmt.Sprintf("--set=%s", rver)); err != nil { + binary, args := utils.SudoCMD("subscription-manager", "release", fmt.Sprintf("--set=%s", rver)) + if err := utils.RunCMD(ctx, "", binary, args...); err != nil { return err } diff --git a/pkg/repo/suse.go b/pkg/repo/suse.go new file mode 100644 index 00000000..86fa43ed --- /dev/null +++ b/pkg/repo/suse.go @@ -0,0 +1,205 @@ +package repo + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "regexp" + "sort" + "strconv" + "strings" + "unicode" + + "golang.org/x/sync/errgroup" + + "github.com/aquasecurity/btfhub/pkg/job" + "github.com/aquasecurity/btfhub/pkg/kernel" + "github.com/aquasecurity/btfhub/pkg/pkg" + "github.com/aquasecurity/btfhub/pkg/utils" +) + +type suseRepo struct { + archs map[string]string + repoAliases map[string]string +} + +func NewSUSERepo() Repository { + return &suseRepo{ + archs: map[string]string{ + "x86_64": "x86_64", + "arm64": "aarch64", + }, + repoAliases: map[string]string{}, + } +} + +func (d *suseRepo) GetKernelPackages(ctx context.Context, dir string, release string, arch string, force bool, jobchan chan<- job.Job) error { + var repos []string + + switch release { + case "12.5": + repos = append(repos, fmt.Sprintf("SUSE_Linux_Enterprise_Server_%s:SLES12-SP5-Debuginfo-Pool", arch)) + repos = append(repos, fmt.Sprintf("SUSE_Linux_Enterprise_Server_%s:SLES12-SP5-Debuginfo-Updates", arch)) + case "15.1": + repos = append(repos, fmt.Sprintf("Basesystem_Module_15_SP1_%s:SLE-Module-Basesystem15-SP1-Debuginfo-Pool", arch)) + repos = append(repos, fmt.Sprintf("Basesystem_Module_15_SP1_%s:SLE-Module-Basesystem15-SP1-Debuginfo-Updates", arch)) + case "15.2": + repos = append(repos, fmt.Sprintf("Basesystem_Module_%s:SLE-Module-Basesystem15-SP2-Debuginfo-Pool", arch)) + repos = append(repos, fmt.Sprintf("Basesystem_Module_%s:SLE-Module-Basesystem15-SP2-Debuginfo-Updates", arch)) + case "15.3": + repos = append(repos, fmt.Sprintf("Basesystem_Module_%s:SLE-Module-Basesystem15-SP3-Debuginfo-Pool", arch)) + repos = append(repos, fmt.Sprintf("Basesystem_Module_%s:SLE-Module-Basesystem15-SP3-Debuginfo-Updates", arch)) + case "15.4": + repos = append(repos, fmt.Sprintf("Basesystem_Module_%s:SLE-Module-Basesystem15-SP4-Debuginfo-Pool", arch)) + repos = append(repos, fmt.Sprintf("Basesystem_Module_%s:SLE-Module-Basesystem15-SP4-Debuginfo-Updates", arch)) + } + for _, r := range repos { + if _, err := utils.RunZypperCMD(ctx, "modifyrepo", "--enable", r); err != nil { + return err + } + } + + if err := d.getRepoAliases(ctx); err != nil { + return fmt.Errorf("repo aliases: %s", err) + } + + // packages are named kernel--debuginfo + // possible types are: default, azure + searchOut, err := zypperSearch(ctx, "kernel-*-debuginfo") + if err != nil { + return err + } + + pkgs, err := d.parseZypperPackages(searchOut, arch) + if err != nil { + return fmt.Errorf("parse package listing: %s", err) + } + + pkgsByKernelType := make(map[string][]pkg.Package) + for _, p := range pkgs { + ks, ok := pkgsByKernelType[p.Flavor] + if !ok { + ks = make([]pkg.Package, 0, 1) + } + ks = append(ks, p) + pkgsByKernelType[p.Flavor] = ks + } + + for kt, ks := range pkgsByKernelType { + sort.Sort(pkg.ByVersion(ks)) + log.Printf("DEBUG: %s %s flavor %d kernels\n", arch, kt, len(ks)) + } + + g, ctx := errgroup.WithContext(ctx) + for kt, ks := range pkgsByKernelType { + ckt := kt + cks := ks + g.Go(func() error { + log.Printf("DEBUG: start kernel type %s %s (%d pkgs)\n", ckt, arch, len(cks)) + err := d.processPackages(ctx, dir, cks, force, jobchan) + log.Printf("DEBUG: end kernel type %s %s\n", ckt, arch) + return err + }) + } + return g.Wait() +} + +func (d *suseRepo) getRepoAliases(ctx context.Context) error { + repos, err := zypperRepos(ctx) + if err != nil { + return err + } + bio := bufio.NewScanner(repos) + for bio.Scan() { + line := bio.Text() + fields := strings.FieldsFunc(line, func(r rune) bool { + return unicode.IsSpace(r) || r == '|' + }) + if len(fields) < 3 { + continue + } + // first field must be a number + if _, err := strconv.Atoi(fields[0]); err != nil { + continue + } + alias, name := fields[1], fields[2] + d.repoAliases[name] = alias + } + return bio.Err() +} + +func (d *suseRepo) processPackages(ctx context.Context, dir string, pkgs []pkg.Package, force bool, jobchan chan<- job.Job) error { + for i, p := range pkgs { + log.Printf("DEBUG: start pkg %s (%d/%d)\n", p, i+1, len(pkgs)) + if err := processPackage(ctx, p, dir, force, jobchan); err != nil { + if errors.Is(err, utils.ErrHasBTF) { + log.Printf("INFO: kernel %s has BTF already, skipping later kernels\n", p) + return nil + } + if errors.Is(err, context.Canceled) { + return nil + } + log.Printf("ERROR: %s: %s\n", p, err) + continue + } + log.Printf("DEBUG: end pkg %s (%d/%d)\n", p, i+1, len(pkgs)) + } + return nil +} + +func (d *suseRepo) parseZypperPackages(rdr io.Reader, arch string) ([]*pkg.SUSEPackage, error) { + var pkgs []*pkg.SUSEPackage + kre := regexp.MustCompile(`^kernel-([^-]+)-debuginfo$`) + bio := bufio.NewScanner(rdr) + for bio.Scan() { + line := bio.Text() + fields := strings.FieldsFunc(line, func(r rune) bool { + return unicode.IsSpace(r) || r == '|' + }) + if len(fields) < 5 { + continue + } + name, ver, pkgarch, repo := fields[0], fields[2], fields[3], fields[4] + if pkgarch != arch { + continue + } + match := kre.FindStringSubmatch(name) + if match != nil { + alias, ok := d.repoAliases[repo] + if !ok { + return nil, fmt.Errorf("unknown repo %s", repo) + } + flavor := match[1] + if flavor == "preempt" { + continue + } + + p := &pkg.SUSEPackage{ + Name: name, + NameOfFile: fmt.Sprintf("%s-%s", ver, match[1]), + KernelVersion: kernel.NewKernelVersion(ver), + Architecture: pkgarch, + Repo: repo, + Flavor: flavor, + Downloaddir: fmt.Sprintf("/var/cache/zypp/packages/%s/%s", alias, arch), + } + pkgs = append(pkgs, p) + } + } + if err := bio.Err(); err != nil { + return nil, err + } + return pkgs, nil +} + +func zypperRepos(ctx context.Context) (*bytes.Buffer, error) { + return utils.RunZypperCMD(ctx, "repos") +} + +func zypperSearch(ctx context.Context, pkg string) (*bytes.Buffer, error) { + return utils.RunZypperCMD(ctx, "search", "-s", pkg) +} diff --git a/pkg/repo/utils.go b/pkg/repo/utils.go index 25149fe9..8c679705 100644 --- a/pkg/repo/utils.go +++ b/pkg/repo/utils.go @@ -12,11 +12,12 @@ import ( "path/filepath" "strings" + "golang.org/x/exp/maps" + "github.com/aquasecurity/btfhub/pkg/job" "github.com/aquasecurity/btfhub/pkg/kernel" "github.com/aquasecurity/btfhub/pkg/pkg" "github.com/aquasecurity/btfhub/pkg/utils" - "golang.org/x/exp/maps" ) func parseYumPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package, error) { @@ -62,7 +63,8 @@ func parseYumPackages(rdr io.Reader, minVersion kernel.Version) ([]pkg.Package, func yumSearch(ctx context.Context, pkg string) (*bytes.Buffer, error) { stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} - cmd := exec.CommandContext(ctx, "sudo", "yum", "search", "--showduplicates", pkg) + binary, args := utils.SudoCMD("yum", "search", "--showduplicates", pkg) + cmd := exec.CommandContext(ctx, binary, args...) cmd.Stdout = stdout cmd.Stderr = stderr if err := cmd.Run(); err != nil { diff --git a/pkg/utils/btf.go b/pkg/utils/btf.go index 07dd7801..07f8b0b8 100644 --- a/pkg/utils/btf.go +++ b/pkg/utils/btf.go @@ -34,3 +34,11 @@ func RunCMD(ctx context.Context, cwd string, binary string, args ...string) erro return nil } + +func SudoCMD(binary string, args ...string) (string, []string) { + _, err := exec.LookPath("sudo") + if err == nil { + return "sudo", append([]string{binary}, args...) + } + return binary, args +} diff --git a/pkg/utils/download.go b/pkg/utils/download.go index afcfc3b8..28aefcaa 100644 --- a/pkg/utils/download.go +++ b/pkg/utils/download.go @@ -47,6 +47,7 @@ func Download(ctx context.Context, url string, dest io.Writer) error { // Create a progress counter reader counter := &ProgressCounter{ + Ctx: ctx, Op: "Download", // operation Name: resp.Request.URL.String(), // file name Size: uint64(resp.ContentLength), // file length @@ -78,11 +79,13 @@ func Download(ctx context.Context, url string, dest io.Writer) error { } // GetLinks returns a list of links from a given URL -func GetLinks(repoURL string) ([]string, error) { - +func GetLinks(ctx context.Context, repoURL string) ([]string, error) { // Read the repo URL - - resp, err := http.Get(repoURL) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, repoURL, nil) + if err != nil { + return nil, fmt.Errorf("http request: %s", err) + } + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("get links from %s: %s", repoURL, err) } @@ -99,6 +102,7 @@ func GetLinks(repoURL string) ([]string, error) { // Create a progress counter reader counter := &ProgressCounter{ + Ctx: ctx, Op: "Download", Name: resp.Request.URL.String(), Size: uint64(resp.ContentLength), diff --git a/pkg/utils/progress.go b/pkg/utils/progress.go index ae23092b..d59d9d4a 100644 --- a/pkg/utils/progress.go +++ b/pkg/utils/progress.go @@ -1,6 +1,7 @@ package utils import ( + "context" "fmt" "time" @@ -8,6 +9,7 @@ import ( ) type ProgressCounter struct { + Ctx context.Context Op string Size uint64 Name string @@ -20,6 +22,10 @@ type ProgressCounter struct { // Write implements the io.Writer interface and is used to count the number of // bytes written to the underlying writer. func (wc *ProgressCounter) Write(p []byte) (int, error) { + if wc.Ctx != nil && wc.Ctx.Err() != nil { + return 0, wc.Ctx.Err() + } + n := len(p) wc.written += uint64(n) diff --git a/pkg/utils/rpm.go b/pkg/utils/rpm.go index 5e4ac5da..b7b5fbd4 100644 --- a/pkg/utils/rpm.go +++ b/pkg/utils/rpm.go @@ -1,6 +1,7 @@ package utils import ( + "compress/bzip2" "compress/gzip" "context" "errors" @@ -44,10 +45,12 @@ func ExtractVmlinuxFromRPM(ctx context.Context, rpmPath string, vmlinuxPath stri case "gzip": grdr, err := gzip.NewReader(file) if err != nil { - return fmt.Errorf("xz reader: %s", err) + return fmt.Errorf("gzip reader: %s", err) } defer grdr.Close() crdr = grdr + case "bzip2": + crdr = bzip2.NewReader(file) default: return fmt.Errorf("unsupported compression: %s", rpmPkg.PayloadCompression()) } @@ -86,6 +89,7 @@ func ExtractVmlinuxFromRPM(ctx context.Context, rpmPath string, vmlinuxPath stri } counter := &ProgressCounter{ + Ctx: ctx, Op: "Extract", Name: cpioHeader.Name, Size: uint64(cpioHeader.Size), diff --git a/pkg/utils/zypper.go b/pkg/utils/zypper.go new file mode 100644 index 00000000..037f3ad0 --- /dev/null +++ b/pkg/utils/zypper.go @@ -0,0 +1,30 @@ +package utils + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "strings" + "sync" +) + +var ( + zypperMtx sync.Mutex +) + +func RunZypperCMD(ctx context.Context, args ...string) (*bytes.Buffer, error) { + zypperMtx.Lock() + defer zypperMtx.Unlock() + + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + binary, args := SudoCMD("zypper", args...) + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("zypper cmd %s %s: %s\n%s", binary, strings.Join(args, " "), err, stderr.String()) + } + return stdout, nil +}