diff --git a/.flake8 b/.flake8 index 8c0ee3c..a0e6355 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,3 @@ [flake8] -max-line-length=130 \ No newline at end of file +max-line-length=130 +ignore=E741 diff --git a/msys2-lint b/msys2-lint new file mode 100755 index 0000000..34bbcc5 --- /dev/null +++ b/msys2-lint @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from msys2_devtools import cmd_lint + +cmd_lint.run() diff --git a/msys2_devtools/cmd_lint.py b/msys2_devtools/cmd_lint.py new file mode 100644 index 0000000..0dd3c8a --- /dev/null +++ b/msys2_devtools/cmd_lint.py @@ -0,0 +1,115 @@ +import requests +import os +import argparse +import sys +import gzip +import json + +from .db import parse_repo +from .srcinfo import parse_srcinfo + + +def check_base_missing(repo): + """Check that all packages have a pkgbase.""" + + for package_id, desc in repo.items(): + if "%BASE%" not in desc: + print(f"pkgbase not found for {package_id}") + + +def check_pycache_missing(repo): + """Check that all .py files have a corresponding .pyc file.""" + + for package_id, desc in repo.items(): + files = desc["%FILES%"] + + mapping = {} + + for f in files: + if f.endswith(".pyc"): + basename = os.path.basename(f) + parent = os.path.dirname(f) + if os.path.basename(parent) == "__pycache__": + grandparentdir = os.path.dirname(parent) + srcname = basename.replace(".opt-1.pyc", ".pyc").replace(".opt-2.pyc", ".pyc").rsplit(".", 2)[0] + srcpath = os.path.join(grandparentdir, srcname + ".py") + else: + srcpath = f[:-1] + mapping[srcpath] = f + + missing = [] + for f in files: + if f.endswith(".py"): + if os.path.basename(os.path.dirname(f)) == "bin": + continue + if f not in mapping: + missing.append(f) + + if missing: + print(f"Missing .pyc files for {package_id}:") + for f in missing: + print(f" {f}") + + +def lint_repos(args): + REPO_URLS = [ + "https://repo.msys2.org/msys/x86_64/msys.files.tar.zst", + "https://repo.msys2.org/mingw/clang32/clang32.files.tar.zst", + "https://repo.msys2.org/mingw/clang64/clang64.files.tar.zst", + "https://repo.msys2.org/mingw/ucrt64/ucrt64.files.tar.zst", + "https://repo.msys2.org/mingw/mingw32/mingw32.files.tar.zst", + "https://repo.msys2.org/mingw/mingw64/mingw64.files.tar.zst", + ] + for url in REPO_URLS: + r = requests.get(url) + r.raise_for_status() + repo = parse_repo(r.content) + + check_base_missing(repo) + check_pycache_missing(repo) + + +def check_srcinfo_same_pkgbase(srcinfo): + for value in srcinfo.values(): + pkgbases = set() + for srcinfo in value["srcinfo"].values(): + base = parse_srcinfo(srcinfo)[0] + pkgbases.add(base["pkgbase"][0]) + if len(pkgbases) > 1: + print(f"Multiple pkgbase values found for {value['path']}: {pkgbases}") + else: + if pkgbases and list(pkgbases)[0] != value['path']: + print(f"pkgbase value does not match path for {value['path']}: {list(pkgbases)[0]}") + + +def lint_srcinfos(args): + SRCINFO_URLS = [ + "https://github.com/msys2/MINGW-packages/releases/download/srcinfo-cache/srcinfo.json.gz", + "https://github.com/msys2/MSYS2-packages/releases/download/srcinfo-cache/srcinfo.json.gz", + ] + for url in SRCINFO_URLS: + r = requests.get(url) + r.raise_for_status() + + data = gzip.decompress(r.content) + srcinfo_cache = json.loads(data) + + check_srcinfo_same_pkgbase(srcinfo_cache) + + +def add_parser(subparsers): + sub = subparsers.add_parser("repos", help="Lint the repos") + sub.set_defaults(func=lint_repos) + + sub = subparsers.add_parser("srcinfos", help="Lint the srcinfos") + sub.set_defaults(func=lint_srcinfos) + + +def run(): + parser = argparse.ArgumentParser(description="Linter", allow_abbrev=False) + parser.set_defaults(func=lambda *x: parser.print_help()) + subparsers = parser.add_subparsers(title="subcommands") + add_parser(subparsers) + + args = parser.parse_args(sys.argv[1:]) + args.func(args) diff --git a/msys2_devtools/db.py b/msys2_devtools/db.py index 061b5c3..470020e 100644 --- a/msys2_devtools/db.py +++ b/msys2_devtools/db.py @@ -28,3 +28,52 @@ def zstdopen(cls, name, mode="r", fileobj=None, cctx=None, dctx=None, **kwargs): return t OPEN_METH = {"zstd": "zstdopen", **tarfile.TarFile.OPEN_METH} + + +def parse_desc(t: str) -> dict[str, list[str]]: + d: dict[str, list[str]] = {} + cat = None + values: list[str] = [] + for l in t.splitlines(): + l = l.strip() + if cat is None: + cat = l + elif not l: + d[cat] = values + cat = None + values = [] + else: + values.append(l) + if cat is not None: + d[cat] = values + return d + + +def parse_repo(data: bytes) -> dict[str, dict[str, list[str]]]: + sources: dict[str, dict[str, list[str]]] = {} + + with io.BytesIO(data) as f: + with ExtTarFile.open(fileobj=f, mode="r") as tar: + packages: dict[str, list] = {} + for info in tar: + package_id = info.name.split("/", 1)[0] + infofile = tar.extractfile(info) + if infofile is None: + continue + with infofile: + packages.setdefault(package_id, []).append( + (info.name, infofile.read())) + + for package_id, infos in sorted(packages.items()): + t = "" + for name, data in sorted(infos): + if name.endswith("/desc"): + t += data.decode("utf-8") + elif name.endswith("/depends"): + t += data.decode("utf-8") + elif name.endswith("/files"): + t += data.decode("utf-8") + desc = parse_desc(t) + sources[package_id] = desc + + return sources diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..9909ca1 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,29 @@ +from msys2_devtools.db import parse_desc + + +def test_parse_desc(): + data = """\ +%FILENAME% +libarchive-3.7.4-1-x86_64.pkg.tar.zst + +%NAME% +libarchive + +%BASE% +libarchive + +%VERSION% +3.7.4-1 + +%DEPENDS% +gcc-libs +libbz2 +libiconv +""" + + desc = parse_desc(data) + assert desc["%FILENAME%"] == ["libarchive-3.7.4-1-x86_64.pkg.tar.zst"] + assert desc["%NAME%"] == ["libarchive"] + assert desc["%BASE%"] == ["libarchive"] + assert desc["%VERSION%"] == ["3.7.4-1"] + assert desc["%DEPENDS%"] == ["gcc-libs", "libbz2", "libiconv"]