diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 318910e..fb4e246 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,16 +10,16 @@ repos: - id: check-yaml - id: debug-statements language_version: python3 + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v2.0.4 + hooks: + - id: autopep8 - repo: https://github.com/PyCQA/flake8 rev: 6.1.0 hooks: - id: flake8 language_version: python3 additional_dependencies: [flake8-typing-imports==1.15.0] - - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.4 - hooks: - - id: autopep8 - repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: diff --git a/config/README.rst b/config/README.rst index e5a67c6..fc3f4a2 100644 --- a/config/README.rst +++ b/config/README.rst @@ -334,7 +334,7 @@ updated. Example: "\"${PYBIN}/tox\" -e py", "cd ..", ] - require-cffi = True + require-cffi = true [zest-releaser] options = [ diff --git a/config/c-code/tests-strategy.j2 b/config/c-code/tests-strategy.j2 index 2a15505..a06c70a 100644 --- a/config/c-code/tests-strategy.j2 +++ b/config/c-code/tests-strategy.j2 @@ -3,7 +3,7 @@ matrix: python-version: {% if with_pypy %} - - "pypy-3.10" + - "pypy-%(pypy_version)s" {% endif %} - "3.7" - "3.8" @@ -15,17 +15,22 @@ - "%(future_python_version)s" {% endif %} {% if with_windows %} - os: [ubuntu-20.04, macos-11, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] {% else %} - os: [ubuntu-20.04, macos-11] + os: [ubuntu-latest, macos-latest] {% endif %} {% if with_pypy or gha_additional_exclude %} exclude: {% endif %} + - os: macos-latest + python-version: "3.7" {% if with_pypy %} - - os: macos-11 - python-version: "pypy-3.10" + - os: macos-latest + python-version: "pypy-%(pypy_version)s" {% endif %} {% for line in gha_additional_exclude %} %(line)s {% endfor %} + include: + - os: macos-12 + python-version: "3.7" diff --git a/config/c-code/tests.yml.j2 b/config/c-code/tests.yml.j2 index ef94837..e19e54a 100644 --- a/config/c-code/tests.yml.j2 +++ b/config/c-code/tests.yml.j2 @@ -322,7 +322,7 @@ jobs: coveralls_finish: needs: test - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Coveralls Finished uses: AndreMiras/coveralls-python-action@develop @@ -335,8 +335,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.9"] - os: [ubuntu-20.04] + python-version: ["%(manylinux_python_version)s"] + os: [ubuntu-latest] steps: {% include 'tests-cache.j2' %} @@ -361,8 +361,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.9"] - os: [ubuntu-20.04] + python-version: ["%(manylinux_python_version)s"] + os: [ubuntu-latest] steps: {% include 'tests-cache.j2' %} @@ -381,13 +381,13 @@ jobs: # python -m pylint --limit-inference-results=1 --rcfile=.pylintrc %(package_name)s -f parseable -r n manylinux: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name # We use a regular Python matrix entry to share as much code as possible. strategy: matrix: - python-version: ["3.9"] - image: [manylinux2014_x86_64, manylinux2014_i686, manylinux2014_aarch64] + python-version: ["%(manylinux_python_version)s"] + image: [%(manylinux_x86_64)s, %(manylinux_i686)s, %(manylinux_aarch64)s] steps: {% set cache_key = "${{ runner.os }}-pip_manylinux-${{ matrix.image }}-${{ matrix.python-version }}" %} @@ -396,7 +396,7 @@ jobs: - name: Update pip run: pip install -U pip - name: Build %(package_name)s (x86_64) - if: matrix.image == 'manylinux2014_x86_64' + if: matrix.image == '%(manylinux_x86_64)s' # An alternate way to do this is to run the container directly with a uses: # and then the script runs inside it. That may work better with caching. # See https://github.com/pyca/bcrypt/blob/f6b5ee2eda76d077c531362ac65e16f045cf1f29/.github/workflows/wheel-builder.yml @@ -405,14 +405,14 @@ jobs: run: | bash .manylinux.sh - name: Build %(package_name)s (i686) - if: matrix.image == 'manylinux2014_i686' + if: matrix.image == '%(manylinux_i686)s' env: DOCKER_IMAGE: quay.io/pypa/${{ matrix.image }} PRE_CMD: linux32 run: | bash .manylinux.sh - name: Build %(package_name)s (aarch64) - if: matrix.image == 'manylinux2014_aarch64' + if: matrix.image == '%(manylinux_aarch64)s' env: DOCKER_IMAGE: quay.io/pypa/${{ matrix.image }} run: | diff --git a/config/config-package.py b/config/config-package.py index e1d6d24..b14809c 100755 --- a/config/config-package.py +++ b/config/config-package.py @@ -12,11 +12,18 @@ # ############################################################################## from functools import cached_property +from set_branch_protection_rules import set_branch_protection from shared.call import abort from shared.call import call from shared.git import get_branch_name from shared.git import get_commit_id from shared.git import git_branch +from shared.packages import FUTURE_PYTHON_VERSION +from shared.packages import MANYLINUX_AARCH64 +from shared.packages import MANYLINUX_I686 +from shared.packages import MANYLINUX_PYTHON_VERSION +from shared.packages import MANYLINUX_X86_64 +from shared.packages import PYPY_VERSION from shared.path import change_dir import argparse import collections @@ -34,7 +41,6 @@ Generated from: https://github.com/zopefoundation/meta/tree/master/config/{config_type} --> """ -FUTURE_PYTHON_VERSION = "3.13.0-alpha - 3.13.0" DEFAULT = object() @@ -502,6 +508,11 @@ def tests_yml(self): with_pypy=self.with_pypy, with_macos=self.with_macos, with_windows=self.with_windows, + manylinux_python_version=MANYLINUX_PYTHON_VERSION, + manylinux_aarch64=MANYLINUX_AARCH64, + manylinux_i686=MANYLINUX_I686, + manylinux_x86_64=MANYLINUX_X86_64, + pypy_version=PYPY_VERSION, ) def manifest_in(self): @@ -625,6 +636,20 @@ def configure(self): call('git', 'push', '--set-upstream', 'origin', self.branch_name) print() + print('If you are an admin and are logged in via `gh auth login`') + print('update branch protection rules? (y/N)?', end=' ') + if input().lower() == 'y': + remote_url = call( + 'git', 'config', '--get', 'remote.origin.url', + capture_output=True).stdout.strip() + package_name = remote_url.rsplit('/')[-1].removesuffix('.git') + success = set_branch_protection( + package_name, self.path / '.meta.toml') + if success: + print('Successfully updated branch protection rules.') + else: + abort(-1) + print() print('If everything went fine up to here:') if updating: print('Updated the previously created PR.') diff --git a/config/default/tests.yml.j2 b/config/default/tests.yml.j2 index 92dd6a6..c80a4c5 100644 --- a/config/default/tests.yml.j2 +++ b/config/default/tests.yml.j2 @@ -26,12 +26,12 @@ jobs: fail-fast: false matrix: os: - - ["ubuntu", "ubuntu-20.04"] + - ["ubuntu", "ubuntu-latest"] {% if with_windows %} - ["windows", "windows-latest"] {% endif %} {% if with_macos %} - - ["macos", "macos-11"] + - ["macos", "macos-latest"] {% endif %} config: # [Python version, tox env] @@ -70,18 +70,18 @@ jobs: - { os: ["windows", "windows-latest"], config: ["3.9", "coverage"] } {% endif %} {% if with_macos %} - - { os: ["macos", "macos-11"], config: ["3.9", "release-check"] } - - { os: ["macos", "macos-11"], config: ["3.9", "lint"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "release-check"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "lint"] } {% if with_docs %} - - { os: ["macos", "macos-11"], config: ["3.9", "docs"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "docs"] } {% endif %} - - { os: ["macos", "macos-11"], config: ["3.9", "coverage"] } + - { os: ["macos", "macos-latest"], config: ["3.9", "coverage"] } # macOS/Python 3.11+ is set up for universal2 architecture # which causes build and package selection issues. - - { os: ["macos", "macos-11"], config: ["3.11", "py311"] } - - { os: ["macos", "macos-11"], config: ["3.12", "py312"] } + - { os: ["macos", "macos-latest"], config: ["3.11", "py311"] } + - { os: ["macos", "macos-latest"], config: ["3.12", "py312"] } {% if with_future_python %} - - { os: ["macos", "macos-11"], config: ["%(future_python_version)s", "py313"] } + - { os: ["macos", "macos-latest"], config: ["%(future_python_version)s", "py313"] } {% endif %} {% endif %} {% for line in gha_additional_exclude %} diff --git a/config/re-enable-actions.py b/config/re-enable-actions.py index 1fe0041..fadc763 100644 --- a/config/re-enable-actions.py +++ b/config/re-enable-actions.py @@ -12,16 +12,14 @@ # ############################################################################## from shared.call import call -from shared.packages import list_packages +from shared.packages import ALL_REPOS +from shared.packages import ORG import argparse -import itertools import pathlib -org = 'zopefoundation' -base_url = f'https://github.com/{org}' -base_path = pathlib.Path(__file__).parent -types = ['buildout-recipe', 'c-code', 'pure-python', 'zope-product', 'toolkit'] +base_url = f'https://github.com/{ORG}' +BASE_PATH = pathlib.Path(__file__).parent parser = argparse.ArgumentParser( @@ -33,9 +31,6 @@ action='store_true') args = parser.parse_args() -repos = itertools.chain( - *[list_packages(base_path / type / 'packages.txt') - for type in types]) def run_workflow(base_url, org, repo): @@ -50,18 +45,18 @@ def run_workflow(base_url, org, repo): return True -for repo in repos: +for repo in ALL_REPOS: print(repo) wfs = call( - 'gh', 'workflow', 'list', '--all', '-R', f'{org}/{repo}', + 'gh', 'workflow', 'list', '--all', '-R', f'{ORG}/{repo}', capture_output=True).stdout test_line = [x for x in wfs.splitlines() if x.startswith('test')][0] if 'disabled_inactivity' not in test_line: print(' ☑️ already enabled') if args.force_run: - run_workflow(base_url, org, repo) + run_workflow(base_url, ORG, repo) continue test_id = test_line.split()[-1] - call('gh', 'workflow', 'enable', test_id, '-R', f'{org}/{repo}') - if run_workflow(base_url, org, repo): + call('gh', 'workflow', 'enable', test_id, '-R', f'{ORG}/{repo}') + if run_workflow(base_url, ORG, repo): print(' ✅ enabled') diff --git a/config/requirements.txt b/config/requirements.txt index b0ec9b3..729bc69 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -3,4 +3,5 @@ Jinja2==3.1.3 pyupgrade==3.3.2 tomlkit==0.12.1 tox==4.8.0 +requests==2.31.0 zest.releaser==8.0.0 diff --git a/config/set_branch_protection_rules.py b/config/set_branch_protection_rules.py new file mode 100644 index 0000000..2de7025 --- /dev/null +++ b/config/set_branch_protection_rules.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +from shared.call import abort +from shared.call import call +from shared.packages import ALL_REPOS +from shared.packages import MANYLINUX_AARCH64 +from shared.packages import MANYLINUX_I686 +from shared.packages import MANYLINUX_PYTHON_VERSION +from shared.packages import MANYLINUX_X86_64 +from shared.packages import NEWEST_PYTHON_VERSION +from shared.packages import OLDEST_PYTHON_VERSION +from shared.packages import ORG +from shared.packages import PYPY_VERSION +import argparse +import json +import os +import pathlib +import requests +import tempfile +import tomllib + + +BASE_URL = f'https://raw.githubusercontent.com/{ORG}' +OLDEST_PYTHON = f'py{OLDEST_PYTHON_VERSION.replace(".", "")}' +NEWEST_PYTHON = f'py{NEWEST_PYTHON_VERSION.replace(".", "")}' +DEFAULT_BRANCH = 'master' + + +def _call_gh( + method, path, repo, *args, capture_output=True, + allowed_return_codes=(0, )): + """Call the gh api command.""" + return call( + 'gh', 'api', + '--method', method, + '-H', 'Accept: application/vnd.github+json', + '-H', 'X-GitHub-Api-Version: 2022-11-28', + f'/repos/{ORG}/{repo}/branches/{DEFAULT_BRANCH}/{path}', + *args, capture_output=capture_output, + allowed_return_codes=allowed_return_codes) + + +def set_branch_protection(repo: str, meta_path: pathlib.Path | None) -> bool: + result = _call_gh( + 'GET', 'protection/required_pull_request_reviews', repo, + allowed_return_codes=(0, 1)) + required_pull_request_reviews = None + if result.returncode == 1: + if json.loads(result.stdout)['message'] != "Branch not protected": + # If there is no branch protection we create it later on using the + # PUT call, but if there is another error we show it: + print(result.stdout) + abort(result.returncode) + else: + required_approving_review_count = json.loads( + result.stdout)['required_approving_review_count'] + required_pull_request_reviews = { + 'required_approving_review_count': required_approving_review_count + } + + if meta_path is None: + response = requests.get( + f'{BASE_URL}/{repo}/{DEFAULT_BRANCH}/.meta.toml', timeout=30) + meta_toml = tomllib.loads(response.text) + else: + with open(meta_path) as f: + meta_toml = tomllib.loads(f.read()) + template = meta_toml['meta']['template'] + with_docs = meta_toml['python'].get('with-docs', False) + with_pypy = meta_toml['python']['with-pypy'] + with_windows = meta_toml['python']['with-windows'] + if template == 'c-code': + required = [ + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_AARCH64})', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_I686})', + f'manylinux ({MANYLINUX_PYTHON_VERSION}, {MANYLINUX_X86_64})', + f'lint ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)', + f'test ({OLDEST_PYTHON_VERSION}, macos-12)', + f'test ({NEWEST_PYTHON_VERSION}, macos-latest)', + f'test ({OLDEST_PYTHON_VERSION}, ubuntu-latest)', + f'test ({NEWEST_PYTHON_VERSION}, ubuntu-latest)', + 'coveralls_finish', + ] + if with_docs: + required.append( + f'docs ({MANYLINUX_PYTHON_VERSION}, ubuntu-latest)') + if with_pypy: + required.append(f'test (pypy-{PYPY_VERSION}, ubuntu-latest)') + required.append(f'test (pypy-{PYPY_VERSION}, windows-latest)') + if with_windows: + required.extend([ + f'test ({OLDEST_PYTHON_VERSION}, windows-latest)', + f'test ({NEWEST_PYTHON_VERSION}, windows-latest)', + ]) + elif with_windows: + required = [ + 'coverage/coveralls', + 'ubuntu-lint', + 'ubuntu-coverage', + f'ubuntu-{OLDEST_PYTHON}', + f'ubuntu-{NEWEST_PYTHON}', + f'windows-{OLDEST_PYTHON}', + f'windows-{NEWEST_PYTHON}', + ] + if with_pypy: + required.extend([ + 'ubuntu-pypy3', + 'windows-pypy3', + ]) + if with_docs: + required.append('ubuntu-docs') + else: # default for most packages + required = ['coverage', 'lint', OLDEST_PYTHON, NEWEST_PYTHON] + if with_docs: + required.append('docs') + if with_pypy: + required.append('pypy3') + + data = { + 'allow_deletions': False, + 'allow_force_pushes': False, + 'allow_fork_syncing': True, + 'lock_branch': False, + 'enforce_admins': None, + 'restrictions': None, + 'required_conversation_resolution': True, + 'required_linear_history': False, + 'required_pull_request_reviews': required_pull_request_reviews, + 'required_status_checks': {'contexts': required, 'strict': False} + } + fd, filename = tempfile.mkstemp('config.json', 'meta', text=True) + try: + file = os.fdopen(fd, 'w') + json.dump(data, file) + file.close() + _call_gh('PUT', 'protection', repo, '--input', filename) + finally: + os.unlink(filename) + return True + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Set the branch protection rules for all known packages.\n' + 'Prerequsites: `gh auth login`.') + parser.add_argument( + '--I-am-authenticated', + help='If you are authenticated via `gh auth login`, use this required' + ' parameter.', + action='store_true', + required=True) + parser.add_argument( + '-r', '--repos', + help='Run the script only for the given repos instead of all.', + metavar='NAME', nargs='*', default=[]) + parser.add_argument( + '-m', '--meta', + help='Use this .meta.toml instead the one on `master` of the repos.', + metavar='PATH', default=None, type=pathlib.Path) + + args = parser.parse_args() + repos = args.repos if args.repos else ALL_REPOS + meta_path = args.meta + + if meta_path and len(repos) > 1: + print('--meta can only be used together with a single repos.') + abort(-1) + + for repo in repos: + print(repo, end="") + set_branch_protection(repo, meta_path) + print(' ✅') diff --git a/config/shared/call.py b/config/shared/call.py index fd16b5c..be5f6df 100644 --- a/config/shared/call.py +++ b/config/shared/call.py @@ -12,6 +12,7 @@ ############################################################################## import subprocess import sys +import textwrap def abort(exitcode): @@ -36,5 +37,12 @@ def call(*args, capture_output=False, cwd=None, allowed_return_codes=(0, )): result = subprocess.run( args, capture_output=capture_output, text=True, cwd=cwd) if result.returncode not in allowed_return_codes: - abort(result.returncode) + if capture_output: + abort_text = textwrap.dedent(f''' + error code: {result.returncode} + stderr: {result.stderr} + stdout: {result.stdout}''') + else: + abort_text = result.returncode + abort(abort_text) return result diff --git a/config/shared/packages.py b/config/shared/packages.py index 5dee802..4a3d992 100644 --- a/config/shared/packages.py +++ b/config/shared/packages.py @@ -10,9 +10,23 @@ # FOR A PARTICULAR PURPOSE. # ############################################################################## +import itertools import pathlib +TYPES = ['buildout-recipe', 'c-code', 'pure-python', 'zope-product', 'toolkit'] +ORG = 'zopefoundation' +BASE_PATH = pathlib.Path(__file__).parent.parent +OLDEST_PYTHON_VERSION = '3.7' +NEWEST_PYTHON_VERSION = '3.12' +FUTURE_PYTHON_VERSION = "3.13.0-alpha - 3.13.0" +PYPY_VERSION = '3.10' +MANYLINUX_PYTHON_VERSION = '3.9' +MANYLINUX_AARCH64 = 'manylinux2014_aarch64' +MANYLINUX_I686 = 'manylinux2014_i686' +MANYLINUX_X86_64 = 'manylinux2014_x86_64' + + def list_packages(path: pathlib.Path) -> list: """List the packages in ``path``. @@ -23,3 +37,8 @@ def list_packages(path: pathlib.Path) -> list: for p in path.read_text().split('\n') if p and not p.startswith('#') ] + + +ALL_REPOS = itertools.chain( + *[list_packages(BASE_PATH / type / 'packages.txt') + for type in TYPES])