From e29afa1844861971a4cbf5fe559069b42161e97a Mon Sep 17 00:00:00 2001 From: Paul Wankadia Date: Fri, 7 Jun 2024 20:37:50 +0000 Subject: [PATCH] Make the Bazel CI workflow use the local Python environment. This is setting up to run mypy as per #496. Change-Id: I48e67387313ac01d493086be8481b34f26bafe82 Reviewed-on: https://code-review.googlesource.com/c/re2/+/63230 Reviewed-by: Alex Chernyakhovsky Reviewed-by: Paul Wankadia --- .github/bazel.sh | 8 ++- .github/workflows/ci-bazel.yml | 10 ++++ .github/workflows/python.yml | 11 ++-- app/build.sh | 4 +- python/setup.py | 87 ------------------------------ python/toolchains/generate.py | 98 ++++++++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 95 deletions(-) create mode 100644 python/toolchains/generate.py diff --git a/.github/bazel.sh b/.github/bazel.sh index 1fe309fab..f9f925cd6 100755 --- a/.github/bazel.sh +++ b/.github/bazel.sh @@ -8,10 +8,14 @@ export MSYS2_ARG_CONV_EXCL='*' for compilation_mode in dbg opt do bazel clean - bazel build --compilation_mode=${compilation_mode} -- \ + bazel build \ + --extra_toolchains=//python/toolchains:all \ + --compilation_mode=${compilation_mode} -- \ //:re2 \ //python:re2 - bazel test --compilation_mode=${compilation_mode} -- \ + bazel test \ + --extra_toolchains=//python/toolchains:all \ + --compilation_mode=${compilation_mode} -- \ //:all \ -//:dfa_test \ -//:exhaustive1_test \ diff --git a/.github/workflows/ci-bazel.yml b/.github/workflows/ci-bazel.yml index b0279bb29..a9b9fc2c3 100644 --- a/.github/workflows/ci-bazel.yml +++ b/.github/workflows/ci-bazel.yml @@ -18,5 +18,15 @@ jobs: - uses: bazel-contrib/setup-bazel@0.8.4 with: bazelisk-version: '1.x' + - uses: actions/setup-python@v5.1.0 + with: + python-version: '3.x' + - name: Prepare Python 3.x environment + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade mypy + python python/toolchains/generate.py + shell: bash - run: .github/bazel.sh shell: bash + # TODO(junyer): Run mypy as per https://github.com/google/re2/issues/496. diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index cd05bcf8a..3c731128d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -46,8 +46,9 @@ jobs: - name: Prepare Python ${{ matrix.ver }} environment run: | "${PYTHON}" -m pip install --upgrade pip - "${PYTHON}" -m pip install --upgrade build wheel auditwheel + "${PYTHON}" -m pip install --upgrade setuptools build wheel auditwheel "${PYTHON}" -m pip install --upgrade absl-py + "${PYTHON}" python/toolchains/generate.py shell: bash - name: Build wheel env: @@ -105,8 +106,9 @@ jobs: - name: Prepare Python ${{ matrix.ver }} environment run: | python -m pip install --upgrade pip - python -m pip install --upgrade build wheel delocate + python -m pip install --upgrade setuptools build wheel delocate python -m pip install --upgrade absl-py + python python/toolchains/generate.py shell: bash - name: Build wheel env: @@ -162,8 +164,9 @@ jobs: - name: Prepare Python ${{ matrix.ver }} environment run: | python -m pip install --upgrade pip - python -m pip install --upgrade build wheel delvewheel + python -m pip install --upgrade setuptools build wheel delvewheel python -m pip install --upgrade absl-py + python python/toolchains/generate.py shell: bash - name: Build wheel env: @@ -205,7 +208,7 @@ jobs: - name: Prepare Python 3.x environment run: | python -m pip install --upgrade pip - python -m pip install --upgrade build wheel + python -m pip install --upgrade setuptools build wheel shell: bash - if: inputs.build == 1 || inputs.force-sdist == true name: Build source diff --git a/app/build.sh b/app/build.sh index a46374446..c8a2ae50e 100755 --- a/app/build.sh +++ b/app/build.sh @@ -7,9 +7,9 @@ DSTDIR=$(mktemp --directory --tmpdir $(basename $0).XXXXXXXXXX) cd ${SRCDIR} # Emscripten doesn't support `-fstack-protector`. AR=emar CC=emcc \ - bazel build --compilation_mode=opt \ + bazel build \ --copt=-fno-stack-protector \ - -- :all + --compilation_mode=opt -- :all cp ../bazel-bin/app/_re2.js ${DSTDIR} bazel clean --expunge cp app.ts index.html _re2.d.ts ${DSTDIR} diff --git a/python/setup.py b/python/setup.py index 7548c6ef6..c65aefa1d 100644 --- a/python/setup.py +++ b/python/setup.py @@ -6,8 +6,6 @@ import setuptools import setuptools.command.build_ext import shutil -import sys -import sysconfig long_description = r"""A drop-in replacement for the re module. @@ -67,10 +65,7 @@ def build_extension(self, ext): except KeyError: pass # Register the local Python toolchains with highest priority. - self.generate_python_toolchains() cmd.append('--extra_toolchains=//python/toolchains:all') - # Print debug information during toolchain resolution. - cmd.append('--toolchain_resolution_debug=.*') cmd += ['--compilation_mode=opt', '--', ':all'] self.spawn(cmd) @@ -82,88 +77,6 @@ def build_extension(self, ext): cmd = ['bazel', 'clean', '--expunge'] self.spawn(cmd) - def generate_python_toolchains(self): - include = sysconfig.get_path('include') - libs = os.path.join(include, '../libs') - - os.makedirs('toolchains') - shutil.copytree(include, 'toolchains/include') - try: - shutil.copytree(libs, 'toolchains/libs') - except FileNotFoundError: - # We must not be running on Windows. :) - pass - - with open('toolchains/BUILD.bazel', 'x') as file: - file.write( - """\ -load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") -load("@rules_python//python:py_runtime.bzl", "py_runtime") -load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") - -package(default_visibility = ["//visibility:public"]) - -toolchain( - name = "py", - toolchain = ":py_toolchain", - toolchain_type = "@rules_python//python:toolchain_type", -) - -py_runtime_pair( - name = "py_toolchain", - py3_runtime = ":interpreter", -) - -py_runtime( - name = "interpreter", - interpreter_path = "{interpreter_path}", - interpreter_version_info = {{ - "major": "{major}", - "minor": "{minor}", - }}, - python_version = "PY3", -) - -toolchain( - name = "py_cc", - toolchain = ":py_cc_toolchain", - toolchain_type = "@rules_python//python/cc:toolchain_type", -) - -py_cc_toolchain( - name = "py_cc_toolchain", - headers = ":headers", - libs = ":libraries", - python_version = "{major}.{minor}", -) - -cc_library( - name = "headers", - hdrs = glob(["include/**/*.h"]), - includes = ["include"], - deps = select({{ - "@platforms//os:windows": [":interface_library"], - "//conditions:default": [], - }}), -) - -cc_import( - name = "interface_library", - interface_library = select({{ - "@platforms//os:windows": "libs/python{major}{minor}.lib", - "//conditions:default": None, - }}), - system_provided = True, -) - -# Not actually necessary for our purposes. :) -cc_library( - name = "libraries", -) -""".format(interpreter_path=sys.executable.replace('\\', '/'), - major=sys.version_info.major, - minor=sys.version_info.minor)) - def options(): bdist_wheel = {} diff --git a/python/toolchains/generate.py b/python/toolchains/generate.py new file mode 100644 index 000000000..84fef7800 --- /dev/null +++ b/python/toolchains/generate.py @@ -0,0 +1,98 @@ +# Copyright 2019 The RE2 Authors. All Rights Reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import os +import shutil +import sys +import sysconfig + + +def generate(): + include = sysconfig.get_path('include') + libs = os.path.join(include, '../libs') + + mydir = os.path.dirname(sys.argv[0]) or '.' + shutil.copytree(include, f'{mydir}/include') + try: + shutil.copytree(libs, f'{mydir}/libs') + except FileNotFoundError: + # We must not be running on Windows. :) + pass + + with open(f'{mydir}/BUILD.bazel', 'x') as file: + file.write( + """\ +load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") +load("@rules_python//python:py_runtime.bzl", "py_runtime") +load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") + +package(default_visibility = ["//visibility:public"]) + +toolchain( + name = "py", + toolchain = ":py_toolchain", + toolchain_type = "@rules_python//python:toolchain_type", +) + +py_runtime_pair( + name = "py_toolchain", + py3_runtime = ":interpreter", +) + +py_runtime( + name = "interpreter", + interpreter_path = "{interpreter_path}", + interpreter_version_info = {{ + "major": "{major}", + "minor": "{minor}", + }}, + python_version = "PY3", +) + +toolchain( + name = "py_cc", + toolchain = ":py_cc_toolchain", + toolchain_type = "@rules_python//python/cc:toolchain_type", +) + +py_cc_toolchain( + name = "py_cc_toolchain", + headers = ":headers", + libs = ":libraries", + python_version = "{major}.{minor}", +) + +cc_library( + name = "headers", + hdrs = glob(["include/**/*.h"]), + includes = ["include"], + deps = select({{ + "@platforms//os:windows": [":interface_library"], + "//conditions:default": [], + }}), +) + +cc_import( + name = "interface_library", + interface_library = select({{ + "@platforms//os:windows": "libs/python{major}{minor}.lib", + "//conditions:default": None, + }}), + system_provided = True, +) + +# Not actually necessary for our purposes. :) +cc_library( + name = "libraries", +) +""".format( + interpreter_path=sys.executable.replace('\\', '/'), + major=sys.version_info.major, + minor=sys.version_info.minor, + ) + ) + + +if __name__ == '__main__': + generate()