Make the Bazel CI workflow use the local Python environment.

This is setting up to run mypy as per #496. Change-Id: I48e67387313ac01d493086be8481b34f26bafe82
google · Jun 7, 2024 · d3e7f57 · d3e7f57
1 parent e38a588
commit d3e7f57
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 95 deletions.
diff --git a/.github/bazel.sh b/.github/bazel.sh
@@ -8,10 +8,14 @@ export MSYS2_ARG_CONV_EXCL='*'
 for compilation_mode in dbg opt
 do
   bazel clean
-  bazel build --compilation_mode=${compilation_mode} -- \
+  bazel build \
+    --extra_toolchains=//python/toolchains:all \
+    --compilation_mode=${compilation_mode} -- \
     //:re2 \
     //python:re2
-  bazel test  --compilation_mode=${compilation_mode} -- \
+  bazel test \
+    --extra_toolchains=//python/toolchains:all \
+    --compilation_mode=${compilation_mode} -- \
     //:all \
     -//:dfa_test \
     -//:exhaustive1_test \

diff --git a/.github/workflows/ci-bazel.yml b/.github/workflows/ci-bazel.yml
@@ -18,5 +18,15 @@ jobs:
       - uses: bazel-contrib/[email protected]
         with:
           bazelisk-version: '1.x'
+      - uses: actions/[email protected]
+        with:
+          python-version: '3.x'
+      - name: Prepare Python 3.x environment
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade mypy
+          python python/toolchains/generate.py
+        shell: bash
       - run: .github/bazel.sh
         shell: bash
+      # TODO(junyer): Run mypy as per https://github.com/google/re2/issues/496.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -46,8 +46,9 @@ jobs:
       - name: Prepare Python ${{ matrix.ver }} environment
         run: |
           "${PYTHON}" -m pip install --upgrade pip
-          "${PYTHON}" -m pip install --upgrade build wheel auditwheel
+          "${PYTHON}" -m pip install --upgrade setuptools build wheel auditwheel
           "${PYTHON}" -m pip install --upgrade absl-py
+          "${PYTHON}" python/toolchains/generate.py
         shell: bash
       - name: Build wheel
         env:
@@ -105,8 +106,9 @@ jobs:
       - name: Prepare Python ${{ matrix.ver }} environment
         run: |
           python -m pip install --upgrade pip
-          python -m pip install --upgrade build wheel delocate
+          python -m pip install --upgrade setuptools build wheel delocate
           python -m pip install --upgrade absl-py
+          python python/toolchains/generate.py
         shell: bash
       - name: Build wheel
         env:
@@ -162,8 +164,9 @@ jobs:
       - name: Prepare Python ${{ matrix.ver }} environment
         run: |
           python -m pip install --upgrade pip
-          python -m pip install --upgrade build wheel delvewheel
+          python -m pip install --upgrade setuptools build wheel delvewheel
           python -m pip install --upgrade absl-py
+          python python/toolchains/generate.py
         shell: bash
       - name: Build wheel
         env:
@@ -205,7 +208,7 @@ jobs:
       - name: Prepare Python 3.x environment
         run: |
           python -m pip install --upgrade pip
-          python -m pip install --upgrade build wheel
+          python -m pip install --upgrade setuptools build wheel
         shell: bash
       - if: inputs.build == 1 || inputs.force-sdist == true
         name: Build source

diff --git a/app/build.sh b/app/build.sh
@@ -7,9 +7,9 @@ DSTDIR=$(mktemp --directory --tmpdir $(basename $0).XXXXXXXXXX)
 cd ${SRCDIR}
 # Emscripten doesn't support `-fstack-protector`.
 AR=emar CC=emcc \
-  bazel build --compilation_mode=opt \
+  bazel build \
   --copt=-fno-stack-protector \
-  -- :all
+  --compilation_mode=opt -- :all
 cp ../bazel-bin/app/_re2.js ${DSTDIR}
 bazel clean --expunge
 cp app.ts index.html _re2.d.ts ${DSTDIR}

diff --git a/python/setup.py b/python/setup.py
@@ -6,8 +6,6 @@
 import setuptools
 import setuptools.command.build_ext
 import shutil
-import sys
-import sysconfig
 
 long_description = r"""A drop-in replacement for the re module.
 
@@ -67,10 +65,7 @@ def build_extension(self, ext):
     except KeyError:
       pass
     # Register the local Python toolchains with highest priority.
-    self.generate_python_toolchains()
     cmd.append('--extra_toolchains=//python/toolchains:all')
-    # Print debug information during toolchain resolution.
-    cmd.append('--toolchain_resolution_debug=.*')
     cmd += ['--compilation_mode=opt', '--', ':all']
     self.spawn(cmd)
 
@@ -82,88 +77,6 @@ def build_extension(self, ext):
     cmd = ['bazel', 'clean', '--expunge']
     self.spawn(cmd)
 
-  def generate_python_toolchains(self):
-    include = sysconfig.get_path('include')
-    libs = os.path.join(include, '../libs')
-
-    os.makedirs('toolchains')
-    shutil.copytree(include, 'toolchains/include')
-    try:
-      shutil.copytree(libs, 'toolchains/libs')
-    except FileNotFoundError:
-      # We must not be running on Windows. :)
-      pass
-
-    with open('toolchains/BUILD.bazel', 'x') as file:
-      file.write(
-          """\
-load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain")
-load("@rules_python//python:py_runtime.bzl", "py_runtime")
-load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair")
-
-package(default_visibility = ["//visibility:public"])
-
-toolchain(
-    name = "py",
-    toolchain = ":py_toolchain",
-    toolchain_type = "@rules_python//python:toolchain_type",
-)
-
-py_runtime_pair(
-    name = "py_toolchain",
-    py3_runtime = ":interpreter",
-)
-
-py_runtime(
-    name = "interpreter",
-    interpreter_path = "{interpreter_path}",
-    interpreter_version_info = {{
-        "major": "{major}",
-        "minor": "{minor}",
-    }},
-    python_version = "PY3",
-)
-
-toolchain(
-    name = "py_cc",
-    toolchain = ":py_cc_toolchain",
-    toolchain_type = "@rules_python//python/cc:toolchain_type",
-)
-
-py_cc_toolchain(
-    name = "py_cc_toolchain",
-    headers = ":headers",
-    libs = ":libraries",
-    python_version = "{major}.{minor}",
-)
-
-cc_library(
-    name = "headers",
-    hdrs = glob(["include/**/*.h"]),
-    includes = ["include"],
-    deps = select({{
-        "@platforms//os:windows": [":interface_library"],
-        "//conditions:default": [],
-    }}),
-)
-
-cc_import(
-    name = "interface_library",
-    interface_library = select({{
-        "@platforms//os:windows": "libs/python{major}{minor}.lib",
-        "//conditions:default": None,
-    }}),
-    system_provided = True,
-)
-
-# Not actually necessary for our purposes. :)
-cc_library(
-    name = "libraries",
-)
-""".format(interpreter_path=sys.executable.replace('\\', '/'),
-           major=sys.version_info.major,
-           minor=sys.version_info.minor))
-
 
 def options():
   bdist_wheel = {}

diff --git a/python/toolchains/generate.py b/python/toolchains/generate.py
@@ -0,0 +1,98 @@
+# Copyright 2019 The RE2 Authors.  All Rights Reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+import os
+import shutil
+import sys
+import sysconfig
+
+
+def generate():
+  include = sysconfig.get_path('include')
+  libs = os.path.join(include, '../libs')
+
+  mydir = os.path.dirname(sys.argv[0]) or '.'
+  shutil.copytree(include, f'{mydir}/include')
+  try:
+    shutil.copytree(libs, f'{mydir}/libs')
+  except FileNotFoundError:
+    # We must not be running on Windows. :)
+    pass
+
+  with open(f'{mydir}/BUILD.bazel', 'x') as file:
+    file.write(
+        """\
+load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain")
+load("@rules_python//python:py_runtime.bzl", "py_runtime")
+load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair")
+
+package(default_visibility = ["//visibility:public"])
+
+toolchain(
+    name = "py",
+    toolchain = ":py_toolchain",
+    toolchain_type = "@rules_python//python:toolchain_type",
+)
+
+py_runtime_pair(
+    name = "py_toolchain",
+    py3_runtime = ":interpreter",
+)
+
+py_runtime(
+    name = "interpreter",
+    interpreter_path = "{interpreter_path}",
+    interpreter_version_info = {{
+        "major": "{major}",
+        "minor": "{minor}",
+    }},
+    python_version = "PY3",
+)
+
+toolchain(
+    name = "py_cc",
+    toolchain = ":py_cc_toolchain",
+    toolchain_type = "@rules_python//python/cc:toolchain_type",
+)
+
+py_cc_toolchain(
+    name = "py_cc_toolchain",
+    headers = ":headers",
+    libs = ":libraries",
+    python_version = "{major}.{minor}",
+)
+
+cc_library(
+    name = "headers",
+    hdrs = glob(["include/**/*.h"]),
+    includes = ["include"],
+    deps = select({{
+        "@platforms//os:windows": [":interface_library"],
+        "//conditions:default": [],
+    }}),
+)
+
+cc_import(
+    name = "interface_library",
+    interface_library = select({{
+        "@platforms//os:windows": "libs/python{major}{minor}.lib",
+        "//conditions:default": None,
+    }}),
+    system_provided = True,
+)
+
+# Not actually necessary for our purposes. :)
+cc_library(
+    name = "libraries",
+)
+""".format(
+            interpreter_path=sys.executable.replace('\\', '/'),
+            major=sys.version_info.major,
+            minor=sys.version_info.minor,
+        )
+    )
+
+
+if __name__ == '__main__':
+  generate()