From f99dbe70ac7a84693cffbd24d576a41e1e1fba30 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Thu, 29 Aug 2024 23:16:14 -0400 Subject: [PATCH 1/3] Make pip packaging work --- .git_archival.txt | 3 + .gitattributes | 3 + .github/workflows/pip.yml | 349 ++++++++++-------- MANIFEST.in | 6 - cmake/HalideGeneratorHelpers.cmake | 16 +- cmake/TargetExportScript.cmake | 5 +- packaging/CMakeLists.txt | 74 +++- packaging/common/HalideConfig.cmake | 16 +- packaging/common/HalideHelpersConfig.cmake | 3 + packaging/pip/CMakeLists.txt | 48 +++ packaging/pip/TrampolineConfig.cmake.in | 6 + pyproject.toml | 100 ++++- python_bindings/CMakeLists.txt | 54 +-- python_bindings/apps/CMakeLists.txt | 14 +- python_bindings/cmake/AddPythonTest.cmake | 27 ++ python_bindings/packaging/CMakeLists.txt | 81 ++++ .../packaging/Halide_PythonConfig.cmake.in | 9 + python_bindings/src/halide/CMakeLists.txt | 201 ++++------ python_bindings/src/halide/__init__.py | 10 + python_bindings/stub/CMakeLists.txt | 4 - python_bindings/tutorial/CMakeLists.txt | 19 +- requirements.txt | 9 +- setup.py | 34 -- src/CMakeLists.txt | 16 +- test/autoschedulers/li2018/CMakeLists.txt | 6 +- 25 files changed, 670 insertions(+), 443 deletions(-) create mode 100644 .git_archival.txt delete mode 100644 MANIFEST.in create mode 100644 packaging/pip/CMakeLists.txt create mode 100644 packaging/pip/TrampolineConfig.cmake.in create mode 100644 python_bindings/cmake/AddPythonTest.cmake create mode 100644 python_bindings/packaging/CMakeLists.txt create mode 100644 python_bindings/packaging/Halide_PythonConfig.cmake.in delete mode 100644 setup.py diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 000000000000..7c5100942aae --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,3 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ diff --git a/.gitattributes b/.gitattributes index 82d52fc38c9e..9841775c5e3b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,3 +11,6 @@ *.png binary *.jpg binary *.tiff binary + +# Freeze version information in git archives +.git_archival.txt export-subst diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index fb2b77da87a9..e45f83ddaaa6 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -6,227 +6,254 @@ name: Build PyPI package on: push: - branches: [ main ] + branches: [ main, build/pip-packaging ] release: types: [ created ] +env: + # TODO: detect this from repo somehow: https://github.com/halide/Halide/issues/8406 + LLVM_VERSION: 18.1.8 + FLATBUFFERS_VERSION: 23.5.26 + WABT_VERSION: 1.0.36 + concurrency: group: '${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true -env: - LLVM_VER: 15.0.7 - permissions: contents: read # to fetch code (actions/checkout) packages: read # to fetch packages (docker) jobs: - # When creating 'dev' (e.g. nightly) PyPI packages, we need to create a unique - # label for each upload. For simplicity, we choose the Unix time-since-epoch in - # UTC form (aka `date +%s`). - pip-labels: - name: Create Label for PyPI Packages - runs-on: ubuntu-latest - outputs: - halide_pypi_label: ${{ steps.make_label.outputs.unix_time_utc }} - steps: - - id: make_label - run: echo "unix_time_utc=$(date +%s)" >> "$GITHUB_OUTPUT" - - pip-linux: - name: Package Halide Python bindings - - runs-on: ubuntu-latest - needs: pip-labels + build-wheels: + name: Build Halide wheels for ${{ matrix.platform_tag }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - arch: [ x86_64, aarch64 ] + include: + - os: ubuntu-latest + platform_tag: manylinux_x86_64 + - os: windows-latest + platform_tag: win_amd64 + - os: macos-13 + platform_tag: macosx_x86_64 + - os: macos-14 + platform_tag: macosx_arm64 + + env: + MACOSX_DEPLOYMENT_TARGET: 11 steps: - - uses: actions/checkout@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v2.1.0 + - name: Login to GitHub Container Registry + if: matrix.os == 'ubuntu-latest' + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v2.1.0 + - uses: actions/checkout@v4 with: - platforms: all + fetch-depth: 0 + fetch-tags: true - - name: Build wheels - uses: pypa/cibuildwheel@v2.11.2 - env: - CIBW_ARCHS_LINUX: "${{ matrix.arch }}" - CIBW_BUILD: "cp38-manylinux* cp39-manylinux* cp310-manylinux*" - CIBW_CONFIG_SETTINGS: "--global-option=egg_info --global-option=-b.dev${{ needs.pip-labels.outputs.halide_pypi_label }} --global-option=--build-number --global-option=${{github.run_id}}" - CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/halide/manylinux2014_x86_64-llvm:${{ env.LLVM_VER }} - # CIBW_MANYLINUX_I686_IMAGE: ghcr.io/halide/manylinux2014_i686-llvm:${{ env.LLVM_VER }} - CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/halide/manylinux2014_aarch64-llvm:${{ env.LLVM_VER }} - CIBW_BEFORE_ALL_LINUX: > - cmake -G Ninja -S . -B build - -DCMAKE_BUILD_TYPE=Release -DWITH_DOCS=NO -DWITH_PYTHON_BINDINGS=NO -DWITH_TESTS=NO - -DWITH_TUTORIALS=NO -DWITH_UTILS=NO -DWITH_PYTHON_STUBS=NO && - cmake --build build --target install - - - uses: actions/upload-artifact@v3 + # See: https://github.com/pypa/setuptools-scm/issues/455 + - name: Suppress git version tag + if: github.event_name == 'push' && github.ref_name == 'main' + run: | + echo 'local_scheme = "no-local-version"' >> pyproject.toml + git update-index --assume-unchanged pyproject.toml + + - uses: ilammy/msvc-dev-cmd@v1 + - uses: lukka/get-cmake@v3.28.4 + + - uses: actions/setup-python@v5 with: - name: wheels - path: ./wheelhouse/*.whl + python-version: 3.8 + if: runner.os == 'macOS' && runner.arch == 'ARM64' - pip-other: - name: Package Halide Python bindings + ######################################################################## + # flatbuffers + ######################################################################## - runs-on: ${{ matrix.runner }} - needs: pip-labels + - name: Cache flatbuffers build folder + if: matrix.os != 'ubuntu-latest' + id: cache-flatbuffers + uses: actions/cache@v4 + with: + path: opt/flatbuffers + key: flatbuffers-${{ env.FLATBUFFERS_VERSION }}-${{ matrix.platform_tag }} - strategy: - fail-fast: false - matrix: - include: - - runner: windows-latest - pytag: win_amd64 - arch: x64 + - uses: actions/checkout@v4 + if: matrix.os != 'ubuntu-latest' && steps.cache-flatbuffers.outputs.cache-hit != 'true' + with: + path: flatbuffers-src + repository: google/flatbuffers + ref: v${{ env.FLATBUFFERS_VERSION }} - - runner: macos-latest - pytag: macosx_universal2 - arch: x86_64;arm64 + - name: Configure flatbuffers + if: matrix.os != 'ubuntu-latest' && steps.cache-flatbuffers.outputs.cache-hit != 'true' + run: > + cmake -G Ninja -S flatbuffers-src -B flatbuffers-build + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/opt/flatbuffers" + "-DFLATBUFFERS_BUILD_TESTS=NO" + + - name: Install flatbuffers + if: matrix.os != 'ubuntu-latest' && steps.cache-flatbuffers.outputs.cache-hit != 'true' + run: | + cmake --build flatbuffers-build --target install + cmake -E rm -rf flatbuffers-src flatbuffers-build + + ######################################################################## + # wabt + ######################################################################## + + - name: Cache wabt build folder + if: matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest' + id: cache-wabt + uses: actions/cache@v4 + with: + path: opt/wabt + key: wabt-${{ env.WABT_VERSION }}-${{ matrix.platform_tag }} - steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + if: > + matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest' + && steps.cache-wabt.outputs.cache-hit != 'true' + with: + submodules: 'true' + path: wabt-src + repository: WebAssembly/wabt + ref: ${{ env.WABT_VERSION }} + + - name: Configure wabt + if: > + matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest' + && steps.cache-wabt.outputs.cache-hit != 'true' + run: > + cmake -G Ninja -S wabt-src -B wabt-build + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/opt/wabt" + "-DWITH_EXCEPTIONS=ON" + "-DBUILD_TESTS=OFF" + "-DBUILD_TOOLS=OFF" + "-DBUILD_LIBWASM=OFF" + "-DUSE_INTERNAL_SHA256=ON" + + - name: Install wabt + if: > + matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest' + && steps.cache-wabt.outputs.cache-hit != 'true' + run: | + cmake --build wabt-build --target install + cmake -E rm -rf wabt-src wabt-build + + ######################################################################## + # LLVM + ######################################################################## - name: Cache LLVM build folder + if: matrix.os != 'ubuntu-latest' id: cache-llvm - uses: actions/cache@v3.0.11 + uses: actions/cache@v4 with: - path: local-llvm - key: llvmorg-${{ env.LLVM_VER }}-${{ runner.os }} - - - uses: ilammy/msvc-dev-cmd@v1 - - uses: lukka/get-cmake@latest + path: opt/llvm + key: llvm-${{ env.LLVM_VERSION }}-${{ matrix.platform_tag }} - - uses: actions/checkout@v3 - if: steps.cache-llvm.outputs.cache-hit != 'true' + - uses: actions/checkout@v4 + if: matrix.os != 'ubuntu-latest' && steps.cache-llvm.outputs.cache-hit != 'true' with: path: llvm-src repository: llvm/llvm-project - ref: llvmorg-${{ env.LLVM_VER }} + ref: llvmorg-${{ env.LLVM_VERSION }} - name: Configure LLVM - if: steps.cache-llvm.outputs.cache-hit != 'true' + if: matrix.os != 'ubuntu-latest' && steps.cache-llvm.outputs.cache-hit != 'true' run: > cmake -G Ninja -S llvm-src/llvm -B llvm-build - -DCMAKE_BUILD_TYPE=Release - "-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64" - "-DLLVM_TARGETS_TO_BUILD=X86;ARM;NVPTX;AArch64;Hexagon;WebAssembly" + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/opt/llvm" + "-DLLVM_ENABLE_ASSERTIONS=ON" + "-DLLVM_ENABLE_BINDINGS=OFF" + "-DLLVM_ENABLE_CURL=OFF" + "-DLLVM_ENABLE_DIA_SDK=OFF" + "-DLLVM_ENABLE_EH=ON" + "-DLLVM_ENABLE_HTTPLIB=OFF" + "-DLLVM_ENABLE_IDE=OFF" + "-DLLVM_ENABLE_LIBEDIT=OFF" + "-DLLVM_ENABLE_LIBXML2=OFF" + "-DLLVM_ENABLE_OCAMLDOC=OFF" "-DLLVM_ENABLE_PROJECTS=clang;lld" - -DLLVM_ENABLE_ASSERTIONS=ON - -DLLVM_ENABLE_RTTI=ON - -DLLVM_ENABLE_EH=ON - -DLLVM_ENABLE_LIBXML2=OFF - -DLLVM_ENABLE_TERMINFO=OFF - -DLLVM_ENABLE_ZSTD=OFF - -DLLVM_ENABLE_ZLIB=OFF - -DLLVM_ENABLE_OCAMLDOC=OFF - -DLLVM_ENABLE_BINDINGS=OFF - -DLLVM_ENABLE_IDE=OFF - - - name: Build LLVM - if: steps.cache-llvm.outputs.cache-hit != 'true' - run: cmake --build llvm-build + "-DLLVM_ENABLE_RTTI=ON" + "-DLLVM_ENABLE_RUNTIMES=compiler-rt" + "-DLLVM_ENABLE_TERMINFO=OFF" + "-DLLVM_ENABLE_WARNINGS=OFF" + "-DLLVM_ENABLE_ZLIB=OFF" + "-DLLVM_ENABLE_ZSTD=OFF" + "-DLLVM_INCLUDE_BENCHMARKS=OFF" + "-DLLVM_INCLUDE_EXAMPLES=OFF" + "-DLLVM_INCLUDE_TESTS=OFF" + "-DLLVM_TARGETS_TO_BUILD=WebAssembly;X86;AArch64;ARM;Hexagon;NVPTX;PowerPC;RISCV" - name: Install LLVM - if: steps.cache-llvm.outputs.cache-hit != 'true' - run: cmake --install llvm-build --prefix local-llvm - - # Remove the LLVM source tree after building it, otherwise we can - # run out of local space while building halide - - name: Clean LLVM Source - if: steps.cache-llvm.outputs.cache-hit != 'true' - shell: bash - run: rm -rf llvm-src - - - name: Configure Halide - if: runner.os == 'Windows' - run: > - cmake -G "Visual Studio 17 2022" -T ClangCL -A "${{ matrix.arch }}" -S . -B halide-build - -DWITH_DOCS=NO - -DWITH_PYTHON_BINDINGS=NO - -DWITH_TESTS=NO - -DWITH_TUTORIALS=NO - -DWITH_UTILS=NO - -DWITH_PYTHON_STUBS=NO - -DLLVM_DIR=${{ github.workspace }}/local-llvm/lib/cmake/llvm - - - name: Configure Halide - if: runner.os != 'Windows' - run: > - cmake -G Ninja -S . -B halide-build - -DCMAKE_BUILD_TYPE=Release - "-DCMAKE_OSX_ARCHITECTURES=${{ matrix.arch }}" - -DWITH_DOCS=NO - -DWITH_PYTHON_BINDINGS=NO - -DWITH_TESTS=NO - -DWITH_TUTORIALS=NO - -DWITH_UTILS=NO - -DWITH_PYTHON_STUBS=NO - -DLLVM_DIR=${{ github.workspace }}/local-llvm/lib/cmake/llvm - - - name: Build Halide - run: cmake --build halide-build --config Release - - - name: Install Halide - run: cmake --install halide-build --config Release --prefix local-halide + if: matrix.os != 'ubuntu-latest' && steps.cache-llvm.outputs.cache-hit != 'true' + run: | + cmake --build llvm-build --target install + cmake -E rm -rf llvm-src llvm-build + + ######################################################################## + # Wheels + ######################################################################## + + #- uses: mxschmitt/action-tmate@v3 - name: Build wheels - uses: pypa/cibuildwheel@v2.10.2 + uses: pypa/cibuildwheel@v2.20.0 env: - CMAKE_PREFIX_PATH: ${{ github.workspace }}/local-halide - CIBW_BUILD: "cp38-${{ matrix.pytag }} cp39-${{ matrix.pytag }} cp310-${{ matrix.pytag }}" - CIBW_CONFIG_SETTINGS: "--global-option=egg_info --global-option=-b.dev${{ needs.pip-labels.outputs.halide_pypi_label }} --global-option=--build-number --global-option=${{github.run_id}}" - CIBW_ARCHS_MACOS: "universal2" - - - uses: actions/upload-artifact@v3 + CIBW_BUILD: "cp3*-${{ matrix.platform_tag }}" + CIBW_SKIP: "cp3{5,6,7}*" + CIBW_ENVIRONMENT_MACOS: > + CMAKE_PREFIX_PATH='${{ github.workspace }}/opt' + Python_ROOT_DIR='' + CIBW_ENVIRONMENT_WINDOWS: > + CMAKE_GENERATOR=Ninja + CMAKE_PREFIX_PATH='${{ github.workspace }}\opt' + CIBW_MANYLINUX_X86_64_IMAGE: "ghcr.io/halide/manylinux_2_28_x86_64-llvm:${{ env.LLVM_VERSION }}" + CIBW_TEST_COMMAND: > + cmake -G Ninja -S {project}/python_bindings/apps -B build -DCMAKE_BUILD_TYPE=Release && + cmake --build build && + ctest --test-dir build --output-on-failure + # Windows puts the Python interpreter in /Scripts, rather than /bin, which CMake doesn't understand. + CIBW_TEST_COMMAND_WINDOWS: > + cmake -G Ninja -S {project}/python_bindings/apps -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=%VIRTUAL_ENV% && + cmake --build build && + ctest --test-dir build --output-on-failure + + - uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-${{ matrix.platform_tag }} path: ./wheelhouse/*.whl - pip-sdist: - name: Make SDist - runs-on: ubuntu-latest - needs: pip-labels - steps: - - uses: actions/checkout@v3 - - run: pipx run build --sdist -C--global-option=egg_info -C--global-option=-b.dev${{ needs.pip-labels.outputs.halide_pypi_label }} - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist/*.tar.gz - publish: name: Publish on PyPI - needs: [ pip-linux, pip-other, pip-sdist ] + needs: build-wheels runs-on: ubuntu-latest + permissions: + id-token: write steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: wheels + pattern: wheels-* + merge-multiple: true path: dist - - uses: pypa/gh-action-pypi-publish@v1.5.1 + - uses: pypa/gh-action-pypi-publish@release/v1 + if: github.event_name == 'push' && github.ref_name == 'main' with: - user: __token__ - password: ${{ secrets.TEST_PYPI_TOKEN }} repository_url: https://test.pypi.org/legacy/ - - uses: pypa/gh-action-pypi-publish@v1.5.1 + - uses: pypa/gh-action-pypi-publish@release/v1 if: github.event_name == 'release' && github.event.action == 'published' - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 2fafbbfc0b1d..000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -graft python_bindings -prune python_bindings/apps -prune python_bindings/test -prune python_bindings/tutorial -prune python_bindings/stub -include README_python.md diff --git a/cmake/HalideGeneratorHelpers.cmake b/cmake/HalideGeneratorHelpers.cmake index d953c952e4b1..dcf802ca162f 100644 --- a/cmake/HalideGeneratorHelpers.cmake +++ b/cmake/HalideGeneratorHelpers.cmake @@ -131,7 +131,7 @@ function(add_halide_generator TARGET) set(stub_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.${GEN_NAME}.${MODULE_NAME}.py_stub_generated.cpp") file(CONFIGURE OUTPUT "${stub_file}" CONTENT "${stub_text}" @ONLY) - Python3_add_library(${TARGET}_pystub MODULE WITH_SOABI "${stub_file}" ${ARG_SOURCES}) + Python_add_library(${TARGET}_pystub MODULE WITH_SOABI "${stub_file}" ${ARG_SOURCES}) set_target_properties(${TARGET}_pystub PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON @@ -398,16 +398,17 @@ function(_Halide_compute_generator_cmd) # TODO: Python Generators need work to support crosscompiling (https://github.com/halide/Halide/issues/7014) if (NOT TARGET Halide::Python) - message(FATAL_ERROR "This version of Halide was built without support for Python bindings; rebuild using WITH_PYTHON_BINDINGS=ON to use this rule with Python Generators.") + message(FATAL_ERROR "Missing Halide::Python. Load the Python component " + "in find_package() or set WITH_PYTHON_BINDINGS=ON if in tree.") endif () - if (NOT TARGET Python3::Interpreter) - message(FATAL_ERROR "You must call find_package(Python3) in your CMake code in order to use this rule with Python Generators.") + if (NOT TARGET Python::Interpreter) + message(FATAL_ERROR "Missing Python::Interpreter. Missing call to find_package(Python 3)?") endif () set("${ARG_OUT_COMMAND}" - ${CMAKE_COMMAND} -E env "PYTHONPATH=$>" -- - ${Halide_PYTHON_LAUNCHER} "$" $ + ${CMAKE_COMMAND} -E env "PYTHONPATH=$/..>" -- + ${Halide_PYTHON_LAUNCHER} "$" $ PARENT_SCOPE) set("${ARG_OUT_DEPENDS}" ${ARG_FROM} Halide::Python ${py_src} PARENT_SCOPE) endfunction() @@ -794,7 +795,7 @@ function(add_halide_python_extension_library TARGET) DEPENDS Halide::GenRT VERBATIM) - Python3_add_library(${TARGET} MODULE WITH_SOABI ${pycpps} ${pyext_module_definition_src}) + Python_add_library(${TARGET} MODULE WITH_SOABI ${pycpps} ${pyext_module_definition_src}) target_link_libraries(${TARGET} PRIVATE ${ARG_HALIDE_LIBRARIES}) target_compile_definitions(${TARGET} PRIVATE # Skip the default module-definition code in each file @@ -802,6 +803,7 @@ function(add_halide_python_extension_library TARGET) # Gotta explicitly specify the module name and function(s) for this mode HALIDE_PYTHON_EXTENSION_MODULE_NAME=${ARG_MODULE_NAME} "HALIDE_PYTHON_EXTENSION_FUNCTIONS=${function_names}") + target_compile_features(${TARGET} PRIVATE cxx_std_17) set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME ${ARG_MODULE_NAME}) _Halide_target_export_single_symbol(${TARGET} "PyInit_${ARG_MODULE_NAME}") endfunction() diff --git a/cmake/TargetExportScript.cmake b/cmake/TargetExportScript.cmake index bc386ba32921..08c1f2d14c48 100644 --- a/cmake/TargetExportScript.cmake +++ b/cmake/TargetExportScript.cmake @@ -42,10 +42,11 @@ function(target_export_script TARGET) endif () ## The Apple linker expects a different flag. - set(EXPORTED_SYMBOLS_FLAG "LINKER:-exported_symbols_list,${ARG_APPLE_LD}") + file(CONFIGURE OUTPUT _target_export_script.apple.ldscript CONTENT [[]]) + set(EXPORTED_SYMBOLS_FLAG "LINKER:-exported_symbols_list,${CMAKE_CURRENT_BINARY_DIR}/_target_export_script.apple.ldscript") check_linker_flag(CXX "${EXPORTED_SYMBOLS_FLAG}" LINKER_HAS_FLAG_EXPORTED_SYMBOLS_LIST) if (LINKER_HAS_FLAG_EXPORTED_SYMBOLS_LIST) - target_link_options(${TARGET} PRIVATE "${EXPORTED_SYMBOLS_FLAG}") + target_link_options(${TARGET} PRIVATE "LINKER:-exported_symbols_list,${ARG_APPLE_LD}") set_property(TARGET ${TARGET} APPEND PROPERTY LINK_DEPENDS "${ARG_APPLE_LD}") return() endif () diff --git a/packaging/CMakeLists.txt b/packaging/CMakeLists.txt index a1cf8b7b7949..ad7c6cd30570 100644 --- a/packaging/CMakeLists.txt +++ b/packaging/CMakeLists.txt @@ -17,6 +17,25 @@ set(Halide_INSTALL_PLUGINDIR "${CMAKE_INSTALL_LIBDIR}" set(Halide_INSTALL_TOOLSDIR "${CMAKE_INSTALL_DATADIR}/tools" CACHE STRING "Path to Halide build-time tools and sources") +## +# RPATH patching helper +## + +function(_Halide_compute_rpath) + cmake_parse_arguments(PARSE_ARGV 0 ARG "" "ORIGIN_DIR;LIB_DIR" "TARGETS") + if (APPLE) + set(rbase @loader_path) + else () + set(rbase $ORIGIN) + endif () + + file(RELATIVE_PATH lib_dir + ${CMAKE_CURRENT_BINARY_DIR}/${ARG_ORIGIN_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/${ARG_LIB_DIR}) + + set_target_properties(${ARG_TARGETS} PROPERTIES INSTALL_RPATH "${rbase};${rbase}/${lib_dir}") +endfunction() + ## # Main library exports ## @@ -30,10 +49,20 @@ install(TARGETS Halide Halide_Generator Halide_GenGen FILE_SET HEADERS COMPONENT Halide_Development) if (WITH_AUTOSCHEDULERS) - install(TARGETS Halide_Adams2019 Halide_Li2018 Halide_Mullapudi2016 Halide_Anderson2021 + set(autoschedulers Halide_Adams2019 Halide_Li2018 Halide_Mullapudi2016 Halide_Anderson2021) + + install(TARGETS ${autoschedulers} EXPORT Halide_Interfaces LIBRARY DESTINATION ${Halide_INSTALL_PLUGINDIR} COMPONENT Halide_Runtime NAMELINK_COMPONENT Halide_Development) + + if (NOT CMAKE_INSTALL_RPATH) + _Halide_compute_rpath( + TARGETS ${autoschedulers} + ORIGIN_DIR "${Halide_INSTALL_PLUGINDIR}" + LIB_DIR "${CMAKE_INSTALL_LIBDIR}" + ) + endif () endif () ## @@ -76,17 +105,11 @@ if (WITH_AUTOSCHEDULERS AND WITH_UTILS) endif () if (NOT CMAKE_INSTALL_RPATH) - file(RELATIVE_PATH lib_dir - ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR} - ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}) - - if (APPLE) - set(rbase @loader_path) - else () - set(rbase $ORIGIN) - endif () - - set_target_properties(${utils} PROPERTIES INSTALL_RPATH "${rbase};${rbase}/${lib_dir}") + _Halide_compute_rpath( + TARGETS ${utils} + ORIGIN_DIR "${CMAKE_INSTALL_BINDIR}" + LIB_DIR "${CMAKE_INSTALL_LIBDIR}" + ) endif () install(TARGETS ${utils} EXPORT Halide_Interfaces COMPONENT Halide_Development) @@ -167,14 +190,19 @@ write_basic_package_version_file(HalideHelpersConfigVersion.cmake COMPATIBILITY SameMajorVersion ARCH_INDEPENDENT) -# Compute a hint to make it easier to find HalideHelpers from find_package(Halide) -# This is read by configure_file below. -file(RELATIVE_PATH HalideHelpers_HINT - "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_CMAKEDIR}" - "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_HELPERSDIR}") +configure_package_config_file( + common/HalideConfig.cmake HalideConfig.cmake + PATH_VARS Halide_INSTALL_HELPERSDIR Halide_Python_INSTALL_CMAKEDIR + INSTALL_DESTINATION "${Halide_INSTALL_CMAKEDIR}" + NO_SET_AND_CHECK_MACRO + NO_CHECK_REQUIRED_COMPONENTS_MACRO +) -configure_file(common/HalideConfig.cmake HalideConfig.cmake @ONLY) -configure_file(common/HalideHelpersConfig.cmake HalideHelpersConfig.cmake @ONLY) +configure_package_config_file( + common/HalideHelpersConfig.cmake HalideHelpersConfig.cmake + INSTALL_DESTINATION "${Halide_INSTALL_HELPERSDIR}" + NO_SET_AND_CHECK_MACRO +) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/HalideConfig.cmake @@ -214,6 +242,14 @@ if (WITH_DOCS) COMPONENT Halide_Documentation) endif () +## +# Pip overrides +## + +if (SKBUILD) + add_subdirectory(pip) +endif () + ## # General packaging variables. ## diff --git a/packaging/common/HalideConfig.cmake b/packaging/common/HalideConfig.cmake index 664479f33a29..383371dd7251 100644 --- a/packaging/common/HalideConfig.cmake +++ b/packaging/common/HalideConfig.cmake @@ -1,4 +1,5 @@ cmake_minimum_required(VERSION 3.28) +@PACKAGE_INIT@ macro(Halide_fail message) set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "${message}") @@ -19,7 +20,7 @@ macro(Halide_find_component_dependency comp dep) endif () endmacro() -set(Halide_known_components Halide PNG JPEG static shared) +set(Halide_known_components Halide Python PNG JPEG static shared) set(Halide_components Halide PNG JPEG) foreach (Halide_comp IN LISTS Halide_known_components) @@ -46,6 +47,7 @@ endif () # Inform downstreams of potential compatibility issues. For instance, exceptions # and RTTI must both be enabled to build Python bindings and ASAN builds should # not be mixed with non-ASAN builds. +set(WITH_AUTOSCHEDULERS "@WITH_AUTOSCHEDULERS@") set(Halide_ENABLE_EXCEPTIONS "@Halide_ENABLE_EXCEPTIONS@") set(Halide_ENABLE_RTTI "@Halide_ENABLE_RTTI@") set(Halide_ASAN_ENABLED "@Halide_ASAN_ENABLED@") @@ -58,7 +60,7 @@ include(CMakeFindDependencyMacro) find_dependency( HalideHelpers "@Halide_VERSION@" EXACT - HINTS "${CMAKE_CURRENT_LIST_DIR}/@HalideHelpers_HINT@" + HINTS "@PACKAGE_Halide_INSTALL_HELPERSDIR@" ) if (Halide_comp_PNG) @@ -110,11 +112,19 @@ else () endif () endif () +## Load Python component +if (Halide_comp_Python OR "@WITH_PYTHON_BINDINGS@") + Halide_find_component_dependency( + Python Halide_Python + HINTS "@PACKAGE_Halide_Python_INSTALL_CMAKEDIR@" + ) +endif () + ## Hide variables and helper macros that are not part of our API. # Delete internal component tracking foreach (comp IN LISTS Halide_known_components) - unset(Halide_comp_${comp}) + unset(Halide_comp_${comp}) endforeach () unset(Halide_components) diff --git a/packaging/common/HalideHelpersConfig.cmake b/packaging/common/HalideHelpersConfig.cmake index aa98ce7847e2..57462d747f68 100644 --- a/packaging/common/HalideHelpersConfig.cmake +++ b/packaging/common/HalideHelpersConfig.cmake @@ -1,4 +1,5 @@ cmake_minimum_required(VERSION 3.28) +@PACKAGE_INIT@ set(Halide_HOST_TARGET @Halide_HOST_TARGET@) @@ -6,3 +7,5 @@ include(${CMAKE_CURRENT_LIST_DIR}/Halide-Interfaces.cmake) include(${CMAKE_CURRENT_LIST_DIR}/HalideTargetHelpers.cmake) include(${CMAKE_CURRENT_LIST_DIR}/HalideGeneratorHelpers.cmake) include(${CMAKE_CURRENT_LIST_DIR}/TargetExportScript.cmake) + +check_required_components(${CMAKE_FIND_PACKAGE_NAME}) diff --git a/packaging/pip/CMakeLists.txt b/packaging/pip/CMakeLists.txt new file mode 100644 index 000000000000..6f12ca6054e2 --- /dev/null +++ b/packaging/pip/CMakeLists.txt @@ -0,0 +1,48 @@ +## +# Create a trampoline to the real *Config.cmake. +# +# The various wheel directories get grafted to one of an unpredictable set +# of paths determined by sysconfig. The trampoline finds platlib via sysconfig +# before jumping to the real *Config.cmake inside our pip package. + +function(configure_trampoline PACKAGE INSTALL_DIR) + configure_file(TrampolineConfig.cmake.in "${PACKAGE}Config.cmake" @ONLY) +endfunction() + +configure_trampoline(Halide "${Halide_INSTALL_CMAKEDIR}") +configure_trampoline(HalideHelpers "${Halide_INSTALL_HELPERSDIR}") + +install( + FILES "${CMAKE_CURRENT_BINARY_DIR}/HalideConfig.cmake" + # It's better to duplicate the version file than to trampoline to it, as + # this would require calling find_package(Python) in the version file. + "${CMAKE_CURRENT_BINARY_DIR}/../HalideConfigVersion.cmake" + # It's okay to hard-code the destination because this code is only + # called by scikit-build-core. Installing to /data ultimately installs + # to the Python root installation directory, whose bin/ directory is + # located on the PATH. On Unix systems, this is directly compatible with + # find_package. Python on Windows unfortunately uses Scripts/ instead of + # bin/, which CMake does not understand. These users can add %VIRTUAL_ENV% + # to their CMAKE_PREFIX_PATH. + DESTINATION "${SKBUILD_DATA_DIR}/share/cmake/Halide" + COMPONENT Halide_Python +) + +# Same thing for HalideHelpers +install( + FILES "${CMAKE_CURRENT_BINARY_DIR}/HalideHelpersConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/../HalideHelpersConfigVersion.cmake" + DESTINATION "${SKBUILD_DATA_DIR}/share/cmake/HalideHelpers" + COMPONENT Halide_Python +) + +## +# Set up RPATH for the Python bindings plugin. + +if (WITH_PYTHON_BINDINGS) + _Halide_compute_rpath( + TARGETS Halide_Python + ORIGIN_DIR "${Halide_INSTALL_PYTHONDIR}" + LIB_DIR "${CMAKE_INSTALL_LIBDIR}" + ) +endif () \ No newline at end of file diff --git a/packaging/pip/TrampolineConfig.cmake.in b/packaging/pip/TrampolineConfig.cmake.in new file mode 100644 index 000000000000..5b05801c2ba3 --- /dev/null +++ b/packaging/pip/TrampolineConfig.cmake.in @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.28) + +include(CMakeFindDependencyMacro) +find_dependency(Python 3) + +include("${Python_SITEARCH}/halide/@INSTALL_DIR@/@PACKAGE@Config.cmake") diff --git a/pyproject.toml b/pyproject.toml index b91c918d1d6e..24b95504da00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,96 @@ [build-system] requires = [ - "setuptools>=43", - "wheel", - "scikit-build", - "pybind11==2.10.4", - "cmake>=3.22", - "ninja; platform_system!='Windows'" + "scikit-build-core==0.10.5", + "pybind11==2.10.4", ] -build-backend = "setuptools.build_meta" +build-backend = "scikit_build_core.build" + +[project] +name = "halide" +authors = [{ name = "The Halide team", email = "halide-dev@lists.csail.mit.edu" }] +description = "Halide is a programming language designed to make it easier to write high-performance image and array processing code." +license = { file = "LICENSE.txt" } +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "numpy", + "imageio", +] +dynamic = ['version'] + +[project.urls] +homepage = "https://halide-lang.org" +documentation = "https://halide-lang.org/docs" +repository = "https://github.com/halide/Halide.git" + +[tool.scikit-build] +cmake.version = ">=3.28" +wheel.install-dir = "halide" +sdist.include = ["dependencies/"] +sdist.exclude = [".github/", "apps/", "test/", "tutorial/"] +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" + +[tool.scikit-build.cmake.define] +CMAKE_DISABLE_FIND_PACKAGE_JPEG = true +CMAKE_DISABLE_FIND_PACKAGE_PNG = true +Halide_ENABLE_EXCEPTIONS = true +Halide_ENABLE_RTTI = true +Halide_INSTALL_PYTHONDIR = "." +Halide_USE_FETCHCONTENT = false +Halide_WASM_BACKEND = "wabt" +WITH_PYTHON_BINDINGS = true +WITH_TESTS = false +WITH_TUTORIALS = false + +## +# Don't version libHalide.so/dylib -- wheels are zip files that do +# not understand symbolic links. Including version information here +# causes the final wheel to have three copies of our library. Not good. +Halide_VERSION_OVERRIDE = "" +Halide_SOVERSION_OVERRIDE = "" + +[[tool.scikit-build.overrides]] +if.platform-system = "^win32" +inherit.cmake.define = "append" +cmake.define.Halide_WASM_BACKEND = "OFF" + +[tool.tbump] +github_url = "https://github.com/halide/Halide/" + +[tool.tbump.version] +current = "19.0.0" +regex = '(?P\d+)\.(?P\d+)\.(?P\d+)' + +[tool.tbump.git] +message_template = "Bump version to {new_version}" +tag_template = "v{new_version}.dev0" + +[[tool.tbump.file]] +src = "CMakeLists.txt" +search = "VERSION {current_version}" + +[[tool.tbump.file]] +src = "python_bindings/CMakeLists.txt" +search = "VERSION {current_version}" + +[[tool.tbump.file]] +src = "vcpkg.json" + +[[tool.tbump.file]] +src = "src/runtime/HalideRuntime.h" +version_template = "{major}" +search = "#define HALIDE_VERSION_MAJOR {current_version}" + +[[tool.tbump.file]] +src = "src/runtime/HalideRuntime.h" +version_template = "{minor}" +search = "#define HALIDE_VERSION_MINOR {current_version}" + +[[tool.tbump.file]] +src = "src/runtime/HalideRuntime.h" +version_template = "{patch}" +search = "#define HALIDE_VERSION_PATCH {current_version}" + +# Must be last table in file since pip.yml appends to it +# See: https://github.com/pypa/setuptools-scm/issues/455 +[tool.setuptools_scm] diff --git a/python_bindings/CMakeLists.txt b/python_bindings/CMakeLists.txt index d3e1422b505d..2d3a7ac77da9 100644 --- a/python_bindings/CMakeLists.txt +++ b/python_bindings/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.28) -project(Halide_Python) +project(Halide_Python VERSION 19.0.0) if (PROJECT_IS_TOP_LEVEL) enable_testing() @@ -16,17 +16,21 @@ set(CMAKE_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard to use") option(CMAKE_CXX_STANDARD_REQUIRED "Prevent CMake C++ standard selection decay" ON) option(CMAKE_CXX_EXTENSIONS "Enable C++ vendor extensions (e.g. GNU)" OFF) +# Duplicated options from parent project +option(WITH_TESTS "Build tests" "${PROJECT_IS_TOP_LEVEL}") +option(WITH_TUTORIALS "Build tutorials" "${PROJECT_IS_TOP_LEVEL}") +option(WITH_PACKAGING "Include install() rules" "${PROJECT_IS_TOP_LEVEL}") + # Support not actually building the bindings, but using the ones we find # via `find_package(Halide)`. This allows running tests against the # installed Halide package. option(WITH_PYTHON_BINDINGS "Build Python bindings" ON) -# Duplicated options from parent project -option(WITH_TESTS "Build tests" ON) -option(WITH_TUTORIALS "Build tutorials" ON) -option(WITH_PYTHON_STUBS "Build Python stubs" ON) +cmake_dependent_option( + WITH_PYTHON_STUBS "Build Python stubs" ON + WITH_PYTHON_BINDINGS OFF +) -# Enable/disable testing cmake_dependent_option( WITH_TEST_PYTHON "Build Python tests" ON WITH_TESTS OFF @@ -40,7 +44,7 @@ cmake_dependent_option( # Development.Module and Development.Embed. We don't need the Embed # part, so only requesting Module avoids failures when Embed is not # available, as is the case in the manylinux Docker images. -find_package(Python3 3.8 REQUIRED Interpreter Development.Module) +find_package(Python 3.8 REQUIRED Interpreter Development.Module) if (WITH_PYTHON_BINDINGS) find_package(pybind11 2.10.4 REQUIRED) @@ -52,34 +56,8 @@ if (NOT Halide_ENABLE_RTTI OR NOT Halide_ENABLE_EXCEPTIONS) message(FATAL_ERROR "Python bindings require RTTI and exceptions to be enabled.") endif () -## -# A helper for creating tests with correct PYTHONPATH and sanitizer preloading -## - -function(add_python_test) - cmake_parse_arguments(ARG "" "FILE;LABEL" "PYTHONPATH;ENVIRONMENT;TEST_ARGS" ${ARGN}) - - list(PREPEND ARG_PYTHONPATH "$>") - list(TRANSFORM ARG_PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:") - - list(PREPEND ARG_ENVIRONMENT "HL_TARGET=${Halide_TARGET};HL_JIT_TARGET=${Halide_TARGET}") - - cmake_path(GET ARG_FILE STEM test_name) - set(test_name "${ARG_LABEL}_${test_name}") - - add_test( - NAME "${test_name}" - COMMAND ${Halide_PYTHON_LAUNCHER} "$" "$" ${ARG_TEST_ARGS} - ) - set_tests_properties( - "${test_name}" - PROPERTIES - LABELS "python" - ENVIRONMENT "${ARG_ENVIRONMENT}" - ENVIRONMENT_MODIFICATION "${ARG_PYTHONPATH}" - ) -endfunction() - +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +include(AddPythonTest) ## # Add our sources to this sub-tree. @@ -89,7 +67,7 @@ if (WITH_PYTHON_BINDINGS) add_subdirectory(src) endif () -if (WITH_PYTHON_BINDINGS AND WITH_PYTHON_STUBS) +if (WITH_PYTHON_STUBS) add_subdirectory(stub) endif () @@ -101,3 +79,7 @@ endif () if (WITH_TUTORIALS) add_subdirectory(tutorial) endif () + +if (WITH_PACKAGING) + add_subdirectory(packaging) +endif () diff --git a/python_bindings/apps/CMakeLists.txt b/python_bindings/apps/CMakeLists.txt index e63b38e8f31e..ebaf96b0de40 100644 --- a/python_bindings/apps/CMakeLists.txt +++ b/python_bindings/apps/CMakeLists.txt @@ -1,3 +1,12 @@ +cmake_minimum_required(VERSION 3.28) +project(Halide_Python_apps) + +if (PROJECT_IS_TOP_LEVEL) + enable_testing() +endif () + +find_package(Halide REQUIRED COMPONENTS Python) + if (Halide_TARGET MATCHES "wasm") message(WARNING "Python apps are skipped under WASM.") return() @@ -13,6 +22,9 @@ if (NOT WITH_AUTOSCHEDULERS) return() endif () +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../cmake") +include(AddPythonTest) + set(TEST_TMPDIR "$") set(TEST_IMAGES_DIR "$") @@ -53,7 +65,7 @@ foreach (app IN LISTS APPS) MODULE_NAME ${G} HALIDE_LIBRARIES app_aot_${G}) list(APPEND DEPS app_ext_${G}) - endforeach() + endforeach () set(app_src "${app}_app.py") add_python_test( diff --git a/python_bindings/cmake/AddPythonTest.cmake b/python_bindings/cmake/AddPythonTest.cmake new file mode 100644 index 000000000000..4cf12fc4316e --- /dev/null +++ b/python_bindings/cmake/AddPythonTest.cmake @@ -0,0 +1,27 @@ +## +# A helper for creating tests with correct PYTHONPATH and sanitizer preloading +## + +function(add_python_test) + cmake_parse_arguments(ARG "" "FILE;LABEL" "PYTHONPATH;ENVIRONMENT;TEST_ARGS" ${ARGN}) + + list(PREPEND ARG_PYTHONPATH "$/..>") + list(TRANSFORM ARG_PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:") + + list(PREPEND ARG_ENVIRONMENT "HL_TARGET=${Halide_TARGET};HL_JIT_TARGET=${Halide_TARGET}") + + cmake_path(GET ARG_FILE STEM test_name) + set(test_name "${ARG_LABEL}_${test_name}") + + add_test( + NAME "${test_name}" + COMMAND ${Halide_PYTHON_LAUNCHER} "$" "$" ${ARG_TEST_ARGS} + ) + set_tests_properties( + "${test_name}" + PROPERTIES + LABELS "python" + ENVIRONMENT "${ARG_ENVIRONMENT}" + ENVIRONMENT_MODIFICATION "${ARG_PYTHONPATH}" + ) +endfunction() diff --git a/python_bindings/packaging/CMakeLists.txt b/python_bindings/packaging/CMakeLists.txt new file mode 100644 index 000000000000..a6816af45d7c --- /dev/null +++ b/python_bindings/packaging/CMakeLists.txt @@ -0,0 +1,81 @@ +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +set(Halide_INSTALL_PYTHONDIR "${CMAKE_INSTALL_LIBDIR}/python3/site-packages/halide" + CACHE STRING "Path to the Python site-packages folder") + +if (WITH_PYTHON_BINDINGS) + install(DIRECTORY "${Halide_Python_SOURCE_DIR}/src/halide/" + DESTINATION "${Halide_INSTALL_PYTHONDIR}" + COMPONENT Halide_Python + FILES_MATCHING + PATTERN "*.py" + PATTERN "halide_" EXCLUDE) + + install(TARGETS Halide_Python + EXPORT Halide_Python-targets + LIBRARY DESTINATION "${Halide_INSTALL_PYTHONDIR}" + COMPONENT Halide_Python) + + get_property(halide_is_imported TARGET Halide::Halide PROPERTY IMPORTED) + get_property(halide_type TARGET Halide::Halide PROPERTY TYPE) + + if ( + NOT CMAKE_INSTALL_RPATH # Honor user overrides + AND NOT halide_is_imported # Imported Halide means user is responsible for RPATH + AND halide_type STREQUAL "SHARED_LIBRARY" # No need to set RPATH if statically linked + ) + if (APPLE) + set(rbase @loader_path) + else () + set(rbase $ORIGIN) + endif () + + file(RELATIVE_PATH lib_dir + "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_PYTHONDIR}" + "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}") + + set_target_properties(Halide_Python PROPERTIES INSTALL_RPATH "${rbase}/${lib_dir}") + endif () +endif () + +if (WITH_PYTHON_STUBS) + install(TARGETS Halide_PyStubs + EXPORT Halide_Python-targets + COMPONENT Halide_Python) +endif () + +if (WITH_TUTORIALS) + install(DIRECTORY ${Halide_Python_SOURCE_DIR}/tutorial/ + DESTINATION ${CMAKE_INSTALL_DOCDIR}/tutorial-python + COMPONENT Halide_Documentation + FILES_MATCHING PATTERN "*.py") +endif () + +if (WITH_PYTHON_BINDINGS OR WITH_PYTHON_STUBS) + set(Halide_Python_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/Halide_Python" + CACHE STRING "Path to Halide_Python CMake files") + + install(EXPORT Halide_Python-targets + DESTINATION ${Halide_Python_INSTALL_CMAKEDIR} + NAMESPACE Halide:: + COMPONENT Halide_Python) + + configure_package_config_file( + Halide_PythonConfig.cmake.in Halide_PythonConfig.cmake + INSTALL_DESTINATION "${Halide_Python_INSTALL_CMAKEDIR}" + NO_SET_AND_CHECK_MACRO + ) + + write_basic_package_version_file( + Halide_PythonConfigVersion.cmake COMPATIBILITY SameMajorVersion + ) + + install( + FILES + "${CMAKE_CURRENT_BINARY_DIR}/Halide_PythonConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/Halide_PythonConfigVersion.cmake" + DESTINATION "${Halide_Python_INSTALL_CMAKEDIR}" + COMPONENT Halide_Python + ) +endif () diff --git a/python_bindings/packaging/Halide_PythonConfig.cmake.in b/python_bindings/packaging/Halide_PythonConfig.cmake.in new file mode 100644 index 000000000000..12a31690805c --- /dev/null +++ b/python_bindings/packaging/Halide_PythonConfig.cmake.in @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.28) +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(Python 3 COMPONENTS Interpreter Development.Module) + +include("${CMAKE_CURRENT_LIST_DIR}/Halide_Python-targets.cmake") + +check_required_components(${CMAKE_FIND_PACKAGE_NAME}) diff --git a/python_bindings/src/halide/CMakeLists.txt b/python_bindings/src/halide/CMakeLists.txt index 9d2523de9169..d5c48e6a3f68 100644 --- a/python_bindings/src/halide/CMakeLists.txt +++ b/python_bindings/src/halide/CMakeLists.txt @@ -1,64 +1,63 @@ -set(native_sources - PyArgument.cpp - PyBoundaryConditions.cpp - PyBuffer.cpp - PyCallable.cpp - PyConciseCasts.cpp - PyDerivative.cpp - PyEnums.cpp - PyError.cpp - PyExpr.cpp - PyExternFuncArgument.cpp - PyFunc.cpp - PyFuncRef.cpp - PyGenerator.cpp - PyHalide.cpp - PyImageParam.cpp - PyInlineReductions.cpp - PyIROperator.cpp - PyLambda.cpp - PyLoopLevel.cpp - PyModule.cpp - PyParam.cpp - PyParameter.cpp - PyPipeline.cpp - PyRDom.cpp - PyStage.cpp - PyTarget.cpp - PyTuple.cpp - PyType.cpp - PyVar.cpp - PyVarOrRVar.cpp - ) -list(TRANSFORM native_sources PREPEND "halide_/") +pybind11_add_module(Halide_Python) +add_library(Halide::Python ALIAS Halide_Python) -set(python_sources - __init__.py - _generator_helpers.py - imageio.py - ) +set_target_properties( + Halide_Python + PROPERTIES + OUTPUT_NAME halide_ + EXPORT_NAME Python +) + +target_sources( + Halide_Python + PRIVATE + halide_/PyArgument.cpp + halide_/PyBoundaryConditions.cpp + halide_/PyBuffer.cpp + halide_/PyCallable.cpp + halide_/PyConciseCasts.cpp + halide_/PyDerivative.cpp + halide_/PyEnums.cpp + halide_/PyError.cpp + halide_/PyExpr.cpp + halide_/PyExternFuncArgument.cpp + halide_/PyFunc.cpp + halide_/PyFuncRef.cpp + halide_/PyGenerator.cpp + halide_/PyHalide.cpp + halide_/PyImageParam.cpp + halide_/PyInlineReductions.cpp + halide_/PyIROperator.cpp + halide_/PyLambda.cpp + halide_/PyLoopLevel.cpp + halide_/PyModule.cpp + halide_/PyParam.cpp + halide_/PyParameter.cpp + halide_/PyPipeline.cpp + halide_/PyRDom.cpp + halide_/PyStage.cpp + halide_/PyTarget.cpp + halide_/PyTuple.cpp + halide_/PyType.cpp + halide_/PyVar.cpp + halide_/PyVarOrRVar.cpp +) # It is technically still possible for a user to override the LIBRARY_OUTPUT_DIRECTORY by setting # CMAKE_LIBRARY_OUTPUT_DIRECTORY_, but they do so at their own peril. If a user needs to # do this, they should use the CMAKE_PROJECT_Halide_Python_INCLUDE_BEFORE variable to override it # just for this project, rather than globally, and they should ensure that the last path component # is `halide`. Otherwise, the tests will break. -pybind11_add_module(Halide_Python MODULE ${native_sources}) -add_library(Halide::Python ALIAS Halide_Python) set_target_properties( - Halide_Python - PROPERTIES - LIBRARY_OUTPUT_NAME halide_ - LIBRARY_OUTPUT_DIRECTORY "$/halide" - EXPORT_NAME Python + Halide_Python PROPERTIES LIBRARY_OUTPUT_DIRECTORY "$/halide" ) + if (Halide_ASAN_ENABLED) set_target_properties( - Halide_Python - PROPERTIES - CMAKE_SHARED_LINKER_FLAGS -shared-libasan + Halide_Python PROPERTIES CMAKE_SHARED_LINKER_FLAGS -shared-libasan ) endif () + target_link_libraries(Halide_Python PRIVATE Halide::Halide) # TODO: There's precious little information about why Python only sometimes prevents DLLs from loading from the PATH @@ -74,97 +73,23 @@ add_custom_command( ) # Copy our Python source files over so that we have a valid package in the binary directory. -# TODO: When upgrading to CMake 3.23 or beyond, investigate the FILE_SET feature. -set(build_tree_pys "") -foreach (pysrc IN LISTS python_sources) - # TODO: CMake 3.22 still doesn't allow target-dependent genex in OUTPUT, but we can hack around this using a stamp - # file. Fix this hack up if and when they ever improve this feature. - set(stamp_file "${CMAKE_CURRENT_BINARY_DIR}/.${pysrc}.stamp") - add_custom_command( - OUTPUT "${stamp_file}" - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/${pysrc}" "$/${pysrc}" - COMMAND ${CMAKE_COMMAND} -E touch "${stamp_file}" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${pysrc}" - VERBATIM - ) - list(APPEND build_tree_pys "${stamp_file}") -endforeach () -add_custom_target(Halide_Python_sources ALL DEPENDS ${build_tree_pys}) -add_dependencies(Halide_Python Halide_Python_sources) - -## -# Packaging -## - -include(CMakeDependentOption) -include(GNUInstallDirs) - -set(Halide_INSTALL_PYTHONDIR "${CMAKE_INSTALL_LIBDIR}/python3/site-packages" - CACHE STRING "Path to the Python site-packages folder") - -install(DIRECTORY "$/" - DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide" - COMPONENT Halide_Python - FILES_MATCHING - PATTERN "*.py" - PATTERN "*/halide_" EXCLUDE - PATTERN "*/CMakeFiles" EXCLUDE - PATTERN "*/__pycache__" EXCLUDE) - -install(TARGETS Halide_Python - EXPORT Halide_Targets - LIBRARY DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide" - COMPONENT Halide_Python) - -get_property(halide_is_imported TARGET Halide::Halide PROPERTY IMPORTED) -get_property(halide_type TARGET Halide::Halide PROPERTY TYPE) -cmake_dependent_option( - Halide_Python_INSTALL_IMPORTED_DEPS "" OFF - "halide_is_imported;halide_type STREQUAL \"SHARED_LIBRARY\"" OFF -) - -if (Halide_Python_INSTALL_IMPORTED_DEPS) - # The following might be a bit confusing, but installing both libHalide - # and its SONAME symbolic link causes the following bad behavior: - # 1. CMake does the right thing and installs libHalide.so.X.Y.Z - # (TARGET_FILE) as a real file and libHalide.so.X - # (TARGET_SONAME_FILE_NAME) as a symbolic link to the former. - # 2. Setuptools dutifully packs both of these into a Python wheel, which - # is a structured zip file. Zip files do not support symbolic links. - # Thus, two independent copies of libHalide are inserted, bloating the - # package. - # The Python module (on Unix systems) links to the SONAME file, and - # installing the symbolic link directly results in a broken link. Hence, - # the renaming dance here. - - if (NOT MSVC) - set(rename_arg RENAME "$") - else () - # DLL systems do not have sonames. - set(rename_arg "") - endif () +set(python_sources + __init__.py + _generator_helpers.py + imageio.py) - # TODO: when we upgrade to CMake 3.22, replace with RUNTIME_DEPENDENCY_SET? - install(FILES "$" - DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide" - COMPONENT Halide_Python - ${rename_arg}) -endif () +list(TRANSFORM python_sources PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" + OUTPUT_VARIABLE python_sources_source_dir) -if ( - NOT CMAKE_INSTALL_RPATH # Honor user overrides - AND NOT halide_is_imported # Imported Halide means user is responsible for RPATH - AND halide_type STREQUAL "SHARED_LIBRARY" # No need to set RPATH if statically linked +set(stamp_file "$/Halide_Python_sources.stamp") +add_custom_command( + OUTPUT "${stamp_file}" + COMMAND "${CMAKE_COMMAND}" -E make_directory $ + COMMAND "${CMAKE_COMMAND}" -E copy -t $ ${python_sources_source_dir} + COMMAND "${CMAKE_COMMAND}" -E touch ${stamp_file} + DEPENDS ${python_sources_source_dir} + VERBATIM ) - if (APPLE) - set(rbase @loader_path) - else () - set(rbase $ORIGIN) - endif () - - file(RELATIVE_PATH lib_dir - "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_PYTHONDIR}/halide" - "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}") - set_target_properties(Halide_Python PROPERTIES INSTALL_RPATH "${rbase}/${lib_dir}") -endif () +add_custom_target(Halide_Python_sources DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${stamp_file}") +add_dependencies(Halide_Python Halide_Python_sources) diff --git a/python_bindings/src/halide/__init__.py b/python_bindings/src/halide/__init__.py index b2abca11e090..3d211c881fd3 100644 --- a/python_bindings/src/halide/__init__.py +++ b/python_bindings/src/halide/__init__.py @@ -1,3 +1,13 @@ +def patch_dll_dirs(): + import os + if hasattr(os, 'add_dll_directory'): + from pathlib import Path + os.add_dll_directory(str(Path(__file__).parent / 'bin')) + + +patch_dll_dirs() +del patch_dll_dirs + from .halide_ import * from .halide_ import _, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9 from ._generator_helpers import ( diff --git a/python_bindings/stub/CMakeLists.txt b/python_bindings/stub/CMakeLists.txt index cc5354470466..85edb47b47cd 100644 --- a/python_bindings/stub/CMakeLists.txt +++ b/python_bindings/stub/CMakeLists.txt @@ -13,7 +13,3 @@ set_target_properties(Halide_PyStubs PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN TRUE POSITION_INDEPENDENT_CODE ON) - -install(TARGETS Halide_PyStubs - EXPORT Halide_Targets - COMPONENT Halide_Python) diff --git a/python_bindings/tutorial/CMakeLists.txt b/python_bindings/tutorial/CMakeLists.txt index 9b17c7516f77..285ed1ae3d08 100644 --- a/python_bindings/tutorial/CMakeLists.txt +++ b/python_bindings/tutorial/CMakeLists.txt @@ -19,7 +19,7 @@ set(tests lesson_12_using_the_gpu.py lesson_13_tuples.py lesson_14_types.py - ) +) set(PYPATH_lesson_10_aot_compilation_run "$") @@ -64,9 +64,9 @@ else () # This target allows CMake to build lesson_10_halide.so (or whatever the correct extension is) as part of the tests # later. It is excluded from ALL since it isn't valid to build outside of this context. - Python3_add_library(lesson_10_halide MODULE EXCLUDE_FROM_ALL - lesson_10_halide.py.cpp - lesson_10_halide.o) + Python_add_library(lesson_10_halide MODULE EXCLUDE_FROM_ALL + lesson_10_halide.py.cpp + lesson_10_halide.o) target_link_libraries(lesson_10_halide PRIVATE Halide::Runtime) @@ -91,14 +91,3 @@ else () set_tests_properties(python_tutorial_lesson_10_aot_compilation_run PROPERTIES FIXTURES_REQUIRED py_lesson_10) endif () - -## -# Packaging -## - -include(GNUInstallDirs) - -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ - DESTINATION ${CMAKE_INSTALL_DOCDIR}/tutorial-python - COMPONENT Halide_Documentation - FILES_MATCHING PATTERN "*.py") diff --git a/requirements.txt b/requirements.txt index 16b9d8ff48b1..129c450c2715 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ -cmake>=3.22 +build +cmake>=3.28 imageio -ninja; platform_system!='Windows' +ninja numpy pillow pybind11==2.10.4 -scikit-build +scikit-build-core==0.10.5 scipy setuptools>=43 +tbump==6.11.0 wheel -build diff --git a/setup.py b/setup.py deleted file mode 100644 index bea2e7f1dd92..000000000000 --- a/setup.py +++ /dev/null @@ -1,34 +0,0 @@ -import pybind11 -from setuptools import find_packages -from skbuild import setup -from pathlib import Path - -this_directory = Path(__file__).parent -long_description = (this_directory / "README_python.md").read_text() - -setup( - name="halide", - version='19.0.0', - author="The Halide team", - author_email="halide-dev@lists.csail.mit.edu", - description="Halide is a programming language designed to make it easier " - "to write high-performance image and array processing code.", - long_description=long_description, - long_description_content_type='text/markdown', - python_requires=">=3.8", - packages=find_packages(where="python_bindings/src"), - package_dir={"": "python_bindings/src"}, - cmake_source_dir="python_bindings", - cmake_args=[ - f"-Dpybind11_ROOT={pybind11.get_cmake_dir()}", - "-DCMAKE_REQUIRE_FIND_PACKAGE_pybind11=YES", - "-DHalide_INSTALL_PYTHONDIR=python_bindings/src", - "-DCMAKE_INSTALL_RPATH=$,@loader_path,$ORIGIN>", - "-DHalide_Python_INSTALL_IMPORTED_DEPS=ON", - "-DWITH_TESTS=NO", - "-DWITH_TUTORIALS=NO", - "-DWITH_PYTHON_STUBS=NO", - "-DCMAKE_PREFIX_PATH=$ENV{CMAKE_PREFIX_PATH}", - "--no-warn-unused-cli", - ], -) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6f0df8f90a65..f5eb3e64f97f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,13 +14,23 @@ if (NOT BUILD_SHARED_LIBS) endif () # Set the (shared) library version +set(Halide_VERSION_OVERRIDE "${Halide_VERSION}" + CACHE STRING "VERSION to set for custom Halide packaging") +mark_as_advanced(Halide_VERSION_OVERRIDE) + +if (Halide_VERSION_OVERRIDE) + # Empty is considered a value distinct from not-defined + set_target_properties(Halide PROPERTIES VERSION "${Halide_VERSION_OVERRIDE}") +endif () + set(Halide_SOVERSION_OVERRIDE "${Halide_VERSION_MAJOR}" CACHE STRING "SOVERSION to set for custom Halide packaging") mark_as_advanced(Halide_SOVERSION_OVERRIDE) -set_target_properties(Halide PROPERTIES - VERSION "${Halide_VERSION}" - SOVERSION "${Halide_SOVERSION_OVERRIDE}") +if (Halide_SOVERSION_OVERRIDE) + # Empty is considered a value distinct from not-defined + set_target_properties(Halide PROPERTIES SOVERSION "${Halide_SOVERSION_OVERRIDE}") +endif () # Always build with PIC, even when static set_target_properties(Halide PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/test/autoschedulers/li2018/CMakeLists.txt b/test/autoschedulers/li2018/CMakeLists.txt index 3eac046f594c..2d4db3070554 100644 --- a/test/autoschedulers/li2018/CMakeLists.txt +++ b/test/autoschedulers/li2018/CMakeLists.txt @@ -26,14 +26,14 @@ if (WITH_PYTHON_BINDINGS) if (Halide_TARGET MATCHES "webgpu") message(WARNING "li2018_gradient_autoscheduler_test_py is not supported with WebGPU.") else() - find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module) + find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development.Module) add_test( NAME li2018_gradient_autoscheduler_test_py - COMMAND ${Halide_PYTHON_LAUNCHER} "$" "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $ + COMMAND ${Halide_PYTHON_LAUNCHER} "$" "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $ ) - set(PYTHONPATH "$>") + set(PYTHONPATH "$/..>") list(TRANSFORM PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:") set_tests_properties(li2018_gradient_autoscheduler_test_py PROPERTIES From 75bf30633dbd0c3486cfb9666210aa8af044bca0 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Tue, 3 Sep 2024 21:10:36 -0400 Subject: [PATCH 2/3] Fix no-python builds --- packaging/CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packaging/CMakeLists.txt b/packaging/CMakeLists.txt index ad7c6cd30570..7f7bb15d2894 100644 --- a/packaging/CMakeLists.txt +++ b/packaging/CMakeLists.txt @@ -190,9 +190,15 @@ write_basic_package_version_file(HalideHelpersConfigVersion.cmake COMPATIBILITY SameMajorVersion ARCH_INDEPENDENT) +if (WITH_PYTHON_BINDINGS) + set(extra_paths Halide_Python_INSTALL_CMAKEDIR) +else () + set(extra_paths "") +endif () + configure_package_config_file( common/HalideConfig.cmake HalideConfig.cmake - PATH_VARS Halide_INSTALL_HELPERSDIR Halide_Python_INSTALL_CMAKEDIR + PATH_VARS Halide_INSTALL_HELPERSDIR ${extra_paths} INSTALL_DESTINATION "${Halide_INSTALL_CMAKEDIR}" NO_SET_AND_CHECK_MACRO NO_CHECK_REQUIRED_COMPONENTS_MACRO From 8a977d8ac15eb9273a099dbe408b2f1df5b5292f Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Tue, 3 Sep 2024 21:50:15 -0400 Subject: [PATCH 3/3] Only add bin/ to DLL paths when exists --- python_bindings/src/halide/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python_bindings/src/halide/__init__.py b/python_bindings/src/halide/__init__.py index 3d211c881fd3..d146b6ffb528 100644 --- a/python_bindings/src/halide/__init__.py +++ b/python_bindings/src/halide/__init__.py @@ -2,7 +2,9 @@ def patch_dll_dirs(): import os if hasattr(os, 'add_dll_directory'): from pathlib import Path - os.add_dll_directory(str(Path(__file__).parent / 'bin')) + bin_dir = Path(__file__).parent / 'bin' + if bin_dir.exists(): + os.add_dll_directory(str(bin_dir)) patch_dll_dirs()