Generate proper, consistent nightly versions for pip and conda packag…

…es (#1347) This PR changes conda C++/Python packages and wheels to all generate a consistent version for nightlies. The nightly version is of the form YY.MM.DDaN, where N is the number of commits from the last tag. The version is embedded in both the package metadata and in the `rmm.__version__` attribute. In addition the commit hash itself is embedded into the package as `rmm.__git_commit__`. These changes ensure that 1) the conda Python package for a given nightly will reliably choose the correct C++ package (previously we relied on build strings and build times, which is more fragile w.r.t. the conda solver); 2) wheels are properly considered nightlies and are treated accordingly by pip (e.g. requiring `--pre` for installation, not conflicting with normal releases, etc); and 3) wheels and conda packages are aligned on versions so that they can be easily compared if necessary. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Lawrence Mitchell (https://github.com/wence-) - AJ Schmidt (https://github.com/ajschmidt8) URL: #1347
rapidsai · Oct 31, 2023 · 207b898 · 207b898
1 parent 5bdcc05
commit 207b898
Show file tree

Hide file tree

Showing 13 changed files with 53 additions and 26 deletions.
diff --git a/VERSION b/VERSION
@@ -0,0 +1 @@
+23.12.00
diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -9,9 +9,11 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+version=$(rapids-generate-version)
+
 rapids-logger "Begin cpp build"
 
 # This calls mambabuild when boa is installed (as is the case in the CI images)
-rapids-conda-retry mambabuild conda/recipes/librmm
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/librmm
 
 rapids-upload-conda-to-s3 cpp
diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -9,11 +9,20 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+package_name="rmm"
+package_dir="python"
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
 rapids-logger "Begin py build"
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 # This calls mambabuild when boa is installed (as is the case in the CI images)
-rapids-conda-retry mambabuild -c "${CPP_CHANNEL}" conda/recipes/rmm
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild -c "${CPP_CHANNEL}" conda/recipes/rmm
 
 rapids-upload-conda-to-s3 python
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -9,9 +9,8 @@ package_dir="python"
 source rapids-configure-sccache
 source rapids-date-string
 
-# Use gha-tools rapids-pip-wheel-version to generate wheel version then
-# update the necessary files
-version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})"
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
@@ -22,17 +21,9 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
 # Patch project metadata files to include the CUDA version suffix and version override.
 pyproject_file="${package_dir}/pyproject.toml"
 
-sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file}
 sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
 
 if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
     sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -37,8 +37,9 @@ sed_runner 's/'"  VERSION .*"'/'"  VERSION ${NEXT_FULL_TAG}"'/g' CMakeLists.txt
 
 # Python update
 sed_runner 's/'"rmm_version .*)"'/'"rmm_version ${NEXT_FULL_TAG})"'/g' python/CMakeLists.txt
-sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/rmm/__init__.py
-sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/pyproject.toml
+
+# Centralized version file update
+echo "${NEXT_FULL_TAG}" > VERSION
 
 # rapids-cmake version
 sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake

diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh
@@ -10,7 +10,7 @@ RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-
 python -m pip install $(echo ./dist/rmm*.whl)[test]
 
 # Run smoke tests for aarch64 pull requests
-if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then
+if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
     python ./ci/wheel_smoke_test.py
 else
     python -m pytest ./python/rmm/tests

diff --git a/conda/recipes/librmm/meta.yaml b/conda/recipes/librmm/meta.yaml
@@ -1,6 +1,6 @@
 # Copyright (c) 2018-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
 {% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0
@@ -10,7 +10,7 @@ package:
   name: librmm-split
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 requirements:
   build:

diff --git a/conda/recipes/rmm/meta.yaml b/conda/recipes/rmm/meta.yaml
@@ -1,6 +1,6 @@
 # Copyright (c) 2019-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
 {% set py_version = environ['CONDA_PY'] %}
@@ -11,7 +11,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -27,7 +27,7 @@ requires = [
 
 [project]
 name = "rmm"
-version = "23.12.00"
+dynamic = ["version"]
 description = "rmm - RAPIDS Memory Manager"
 readme = { file = "README.md", content-type = "text/markdown" }
 authors = [
@@ -114,3 +114,6 @@ skip = [
 
 [tool.setuptools]
 license-files = ["LICENSE"]
+
+[tool.setuptools.dynamic]
+version = {file = "rmm/VERSION"}
diff --git a/python/rmm/VERSION b/python/rmm/VERSION
@@ -0,0 +1 @@
+../../VERSION
diff --git a/python/rmm/__init__.py b/python/rmm/__init__.py
@@ -23,6 +23,7 @@
     set_logging_level,
     should_log,
 )
+from rmm._version import __git_commit__, __version__
 from rmm.mr import disable_logging, enable_logging, get_log_filenames
 from rmm.rmm import (
     RMMError,
@@ -51,5 +52,3 @@
     "should_log",
     "unregister_reinitialize_hook",
 ]
-
-__version__ = "23.12.00"
diff --git a/python/rmm/_version.py b/python/rmm/_version.py
@@ -0,0 +1,20 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.resources
+
+__version__ = (
+    importlib.resources.files("rmm").joinpath("VERSION").read_text().strip()
+)
+__git_commit__ = ""
diff --git a/python/setup.py b/python/setup.py
@@ -6,6 +6,6 @@
 packages = find_packages(include=["rmm*"])
 setup(
     packages=packages,
-    package_data={key: ["*.pxd"] for key in packages},
+    package_data={key: ["VERSION", "*.pxd"] for key in packages},
     zip_safe=False,
 )