Change scripts to accommodate conda prefix to work with Nova (pytorch…

…#2022) Summary: Pull Request resolved: pytorch#2022 Pull Request resolved: pytorch#2021 Nova creates conda environments with `--prefix` where the environments are outside of the default `envs` folder. Current script only works with `--name`. This diff changes the scripts to accommodate conda prefix. https://github.com/pytorch/FBGEMM/actions/runs/6189731547 Reviewed By: spcyppt Differential Revision: D49306286 fbshipit-source-id: 2f2ca00645526639369de3f555dbab30da56739e
q10 · Sep 15, 2023 · bbc676e · bbc676e
1 parent f9e7ba7
commit bbc676e
Show file tree

Hide file tree

Showing 12 changed files with 198 additions and 63 deletions.
diff --git a/.github/scripts/fbgemm_gpu_build.bash b/.github/scripts/fbgemm_gpu_build.bash
@@ -40,10 +40,16 @@ prepare_fbgemm_gpu_build () {
   git submodule sync
   git submodule update --init --recursive
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   echo "[BUILD] Installing other build dependencies ..."
-  (exec_with_retries conda run --no-capture-output -n "${env_name}" python -m pip install -r requirements.txt) || return 1
+  # shellcheck disable=SC2086
+  (exec_with_retries conda run --no-capture-output ${env_prefix} python -m pip install -r requirements.txt) || return 1
 
+  # shellcheck disable=SC2086
   (test_python_import_package "${env_name}" numpy) || return 1
+  # shellcheck disable=SC2086
   (test_python_import_package "${env_name}" skbuild) || return 1
 
   echo "[BUILD] Successfully ran git submodules update"
@@ -78,7 +84,8 @@ __configure_fbgemm_gpu_build_rocm () {
   fi
 
   echo "[BUILD] Setting the following ROCm targets: ${arch_list}"
-  print_exec conda env config vars set -n "${env_name}" PYTORCH_ROCM_ARCH="${arch_list}"
+  # shellcheck disable=SC2086
+  print_exec conda env config vars set ${env_prefix} PYTORCH_ROCM_ARCH="${arch_list}"
 
   echo "[BUILD] Setting ROCm build args ..."
   build_args=()
@@ -116,8 +123,8 @@ __configure_fbgemm_gpu_build_cuda () {
 
   # Build only CUDA 7.0 and 8.0 (i.e. V100 and A100) because of 100 MB binary size limits from PyPI.
   echo "[BUILD] Setting CUDA build args ..."
-  # shellcheck disable=SC2155
-  local nvml_lib_path=$(conda run --no-capture-output -n "${env_name}" printenv NVML_LIB_PATH)
+  # shellcheck disable=SC2155,SC2086
+  local nvml_lib_path=$(conda run --no-capture-output ${env_prefix} printenv NVML_LIB_PATH)
   build_args=(
     --nvml_lib_path="${nvml_lib_path}"
     -DTORCH_CUDA_ARCH_LIST="'${arch_list}'"
@@ -187,16 +194,17 @@ __build_fbgemm_gpu_common_pre_steps () {
   echo "[BUILD] Determined Python package name to use: ${package_name}"
 
   # Extract the Python tag
-  # shellcheck disable=SC2207
-  python_version=($(conda run --no-capture-output -n "${env_name}" python --version))
+  # shellcheck disable=SC2207,SC2086
+  python_version=($(conda run --no-capture-output ${env_prefix} python --version))
   # shellcheck disable=SC2206
   python_version_arr=(${python_version[1]//./ })
   python_tag="py${python_version_arr[0]}${python_version_arr[1]}"
   echo "[BUILD] Extracted Python tag: ${python_tag}"
 
   echo "[BUILD] Running pre-build cleanups ..."
   print_exec rm -rf dist
-  print_exec conda run --no-capture-output -n "${env_name}" python setup.py clean
+  # shellcheck disable=SC2086
+  print_exec conda run --no-capture-output ${env_prefix} python setup.py clean
 
   echo "[BUILD] Printing git status ..."
   print_exec git status
@@ -285,6 +293,9 @@ build_fbgemm_gpu_package () {
     return 1
   fi
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   # Set up and configure the build
   __build_fbgemm_gpu_common_pre_steps || return 1
   __configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1
@@ -315,7 +326,8 @@ build_fbgemm_gpu_package () {
 
   # Distribute Python extensions as wheels on Linux
   echo "[BUILD] Building FBGEMM-GPU wheel (VARIANT=${fbgemm_variant}) ..."
-  print_exec conda run --no-capture-output -n "${env_name}" \
+  # shellcheck disable=SC2086
+  print_exec conda run --no-capture-output ${env_prefix} \
     python setup.py "${run_multicore}" bdist_wheel \
       --package_name="${package_name}" \
       --python-tag="${python_tag}" \
@@ -351,6 +363,9 @@ build_fbgemm_gpu_install () {
     return 1
   fi
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   # Set up and configure the build
   __build_fbgemm_gpu_common_pre_steps || return 1
   __configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1
@@ -365,7 +380,8 @@ build_fbgemm_gpu_install () {
   # Parallelism may need to be limited to prevent the build from being
   # canceled for going over ulimits
   echo "[BUILD] Building + installing FBGEMM-GPU (VARIANT=${fbgemm_variant}) ..."
-  print_exec conda run --no-capture-output -n "${env_name}" \
+  # shellcheck disable=SC2086
+  print_exec conda run --no-capture-output ${env_prefix} \
     python setup.py install "${build_args[@]}"
 
   # Run checks on the built libraries
@@ -395,6 +411,9 @@ build_fbgemm_gpu_develop () {
     return 1
   fi
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   # Set up and configure the build
   __build_fbgemm_gpu_common_pre_steps || return 1
   __configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1
@@ -409,7 +428,8 @@ build_fbgemm_gpu_develop () {
   # Parallelism may need to be limited to prevent the build from being
   # canceled for going over ulimits
   echo "[BUILD] Building (develop) FBGEMM-GPU (VARIANT=${fbgemm_variant}) ..."
-  print_exec conda run --no-capture-output -n "${env_name}" \
+  # shellcheck disable=SC2086
+  print_exec conda run --no-capture-output ${env_prefix} \
     python setup.py build develop "${build_args[@]}"
 
   # Run checks on the built libraries

diff --git a/.github/scripts/fbgemm_gpu_docs.bash b/.github/scripts/fbgemm_gpu_docs.bash
@@ -31,11 +31,15 @@ install_docs_tools () {
 
   test_network_connection || return 1
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   echo "[INSTALL] Installing docs tools ..."
-  (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
+  # shellcheck disable=SC2086
+  (exec_with_retries conda install ${env_prefix} -c conda-forge -y \
     doxygen) || return 1
 
-  # Check binaries are visible in the PAATH
+  # Check binaries are visible in the PATH
   (test_binpath "${env_name}" doxygen) || return 1
 
   echo "[INSTALL] Successfully installed all the docs tools"
@@ -62,14 +66,20 @@ build_fbgemm_gpu_docs () {
     echo ""
   fi
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   echo "[BUILD] Installing docs-build dependencies ..."
-  (exec_with_retries conda run -n "${env_name}" python -m pip install -r requirements.txt) || return 1
+  # shellcheck disable=SC2086
+  (exec_with_retries conda run ${env_prefix} python -m pip install -r requirements.txt) || return 1
 
   echo "[BUILD] Running Doxygen build ..."
-  (exec_with_retries conda run -n "${env_name}" doxygen Doxyfile.in) || return 1
+  # shellcheck disable=SC2086
+  (exec_with_retries conda run ${env_prefix} doxygen Doxyfile.in) || return 1
 
   echo "[BUILD] Building HTML pages ..."
-  (exec_with_retries conda run -n "${env_name}" make html) || return 1
+  # shellcheck disable=SC2086
+  (exec_with_retries conda run ${env_prefix} make html) || return 1
 
   echo "[INSTALL] FBGEMM-GPU documentation build completed"
 }
diff --git a/.github/scripts/fbgemm_gpu_install.bash b/.github/scripts/fbgemm_gpu_install.bash
@@ -20,7 +20,8 @@ __fbgemm_gpu_post_install_checks () {
   (test_python_import_symbol "${env_name}" fbgemm_gpu __version__) || return 1
 
   echo "[CHECK] Printing out the FBGEMM-GPU version ..."
-  installed_fbgemm_gpu_version=$(conda run -n "${env_name}" python -c "import fbgemm_gpu; print(fbgemm_gpu.__version__)")
+  # shellcheck disable=SC2086
+  installed_fbgemm_gpu_version=$(conda run ${env_prefix} python -c "import fbgemm_gpu; print(fbgemm_gpu.__version__)")
   echo "[CHECK] The installed version is: ${installed_fbgemm_gpu_version}"
 }
 
@@ -46,8 +47,12 @@ install_fbgemm_gpu_wheel () {
   print_exec sha256sum "${wheel_path}"
   print_exec md5sum "${wheel_path}"
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   echo "[INSTALL] Installing FBGEMM-GPU wheel: ${wheel_path} ..."
-  (exec_with_retries conda run -n "${env_name}" python -m pip install "${wheel_path}") || return 1
+  # shellcheck disable=SC2086
+  (exec_with_retries conda run ${env_prefix} python -m pip install "${wheel_path}") || return 1
 
   __fbgemm_gpu_post_install_checks || return 1
 
@@ -131,9 +136,12 @@ install_fbgemm_gpu_pip () {
     fi
   fi
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   echo "[INSTALL] Attempting to install FBGEMM-GPU ${fbgemm_gpu_version}+${fbgemm_gpu_variant} through PIP ..."
   # shellcheck disable=SC2086
-  (exec_with_retries conda run -n "${env_name}" pip install ${fbgemm_gpu_package}) || return 1
+  (exec_with_retries conda run ${env_prefix} pip install ${fbgemm_gpu_package}) || return 1
 
   __fbgemm_gpu_post_install_checks || return 1
 

diff --git a/.github/scripts/fbgemm_gpu_lint.bash b/.github/scripts/fbgemm_gpu_lint.bash
@@ -31,8 +31,12 @@ install_lint_tools () {
 
   test_network_connection || return 1
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   echo "[INSTALL] Installing lint tools ..."
-  (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
+  # shellcheck disable=SC2086
+  (exec_with_retries conda install ${env_prefix} -c conda-forge -y \
     click \
     flake8 \
     ufmt) || return 1
@@ -72,10 +76,14 @@ lint_fbgemm_gpu_flake8 () {
 
   echo "::add-matcher::fbgemm_gpu/test/lint/flake8_problem_matcher.json"
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   # E501 = line too long
   # W503 = line break before binary operator (deprecated)
   # E203 = whitespace before ":"
-  (print_exec conda run -n "${env_name}" flake8 --ignore=E501,W503,E203 .) || return 1
+  # shellcheck disable=SC2086
+  (print_exec conda run ${env_prefix} flake8 --ignore=E501,W503,E203 .) || return 1
 
   echo "[TEST] Finished running flake8 lint checks"
 }
@@ -102,8 +110,12 @@ lint_fbgemm_gpu_ufmt () {
     fbgemm_gpu/bench
   )
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   for p in "${lint_paths[@]}"; do
-    (print_exec conda run -n "${env_name}" ufmt diff "${p}") || return 1
+    # shellcheck disable=SC2086
+    (print_exec conda run ${env_prefix} ufmt diff "${p}") || return 1
   done
 
   echo "[TEST] Finished running ufmt lint checks"
@@ -131,8 +143,12 @@ lint_fbgemm_gpu_copyright () {
     fbgemm_gpu/bench
   )
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   for p in "${lint_paths[@]}"; do
-    (print_exec conda run -n "${env_name}" python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1
+    # shellcheck disable=SC2086
+    (print_exec conda run ${env_prefix} python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1
   done
 
   echo "[TEST] Finished running Meta Copyright Header checks"

diff --git a/.github/scripts/fbgemm_gpu_test.bash b/.github/scripts/fbgemm_gpu_test.bash
@@ -28,7 +28,11 @@ run_python_test () {
     echo "################################################################################"
   fi
 
-  if print_exec conda run --no-capture-output -n "${env_name}" python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning "${python_test_file}"; then
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
+  # shellcheck disable=SC2086
+  if print_exec conda run --no-capture-output ${env_prefix} python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning "${python_test_file}"; then
     echo "[TEST] Python test suite PASSED: ${python_test_file}"
     echo ""
   else
@@ -62,10 +66,14 @@ run_fbgemm_gpu_tests () {
     echo ""
   fi
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   # Enable ROCM testing if specified
   if [ "$fbgemm_variant" == "rocm" ]; then
     echo "[TEST] Set environment variable FBGEMM_TEST_WITH_ROCM to enable ROCm tests ..."
-    print_exec conda env config vars set -n "${env_name}" FBGEMM_TEST_WITH_ROCM=1
+    # shellcheck disable=SC2086
+    print_exec conda env config vars set ${env_prefix} FBGEMM_TEST_WITH_ROCM=1
   fi
 
   # These are either non-tests or currently-broken tests in both FBGEMM_GPU and FBGEMM_GPU-CPU
@@ -90,7 +98,8 @@ run_fbgemm_gpu_tests () {
   fi
 
   echo "[TEST] Installing pytest ..."
-  print_exec conda install -n "${env_name}" -y pytest
+  # shellcheck disable=SC2086
+  print_exec conda install ${env_prefix} -y pytest
 
   echo "[TEST] Checking imports ..."
   (test_python_import_package "${env_name}" fbgemm_gpu) || return 1

diff --git a/.github/scripts/nova_postscript.bash b/.github/scripts/nova_postscript.bash
@@ -8,7 +8,7 @@
 echo "Current working directory: $(pwd)"
 cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}"
 PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash"
-BUILD_ENV_NAME=base
+BUILD_ENV_NAME=${CONDA_ENV}
 GITHUB_ENV=TRUE
 export GITHUB_ENV
 

diff --git a/.github/scripts/nova_prescript.bash b/.github/scripts/nova_prescript.bash
@@ -8,7 +8,7 @@
 echo "Current working directory: $(pwd)"
 cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}"
 PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash"
-BUILD_ENV_NAME=base
+BUILD_ENV_NAME=${CONDA_ENV}
 echo "--------------------------"
 echo "----- conda env list -----"
 conda env list