Use cron schedule for PIP install workflows (pytorch#2027)

Summary: - Use a cron schedule for the PIP install workflows - Activate building wheels for ROCm variant Pull Request resolved: pytorch#2027 Reviewed By: spcyppt Differential Revision: D49484270 Pulled By: q10 fbshipit-source-id: 72a50b89c50d6f82c02f4a673f51abbc683e45ca
q10 · Sep 21, 2023 · 8f02dd3 · 8f02dd3
1 parent a511f9d
commit 8f02dd3
Show file tree

Hide file tree

Showing 18 changed files with 195 additions and 98 deletions.
diff --git a/.github/scripts/fbgemm_gpu_build.bash b/.github/scripts/fbgemm_gpu_build.bash
@@ -24,7 +24,7 @@ prepare_fbgemm_gpu_build () {
     echo "################################################################################"
     echo "# Prepare FBGEMM-GPU Build"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -154,7 +154,7 @@ __configure_fbgemm_gpu_build () {
     echo "################################################################################"
     echo "# Configure FBGEMM-GPU Build"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -310,7 +310,7 @@ build_fbgemm_gpu_package () {
   echo "################################################################################"
   echo "# Build FBGEMM-GPU Package (Wheel)"
   echo "#"
-  echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+  echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
   echo "################################################################################"
   echo ""
 
@@ -322,13 +322,15 @@ build_fbgemm_gpu_package () {
   echo "[BUILD] Checking build_args:"
   echo "${build_args[@]}"
 
-  core=$(lscpu | grep "Core(s)" | awk '{print $NF}') && echo "core = ${core}" || echo "core not found"
-  sockets=$(lscpu | grep "Socket(s)" | awk '{print $NF}') && echo "sockets = ${sockets}" || echo "sockets not found"
-  re='^[0-9]+$'
-  run_multicore=""
+  # shellcheck disable=SC2155
+  local core=$(lscpu | grep "Core(s)" | awk '{print $NF}') && echo "core = ${core}" || echo "core not found"
+  # shellcheck disable=SC2155
+  local sockets=$(lscpu | grep "Socket(s)" | awk '{print $NF}') && echo "sockets = ${sockets}" || echo "sockets not found"
+  local re='^[0-9]+$'
+  local run_multicore=""
   if [[ $core =~ $re && $sockets =~ $re ]] ; then
-    n_core=$((core * sockets))
-    run_multicore=" -j ${n_core}"
+    local n_core=$((core * sockets))
+    local run_multicore=" -j ${n_core}"
   fi
 
   # Distribute Python extensions as wheels on Linux
@@ -380,7 +382,7 @@ build_fbgemm_gpu_install () {
   echo "################################################################################"
   echo "# Build + Install FBGEMM-GPU Package"
   echo "#"
-  echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+  echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
   echo "################################################################################"
   echo ""
 
@@ -426,9 +428,9 @@ build_fbgemm_gpu_develop () {
   __configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1
 
   echo "################################################################################"
-  echo "# Build + Install FBGEMM-GPU Package"
+  echo "# Build + Install FBGEMM-GPU Package (Develop)"
   echo "#"
-  echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+  echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
   echo "################################################################################"
   echo ""
 

diff --git a/.github/scripts/fbgemm_gpu_docs.bash b/.github/scripts/fbgemm_gpu_docs.bash
@@ -24,7 +24,7 @@ install_docs_tools () {
     echo "################################################################################"
     echo "# Install Documentation Tools"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -61,7 +61,7 @@ build_fbgemm_gpu_docs () {
     echo "################################################################################"
     echo "# Build FBGEMM-GPU Documentation"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi

diff --git a/.github/scripts/fbgemm_gpu_install.bash b/.github/scripts/fbgemm_gpu_install.bash
@@ -43,7 +43,7 @@ install_fbgemm_gpu_wheel () {
     echo "################################################################################"
     echo "# Install FBGEMM-GPU from Wheel"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -79,7 +79,7 @@ install_fbgemm_gpu_pip () {
     echo "################################################################################"
     echo "# Install FBGEMM-GPU Package from PIP"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi

diff --git a/.github/scripts/fbgemm_gpu_lint.bash b/.github/scripts/fbgemm_gpu_lint.bash
@@ -24,7 +24,7 @@ install_lint_tools () {
     echo "################################################################################"
     echo "# Install Lint Tools"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -69,7 +69,7 @@ lint_fbgemm_gpu_flake8 () {
     echo "################################################################################"
     echo "# Run FBGEMM_GPU Lint: flake8"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -99,7 +99,7 @@ lint_fbgemm_gpu_ufmt () {
     echo "################################################################################"
     echo "# Run FBGEMM_GPU Lint: ufmt"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -132,7 +132,7 @@ lint_fbgemm_gpu_copyright () {
     echo "################################################################################"
     echo "# Run FBGEMM_GPU Lint: Meta Copyright Headers"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi

diff --git a/.github/scripts/fbgemm_gpu_test.bash b/.github/scripts/fbgemm_gpu_test.bash
@@ -61,7 +61,7 @@ run_fbgemm_gpu_tests () {
     echo "################################################################################"
     echo "# Run FBGEMM-GPU Tests"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -143,7 +143,7 @@ test_setup_conda_environment () {
     echo "################################################################################"
     echo "# Setup FBGEMM-GPU Build Container (All Steps)"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi

diff --git a/.github/scripts/nova_postscript.bash b/.github/scripts/nova_postscript.bash
@@ -5,27 +5,29 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-echo "Current working directory: $(pwd)"
-cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}"
+echo "[NOVA] Current working directory: $(pwd)"
+cd "${FBGEMM_REPO}" || echo "[NOVA] Failed to cd to ${FBGEMM_REPO}"
 PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash"
 BUILD_ENV_NAME=${CONDA_ENV}
 GITHUB_ENV=TRUE
 export GITHUB_ENV
 
 # Install FBGEMM_GPU Nightly
-echo "Current working directory: $(pwd)"
+echo "[NOVA] Current working directory: $(pwd)"
+
+# Load the FBGEMM_GPU build scripts infrastructure
 # shellcheck disable=SC1091
 # shellcheck source=.github/scripts/setup_env.bash
 . "${PRELUDE}";
 
 install_fbgemm_gpu_wheel "${BUILD_ENV_NAME}" fbgemm_gpu/dist/*.whl
 
 # Test with PyTest
-echo "Current working directory: $(pwd)"
+echo "[NOVA] Current working directory: $(pwd)"
 CPU_GPU="${CU_VERSION}"
 if [ "${CU_VERSION}" != 'cpu' ]; then
     CPU_GPU=""
 fi
 $CONDA_RUN python3 -c "import torch; print('cuda.is_available() ', torch.cuda.is_available()); print ('device_count() ',torch.cuda.device_count());"
-cd "${FBGEMM_REPO}/fbgemm_gpu/test" || { echo "Failed to cd to fbgemm_gpu/test from $(pwd)"; };
+cd "${FBGEMM_REPO}/fbgemm_gpu/test" || { echo "[NOVA] Failed to cd to fbgemm_gpu/test from $(pwd)"; };
 run_fbgemm_gpu_tests "${BUILD_ENV_NAME}" "${CPU_GPU}"
diff --git a/.github/scripts/nova_prescript.bash b/.github/scripts/nova_prescript.bash
@@ -5,71 +5,91 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-echo "Current working directory: $(pwd)"
-cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}"
+export PATH="${PATH}:/usr/sbin:/sbin"
+
+echo "[NOVA] Current working directory: $(pwd)"
+cd "${FBGEMM_REPO}" || exit 1
+
 PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash"
 BUILD_ENV_NAME=${CONDA_ENV}
-echo "--------------------------"
-echo "----- conda env list -----"
-conda env list
-echo "--------------------------"
-echo "PRELUDE = $PRELUDE"
-export PATH="${PATH}:/usr/sbin:/sbin"
-echo "CU_VERSION = ${CU_VERSION}"
-echo "PYTHON_VERSION = ${PYTHON_VERSION}"
-echo "python3 --version = $(python3 --version)"
-echo "ARCH = ${ARCH}"
-echo "---------------------------"
+
+# Load the FBGEMM_GPU build scripts infrastructure
 # shellcheck disable=SC1091
 # shellcheck source=.github/scripts/setup_env.bash
 . "${PRELUDE}";
 
-## Display System Info
+# Display System Info
 print_system_info
 
-## Display GPU Info
+# Display Conda information
+print_conda_info
+
+# Display GPU Info
 print_gpu_info
 
-## Install C/C++ Compilers
+# Install C/C++ Compilers
 install_cxx_compiler "${BUILD_ENV_NAME}"
 
-## Install Build Tools
+# Install Build Tools
 install_build_tools "${BUILD_ENV_NAME}"
 
-## Install cuDNN
-CPU_GPU=${CU_VERSION}
-if [ "${CU_VERSION}" != 'cpu' ]; then
-    ## Nova $CU_VERSION is e.g., cu118
-    cuda_version_num=$(echo "$CU_VERSION" | cut -c 3-)
-    install_cudnn "${BUILD_ENV_NAME}" "$(pwd)/build_only/cudnn" "$cuda_version_num"
-    echo "-------- Finding NVML_LIB_PATH -----------"
-    echo "NVML_LIB_PATH = ${NVML_LIB_PATH}"
-    echo "CONDA_ENV = ${CONDA_ENV}, CUDA_HOME = ${CUDA_HOME}"
-    if [[ ${NVML_LIB_PATH} == "" ]]; then NVML_LIB_PATH=$(find "${CUDA_HOME}" -name libnvidia-ml.so) && export NVML_LIB_PATH && echo "looking in ${CUDA_HOME}" || echo "libnvidia-ml.so not found in ${CUDA_HOME}"; fi
-    if [[ ${NVML_LIB_PATH} == "" ]]; then NVML_LIB_PATH=$(find "${CONDA_ENV}" -name libnvidia-ml.so) && export NVML_LIB_PATH && echo "looking in ${CONDA_ENV}" || echo "libnvidia-ml.so not found in ${CONDA_ENV}"; fi
-    echo "NVML_LIB_PATH = ${NVML_LIB_PATH}"
-    echo "------------------------------------------"
-    CPU_GPU="cuda"
+if [[ $CU_VERSION = cu* ]]; then
+  # Extract the CUDA version number from CU_VERSION
+  cuda_version=$(echo "[NOVA] ${CU_VERSION}" | cut -c 3-)
+  install_cudnn "${BUILD_ENV_NAME}" "$(pwd)/build_only/cudnn" "${cuda_version}"
+
+  echo "[NOVA] -------- Finding NVML_LIB_PATH -----------"
+  if [[ ${NVML_LIB_PATH} == "" ]]; then
+    NVML_LIB_PATH=$(find "${CUDA_HOME}" -name libnvidia-ml.so) &&
+    export NVML_LIB_PATH &&
+    echo "[NOVA] looking in ${CUDA_HOME}" ||
+    echo "[NOVA] libnvidia-ml.so not found in ${CUDA_HOME}";
+  fi
+
+  if [[ ${NVML_LIB_PATH} == "" ]]; then
+    NVML_LIB_PATH=$(find "${CONDA_ENV}" -name libnvidia-ml.so) &&
+    export NVML_LIB_PATH &&
+    echo "[NOVA] looking in ${CONDA_ENV}" ||
+    echo "[NOVA] libnvidia-ml.so not found in ${CONDA_ENV}";
+  fi
+
+  echo "[NOVA] NVML_LIB_PATH = ${NVML_LIB_PATH}"
+  echo "[NOVA] ------------------------------------------"
+
+  echo "[NOVA] Building the CUDA variant of FBGEMM_GPU ..."
+  fbgemm_variant="cuda"
+
+elif [[ $CU_VERSION = rocm* ]]; then
+  echo "[NOVA] Building the ROCm variant of FBGEMM_GPU ..."
+  fbgemm_variant="rocm"
+
+else
+  echo "[NOVA] Building the CPU variant of FBGEMM_GPU ..."
+  fbgemm_variant="cpu"
 fi
 
-cd "${FBGEMM_REPO}/fbgemm_gpu" || { echo "Failed to cd to fbgemm_gpu from $(pwd)"; }
+# Install the necessary Python eggs for building
+cd "${FBGEMM_REPO}/fbgemm_gpu" || exit 1
 prepare_fbgemm_gpu_build "${BUILD_ENV_NAME}"
 
-# reset NOVA flag to run setup.py
+# Reset the BUILD_FROM_NOVA flag to run setup.py for the actual build
 BUILD_FROM_NOVA=0
 export BUILD_FROM_NOVA
 
-## Build FBGEMM_GPU Nightly
-cd "${FBGEMM_REPO}/fbgemm_gpu" || echo "Failed to cd to ${FBGEMM_REPO}/fbgemm_gpu from $(pwd)"
-if [[ ${CHANNEL} == "" ]]; then CHANNEL="nightly"; fi #set nightly by default
-echo "----------------------------------------------"
-echo "build_fbgemm_gpu_package ${BUILD_ENV_NAME} ${CHANNEL} ${CPU_GPU}"
-build_fbgemm_gpu_package "${BUILD_ENV_NAME}" "${CHANNEL}" "${CPU_GPU}"
-echo "----------------------------------------------"
-
-## Temporary workaround - copy dist/ to root repo for smoke test
-echo "Copying dist folder to root repo.."
-(cp -r "${FBGEMM_REPO}/fbgemm_gpu/dist" "${FBGEMM_REPO}") && (echo "dist folder has been copied to ${FBGEMM_REPO}") || echo "Failed to copy dist/ folder to ${FBGEMM_REPO}"
-echo "----------------------------------"
-ls -al "${FBGEMM_REPO}/dist"
-echo "----------------------------------"
+# Build FBGEMM_GPU nightly by default
+if [[ ${CHANNEL} == "" ]]; then
+  CHANNEL="nightly"
+fi
+
+# Build the wheel
+build_fbgemm_gpu_package "${BUILD_ENV_NAME}" "${CHANNEL}" "${fbgemm_variant}"
+
+# Temporary workaround - copy dist/ to root repo for smoke test
+echo "[NOVA] Copying dist folder to root repo ..."
+if print_exec cp -r "${FBGEMM_REPO}/fbgemm_gpu/dist" "${FBGEMM_REPO}"; then
+  echo "[NOVA] dist folder has been copied to ${FBGEMM_REPO}"
+  ls -al "${FBGEMM_REPO}/dist"
+else
+  echo "[NOVA] Failed to copy dist/ folder to ${FBGEMM_REPO}"
+  exit 1
+fi
diff --git a/.github/scripts/utils_build.bash b/.github/scripts/utils_build.bash
@@ -18,7 +18,7 @@ setup_bazel () {
   echo "################################################################################"
   echo "# Setup Bazel"
   echo "#"
-  echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+  echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
   echo "################################################################################"
   echo ""
 
@@ -54,7 +54,7 @@ install_cxx_compiler () {
     echo "################################################################################"
     echo "# Install C/C++ Compilers"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -149,7 +149,7 @@ install_build_tools () {
     echo "################################################################################"
     echo "# Install Build Tools"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi

diff --git a/.github/scripts/utils_conda.bash b/.github/scripts/utils_conda.bash
@@ -30,7 +30,7 @@ setup_miniconda () {
     echo "################################################################################"
     echo "# Setup Miniconda"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -88,7 +88,7 @@ create_conda_environment () {
     echo "################################################################################"
     echo "# Create Conda Environment"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -132,3 +132,19 @@ create_conda_environment () {
   echo "[SETUP] Installed Python version: $(conda run ${env_prefix} python --version)"
   echo "[SETUP] Successfully created Conda environment: ${env_name}"
 }
+
+print_conda_info () {
+  echo "################################################################################"
+  echo "# Print Conda Environment Info"
+  echo "#"
+  echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
+  echo "################################################################################"
+  echo ""
+  print_exec conda info
+  echo ""
+  print_exec conda info --envs
+  echo ""
+  # shellcheck disable=SC2153
+  echo "PYTHON_VERSION:     ${PYTHON_VERSION}"
+  echo "python3 --version:  $(python3 --version)"
+}