From 8f02dd3a6523f139090cdc6b9eaa0b993705895f Mon Sep 17 00:00:00 2001 From: Benson Ma Date: Thu, 21 Sep 2023 04:47:10 -0700 Subject: [PATCH] Use cron schedule for PIP install workflows (#2027) Summary: - Use a cron schedule for the PIP install workflows - Activate building wheels for ROCm variant Pull Request resolved: https://github.com/pytorch/FBGEMM/pull/2027 Reviewed By: spcyppt Differential Revision: D49484270 Pulled By: q10 fbshipit-source-id: 72a50b89c50d6f82c02f4a673f51abbc683e45ca --- .github/scripts/fbgemm_gpu_build.bash | 26 ++-- .github/scripts/fbgemm_gpu_docs.bash | 4 +- .github/scripts/fbgemm_gpu_install.bash | 4 +- .github/scripts/fbgemm_gpu_lint.bash | 8 +- .github/scripts/fbgemm_gpu_test.bash | 4 +- .github/scripts/nova_postscript.bash | 12 +- .github/scripts/nova_prescript.bash | 114 ++++++++++-------- .github/scripts/utils_build.bash | 6 +- .github/scripts/utils_conda.bash | 20 ++- .github/scripts/utils_cuda.bash | 4 +- .github/scripts/utils_pip.bash | 4 +- .github/scripts/utils_pytorch.bash | 4 +- .github/scripts/utils_rocm.bash | 2 +- .github/scripts/utils_system.bash | 6 +- .../workflows/build_wheels_linux_aarch64.yml | 4 +- .github/workflows/build_wheels_linux_x86.yml | 4 + .../workflows/build_wheels_linux_x86_rocm.yml | 51 ++++++++ .github/workflows/fbgemm_gpu_pip.yml | 16 +-- 18 files changed, 195 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/build_wheels_linux_x86_rocm.yml diff --git a/.github/scripts/fbgemm_gpu_build.bash b/.github/scripts/fbgemm_gpu_build.bash index 12b3c49ca..5354c24ab 100644 --- a/.github/scripts/fbgemm_gpu_build.bash +++ b/.github/scripts/fbgemm_gpu_build.bash @@ -24,7 +24,7 @@ prepare_fbgemm_gpu_build () { echo "################################################################################" echo "# Prepare FBGEMM-GPU Build" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -154,7 +154,7 @@ __configure_fbgemm_gpu_build () { echo "################################################################################" echo "# Configure FBGEMM-GPU Build" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -310,7 +310,7 @@ build_fbgemm_gpu_package () { echo "################################################################################" echo "# Build FBGEMM-GPU Package (Wheel)" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" @@ -322,13 +322,15 @@ build_fbgemm_gpu_package () { echo "[BUILD] Checking build_args:" echo "${build_args[@]}" - core=$(lscpu | grep "Core(s)" | awk '{print $NF}') && echo "core = ${core}" || echo "core not found" - sockets=$(lscpu | grep "Socket(s)" | awk '{print $NF}') && echo "sockets = ${sockets}" || echo "sockets not found" - re='^[0-9]+$' - run_multicore="" + # shellcheck disable=SC2155 + local core=$(lscpu | grep "Core(s)" | awk '{print $NF}') && echo "core = ${core}" || echo "core not found" + # shellcheck disable=SC2155 + local sockets=$(lscpu | grep "Socket(s)" | awk '{print $NF}') && echo "sockets = ${sockets}" || echo "sockets not found" + local re='^[0-9]+$' + local run_multicore="" if [[ $core =~ $re && $sockets =~ $re ]] ; then - n_core=$((core * sockets)) - run_multicore=" -j ${n_core}" + local n_core=$((core * sockets)) + local run_multicore=" -j ${n_core}" fi # Distribute Python extensions as wheels on Linux @@ -380,7 +382,7 @@ build_fbgemm_gpu_install () { echo "################################################################################" echo "# Build + Install FBGEMM-GPU Package" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" @@ -426,9 +428,9 @@ build_fbgemm_gpu_develop () { __configure_fbgemm_gpu_build "${fbgemm_variant}" "${fbgemm_variant_targets}" || return 1 echo "################################################################################" - echo "# Build + Install FBGEMM-GPU Package" + echo "# Build + Install FBGEMM-GPU Package (Develop)" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" diff --git a/.github/scripts/fbgemm_gpu_docs.bash b/.github/scripts/fbgemm_gpu_docs.bash index d2b21f564..0e923afb1 100644 --- a/.github/scripts/fbgemm_gpu_docs.bash +++ b/.github/scripts/fbgemm_gpu_docs.bash @@ -24,7 +24,7 @@ install_docs_tools () { echo "################################################################################" echo "# Install Documentation Tools" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -61,7 +61,7 @@ build_fbgemm_gpu_docs () { echo "################################################################################" echo "# Build FBGEMM-GPU Documentation" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/fbgemm_gpu_install.bash b/.github/scripts/fbgemm_gpu_install.bash index e78ca79bd..02e60e8e9 100644 --- a/.github/scripts/fbgemm_gpu_install.bash +++ b/.github/scripts/fbgemm_gpu_install.bash @@ -43,7 +43,7 @@ install_fbgemm_gpu_wheel () { echo "################################################################################" echo "# Install FBGEMM-GPU from Wheel" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -79,7 +79,7 @@ install_fbgemm_gpu_pip () { echo "################################################################################" echo "# Install FBGEMM-GPU Package from PIP" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/fbgemm_gpu_lint.bash b/.github/scripts/fbgemm_gpu_lint.bash index 122d54786..fc2ab7d25 100644 --- a/.github/scripts/fbgemm_gpu_lint.bash +++ b/.github/scripts/fbgemm_gpu_lint.bash @@ -24,7 +24,7 @@ install_lint_tools () { echo "################################################################################" echo "# Install Lint Tools" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -69,7 +69,7 @@ lint_fbgemm_gpu_flake8 () { echo "################################################################################" echo "# Run FBGEMM_GPU Lint: flake8" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -99,7 +99,7 @@ lint_fbgemm_gpu_ufmt () { echo "################################################################################" echo "# Run FBGEMM_GPU Lint: ufmt" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -132,7 +132,7 @@ lint_fbgemm_gpu_copyright () { echo "################################################################################" echo "# Run FBGEMM_GPU Lint: Meta Copyright Headers" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/fbgemm_gpu_test.bash b/.github/scripts/fbgemm_gpu_test.bash index b593caa95..be5a95ab1 100644 --- a/.github/scripts/fbgemm_gpu_test.bash +++ b/.github/scripts/fbgemm_gpu_test.bash @@ -61,7 +61,7 @@ run_fbgemm_gpu_tests () { echo "################################################################################" echo "# Run FBGEMM-GPU Tests" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -143,7 +143,7 @@ test_setup_conda_environment () { echo "################################################################################" echo "# Setup FBGEMM-GPU Build Container (All Steps)" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/nova_postscript.bash b/.github/scripts/nova_postscript.bash index c1e5d5bfd..4602d6bd2 100644 --- a/.github/scripts/nova_postscript.bash +++ b/.github/scripts/nova_postscript.bash @@ -5,15 +5,17 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -echo "Current working directory: $(pwd)" -cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}" +echo "[NOVA] Current working directory: $(pwd)" +cd "${FBGEMM_REPO}" || echo "[NOVA] Failed to cd to ${FBGEMM_REPO}" PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash" BUILD_ENV_NAME=${CONDA_ENV} GITHUB_ENV=TRUE export GITHUB_ENV # Install FBGEMM_GPU Nightly -echo "Current working directory: $(pwd)" +echo "[NOVA] Current working directory: $(pwd)" + +# Load the FBGEMM_GPU build scripts infrastructure # shellcheck disable=SC1091 # shellcheck source=.github/scripts/setup_env.bash . "${PRELUDE}"; @@ -21,11 +23,11 @@ echo "Current working directory: $(pwd)" install_fbgemm_gpu_wheel "${BUILD_ENV_NAME}" fbgemm_gpu/dist/*.whl # Test with PyTest -echo "Current working directory: $(pwd)" +echo "[NOVA] Current working directory: $(pwd)" CPU_GPU="${CU_VERSION}" if [ "${CU_VERSION}" != 'cpu' ]; then CPU_GPU="" fi $CONDA_RUN python3 -c "import torch; print('cuda.is_available() ', torch.cuda.is_available()); print ('device_count() ',torch.cuda.device_count());" -cd "${FBGEMM_REPO}/fbgemm_gpu/test" || { echo "Failed to cd to fbgemm_gpu/test from $(pwd)"; }; +cd "${FBGEMM_REPO}/fbgemm_gpu/test" || { echo "[NOVA] Failed to cd to fbgemm_gpu/test from $(pwd)"; }; run_fbgemm_gpu_tests "${BUILD_ENV_NAME}" "${CPU_GPU}" diff --git a/.github/scripts/nova_prescript.bash b/.github/scripts/nova_prescript.bash index 89106525f..9cb9b8400 100644 --- a/.github/scripts/nova_prescript.bash +++ b/.github/scripts/nova_prescript.bash @@ -5,71 +5,91 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -echo "Current working directory: $(pwd)" -cd "${FBGEMM_REPO}" || echo "Failed to cd to ${FBGEMM_REPO}" +export PATH="${PATH}:/usr/sbin:/sbin" + +echo "[NOVA] Current working directory: $(pwd)" +cd "${FBGEMM_REPO}" || exit 1 + PRELUDE="${FBGEMM_REPO}/.github/scripts/setup_env.bash" BUILD_ENV_NAME=${CONDA_ENV} -echo "--------------------------" -echo "----- conda env list -----" -conda env list -echo "--------------------------" -echo "PRELUDE = $PRELUDE" -export PATH="${PATH}:/usr/sbin:/sbin" -echo "CU_VERSION = ${CU_VERSION}" -echo "PYTHON_VERSION = ${PYTHON_VERSION}" -echo "python3 --version = $(python3 --version)" -echo "ARCH = ${ARCH}" -echo "---------------------------" + +# Load the FBGEMM_GPU build scripts infrastructure # shellcheck disable=SC1091 # shellcheck source=.github/scripts/setup_env.bash . "${PRELUDE}"; -## Display System Info +# Display System Info print_system_info -## Display GPU Info +# Display Conda information +print_conda_info + +# Display GPU Info print_gpu_info -## Install C/C++ Compilers +# Install C/C++ Compilers install_cxx_compiler "${BUILD_ENV_NAME}" -## Install Build Tools +# Install Build Tools install_build_tools "${BUILD_ENV_NAME}" -## Install cuDNN -CPU_GPU=${CU_VERSION} -if [ "${CU_VERSION}" != 'cpu' ]; then - ## Nova $CU_VERSION is e.g., cu118 - cuda_version_num=$(echo "$CU_VERSION" | cut -c 3-) - install_cudnn "${BUILD_ENV_NAME}" "$(pwd)/build_only/cudnn" "$cuda_version_num" - echo "-------- Finding NVML_LIB_PATH -----------" - echo "NVML_LIB_PATH = ${NVML_LIB_PATH}" - echo "CONDA_ENV = ${CONDA_ENV}, CUDA_HOME = ${CUDA_HOME}" - if [[ ${NVML_LIB_PATH} == "" ]]; then NVML_LIB_PATH=$(find "${CUDA_HOME}" -name libnvidia-ml.so) && export NVML_LIB_PATH && echo "looking in ${CUDA_HOME}" || echo "libnvidia-ml.so not found in ${CUDA_HOME}"; fi - if [[ ${NVML_LIB_PATH} == "" ]]; then NVML_LIB_PATH=$(find "${CONDA_ENV}" -name libnvidia-ml.so) && export NVML_LIB_PATH && echo "looking in ${CONDA_ENV}" || echo "libnvidia-ml.so not found in ${CONDA_ENV}"; fi - echo "NVML_LIB_PATH = ${NVML_LIB_PATH}" - echo "------------------------------------------" - CPU_GPU="cuda" +if [[ $CU_VERSION = cu* ]]; then + # Extract the CUDA version number from CU_VERSION + cuda_version=$(echo "[NOVA] ${CU_VERSION}" | cut -c 3-) + install_cudnn "${BUILD_ENV_NAME}" "$(pwd)/build_only/cudnn" "${cuda_version}" + + echo "[NOVA] -------- Finding NVML_LIB_PATH -----------" + if [[ ${NVML_LIB_PATH} == "" ]]; then + NVML_LIB_PATH=$(find "${CUDA_HOME}" -name libnvidia-ml.so) && + export NVML_LIB_PATH && + echo "[NOVA] looking in ${CUDA_HOME}" || + echo "[NOVA] libnvidia-ml.so not found in ${CUDA_HOME}"; + fi + + if [[ ${NVML_LIB_PATH} == "" ]]; then + NVML_LIB_PATH=$(find "${CONDA_ENV}" -name libnvidia-ml.so) && + export NVML_LIB_PATH && + echo "[NOVA] looking in ${CONDA_ENV}" || + echo "[NOVA] libnvidia-ml.so not found in ${CONDA_ENV}"; + fi + + echo "[NOVA] NVML_LIB_PATH = ${NVML_LIB_PATH}" + echo "[NOVA] ------------------------------------------" + + echo "[NOVA] Building the CUDA variant of FBGEMM_GPU ..." + fbgemm_variant="cuda" + +elif [[ $CU_VERSION = rocm* ]]; then + echo "[NOVA] Building the ROCm variant of FBGEMM_GPU ..." + fbgemm_variant="rocm" + +else + echo "[NOVA] Building the CPU variant of FBGEMM_GPU ..." + fbgemm_variant="cpu" fi -cd "${FBGEMM_REPO}/fbgemm_gpu" || { echo "Failed to cd to fbgemm_gpu from $(pwd)"; } +# Install the necessary Python eggs for building +cd "${FBGEMM_REPO}/fbgemm_gpu" || exit 1 prepare_fbgemm_gpu_build "${BUILD_ENV_NAME}" -# reset NOVA flag to run setup.py +# Reset the BUILD_FROM_NOVA flag to run setup.py for the actual build BUILD_FROM_NOVA=0 export BUILD_FROM_NOVA -## Build FBGEMM_GPU Nightly -cd "${FBGEMM_REPO}/fbgemm_gpu" || echo "Failed to cd to ${FBGEMM_REPO}/fbgemm_gpu from $(pwd)" -if [[ ${CHANNEL} == "" ]]; then CHANNEL="nightly"; fi #set nightly by default -echo "----------------------------------------------" -echo "build_fbgemm_gpu_package ${BUILD_ENV_NAME} ${CHANNEL} ${CPU_GPU}" -build_fbgemm_gpu_package "${BUILD_ENV_NAME}" "${CHANNEL}" "${CPU_GPU}" -echo "----------------------------------------------" - -## Temporary workaround - copy dist/ to root repo for smoke test -echo "Copying dist folder to root repo.." -(cp -r "${FBGEMM_REPO}/fbgemm_gpu/dist" "${FBGEMM_REPO}") && (echo "dist folder has been copied to ${FBGEMM_REPO}") || echo "Failed to copy dist/ folder to ${FBGEMM_REPO}" -echo "----------------------------------" -ls -al "${FBGEMM_REPO}/dist" -echo "----------------------------------" +# Build FBGEMM_GPU nightly by default +if [[ ${CHANNEL} == "" ]]; then + CHANNEL="nightly" +fi + +# Build the wheel +build_fbgemm_gpu_package "${BUILD_ENV_NAME}" "${CHANNEL}" "${fbgemm_variant}" + +# Temporary workaround - copy dist/ to root repo for smoke test +echo "[NOVA] Copying dist folder to root repo ..." +if print_exec cp -r "${FBGEMM_REPO}/fbgemm_gpu/dist" "${FBGEMM_REPO}"; then + echo "[NOVA] dist folder has been copied to ${FBGEMM_REPO}" + ls -al "${FBGEMM_REPO}/dist" +else + echo "[NOVA] Failed to copy dist/ folder to ${FBGEMM_REPO}" + exit 1 +fi diff --git a/.github/scripts/utils_build.bash b/.github/scripts/utils_build.bash index 12febee99..0d4b886f7 100644 --- a/.github/scripts/utils_build.bash +++ b/.github/scripts/utils_build.bash @@ -18,7 +18,7 @@ setup_bazel () { echo "################################################################################" echo "# Setup Bazel" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" @@ -54,7 +54,7 @@ install_cxx_compiler () { echo "################################################################################" echo "# Install C/C++ Compilers" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -149,7 +149,7 @@ install_build_tools () { echo "################################################################################" echo "# Install Build Tools" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/utils_conda.bash b/.github/scripts/utils_conda.bash index 96aae5dfe..63bf64d0c 100644 --- a/.github/scripts/utils_conda.bash +++ b/.github/scripts/utils_conda.bash @@ -30,7 +30,7 @@ setup_miniconda () { echo "################################################################################" echo "# Setup Miniconda" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -88,7 +88,7 @@ create_conda_environment () { echo "################################################################################" echo "# Create Conda Environment" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -132,3 +132,19 @@ create_conda_environment () { echo "[SETUP] Installed Python version: $(conda run ${env_prefix} python --version)" echo "[SETUP] Successfully created Conda environment: ${env_name}" } + +print_conda_info () { + echo "################################################################################" + echo "# Print Conda Environment Info" + echo "#" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" + echo "################################################################################" + echo "" + print_exec conda info + echo "" + print_exec conda info --envs + echo "" + # shellcheck disable=SC2153 + echo "PYTHON_VERSION: ${PYTHON_VERSION}" + echo "python3 --version: $(python3 --version)" +} diff --git a/.github/scripts/utils_cuda.bash b/.github/scripts/utils_cuda.bash index 705ef8dc2..285c9a5ea 100644 --- a/.github/scripts/utils_cuda.bash +++ b/.github/scripts/utils_cuda.bash @@ -25,7 +25,7 @@ install_cuda () { echo "################################################################################" echo "# Install CUDA" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -93,7 +93,7 @@ install_cudnn () { echo "################################################################################" echo "# Install cuDNN" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/utils_pip.bash b/.github/scripts/utils_pip.bash index 4782632a3..686485711 100644 --- a/.github/scripts/utils_pip.bash +++ b/.github/scripts/utils_pip.bash @@ -31,7 +31,7 @@ install_from_pytorch_pip () { echo "################################################################################" echo "# Install ${package_name_raw} (PyTorch PIP)" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -117,7 +117,7 @@ publish_to_pypi () { echo "################################################################################" echo "# Publish to PyPI" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/utils_pytorch.bash b/.github/scripts/utils_pytorch.bash index 8aaea9f4f..1fb743b65 100644 --- a/.github/scripts/utils_pytorch.bash +++ b/.github/scripts/utils_pytorch.bash @@ -33,7 +33,7 @@ install_pytorch_conda () { echo "################################################################################" echo "# Install PyTorch (Conda)" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi @@ -120,7 +120,7 @@ install_pytorch_pip () { echo "################################################################################" echo "# Install PyTorch (PIP)" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/utils_rocm.bash b/.github/scripts/utils_rocm.bash index 9802fb80f..8efb8128f 100644 --- a/.github/scripts/utils_rocm.bash +++ b/.github/scripts/utils_rocm.bash @@ -28,7 +28,7 @@ install_rocm_ubuntu () { echo "################################################################################" echo "# Install ROCm (Ubuntu)" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" fi diff --git a/.github/scripts/utils_system.bash b/.github/scripts/utils_system.bash index 297559d09..de37ec80e 100644 --- a/.github/scripts/utils_system.bash +++ b/.github/scripts/utils_system.bash @@ -56,7 +56,7 @@ free_disk_space () { echo "################################################################################" echo "# Free Disk Space" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" @@ -168,7 +168,7 @@ print_system_info () { echo "################################################################################" echo "# Print System Info" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" @@ -187,7 +187,7 @@ print_ec2_info () { echo "################################################################################" echo "# Print EC2 Instance Info" echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}" echo "################################################################################" echo "" diff --git a/.github/workflows/build_wheels_linux_aarch64.yml b/.github/workflows/build_wheels_linux_aarch64.yml index 35bb2f42f..87331980d 100644 --- a/.github/workflows/build_wheels_linux_aarch64.yml +++ b/.github/workflows/build_wheels_linux_aarch64.yml @@ -15,8 +15,8 @@ on: workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: generate-matrix: diff --git a/.github/workflows/build_wheels_linux_x86.yml b/.github/workflows/build_wheels_linux_x86.yml index dfb141d6f..76b7db4ab 100644 --- a/.github/workflows/build_wheels_linux_x86.yml +++ b/.github/workflows/build_wheels_linux_x86.yml @@ -14,6 +14,10 @@ on: - v[0-9]+.[0-9]+.[0-9]+ workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main diff --git a/.github/workflows/build_wheels_linux_x86_rocm.yml b/.github/workflows/build_wheels_linux_x86_rocm.yml new file mode 100644 index 000000000..6139e9dcc --- /dev/null +++ b/.github/workflows/build_wheels_linux_x86_rocm.yml @@ -0,0 +1,51 @@ +name: Build x86 Linux Wheels (ROCm) + +on: + pull_request: + push: + branches: + - nightly + - main + # Release candidate branch look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-release+ + tags: + # Release candidate tag look like: v1.11.0-rc1 + - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ + - v[0-9]+.[0-9]+.[0-9]+ + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + generate-matrix: + if: ${{ github.event_name == 'pull_request' || (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) }} + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + with-cuda: disable + with-rocm: enable + with-cpu: disable + build: + needs: generate-matrix + name: pytorch/FBGEMM + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: pytorch/FBGEMM + ref: "" + pre-script: ../.github/scripts/nova_prescript.bash + post-script: ../.github/scripts/nova_postscript.bash + smoke-test-script: "" + env-var-script: .github/scripts/nova_dir.bash + package-name: fbgemm_gpu + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.generate-matrix.outputs.matrix }} + trigger-event: ${{ github.event_name }} + secrets: + AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} + AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/fbgemm_gpu_pip.yml b/.github/workflows/fbgemm_gpu_pip.yml index fafab2949..d0ab55458 100644 --- a/.github/workflows/fbgemm_gpu_pip.yml +++ b/.github/workflows/fbgemm_gpu_pip.yml @@ -6,11 +6,13 @@ name: FBGEMM_GPU PIP Install + Test on: - # PR Trigger (enabled for regression checks and debugging) + # Cron Trigger (UTC) # - pull_request: - branches: - - main + # Based on the the nightly releases schedule in PyTorch infrastructure, the + # wheels are published to PyTorch PIP at around 11:30 UTC every day. + # + schedule: + - cron: '30 12 * * *' # Manual Trigger # @@ -34,7 +36,7 @@ on: jobs: test_pypi_install_cpu: - if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.event.inputs.fbgemm_gpu_variant_type == 'cpu') }} + if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.fbgemm_gpu_variant_type == 'cpu') }} runs-on: ${{ matrix.host-machine.instance }} container: image: amazonlinux:2023 @@ -88,7 +90,7 @@ jobs: test_pypi_install_cuda: - if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.event.inputs.fbgemm_gpu_variant_type == 'cuda') }} + if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.fbgemm_gpu_variant_type == 'cuda') }} runs-on: ${{ matrix.host-machine.instance }} defaults: run: @@ -145,7 +147,7 @@ jobs: test_pypi_install_rocm: - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.fbgemm_gpu_variant_type == 'rocm' }} + if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.fbgemm_gpu_variant_type == 'rocm') }} runs-on: ${{ matrix.host-machine.instance }} container: image: "rocm/dev-ubuntu-20.04:${{ matrix.rocm-version }}-complete"