Skip to content

Commit

Permalink
Use PyTorch PIP for workflows (pytorch#2025)
Browse files Browse the repository at this point in the history 
Summary:
- Re-organize bash scripts for easier readability

- Use PyTorch PIP to install fbgemm_gpu for PIP-install workflows

Pull Request resolved: pytorch#2025

Reviewed By: spcyppt

Differential Revision: D49342654

Pulled By: q10

fbshipit-source-id: fe9f85f568dcba6b2992f51df6a4a094009364d7
  • Loading branch information
@q10 @facebook-github-bot
q10 authored and facebook-github-bot committed Sep 16, 2023
1 parent 8e588e6 commit eb1103b
Show file tree
Hide file tree
Showing 5 changed files with 348 additions and 343 deletions.
68 changes: 6 additions & 62 deletions .github/scripts/fbgemm_gpu_install.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_pip.bash"

################################################################################
# FBGEMM_GPU Install Functions
Expand Down Expand Up @@ -78,72 +80,14 @@ install_fbgemm_gpu_pip () {
echo ""
fi

test_network_connection || return 1

# Set the package variant
if [ "$fbgemm_gpu_variant_type" == "cuda" ]; then
# Extract the CUDA version or default to 11.8.0
local cuda_version="${fbgemm_gpu_variant_version:-11.8.0}"
# shellcheck disable=SC2206
local cuda_version_arr=(${cuda_version//./ })
# Convert, i.e. cuda 11.7.1 => cu117
local fbgemm_gpu_variant="cu${cuda_version_arr[0]}${cuda_version_arr[1]}"
elif [ "$fbgemm_gpu_variant_type" == "rocm" ]; then
# Extract the ROCM version or default to 5.5.1
local rocm_version="${fbgemm_gpu_variant_version:-5.5.1}"
# shellcheck disable=SC2206
local rocm_version_arr=(${rocm_version//./ })
# Convert, i.e. rocm 5.5.1 => rocm5.5
local fbgemm_gpu_variant="rocm${rocm_version_arr[0]}.${rocm_version_arr[1]}"
else
local fbgemm_gpu_variant_type="cpu"
local fbgemm_gpu_variant="cpu"
fi
echo "[INSTALL] Extracted FBGEMM-GPU variant: ${fbgemm_gpu_variant}"

# Set the package name and installation channel
# if [ "$fbgemm_gpu_version" == "nightly" ] || [ "$fbgemm_gpu_version" == "test" ]; then
# local fbgemm_gpu_package="--pre fbgemm-gpu"
# local fbgemm_gpu_channel="https://download.pytorch.org/whl/${fbgemm_gpu_version}/${fbgemm_gpu_variant}/"
# elif [ "$fbgemm_gpu_version" == "latest" ]; then
# local fbgemm_gpu_package="fbgemm-gpu"
# local fbgemm_gpu_channel="https://download.pytorch.org/whl/${fbgemm_gpu_variant}/"
# else
# local fbgemm_gpu_package="fbgemm-gpu==${fbgemm_gpu_version}+${fbgemm_gpu_variant}"
# local fbgemm_gpu_channel="https://download.pytorch.org/whl/${fbgemm_gpu_variant}/"
# fi

if [ "$fbgemm_gpu_variant_type" == "cuda" ]; then
if [ "$fbgemm_gpu_version" == "nightly" ]; then
local fbgemm_gpu_package="fbgemm-gpu-nightly"
elif [ "$fbgemm_gpu_version" == "latest" ]; then
local fbgemm_gpu_package="fbgemm-gpu"
else
local fbgemm_gpu_package="fbgemm-gpu==${fbgemm_gpu_version}"
fi

elif [ "$fbgemm_gpu_variant_type" == "rocm" ]; then
echo "ROCm is currently not supported in PyPI!"
return 1

else
if [ "$fbgemm_gpu_version" == "nightly" ]; then
local fbgemm_gpu_package="fbgemm-gpu-nightly-cpu"
elif [ "$fbgemm_gpu_version" == "latest" ]; then
local fbgemm_gpu_package="fbgemm-gpu-cpu"
else
local fbgemm_gpu_package="fbgemm-gpu-cpu==${fbgemm_gpu_version}"
fi
fi

# shellcheck disable=SC2155
local env_prefix=$(env_name_or_prefix "${env_name}")

echo "[INSTALL] Attempting to install FBGEMM-GPU ${fbgemm_gpu_version}+${fbgemm_gpu_variant} through PIP ..."
# shellcheck disable=SC2086
(exec_with_retries conda run ${env_prefix} pip install ${fbgemm_gpu_package}) || return 1
# Install the package from PyTorch PIP (not PyPI)
install_from_pytorch_pip "${env_name}" fbgemm_gpu "${fbgemm_gpu_version}" "${fbgemm_gpu_variant_type}" "${fbgemm_gpu_variant_version}" || return 1

# Run post-installation checks
__fbgemm_gpu_post_install_checks || return 1

echo "[INSTALL] FBGEMM-GPU installation through PIP completed ..."
echo "[INSTALL] Successfully installed FBGEMM-GPU through PyTorch PIP"
}
229 changes: 4 additions & 225 deletions .github/scripts/setup_env.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,14 @@
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_system.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_build.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_conda.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_cuda.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_pip.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_rocm.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/utils_pytorch.bash"
Expand All @@ -27,228 +31,3 @@
. "$( dirname -- "$BASH_SOURCE"; )/fbgemm_gpu_lint.bash"
# shellcheck disable=SC1091,SC2128
. "$( dirname -- "$BASH_SOURCE"; )/fbgemm_gpu_test.bash"

################################################################################
# Bazel Setup Functions
################################################################################

setup_bazel () {
local bazel_version="${1:-6.1.1}"
echo "################################################################################"
echo "# Setup Bazel"
echo "#"
echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
echo "################################################################################"
echo ""

test_network_connection || return 1

local bazel_variant="$PLATFORM_NAME_LC"
echo "[SETUP] Downloading installer Bazel ${bazel_version} (${bazel_variant}) ..."
print_exec wget -q "https://github.com/bazelbuild/bazel/releases/download/${bazel_version}/bazel-${bazel_version}-installer-${bazel_variant}.sh" -O install-bazel.sh

echo "[SETUP] Installing Bazel ..."
print_exec bash install-bazel.sh
print_exec rm -f install-bazel.sh

print_exec bazel --version
echo "[SETUP] Successfully set up Bazel"
}


################################################################################
# Build Tools Setup Functions
################################################################################

install_cxx_compiler () {
local env_name="$1"
local use_system_package_manager="$2"
if [ "$env_name" == "" ]; then
echo "Usage: ${FUNCNAME[0]} ENV_NAME [USE_YUM]"
echo "Example(s):"
echo " ${FUNCNAME[0]} build_env # Install C/C++ compilers through Conda"
echo " ${FUNCNAME[0]} build_env 1 # Install C/C++ compilers through the system package manager"
return 1
else
echo "################################################################################"
echo "# Install C/C++ Compilers"
echo "#"
echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
echo "################################################################################"
echo ""
fi

test_network_connection || return 1

if [ "$use_system_package_manager" != "" ]; then
echo "[INSTALL] Installing C/C++ compilers through the system package manager ..."
install_system_packages gcc gcc-c++

else
# Install gxx_linux-<arch> from conda-forge instead of from anaconda channel.
# sysroot_linux-<arch> needs to be installed alongside this:
#
# https://root-forum.cern.ch/t/error-timespec-get-has-not-been-declared-with-conda-root-package/45712/6
# https://github.com/conda-forge/conda-forge.github.io/issues/1625
# https://conda-forge.org/docs/maintainer/knowledge_base.html#using-centos-7
# https://github.com/conda/conda-build/issues/4371
#
# NOTE: We install g++ 10.x instead of 11.x becaue 11.x builds binaries that
# reference GLIBCXX_3.4.29, which may not be available on systems with older
# versions of libstdc++.so.6 such as CentOS Stream 8 and Ubuntu 20.04
local archname=""
if [ "$MACHINE_NAME_LC" = "x86_64" ]; then
archname="64"
elif [ "$MACHINE_NAME_LC" = "aarch64" ] || [ "$MACHINE_NAME_LC" = "arm64" ]; then
archname="aarch64"
else
archname="$MACHINE_NAME_LC"
fi

# shellcheck disable=SC2155
local env_prefix=$(env_name_or_prefix "${env_name}")

echo "[INSTALL] Installing C/C++ compilers through Conda (architecture = ${archname}) ..."
# shellcheck disable=SC2086
(exec_with_retries conda install ${env_prefix} -y "gxx_linux-${archname}"=10.4.0 "sysroot_linux-${archname}"=2.17 -c conda-forge) || return 1

# The compilers are visible in the PATH as `x86_64-conda-linux-gnu-cc` and
# `x86_64-conda-linux-gnu-c++`, so symlinks will need to be created
echo "[INSTALL] Setting the C/C++ compiler symlinks ..."
# shellcheck disable=SC2155,SC2086
local cc_path=$(conda run ${env_prefix} printenv CC)
# shellcheck disable=SC2155,SC2086
local cxx_path=$(conda run ${env_prefix} printenv CXX)

print_exec ln -s "${cc_path}" "$(dirname "$cc_path")/cc"
print_exec ln -s "${cc_path}" "$(dirname "$cc_path")/gcc"
print_exec ln -s "${cxx_path}" "$(dirname "$cxx_path")/c++"
print_exec ln -s "${cxx_path}" "$(dirname "$cxx_path")/g++"
fi

# Check C/C++ compilers are visible
(test_binpath "${env_name}" cc) || return 1
(test_binpath "${env_name}" gcc) || return 1
(test_binpath "${env_name}" c++) || return 1
(test_binpath "${env_name}" g++) || return 1

# https://stackoverflow.com/questions/2224334/gcc-dump-preprocessor-defines
echo "[INFO] Printing out all preprocessor defines in the C compiler ..."
# shellcheck disable=SC2086
print_exec conda run ${env_prefix} cc -dM -E -

# https://stackoverflow.com/questions/2224334/gcc-dump-preprocessor-defines
echo "[INFO] Printing out all preprocessor defines in the C++ compiler ..."
# shellcheck disable=SC2086
print_exec conda run ${env_prefix} c++ -dM -E -x c++ -

# Print out the C++ version
# shellcheck disable=SC2086
print_exec conda run ${env_prefix} c++ --version

# https://stackoverflow.com/questions/4991707/how-to-find-my-current-compilers-standard-like-if-it-is-c90-etc
echo "[INFO] Printing the default version of the C standard used by the compiler ..."
print_exec "conda run ${env_prefix} cc -dM -E - | grep __STDC_VERSION__"

# https://stackoverflow.com/questions/2324658/how-to-determine-the-version-of-the-c-standard-used-by-the-compiler
echo "[INFO] Printing the default version of the C++ standard used by the compiler ..."
print_exec "conda run ${env_prefix} c++ -dM -E -x c++ - | grep __cplusplus"

echo "[INSTALL] Successfully installed C/C++ compilers"
}

install_build_tools () {
local env_name="$1"
if [ "$env_name" == "" ]; then
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
echo "Example(s):"
echo " ${FUNCNAME[0]} build_env"
return 1
else
echo "################################################################################"
echo "# Install Build Tools"
echo "#"
echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
echo "################################################################################"
echo ""
fi

test_network_connection || return 1

# shellcheck disable=SC2155
local env_prefix=$(env_name_or_prefix "${env_name}")

echo "[INSTALL] Installing build tools ..."
# shellcheck disable=SC2086
(exec_with_retries conda install ${env_prefix} -y \
click \
cmake \
hypothesis \
jinja2 \
ninja \
numpy \
scikit-build \
wheel) || return 1

# Check binaries are visible in the PAATH
(test_binpath "${env_name}" cmake) || return 1
(test_binpath "${env_name}" ninja) || return 1

# Check Python packages are importable
local import_tests=( click hypothesis jinja2 numpy skbuild wheel )
for p in "${import_tests[@]}"; do
(test_python_import_package "${env_name}" "${p}") || return 1
done

echo "[INSTALL] Successfully installed all the build tools"
}


################################################################################
# PyPI Publish Functions
################################################################################

publish_to_pypi () {
local env_name="$1"
local package_name="$2"
local pypi_token="$3"
if [ "$pypi_token" == "" ]; then
echo "Usage: ${FUNCNAME[0]} ENV_NAME PACKAGE_NAME PYPI_TOKEN"
echo "Example(s):"
echo " ${FUNCNAME[0]} build_env fbgemm_gpu_nightly-*.whl MY_TOKEN"
echo ""
echo "PYPI_TOKEN is missing!"
return 1
else
echo "################################################################################"
echo "# Publish to PyPI"
echo "#"
echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
echo "################################################################################"
echo ""
fi

test_network_connection || return 1

# shellcheck disable=SC2155
local env_prefix=$(env_name_or_prefix "${env_name}")

echo "[INSTALL] Installing twine ..."
# shellcheck disable=SC2086
print_exec conda install ${env_prefix} -y twine
(test_python_import_package "${env_name}" twine) || return 1
(test_python_import_package "${env_name}" OpenSSL) || return 1

echo "[PUBLISH] Uploading package(s) to PyPI: ${package_name} ..."
# shellcheck disable=SC2086
conda run ${env_prefix} \
python -m twine upload \
--username __token__ \
--password "${pypi_token}" \
--skip-existing \
--verbose \
"${package_name}"

echo "[PUBLISH] Successfully published package(s) to PyPI: ${package_name}"
echo "[PUBLISH] NOTE: The publish command is a successful no-op if the wheel version already existed in PyPI; please double check!"
}
Loading

0 comments on commit eb1103b

Please sign in to comment.