Bazel and docs fixes (pytorch#2823)

Summary: X-link: fairinternal/wav2letter#12 X-link: flashlight/wav2letter#1023 X-link: facebookresearch/FBGEMM#23 Pull Request resolved: pytorch#2823 - Fix Bazel build for FBGEMM - Update documentation for FBGEMM_GPU Reviewed By: spcyppt Differential Revision: D59620856 fbshipit-source-id: 9ec0920bbd8096ec1875a9b1b9969caf75fb8e65
q10 · Jul 11, 2024 · 98fa998 · 98fa998
1 parent 8cf6133
commit 98fa998
Show file tree

Hide file tree

Showing 16 changed files with 35 additions and 36 deletions.
diff --git a/.github/scripts/fbgemm_gpu_install.bash b/.github/scripts/fbgemm_gpu_install.bash
@@ -178,9 +178,9 @@ install_fbgemm_gpu_pip () {
     echo "Usage: ${FUNCNAME[0]} ENV_NAME FBGEMM_GPU_CHANNEL[/VERSION] FBGEMM_GPU_VARIANT_TYPE[/VARIANT_VERSION]"
     echo "Example(s):"
     echo "    ${FUNCNAME[0]} build_env 0.5.0 cpu                  # Install the CPU variant, specific version from release channel"
-    echo "    ${FUNCNAME[0]} build_env release cuda 12.4.1        # Install the CUDA variant, latest version from release channel"
-    echo "    ${FUNCNAME[0]} build_env test/0.8.0rc0 cuda 12.4.1  # Install the CUDA 12.4 variant, specific version from test channel"
-    echo "    ${FUNCNAME[0]} build_env nightly rocm 5.3           # Install the ROCM 5.3 variant, latest version from nightly channel"
+    echo "    ${FUNCNAME[0]} build_env release cuda/12.4.1        # Install the CUDA variant, latest version from release channel"
+    echo "    ${FUNCNAME[0]} build_env test/0.8.0rc0 cuda/12.4.1  # Install the CUDA 12.4 variant, specific version from test channel"
+    echo "    ${FUNCNAME[0]} build_env nightly rocm/5.3           # Install the ROCM 5.3 variant, latest version from nightly channel"
     return 1
   else
     echo "################################################################################"

diff --git a/.gitmodules b/.gitmodules
@@ -1,15 +1,15 @@
-[submodule "third_party/asmjit"]
-	path = third_party/asmjit
+[submodule "external/asmjit"]
+	path = external/asmjit
 	url = https://github.com/asmjit/asmjit.git
-[submodule "third_party/cpuinfo"]
-	path = third_party/cpuinfo
+[submodule "external/cpuinfo"]
+	path = external/cpuinfo
 	url = https://github.com/pytorch/cpuinfo
-[submodule "third_party/googletest"]
-	path = third_party/googletest
+[submodule "external/googletest"]
+	path = external/googletest
 	url = https://github.com/google/googletest
-[submodule "third_party/hipify_torch"]
-	path = third_party/hipify_torch
+[submodule "external/hipify_torch"]
+	path = external/hipify_torch
 	url = https://github.com/ROCmSoftwarePlatform/hipify_torch.git
-[submodule "third_party/cutlass"]
-	path = third_party/cutlass
+[submodule "external/cutlass"]
+	path = external/cutlass
 	url = https://github.com/NVIDIA/cutlass.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -112,7 +112,7 @@ endif()
 
 set(FBGEMM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(FBGEMM_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FBGEMM_THIRDPARTY_DIR ${FBGEMM_BINARY_DIR}/third_party)
+set(FBGEMM_THIRDPARTY_DIR ${FBGEMM_BINARY_DIR}/external)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # Add address sanitizer
@@ -249,7 +249,7 @@ message(WARNING "==========")
 if(NOT TARGET asmjit)
   # Download asmjit from github if ASMJIT_SRC_DIR is not specified.
   if(NOT DEFINED ASMJIT_SRC_DIR)
-    set(ASMJIT_SRC_DIR "${FBGEMM_SOURCE_DIR}/third_party/asmjit"
+    set(ASMJIT_SRC_DIR "${FBGEMM_SOURCE_DIR}/external/asmjit"
       CACHE STRING "asmjit source directory from submodules")
   endif()
 
@@ -282,7 +282,7 @@ endif()
 if(NOT TARGET cpuinfo)
   #Download cpuinfo from github if CPUINFO_SOURCE_DIR is not specified.
   if(NOT DEFINED CPUINFO_SOURCE_DIR)
-    set(CPUINFO_SOURCE_DIR "${FBGEMM_SOURCE_DIR}/third_party/cpuinfo"
+    set(CPUINFO_SOURCE_DIR "${FBGEMM_SOURCE_DIR}/external/cpuinfo"
       CACHE STRING "cpuinfo source directory from submodules")
   endif()
 

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -0,0 +1,8 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+module(name = "fbgemm")
+
+bazel_dep(name = "bazel_skylib", version = "1.5.0")
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
@@ -7,13 +7,6 @@ workspace(name = "fbgemm")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-http_archive(
-    name = "bazel_skylib",
-    urls = [
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
-    ],
-)
-
 http_archive(
     name = "com_google_googletest",
     strip_prefix = "googletest-1.14.0",
@@ -22,14 +15,17 @@ http_archive(
     ],
 )
 
+# NOTE: Starting with Bazel 7, third-party libraries need to be in external/
+# https://github.com/bazelbuild/bazel/issues/19963
+
 new_local_repository(
     name = "cpuinfo",
-    build_file = "third_party/cpuinfo.BUILD",
-    path = "third_party/cpuinfo",
+    build_file = "@//external:cpuinfo.BUILD",
+    path = "external/cpuinfo",
 )
 
 new_local_repository(
     name = "asmjit",
-    build_file = "third_party/asmjit.BUILD",
-    path = "third_party/asmjit",
+    build_file = "@//external:asmjit.BUILD",
+    path = "external/asmjit",
 )
diff --git a/third_party/asmjit → external/asmjit b/third_party/asmjit → external/asmjit
diff --git a/third_party/asmjit.BUILD → external/asmjit.BUILD b/third_party/asmjit.BUILD → external/asmjit.BUILD
diff --git a/third_party/cpuinfo → external/cpuinfo b/third_party/cpuinfo → external/cpuinfo
diff --git a/third_party/cpuinfo.BUILD → external/cpuinfo.BUILD b/third_party/cpuinfo.BUILD → external/cpuinfo.BUILD
diff --git a/third_party/cutlass → external/cutlass b/third_party/cutlass → external/cutlass
diff --git a/third_party/googletest → external/googletest b/third_party/googletest → external/googletest
diff --git a/third_party/hipify_torch → external/hipify_torch b/third_party/hipify_torch → external/hipify_torch
diff --git a/fbgemm_gpu/CMakeLists.txt b/fbgemm_gpu/CMakeLists.txt
@@ -12,7 +12,7 @@ cmake_minimum_required(VERSION 3.25.0 FATAL_ERROR)
 
 set(CMAKEMODULES ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules)
 set(FBGEMM ${CMAKE_CURRENT_SOURCE_DIR}/..)
-set(THIRDPARTY ${FBGEMM}/third_party)
+set(THIRDPARTY ${FBGEMM}/external)
 
 include(${CMAKEMODULES}/Utilities.cmake)
 
@@ -106,7 +106,7 @@ endif()
 if(NOT FBGEMM_CPU_ONLY AND NOT USE_ROCM)
   # CUTLASS currently doesn't build on ROCm and CK hasnt yet been added:
   #
-  # 2024-05-06T23:09:35.5730483Z /__w/FBGEMM/FBGEMM/fbgemm_gpu/../third_party/cutlass/include/cutlass/half.h:73:10: fatal error: 'cuda_fp16.h' file not found
+  # 2024-05-06T23:09:35.5730483Z /__w/FBGEMM/FBGEMM/fbgemm_gpu/../external/cutlass/include/cutlass/half.h:73:10: fatal error: 'cuda_fp16.h' file not found
   # #include <cuda_fp16.h>
   #
   add_subdirectory(experimental/gen_ai)

diff --git a/fbgemm_gpu/FbgemmGpu.cmake b/fbgemm_gpu/FbgemmGpu.cmake
@@ -39,7 +39,7 @@ set(fbgemm_sources_include_directories
 ################################################################################
 
 file(GLOB_RECURSE asmjit_sources
-  "${CMAKE_CURRENT_SOURCE_DIR}/../third_party/asmjit/src/asmjit/*/*.cpp")
+  "${CMAKE_CURRENT_SOURCE_DIR}/../external/asmjit/src/asmjit/*/*.cpp")
 
 set(third_party_include_directories
   ${THIRDPARTY}/asmjit/src

diff --git a/fbgemm_gpu/docs/src/fbgemm_gpu-development/BuildInstructions.rst b/fbgemm_gpu/docs/src/fbgemm_gpu-development/BuildInstructions.rst
@@ -528,7 +528,6 @@ For CPU-only builds, the ``--cpu_only`` flag needs to be specified.
   # Build the wheel artifact only
   python setup.py bdist_wheel \
       --package_variant=cpu \
-      --package_name="${package_name}" \
       --python-tag="${python_tag}" \
       --plat-name="${python_plat_name}"
 
@@ -546,7 +545,6 @@ To build using Clang + ``libstdc++`` instead of GCC, simply append the
   # Build the wheel artifact only
   python setup.py bdist_wheel \
       --package_variant=cpu \
-      --package_name="${package_name}" \
       --python-tag="${python_tag}" \
       --plat-name="${python_plat_name}" \
       --cxxprefix=$CONDA_PREFIX
@@ -607,7 +605,6 @@ toolchains have been properly installed.
   # Build the wheel artifact only
   python setup.py bdist_wheel \
       --package_variant=cuda \
-      --package_name="${package_name}" \
       --python-tag="${python_tag}" \
       --plat-name="${python_plat_name}" \
       --nvml_lib_path=${NVML_LIB_PATH} \
@@ -636,7 +633,6 @@ experimental modules are the same as those for a CUDA build, but with specifying
   # Build the wheel artifact only
   python setup.py bdist_wheel \
       --package_variant=genai \
-      --package_name="${package_name}" \
       --python-tag="${python_tag}" \
       --plat-name="${python_plat_name}" \
       --nvml_lib_path=${NVML_LIB_PATH} \
@@ -678,7 +674,6 @@ presuming the toolchains have been properly installed.
   # Build the wheel artifact only
   python setup.py bdist_wheel \
       --package_variant=rocm \
-      --package_name="${package_name}" \
       --python-tag="${python_tag}" \
       --plat-name="${python_plat_name}" \
       -DHIP_ROOT_DIR="${ROCM_PATH}" \

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -18,7 +18,7 @@ if(FBGEMM_BUILD_TESTS AND NOT TARGET gtest)
   #Download Googletest framework from github if
   #GOOGLETEST_SOURCE_DIR is not specified.
   if(NOT DEFINED GOOGLETEST_SOURCE_DIR)
-    set(GOOGLETEST_SOURCE_DIR "${FBGEMM_SOURCE_DIR}/third_party/googletest"
+    set(GOOGLETEST_SOURCE_DIR "${FBGEMM_SOURCE_DIR}/external/googletest"
       CACHE STRING "googletest source directory from submodules")
   endif()