Draft: FoundationDB 7.3.x released

CMakeLists.txt

@@ -1,4 +1,5 @@
 project(fdb_c C CXX ASM)
+cmake_minimum_required(VERSION 3.27)
 
 function(glob_flow_source VAR_NAME)
     file(GLOB SOURCE_FILES *.c *.cpp *.h)
@@ -13,17 +14,21 @@ function(glob_flow_source VAR_NAME)
     set(${VAR_NAME} "${${VAR_NAME}}" PARENT_SCOPE)
 endfunction()
 
-find_package(Boost REQUIRED)
+find_package(Boost REQUIRED COMPONENTS filesystem)
 find_package(fmt CONFIG REQUIRED)
 find_package(msgpack-cxx CONFIG REQUIRED)
 find_package(OpenSSL REQUIRED)
 find_package(absl CONFIG REQUIRED)
+find_package(RapidJSON CONFIG REQUIRED)
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_CXX_STANDARD 20)
 
 add_compile_definitions(NO_INTELLISENSE)
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}/src)
+
+add_subdirectory(src/contrib/SimpleOpt)
+add_subdirectory(src/contrib/crc32)
+add_subdirectory(src/contrib/libb64)
 add_subdirectory(src/flow)
 add_subdirectory(src/fdbrpc)
 add_subdirectory(src/fdbclient)

examples/basic_rw.cpp

@@ -47,7 +47,7 @@ void wait(std::shared_ptr<FDBFuture> future) {
 int main() {
   using namespace std::chrono_literals;
 
-  assert_fdb_error(fdb_select_api_version(FDB_API_VERSION));
+  assert_fdb_error(fdb_select_api_version(710));
 
   // Start network
   assert_fdb_error(fdb_setup_network());

extractor/Dockerfile

@@ -43,5 +43,22 @@ RUN apt-get update \
         --yes --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
+# Install latest cmake
+ENV CMAKE_VERSION 3.26.3
+ENV CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v3.27.0-rc4/cmake-3.27.0-rc4-linux-x86_64.sh
+RUN wget -O /tmp/cmake-installer "$CMAKE_URL" \
+    && chmod +x /tmp/cmake-installer \
+    && /tmp/cmake-installer --prefix=/usr/local --skip-license \
+    && rm /tmp/cmake-installer
+
+# Install build depends
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+    && apt-get install \
+        liblz4-dev \
+        python3-sphinx \
+        --yes --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+
 ENV CC=clang-15
 ENV CXX=clang++-15

extractor/Makefile

@@ -5,8 +5,7 @@ OUTPUT_DIR_REAL = $(realpath $(OUTPUT_DIR))
 
 BUILDER_DOCKER_FLAGS = \
 	-u $(shell id -u):$(shell id -g) \
-	-v "$(CURDIR)/$(EXTRACT_WORKDIR):$(CURDIR)/$(EXTRACT_WORKDIR)" \
-	-e FOUNDATIONDB_TAG="7.1.41"
+	-v "$(CURDIR)/$(EXTRACT_WORKDIR):$(CURDIR)/$(EXTRACT_WORKDIR)"
 
 .PHONY: all docker-image extract build-tests clean
 

extractor/workdir/cmake/fdbclient.cmake

@@ -3,6 +3,7 @@ glob_flow_source(FDBCLIENT_SRC)
 list(APPEND FDBCLIENT_SRC sha1/SHA1.cpp)
 
 add_library(fdbclient STATIC ${FDBCLIENT_SRC})
+target_include_directories(fdbclient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 target_link_libraries(fdbclient
     PRIVATE
         fdbrpc

extractor/workdir/cmake/fdbrpc.cmake

@@ -4,12 +4,14 @@ add_library(fdbrpc STATIC ${FDBRPC_SRC})
 target_link_libraries(fdbrpc
     PRIVATE
         flow
+        rapidjson
+        crc32
+        libb64
 )
-
-add_library(fdbb64 STATIC libb64/cdecode.c libb64/cencode.c)
-target_link_libraries(fdbrpc PRIVATE fdbb64)
+target_include_directories(fdbrpc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
 add_library(fdbeio STATIC libeio/eio.c)
+target_include_directories(fdbeio PUBLIC libeio/)
 target_compile_definitions(fdbeio PRIVATE USE_UCONTEXT)
 target_compile_options(fdbeio BEFORE PRIVATE -w) # disable warnings for eio
 target_link_libraries(fdbrpc PRIVATE fdbeio)

extractor/workdir/cmake/flow.cmake

@@ -4,9 +4,15 @@ add_library(flow STATIC ${FLOW_SRC})
 target_link_libraries(flow
     PRIVATE
         Boost::headers
+        Boost::filesystem
         fmt::fmt
         absl::debugging
         OpenSSL::SSL
         OpenSSL::Crypto
+        crc32
+        SimpleOpt
 )
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h)
+target_include_directories(flow PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/include)
+
+make_directory(${CMAKE_CURRENT_BINARY_DIR}/include/flow)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/include/flow/config.h)

extractor/workdir/extract.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
 
-set -ex
+set -exu
 
 OUTPUT_DIR="$1"
 [ -d "$OUTPUT_DIR" ] || exit 1
@@ -24,28 +24,53 @@ function convert_fdb_source() {
     rm -rf "${OUTPUT_DIR}/${_subdir}"
     mkdir -p "${OUTPUT_DIR}/${_subdir}"
 
-    for path in "${FOUNDATIONDB_REPO}/${_subdir}/"*; do
+    find "${FOUNDATIONDB_REPO}/${_subdir}/" -type f -print0 | 
+    while IFS= read -r -d '' path; do
+        dir="$(realpath --relative-to="${FOUNDATIONDB_REPO}/${_subdir}" "$(dirname "$path")")/"
+        [ "$dir" = "./" ] && dir=
         file="$(basename "$path")"
-        if [[ " ${_skip_source[@]} " =~ " $file " ]]; then
-            echo "Skip $file"
-        elif [[ " ${_copy_source[@]} " =~ " $file " ]]; then
-            cp "$path" "${OUTPUT_DIR}/${_subdir}/${file}"
+
+        skip=N
+        copy=N
+
+        parent_dir="${dir}${file}"
+        while [ "$parent_dir" != "." ]; do
+            if [[ " ${_skip_source[@]} " =~ " $parent_dir " ]]; then
+                skip=Y
+                break;
+            fi
+            parent_dir="$(dirname "$parent_dir")"
+        done
+
+        if [ "$skip" = "Y" ]; then
+            echo "Skip $dir$file"
+        elif [[ " ${_copy_source[@]} " =~ " $dir$file " ]]; then
+            copy=Y
         elif [[ "$file" =~ ^(.*)\.(h|cpp)$ ]]; then
-            _target="${OUTPUT_DIR}/${_subdir}/${file}"
+            _target="${OUTPUT_DIR}/${_subdir}/${dir}${file}"
+
+            mkdir -p "$(dirname "$_target")"
             sed  "$path" \
                 -e 's|contrib/fmt-8.1.1/include/||' \
-                -e 's|flow/stacktrace.h|absl/debugging/stacktrace.h|' \
-                -e 's|fdbrpc/md5/md5.h|openssl/md5.h|;' \
+                -e 's|stacktrace/stacktrace.h|absl/debugging/stacktrace.h|' \
+                -e 's|md5/md5.h|openssl/md5.h|;' \
                 > "$_target"
 
             if [[ "$file" =~ ^(.*)\.actor\.(h|cpp)$ ]]; then
-                _actor_target="${OUTPUT_DIR}/${_subdir}/${BASH_REMATCH[1]}.actor.g.${BASH_REMATCH[2]}"
+                _actor_target="${OUTPUT_DIR}/${_subdir}/${dir}${BASH_REMATCH[1]}.actor.g.${BASH_REMATCH[2]}"
                 $ACTORCOMPILER "$_target" "$_actor_target" --generate-probes
             fi
         elif [[ "$file" =~ ^(.*).h.cmake$ ]]; then
-            cp "$FOUNDATIONDB_BUILDDIR/${_subdir}/${BASH_REMATCH[1]}.h" "${OUTPUT_DIR}/${_subdir}/${BASH_REMATCH[1]}.h"
+            _incdir="${OUTPUT_DIR}/${_subdir}/include/${_subdir}"
+            mkdir -p "$_incdir"
+            cp "$FOUNDATIONDB_BUILDDIR/${_subdir}/include/${_subdir}/${BASH_REMATCH[1]}.h" "${_incdir}/${BASH_REMATCH[1]}.h"
         else
-            cp -R "$path" "${OUTPUT_DIR}/${_subdir}/${file}"
+            copy=Y
+        fi
+
+        if [ "$copy" = "Y" ]; then
+            mkdir -p "${OUTPUT_DIR}/${_subdir}/${dir}"
+            cp "$path" "${OUTPUT_DIR}/${_subdir}/${dir}${file}"
         fi
     done
     cp "${WORKDIR}/cmake/${_subdir}.cmake" "${OUTPUT_DIR}/${_subdir}/CMakeLists.txt"
@@ -59,9 +84,7 @@ _skip_source=(
     coveragetool
     actorcompiler
     no_intellisense.opt
-    stacktrace_internal
-    stacktrace.amalgamation.cpp
-    stacktrace.h
+    ApiVersions.cmake
 )
 _copy_source=(
     config.h.cmake
@@ -74,19 +97,20 @@ _skip_source=(
     CMakeLists.txt
     README.md
     actorFuzz.py
+    tests
+
+    TokenSign
 
     # Depends
     libcoroutine # use boost coro impl
-    md5          # use openssl impl
-    # libb64
     # libeio
 )
 _copy_source=()
 convert_fdb_source
 
 # fdbrpc requires fdbserver/Knobs.h
-mkdir "${OUTPUT_DIR}/fdbserver"
-cp "${FOUNDATIONDB_REPO}/fdbserver/Knobs.h" "${OUTPUT_DIR}/fdbserver/Knobs.h"
+mkdir -p "${OUTPUT_DIR}/fdbserver/include/fdbserver"
+cp "${FOUNDATIONDB_REPO}/fdbserver/include/fdbserver/Knobs.h" "${OUTPUT_DIR}/fdbserver/include/fdbserver/Knobs.h"
 
 # fdbclient
 _subdir=fdbclient
@@ -96,26 +120,26 @@ _skip_source=(
     azurestorage.cmake
     vexillographer  
 
-    # Depends
-    rapidxml        # required by s3
-    rapidjson       # required by unit test
-    # json_spirit
-    # sha1
-
     # Disable AZURE
-    BackupContainerAzureBlobStore.actor.cpp
+    azure_backup
 
     # Disable S3
     S3BlobStore.actor.cpp
 )
 _copy_source=()
 convert_fdb_source
 $VEXILLOGRAPHER "${FOUNDATIONDB_REPO}/fdbclient/vexillographer/fdb.options" cpp "${OUTPUT_DIR}/fdbclient/FDBOptions.g"
+mv "${OUTPUT_DIR}"/fdbclient{,/include/fdbclient}/FDBOptions.g.h
+$VEXILLOGRAPHER "${FOUNDATIONDB_REPO}/fdbclient/vexillographer/fdb.options" c "${OUTPUT_DIR}/fdbclient/include/fdbclient/fdb_c_options.g.h"
 
 # bindings/c
 mkdir -p "${OUTPUT_DIR}/bindings/c"
 cp "${FOUNDATIONDB_REPO}/bindings/c/fdb_c.cpp" "${OUTPUT_DIR}/bindings/c/fdb_c.cpp"
 cp -r "${FOUNDATIONDB_REPO}/bindings/c/foundationdb" "${OUTPUT_DIR}/bindings/c/foundationdb"
+
+rm "${OUTPUT_DIR}/bindings/c/foundationdb/fdb_c_apiversion.h.cmake"
+cp {"${FOUNDATIONDB_BUILDDIR}","${OUTPUT_DIR}"}/bindings/c/foundationdb/fdb_c_apiversion.g.h
+
 $VEXILLOGRAPHER "${FOUNDATIONDB_REPO}/fdbclient/vexillographer/fdb.options" c "${OUTPUT_DIR}/bindings/c/foundationdb/fdb_c_options.g.h"
 function generate_fdb_c_asm() {
     _os="${1}"
@@ -134,3 +158,8 @@ generate_fdb_c_asm linux aarch64
 generate_fdb_c_asm osx aarch64
 cp "${WORKDIR}/cmake/fdb_c.cmake" "${OUTPUT_DIR}/bindings/c/CMakeLists.txt"
 
+mkdir "${OUTPUT_DIR}/contrib"
+for dep in crc32 SimpleOpt libb64
+do
+    cp -R "${FOUNDATIONDB_REPO}/contrib/${dep}" "${OUTPUT_DIR}/contrib/${dep}"
+done

extractor/workdir/patches/fix-fmt-cxx20.patch

@@ -0,0 +1,36 @@
+diff --git a/flow/TLSTest.cpp b/flow/TLSTest.cpp
+index ebbbdacd3..d988dd899 100644
+--- a/flow/TLSTest.cpp
++++ b/flow/TLSTest.cpp
+@@ -31,25 +31,25 @@
+ std::FILE* outp = stdout;
+
+ template <class... Args>
+-void log(Args&&... args) {
++void log(fmt::format_string<Args...> fmt, Args&&... args) {
+ 	auto buf = fmt::memory_buffer{};
+-	fmt::format_to(std::back_inserter(buf), std::forward<Args>(args)...);
++	fmt::format_to(std::back_inserter(buf), fmt, std::forward<Args>(args)...);
+ 	fmt::print(outp, "{}\n", std::string_view(buf.data(), buf.size()));
+ }
+
+ template <class... Args>
+-void logc(Args&&... args) {
++void logc(fmt::format_string<Args...> fmt, Args&&... args) {
+ 	auto buf = fmt::memory_buffer{};
+ 	fmt::format_to(std::back_inserter(buf), "[CLIENT] ");
+-	fmt::format_to(std::back_inserter(buf), std::forward<Args>(args)...);
++	fmt::format_to(std::back_inserter(buf), fmt, std::forward<Args>(args)...);
+ 	fmt::print(outp, "{}\n", std::string_view(buf.data(), buf.size()));
+ }
+
+ template <class... Args>
+-void logs(Args&&... args) {
++void logs(fmt::format_string<Args...> fmt, Args&&... args) {
+ 	auto buf = fmt::memory_buffer{};
+ 	fmt::format_to(std::back_inserter(buf), "[SERVER] ");
+-	fmt::format_to(std::back_inserter(buf), std::forward<Args>(args)...);
++	fmt::format_to(std::back_inserter(buf), fmt, std::forward<Args>(args)...);
+ 	fmt::print(outp, "{}\n", std::string_view(buf.data(), buf.size()));
+ }
+

extractor/workdir/prepare-build.sh

@@ -5,7 +5,7 @@ set -ex
 WORKDIR="$(realpath "$(dirname "$0")")"
 
 FOUNDATIONDB_GIT=https://github.com/apple/foundationdb.git
-FOUNDATIONDB_TAG=${FOUNDATIONDB_TAG:?FOUNDATIONDB_TAG is required}
+FOUNDATIONDB_TAG=7.3.25
 FOUNDATIONDB_REPO=${FOUNDATIONDB_REPO:?FOUNDATIONDB_REPO is required}
 FOUNDATIONDB_BUILDDIR=${FOUNDATIONDB_BUILDDIR:?FOUNDATIONDB_BUILDDIR is required}
 
@@ -21,11 +21,11 @@ git reset HEAD --hard
 git clean -dffx
 
 # Apply patchs
-for patch in fix-fmt-format.patch build-c-binding-only.patch backport-doctest.patch
+for patch in fix-fmt-cxx20.patch # build-c-binding-only.patch fix-fmt-format.patch backport-doctest.patch
 do
     git apply "${WORKDIR}/patches/${patch}"
 done
 
 # Configure build
 # rm -rf "${FOUNDATIONDB_BUILDDIR}"
-cmake -B "${FOUNDATIONDB_BUILDDIR}" -DCMAKE_BUILD_TYPE=Debug -G Ninja "${FOUNDATIONDB_REPO}"
+cmake -B "${FOUNDATIONDB_BUILDDIR}" -DCMAKE_BUILD_TYPE=Debug -DUSE_JEMALLOC=OFF -G Ninja "${FOUNDATIONDB_REPO}"