From 6b7869a9daf4a4a383f9733c386deb0372869a00 Mon Sep 17 00:00:00 2001 From: tobtoht Date: Thu, 12 Dec 2024 14:44:16 +0100 Subject: [PATCH] Bootstrappable FCMP++ --- .github/workflows/guix.yml | 2 +- contrib/guix/guix-build | 24 ++++++++ contrib/guix/libexec/build.sh | 41 +++++++++++++- contrib/guix/manifest.scm | 5 +- contrib/guix/rust/cargo.scm | 75 +++++++++++++++++++++++++ contrib/guix/rust/cargo.sh | 70 +++++++++++++++++++++++ contrib/guix/rust/config.toml | 14 +++++ contrib/guix/rust/stubs | 30 ++++++++++ src/fcmp_pp/fcmp_pp_rust/CMakeLists.txt | 4 +- 9 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 contrib/guix/rust/cargo.scm create mode 100644 contrib/guix/rust/cargo.sh create mode 100644 contrib/guix/rust/config.toml create mode 100644 contrib/guix/rust/stubs diff --git a/.github/workflows/guix.yml b/.github/workflows/guix.yml index 47e2eccf753..e2eb3d9d096 100644 --- a/.github/workflows/guix.yml +++ b/.github/workflows/guix.yml @@ -83,7 +83,7 @@ jobs: sudo aa-enforce guix || true sudo apt purge apparmor - name: build - run: SUBSTITUTE_URLS='http://bordeaux.guix.gnu.org' HOSTS="${{ matrix.toolchain.target }}" ./contrib/guix/guix-build + run: ADDITIONAL_GUIX_TIMEMACHINE_FLAGS="--disable-authentication" SUBSTITUTE_URLS='http://bordeaux.guix.gnu.org' HOSTS="${{ matrix.toolchain.target }}" ./contrib/guix/guix-build - uses: actions/upload-artifact@v4 with: name: ${{ matrix.toolchain.target }} diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 76b7b03d3df..bf88d58934a 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -264,6 +264,29 @@ mkdir -p "$OUTDIR_BASE" LOGDIR_BASE="${LOGDIR_BASE:-${VERSION_BASE}/logs}" mkdir -p "$LOGDIR_BASE" +# Download and archive Rust dependencies. +# Version and Hash need to be updated when: +# - we bump the time-machine and Guix has a new version of Rust. +# - Cargo.lock in src/fcmp_pp/fcmp_pp_rust is updated. +RUST_DEPS_VERSION=0 +RUST_DEPS_HASH="hchq2aqq68vxp3hm1hhl74b90vsqb9xv" +RUST_DEPS_ARCHIVE="rust_deps-${RUST_DEPS_VERSION}.tar.gz" +RUST_DEPS_STORE_ITEM="/gnu/store/${RUST_DEPS_HASH}-${RUST_DEPS_ARCHIVE}" +if [ ! -f "${RUST_DEPS_STORE_ITEM}" ]; then + time-machine environment --manifest="${PWD}/contrib/guix/rust/cargo.scm" \ + --container \ + --pure \ + --network \ + --no-cwd \ + --user="user" \ + --share="$PWD"=/monero \ + -- env RUST_DEPS_ARCHIVE="$RUST_DEPS_ARCHIVE" \ + bash /monero/contrib/guix/rust/cargo.sh + + time-machine download ${RUST_DEPS_ARCHIVE} + rm ${RUST_DEPS_ARCHIVE} +fi + # Download the depends sources now as we won't have internet access in the build # container for host in $HOSTS; do @@ -438,6 +461,7 @@ EOF --share="$DISTSRC_BASE"=/distsrc-base \ --share="$OUTDIR_BASE"=/outdir-base \ --share="$LOGDIR_BASE"=/logdir-base \ + --share="$RUST_DEPS_STORE_ITEM"=/rust-deps \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index be2c937fb64..f1c3b4ad4e9 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -103,6 +103,11 @@ prepend_to_search_env_var() { export "${1}=${2}${!1:+:}${!1}" } +# error: failed to run custom build command for `compiler_builtins` +# +# error while loading shared libraries: libgcc_s.so.1: cannot open shared object file: No such file or directory +export LD_LIBRARY_PATH="${NATIVE_GCC}/lib" + # Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in @@ -132,7 +137,7 @@ case "$HOST" in # See depends/hosts/darwin.mk for more details. ;; *android*) - export LD_LIBRARY_PATH="$(find /gnu/store -maxdepth 1 -name "*zlib*" | sort | head -n 1)/lib:$(find /gnu/store -maxdepth 1 -name "*gcc-11*-lib" | sort | head -n 1)/lib" + export LD_LIBRARY_PATH="${NATIVE_GCC}/lib:$(find /gnu/store -maxdepth 1 -name "*zlib*" | sort | head -n 1)/lib:$(find /gnu/store -maxdepth 1 -name "*gcc-11*-lib" | sort | head -n 1)/lib" ;; *linux-gnu*) CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" @@ -320,6 +325,30 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac +# error: "/gnu/store/<...>-rust-1.82.0/lib/rustlib/src/rust/library/Cargo.lock" does not exist, +# unable to build with the standard library +# +# The standard library does not exist at the location Cargo expects. +# +# We can override the path to the Rust source by setting the __CARGO_TESTS_ONLY_SRC_ROOT environment variable. +# See: https://github.com/rust-lang/cargo/blob/rust-1.82.0/src/cargo/core/compiler/standard_lib.rs#L183 +export __CARGO_TESTS_ONLY_SRC_ROOT=/rust/library + +# error: the `-Z` flag is only accepted on the nightly channel of Cargo, but this is the `stable` channel +# +# Since we don't have access to the nightly channel, we need to bypass the check with RUSTC_BOOTSTRAP. +# +# We could avoid using `-Z build-std` by cross-compiling the full standard library for each target. This approach +# adds hours to our build time and greatly increases the amount of foreign source code that is compiled as part of +# our build process. +export RUSTC_BOOTSTRAP=1 + +# See: https://rust-lang.github.io/rust-project-goals/2025h1/build-std.html +CARGO_OPTIONS="-Zbuild-std=std,panic_abort;" + +# TODO: add doc +CARGO_OPTIONS+="-Zbuild-std-features=panic_immediate_abort;" + export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 # Force Trezor support for release binaries export USE_DEVICE_TREZOR_MANDATORY=1 @@ -347,6 +376,15 @@ mkdir -p "$DISTSRC" # checked out before starting a build. CMAKEFLAGS+=" -DMANUAL_SUBMODULES=1" + # Make sure cargo knows where to find our vendored sources. + mkdir -p /home/user/.cargo + cp contrib/guix/rust/config.toml /home/user/.cargo/ + sed -i "s/TARGET/${HOST}/g" /home/user/.cargo/config.toml + + # Unpack rust dependencies + mkdir -p /rust + tar xf /rust-deps -C /rust + # Configure this DISTSRC for $HOST # shellcheck disable=SC2086 env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \ @@ -354,6 +392,7 @@ mkdir -p "$DISTSRC" -DCMAKE_INSTALL_PREFIX="${INSTALLPATH}" \ -DCMAKE_EXE_LINKER_FLAGS="${HOST_LDFLAGS}" \ -DCMAKE_SHARED_LINKER_FLAGS="${HOST_LDFLAGS}" \ + -DCARGO_OPTIONS="${CARGO_OPTIONS}" \ ${CMAKEFLAGS} make -C build --jobs="$JOBS" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 18da9b12de0..ec88019d294 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -18,6 +18,7 @@ (gnu packages perl) (gnu packages pkg-config) ((gnu packages python) #:select (python-minimal)) + (gnu packages rust) ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) (guix build-system gnu) @@ -278,6 +279,8 @@ chain for " target " development.")) pkg-config gperf ; required to build eudev in depends cmake-minimal + rust + (list rust "cargo") ;; Scripting perl ; required to build openssl in depends @@ -307,7 +310,7 @@ chain for " target " development.")) (list gcc-toolchain-12 "static"))) ((string-contains target "darwin") (list - gcc-toolchain-10 + gcc-toolchain-11 clang-toolchain-11 binutils)) (else '()))))) diff --git a/contrib/guix/rust/cargo.scm b/contrib/guix/rust/cargo.scm new file mode 100644 index 00000000000..29adfedee94 --- /dev/null +++ b/contrib/guix/rust/cargo.scm @@ -0,0 +1,75 @@ +(use-modules (gnu packages) + ((gnu packages bash) #:select (bash-minimal)) + ((gnu packages certs) #:select (nss-certs)) + (gnu packages compression) + (gnu packages curl) + (gnu packages moreutils) + (gnu packages rust) + ((gnu packages tls) #:select (openssl)) + ((gnu packages web) #:select (jq)) + (guix build-system trivial) + (guix download) + ((guix licenses) #:prefix license:) + (guix packages)) + +;; We don't use (list rust "rust-src") for two reasons: +;; +;; - Hashes in Cargo.lock are replaced, resulting in: +;; error: checksum for ` ` changed between lock files +;; +;; - It drags in a bunch of unnecessary deps, including python. +;; See: guix graph --type=references rust | xdot - + +;; Instead, we create a new package with the unaltered rust source +;; and vendor the standard library in cargo.sh + +;; TODO: can we use inherit here? + +(define-public rust-std + (package + (name "rust-std") + (version (package-version rust)) + ;; You'd expect (source (package-source (rust)) to work here, + ;; but it refers to the source store item and NOT the .tar.gz archive + (source (origin + (method url-fetch) + (uri (origin-uri (package-source rust))) + (sha256 + (content-hash-value (origin-hash (package-source rust)))))) + (build-system trivial-build-system) + (native-inputs (list tar gzip)) + (arguments + `(#:modules ((guix build utils)) + #:builder + (begin + (use-modules (guix build utils)) + (let ((out (assoc-ref %outputs "out")) + (source (assoc-ref %build-inputs "source")) + (tar (search-input-file %build-inputs "/bin/tar")) + (gzip (search-input-file %build-inputs "/bin/gzip")) + (gzip-path (string-append (assoc-ref %build-inputs "gzip") "/bin"))) + (setenv "PATH" gzip-path) + (mkdir out) + (invoke tar "xvf" source "-C" out "--strip-components=1"))))) + (synopsis (package-synopsis rust)) + (description (package-description rust)) + (home-page (package-home-page rust)) + (license (package-license rust)))) + +(packages->manifest + (append + (list + bash-minimal + coreutils-minimal + curl + findutils ;; find + grep + gzip + jq + moreutils + nss-certs + openssl + sed + tar + (list rust "cargo") + rust-std))) diff --git a/contrib/guix/rust/cargo.sh b/contrib/guix/rust/cargo.sh new file mode 100644 index 00000000000..7ff5dff037b --- /dev/null +++ b/contrib/guix/rust/cargo.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -e -o pipefail + +# Environment variables for determinism +export LC_ALL=C +export SOURCE_DATE_EPOCH=1397818193 +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ="UTC" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + +echo "Fetching rust dependencies.." + +cd /monero/src/fcmp_pp/fcmp_pp_rust + +# error: the cargo feature `public-dependency` requires a nightly version of Cargo, but this is the `stable` channel +# +# https://doc.rust-lang.org/cargo/reference/unstable.html#public-dependency +export RUSTC_BOOTSTRAP=1 + +# Assert that `Cargo.lock` will remain unchanged +CARGO_OPTIONS="--locked" + +# https://github.com/rust-lang/wg-cargo-std-aware/issues/23#issuecomment-1445119470 +# If we don't vendor std, we'll run into: 'error: no matching package named `compiler_builtins` found' during build. +CARGO_OPTIONS+=" --sync $(store_path rust-std)/library/Cargo.toml" + +# Vendor fcmp_pp_rust + std library deps +cargo vendor ${CARGO_OPTIONS} /rust/vendor + +cp -r "$(store_path rust-std)/library" /rust/library + +cd /rust/vendor + +# `cargo vendor` includes dozens of packages that aren't needed to build the standard library. +# We can't simply remove these packages, because cargo expects them to be there. +# Instead, we replace the packages with a stub, which is sufficient to pass cargo's checks. +while IFS= read -r line; do + cd "$line" + find . -not -path "." -not -name "Cargo.toml" -not -name ".cargo-checksum.json" -delete + mkdir src + touch src/lib.rs + + # Cargo.toml must remain unaltered. + # src/lib.rs must exist, but may be empty. + # 'e3b0...b855' is equivalent to `echo -n "" | sha256sum -` + # We can't set 'package' to the empty hash, as this might conflict with fcmp_pp_rust's Cargo.lock, resulting + # in the same error. + cat .cargo-checksum.json \ + | jq '{files: {"Cargo.toml": .files["Cargo.toml"]}, package}' \ + | jq '.files += {"src/lib.rs": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}' \ + | sponge .cargo-checksum.json + cd .. +done < "/monero/contrib/guix/rust/stubs" + +cd /rust + +# Create deterministic archive +find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "/monero/$RUST_DEPS_ARCHIVE" diff --git a/contrib/guix/rust/config.toml b/contrib/guix/rust/config.toml new file mode 100644 index 00000000000..68cb79fb9b6 --- /dev/null +++ b/contrib/guix/rust/config.toml @@ -0,0 +1,14 @@ +[source.crates-io] +replace-with = "vendored-sources" + +[source."git+https://github.com/kayabaNerve/crypto-bigint?branch=c-repr"] +git = "https://github.com/kayabaNerve/crypto-bigint" +branch = "c-repr" +replace-with = "vendored-sources" + +[source."git+https://github.com/kayabaNerve/fcmp-plus-plus"] +git = "https://github.com/kayabaNerve/fcmp-plus-plus" +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "/rust/vendor" diff --git a/contrib/guix/rust/stubs b/contrib/guix/rust/stubs new file mode 100644 index 00000000000..4ae6318769c --- /dev/null +++ b/contrib/guix/rust/stubs @@ -0,0 +1,30 @@ +addr2line +adler +allocator-api2 +cc +dlmalloc +fortanix-sgx-abi +getopts +gimli-0.28.1 +gimli +hermit-abi +memchr +miniz_oxide +object +r-efi +r-efi-alloc +rand +rand_xorshift +unicode-width +unwinding +wasi +windows-sys +windows-targets +windows_aarch64_gnullvm +windows_aarch64_msvc +windows_i686_gnu +windows_i686_gnullvm +windows_i686_msvc +windows_x86_64_gnu +windows_x86_64_gnullvm +windows_x86_64_msvc diff --git a/src/fcmp_pp/fcmp_pp_rust/CMakeLists.txt b/src/fcmp_pp/fcmp_pp_rust/CMakeLists.txt index 646fc6981d3..9d2df657e97 100644 --- a/src/fcmp_pp/fcmp_pp_rust/CMakeLists.txt +++ b/src/fcmp_pp/fcmp_pp_rust/CMakeLists.txt @@ -86,10 +86,10 @@ endif() set(RUST_TARGET "${RUST_ARCH}-${RUST_PLATFORM}${RUST_TOOLCHAIN}") if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CARGO_CMD cargo build --target "${RUST_TARGET}") + set(CARGO_CMD cargo build --target "${RUST_TARGET}" ${CARGO_OPTIONS}) set(TARGET_DIR "debug") else () - set(CARGO_CMD cargo build --target "${RUST_TARGET}" --release) + set(CARGO_CMD cargo build --target "${RUST_TARGET}" --release ${CARGO_OPTIONS}) set(TARGET_DIR "release") endif ()