Skip to content

Commit

Permalink
Bootstrappable FCMP++
Browse files Browse the repository at this point in the history
  • Loading branch information
tobtoht committed Dec 14, 2024
1 parent 8a7cdda commit e89ffb1
Show file tree
Hide file tree
Showing 9 changed files with 268 additions and 8 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/guix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:
sudo aa-enforce guix || true
sudo apt purge apparmor
- name: build
run: SUBSTITUTE_URLS='http://bordeaux.guix.gnu.org' HOSTS="${{ matrix.toolchain.target }}" ./contrib/guix/guix-build
run: ADDITIONAL_GUIX_TIMEMACHINE_FLAGS="--disable-authentication" SUBSTITUTE_URLS='http://bordeaux.guix.gnu.org' HOSTS="${{ matrix.toolchain.target }}" ./contrib/guix/guix-build
- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.toolchain.target }}
Expand Down
35 changes: 32 additions & 3 deletions contrib/guix/guix-build
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,11 @@ contains() {
return 1
}

# Usage: depends_print DEPENDS-VARIABLE
depends_print() {
make -C "${PWD}/contrib/depends" --no-print-directory "print-$1"
}

# If the user explicitly specified a precious directory, create it so we
# can map it into the container
for precious_dir_name in $precious_dir_names; do
Expand Down Expand Up @@ -250,7 +255,7 @@ mkdir -p "$VAR_BASE"
# Get precious dir definitions from depends and the environment
for precious_dir_name in $precious_dir_names; do
if contains "$depends_precious_dir_names" "$precious_dir_name"; then
precious_dir_path="$(make -C "${PWD}/contrib/depends" --no-print-directory print-${precious_dir_name})"
precious_dir_path="$(depends_print ${precious_dir_name})"
else
precious_dir_path="${!precious_dir_name}"
fi
Expand All @@ -264,6 +269,29 @@ mkdir -p "$OUTDIR_BASE"
LOGDIR_BASE="${LOGDIR_BASE:-${VERSION_BASE}/logs}"
mkdir -p "$LOGDIR_BASE"

# Download and archive Rust dependencies.
SOURCES_PATH="${SOURCES_PATH:-$(depends_print SOURCES_PATH)}"
mkdir -p "$SOURCES_PATH"
# Version and Hash need to be updated when:
# - we bump the time-machine and Guix has a new version of Rust.
# - Cargo.lock in src/fcmp_pp/fcmp_pp_rust is updated.
RUST_DEPS_VERSION=0
RUST_DEPS_HASH="6abbac8d3d96db1fa64bd4b72d0b210c30db754ad95efa03d4de5437e2041f64"
RUST_DEPS_ARCHIVE="rust_deps-${RUST_DEPS_VERSION}.tar.gz"
if [ ! -f "${SOURCES_PATH}/${RUST_DEPS_ARCHIVE}" ]; then
time-machine environment --manifest="${PWD}/contrib/guix/rust/cargo.scm" \
--container \
--pure \
--network \
--no-cwd \
--user="user" \
--share="$PWD"=/monero \
--share="$SOURCES_PATH"=/sources \
-- env RUST_DEPS_ARCHIVE="$RUST_DEPS_ARCHIVE" \
bash /monero/contrib/guix/rust/cargo.sh
fi
echo "${RUST_DEPS_HASH} ${SOURCES_PATH}/${RUST_DEPS_ARCHIVE}" | sha256sum -c

# Download the depends sources now as we won't have internet access in the build
# container
for host in $HOSTS; do
Expand Down Expand Up @@ -439,7 +467,7 @@ EOF
--share="$OUTDIR_BASE"=/outdir-base \
--share="$LOGDIR_BASE"=/logdir-base \
--expose="$(git rev-parse --git-common-dir)" \
${SOURCES_PATH:+--share="$SOURCES_PATH"} \
--share="$SOURCES_PATH"=/sources \
${BASE_CACHE:+--share="$BASE_CACHE"} \
--cores="$JOBS" \
--keep-failed \
Expand All @@ -455,8 +483,9 @@ EOF
COMMIT_TIMESTAMP="${COMMIT_TIMESTAMP:?unable to determine value}" \
${V:+V=1} \
${DEPENDS_ONLY:+DEPENDS_ONLY=1} \
${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \
SOURCES_PATH="/sources" \
${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \
RUST_DEPS_ARCHIVE="$RUST_DEPS_ARCHIVE" \
DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$host")" \
OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$host")" \
LOGDIR="$(LOGDIR_BASE=/logdir-base && logdir_for_host "$host")" \
Expand Down
41 changes: 40 additions & 1 deletion contrib/guix/libexec/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ prepend_to_search_env_var() {
export "${1}=${2}${!1:+:}${!1}"
}

# error: failed to run custom build command for `compiler_builtins`
#
# error while loading shared libraries: libgcc_s.so.1: cannot open shared object file: No such file or directory
export LD_LIBRARY_PATH="${NATIVE_GCC}/lib"

# Set environment variables to point the CROSS toolchain to the right
# includes/libs for $HOST
case "$HOST" in
Expand Down Expand Up @@ -132,7 +137,7 @@ case "$HOST" in
# See depends/hosts/darwin.mk for more details.
;;
*android*)
export LD_LIBRARY_PATH="$(find /gnu/store -maxdepth 1 -name "*zlib*" | sort | head -n 1)/lib:$(find /gnu/store -maxdepth 1 -name "*gcc-11*-lib" | sort | head -n 1)/lib"
export LD_LIBRARY_PATH="${NATIVE_GCC}/lib:$(find /gnu/store -maxdepth 1 -name "*zlib*" | sort | head -n 1)/lib:$(find /gnu/store -maxdepth 1 -name "*gcc-11*-lib" | sort | head -n 1)/lib"
;;
*linux-gnu*)
CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")"
Expand Down Expand Up @@ -320,6 +325,30 @@ case "$HOST" in
*mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;;
esac

# error: "/gnu/store/<...>-rust-1.82.0/lib/rustlib/src/rust/library/Cargo.lock" does not exist,
# unable to build with the standard library
#
# The standard library does not exist at the location Cargo expects.
#
# We can override the path to the Rust source by setting the __CARGO_TESTS_ONLY_SRC_ROOT environment variable.
# See: https://github.com/rust-lang/cargo/blob/rust-1.82.0/src/cargo/core/compiler/standard_lib.rs#L183
export __CARGO_TESTS_ONLY_SRC_ROOT=/rust/library

# error: the `-Z` flag is only accepted on the nightly channel of Cargo, but this is the `stable` channel
#
# Since we don't have access to the nightly channel, we need to bypass the check with RUSTC_BOOTSTRAP.
#
# We could avoid using `-Z build-std` by cross-compiling the full standard library for each target. This approach
# adds hours to our build time and greatly increases the amount of foreign source code that is compiled as part of
# our build process.
export RUSTC_BOOTSTRAP=1

# See: https://rust-lang.github.io/rust-project-goals/2025h1/build-std.html
CARGO_OPTIONS="-Zbuild-std=std,panic_abort;"

# TODO: add doc
CARGO_OPTIONS+="-Zbuild-std-features=panic_immediate_abort;"

export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
# Force Trezor support for release binaries
export USE_DEVICE_TREZOR_MANDATORY=1
Expand Down Expand Up @@ -347,13 +376,23 @@ mkdir -p "$DISTSRC"
# checked out before starting a build.
CMAKEFLAGS+=" -DMANUAL_SUBMODULES=1"

# Make sure cargo knows where to find our vendored sources.
mkdir -p /home/user/.cargo
cp contrib/guix/rust/config.toml /home/user/.cargo/
sed -i "s/TARGET/${HOST}/g" /home/user/.cargo/config.toml

# Unpack rust dependencies
mkdir -p /rust
tar xf "$SOURCES_PATH/$RUST_DEPS_ARCHIVE" -C /rust

# Configure this DISTSRC for $HOST
# shellcheck disable=SC2086
env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
cmake --toolchain "${BASEPREFIX}/${HOST}/share/toolchain.cmake" -S . -B build \
-DCMAKE_INSTALL_PREFIX="${INSTALLPATH}" \
-DCMAKE_EXE_LINKER_FLAGS="${HOST_LDFLAGS}" \
-DCMAKE_SHARED_LINKER_FLAGS="${HOST_LDFLAGS}" \
-DCARGO_OPTIONS="${CARGO_OPTIONS}" \
${CMAKEFLAGS}

make -C build --jobs="$JOBS"
Expand Down
5 changes: 4 additions & 1 deletion contrib/guix/manifest.scm
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
(gnu packages perl)
(gnu packages pkg-config)
((gnu packages python) #:select (python-minimal))
(gnu packages rust)
((gnu packages tls) #:select (openssl))
((gnu packages version-control) #:select (git-minimal))
(guix build-system gnu)
Expand Down Expand Up @@ -278,6 +279,8 @@ chain for " target " development."))
pkg-config
gperf ; required to build eudev in depends
cmake-minimal
rust
(list rust "cargo")

;; Scripting
perl ; required to build openssl in depends
Expand Down Expand Up @@ -307,7 +310,7 @@ chain for " target " development."))
(list gcc-toolchain-12 "static")))
((string-contains target "darwin")
(list
gcc-toolchain-10
gcc-toolchain-11
clang-toolchain-11
binutils))
(else '())))))
73 changes: 73 additions & 0 deletions contrib/guix/rust/cargo.scm
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
(use-modules (gnu packages)
((gnu packages bash) #:select (bash-minimal))
((gnu packages certs) #:select (nss-certs))
(gnu packages compression)
(gnu packages curl)
(gnu packages moreutils)
(gnu packages rust)
((gnu packages tls) #:select (openssl))
((gnu packages web) #:select (jq))
(guix build-system trivial)
(guix download)
((guix licenses) #:prefix license:)
(guix packages))

;; We don't use (list rust "rust-src") for two reasons:
;;
;; - Hashes in Cargo.lock are replaced, resulting in:
;; error: checksum for `<package> <version>` changed between lock files
;;
;; - It drags in a bunch of unnecessary deps, including python.
;; See: guix graph --type=references rust | xdot -

;; Instead, we create a new package with the unaltered rust source
;; and vendor the standard library in cargo.sh

(define-public rust-std
(package
(name "rust-std")
(version (package-version rust))
;; You'd expect (source (package-source (rust)) to work here,
;; but it refers to the source store item and NOT the .tar.gz archive
(source (origin
(method url-fetch)
(uri (origin-uri (package-source rust)))
(sha256
(content-hash-value (origin-hash (package-source rust))))))
(build-system trivial-build-system)
(native-inputs (list tar gzip))
(arguments
`(#:modules ((guix build utils))
#:builder
(begin
(use-modules (guix build utils))
(let ((out (assoc-ref %outputs "out"))
(source (assoc-ref %build-inputs "source"))
(tar (search-input-file %build-inputs "/bin/tar"))
(gzip (search-input-file %build-inputs "/bin/gzip"))
(gzip-path (string-append (assoc-ref %build-inputs "gzip") "/bin")))
(setenv "PATH" gzip-path)
(mkdir out)
(invoke tar "xvf" source "-C" out "--strip-components=1")))))
(synopsis (package-synopsis rust))
(description (package-description rust))
(home-page (package-home-page rust))
(license (package-license rust))))

(packages->manifest
(append
(list
bash-minimal
coreutils-minimal
curl
findutils ;; find
grep
gzip
jq
moreutils
nss-certs
openssl
sed
tar
(list rust "cargo")
rust-std)))
72 changes: 72 additions & 0 deletions contrib/guix/rust/cargo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env bash
set -e -o pipefail

# Environment variables for determinism
export LC_ALL=C
export SOURCE_DATE_EPOCH=1397818193
export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name"
export TZ="UTC"

# Given a package name and an output name, return the path of that output in our
# current guix environment
store_path() {
grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \
| head --lines=1 \
| sed --expression='s|\x29*$||' \
--expression='s|^[[:space:]]*"||' \
--expression='s|"[[:space:]]*$||'
}

echo "Fetching rust dependencies.."

cd /monero/src/fcmp_pp/fcmp_pp_rust

# error: the cargo feature `public-dependency` requires a nightly version of Cargo, but this is the `stable` channel
#
# https://doc.rust-lang.org/cargo/reference/unstable.html#public-dependency
export RUSTC_BOOTSTRAP=1

# Assert that `Cargo.lock` will remain unchanged
CARGO_OPTIONS="--locked"

# https://github.com/rust-lang/wg-cargo-std-aware/issues/23#issuecomment-1445119470
# If we don't vendor std, we'll run into: 'error: no matching package named `compiler_builtins` found' during build.
CARGO_OPTIONS+=" --sync $(store_path rust-std)/library/Cargo.toml"

# Vendor fcmp_pp_rust + std library deps
cargo vendor ${CARGO_OPTIONS} /rust/vendor

cp -r "$(store_path rust-std)/library" /rust/library

cd /rust/vendor

# `cargo vendor` includes dozens of packages that aren't needed to build the standard library.
# We can't simply remove these packages, because cargo expects them to be there.
# Instead, we replace the packages with a stub, which is sufficient to pass cargo's checks.
while IFS= read -r line; do
cd "$line"
find . -not -path "." -not -name "Cargo.toml" -not -name ".cargo-checksum.json" -delete
mkdir src
touch src/lib.rs

# Cargo.toml must remain unaltered.
# src/lib.rs must exist, but may be empty.
# 'e3b0...b855' is equivalent to `echo -n "" | sha256sum -`
# We can't set 'package' to the empty hash, as this might conflict with fcmp_pp_rust's Cargo.lock, resulting
# in the same error.
cat .cargo-checksum.json \
| jq '{files: {"Cargo.toml": .files["Cargo.toml"]}, package}' \
| jq '.files += {"src/lib.rs": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}' \
| sponge .cargo-checksum.json
cd ..
done < "/monero/contrib/guix/rust/stubs"

cd /rust

# Create deterministic archive
find . -print0 \
| sort --zero-terminated \
| tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \
| gzip -9n > "/sources/$RUST_DEPS_ARCHIVE"

sha256sum "/sources/$RUST_DEPS_ARCHIVE"
14 changes: 14 additions & 0 deletions contrib/guix/rust/config.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
[source.crates-io]
replace-with = "vendored-sources"

[source."git+https://github.com/kayabaNerve/crypto-bigint?branch=c-repr"]
git = "https://github.com/kayabaNerve/crypto-bigint"
branch = "c-repr"
replace-with = "vendored-sources"

[source."git+https://github.com/kayabaNerve/fcmp-plus-plus"]
git = "https://github.com/kayabaNerve/fcmp-plus-plus"
replace-with = "vendored-sources"

[source.vendored-sources]
directory = "/rust/vendor"
30 changes: 30 additions & 0 deletions contrib/guix/rust/stubs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
addr2line
adler
allocator-api2
cc
dlmalloc
fortanix-sgx-abi
getopts
gimli-0.28.1
gimli
hermit-abi
memchr
miniz_oxide
object
r-efi
r-efi-alloc
rand
rand_xorshift
unicode-width
unwinding
wasi
windows-sys
windows-targets
windows_aarch64_gnullvm
windows_aarch64_msvc
windows_i686_gnu
windows_i686_gnullvm
windows_i686_msvc
windows_x86_64_gnu
windows_x86_64_gnullvm
windows_x86_64_msvc
4 changes: 2 additions & 2 deletions src/fcmp_pp/fcmp_pp_rust/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,10 @@ endif()
set(RUST_TARGET "${RUST_ARCH}-${RUST_PLATFORM}${RUST_TOOLCHAIN}")

if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CARGO_CMD cargo build --target "${RUST_TARGET}")
set(CARGO_CMD cargo build --target "${RUST_TARGET}" ${CARGO_OPTIONS})
set(TARGET_DIR "debug")
else ()
set(CARGO_CMD cargo build --target "${RUST_TARGET}" --release)
set(CARGO_CMD cargo build --target "${RUST_TARGET}" --release ${CARGO_OPTIONS})
set(TARGET_DIR "release")
endif ()

Expand Down

0 comments on commit e89ffb1

Please sign in to comment.