Merge pull request #49557 from symphorien/rust-system-libs

rustc: build with system llvm and jemalloc

pkgs/development/compilers/rust/default.nix

@@ -29,7 +29,7 @@ in rec {
       ./patches/disable-test-inherit-env.patch
     ];
 
-    forceBundledLLVM = true;
+    withBundledLLVM = false;
 
     configureFlags = [ "--release-channel=stable" ];
 

pkgs/development/compilers/rust/rustc.nix

@@ -1,9 +1,9 @@
-{ stdenv, targetPackages
+{ stdenv, targetPackages, removeReferencesTo
 , fetchurl, fetchgit, fetchzip, file, python2, tzdata, ps
 , llvm, jemalloc, ncurses, darwin, rustPlatform, git, cmake, curl
 , which, libffi, gdb
 , version
-, forceBundledLLVM ? false
+, withBundledLLVM ? false
 , src
 , configureFlags ? []
 , patches
@@ -20,6 +20,8 @@ let
 
   llvmShared = llvm.override { enableSharedLibraries = true; };
 
+  prefixedJemalloc = jemalloc.override { stripPrefix = false; };
+
   target = builtins.replaceStrings [" "] [","] (builtins.toString targets);
 in
 
@@ -40,7 +42,11 @@ stdenv.mkDerivation {
   # See https://github.com/NixOS/nixpkgs/pull/34227
   stripDebugList = if stdenv.isDarwin then [ "bin" ] else null;
 
-  NIX_LDFLAGS = optionalString stdenv.isDarwin "-rpath ${llvmShared}/lib";
+  NIX_LDFLAGS =
+       # when linking stage1 libstd: cc: undefined reference to `__cxa_begin_catch'
+       optional (stdenv.isLinux && !withBundledLLVM) "--push-state --as-needed -lstdc++ --pop-state"
+    ++ optional (stdenv.isDarwin && !withBundledLLVM) "-lc++"
+    ++ optional stdenv.isDarwin "-rpath ${llvmShared}/lib";
 
   # Enable nightly features in stable compiles (used for
   # bootstrapping, see https://github.com/rust-lang/rust/pull/37265).
@@ -54,13 +60,12 @@ stdenv.mkDerivation {
   # We need rust to build rust. If we don't provide it, configure will try to download it.
   # Reference: https://github.com/rust-lang/rust/blob/master/src/bootstrap/configure.py
   configureFlags = configureFlags
-                ++ [ "--enable-local-rust" "--local-rust-root=${rustPlatform.rust.rustc}" "--enable-rpath" ]
-                ++ [ "--enable-vendor" ]
-                # ++ [ "--jemalloc-root=${jemalloc}/lib"
-                ++ [ "--default-linker=${targetPackages.stdenv.cc}/bin/cc" ]
-                ++ optional (!forceBundledLLVM) [ "--enable-llvm-link-shared" ]
-                ++ optional (targets != []) "--target=${target}"
-                ++ optional (!forceBundledLLVM) "--llvm-root=${llvmShared}";
+                ++ [ "--enable-local-rust" "--local-rust-root=${rustPlatform.rust.rustc}" "--enable-rpath"
+                     "--enable-vendor"
+                     "--jemalloc-root=${prefixedJemalloc}/lib"
+                     "--default-linker=${targetPackages.stdenv.cc}/bin/cc" ]
+                ++ optional (!withBundledLLVM) [ "--enable-llvm-link-shared" "--llvm-root=${llvmShared}" ]
+                ++ optional (targets != []) "--target=${target}";
 
   # The bootstrap.py will generated a Makefile that then executes the build.
   # The BOOTSTRAP_ARGS used by this Makefile must include all flags to pass
@@ -79,29 +84,13 @@ stdenv.mkDerivation {
   postPatch = ''
     patchShebangs src/etc
 
-    # Fix dynamic linking against llvm
-    #${optionalString (!forceBundledLLVM) ''sed -i 's/, kind = \\"static\\"//g' src/etc/mklldeps.py''}
+    ${optionalString (!withBundledLLVM) ''rm -rf src/llvm''}
+    rm -rf src/jemalloc
 
     # Fix the configure script to not require curl as we won't use it
     sed -i configure \
       -e '/probe_need CFG_CURL curl/d'
 
-    # Fix the use of jemalloc prefixes which our jemalloc doesn't have
-    # TODO: reenable if we can figure out how to get our jemalloc to work
-    #[ -f src/liballoc_jemalloc/lib.rs ] && sed -i 's,je_,,g' src/liballoc_jemalloc/lib.rs
-    #[ -f src/liballoc/heap.rs ] && sed -i 's,je_,,g' src/liballoc/heap.rs # Remove for 1.4.0+
-
-    # Disable fragile tests.
-    rm -vr src/test/run-make-fulldeps/linker-output-non-utf8 || true
-    rm -vr src/test/run-make-fulldeps/issue-26092 || true
-
-    # Remove test targeted at LLVM 3.9 - https://github.com/rust-lang/rust/issues/36835
-    rm -vr src/test/ui/run-pass/issue-36023.rs || true
-
-    # Disable test getting stuck on hydra - possible fix:
-    # https://reviews.llvm.org/rL281650
-    rm -vr src/test/ui/run-pass/issue-36474.rs || true
-
     # On Hydra: `TcpListener::bind(&addr)`: Address already in use (os error 98)'
     sed '/^ *fn fast_rebind()/i#[ignore]' -i src/libstd/net/tcp.rs
 
@@ -137,14 +126,14 @@ stdenv.mkDerivation {
   # ps is needed for one of the test cases
   nativeBuildInputs =
     [ file python2 ps rustPlatform.rust.rustc git cmake
-      which libffi
+      which libffi removeReferencesTo
     ]
     # Only needed for the debuginfo tests
     ++ optional (!stdenv.isDarwin) gdb;
 
-  buildInputs = [ ncurses ] ++ targetToolchains
+  buildInputs = targetToolchains
     ++ optional stdenv.isDarwin Security
-    ++ optional (!forceBundledLLVM) llvmShared;
+    ++ optional (!withBundledLLVM) llvmShared;
 
   outputs = [ "out" "man" "doc" ];
   setOutputFlags = false;
@@ -165,6 +154,12 @@ stdenv.mkDerivation {
 
   inherit doCheck;
 
+  # remove references to llvm-config in lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/librustc_codegen_llvm-llvm.so
+  # and thus a transitive dependency on ncurses
+  postInstall = ''
+    find $out/lib -name "*.so" -type f -exec remove-references-to -t ${llvmShared} '{}' '+'
+  '';
+
   configurePlatforms = [];
 
   # https://github.com/NixOS/nixpkgs/pull/21742#issuecomment-272305764

pkgs/development/libraries/jemalloc/common.nix

@@ -1,6 +1,13 @@
-{ stdenv, fetchurl, version, sha256, ... }@args:
+{ version, sha256 }:
+{ stdenv, fetchurl,
+# By default, jemalloc puts a je_ prefix onto all its symbols on OSX, which
+# then stops downstream builds (mariadb in particular) from detecting it. This
+# option should remove the prefix and give us a working jemalloc.
+# Causes segfaults with some software (ex. rustc), but defaults to true for backward
+# compatibility. Ignored on non OSX.
+stripPrefix ? true }:
 
-stdenv.mkDerivation (rec {
+stdenv.mkDerivation rec {
   name = "jemalloc-${version}";
   inherit version;
 
@@ -9,10 +16,8 @@ stdenv.mkDerivation (rec {
     inherit sha256;
   };
 
-  # By default, jemalloc puts a je_ prefix onto all its symbols on OSX, which
-  # then stops downstream builds (mariadb in particular) from detecting it. This
-  # option should remove the prefix and give us a working jemalloc.
-  configureFlags = stdenv.lib.optional stdenv.isDarwin "--with-jemalloc-prefix=";
+  # see the comment on stripPrefix
+  configureFlags = stdenv.lib.optional (stdenv.isDarwin && stripPrefix) "--with-jemalloc-prefix=";
   doCheck = true;
 
   enableParallelBuilding = true;
@@ -28,4 +33,4 @@ stdenv.mkDerivation (rec {
     platforms = platforms.all;
     maintainers = with maintainers; [ wkennington ];
   };
-} // (builtins.removeAttrs args [ "stdenv" "fetchurl" "version" "sha256" ]))
+}

pkgs/development/libraries/jemalloc/default.nix

@@ -1,6 +1,4 @@
-{ stdenv, fetchurl, fetchpatch }:
 import ./common.nix {
-  inherit stdenv fetchurl;
   version = "5.1.0";
   sha256 = "0s3jpcyhzia8d4k0xyc67is78kg416p9yc3c2f9w6fhhqqffd5jk";
 }

pkgs/development/libraries/jemalloc/jemalloc450.nix

@@ -1,6 +1,4 @@
-{ stdenv, fetchurl }:
 import ./common.nix {
-  inherit stdenv fetchurl;
   version = "4.5.0";
   sha256 = "10373xhpc10pgmai9fkc1z0rs029qlcb3c0qfnvkbwdlcibdh2cl";
-}
+} 

pkgs/top-level/all-packages.nix

@@ -7363,6 +7363,7 @@ with pkgs;
   # For beta and nightly releases use the nixpkgs-mozilla overlay
   rust = callPackage ../development/compilers/rust ({
     inherit (darwin.apple_sdk.frameworks) CoreFoundation Security;
+    llvm = llvm_7;
   } // stdenv.lib.optionalAttrs (stdenv.cc.isGNU && stdenv.hostPlatform.isi686) {
     stdenv = overrideCC stdenv gcc6; # with gcc-7: undefined reference to `__divmoddi4'
   });