stdenv: add minimal flag

pkgs/development/compilers/gcc/11/default.nix

@@ -190,7 +190,7 @@ stdenv.mkDerivation ({
   preConfigure = (import ../common/pre-configure.nix {
     inherit lib;
     inherit version targetPlatform hostPlatform gnatboot langAda langGo langJit crossStageStatic enableMultilib;
-  }) + ''
+  }) + lib.optionalString (libxcrypt!=null) ''
     ln -sf ${libxcrypt}/include/crypt.h libsanitizer/sanitizer_common/crypt.h
   '';
 

pkgs/development/interpreters/perl/default.nix

@@ -1,7 +1,7 @@
 { config, lib, stdenv, fetchurl, fetchFromGitHub, pkgs, buildPackages
 , callPackage
 , enableThreading ? true, coreutils, makeWrapper
-, enableCrypt ? true, libxcrypt ? null
+, enableCrypt ? libxcrypt!=null, libxcrypt ? null
 , zlib
 }:
 

pkgs/development/interpreters/python/cpython/default.nix

@@ -384,7 +384,7 @@ in with passthru; stdenv.mkDerivation {
 
   postInstall = let
     # References *not* to nuke from (sys)config files
-    keep-references = concatMapStringsSep " " (val: "-e ${val}") ([
+    keep-references = concatMapStringsSep " " (val: "-e ${val}") (optionals (libxcrypt!=null) [
       (placeholder "out") libxcrypt
     ] ++ optionals tzdataSupport [
       tzdata

pkgs/stdenv/cross/default.nix

@@ -1,7 +1,10 @@
 { lib
 , localSystem, crossSystem, config, overlays, crossOverlays ? []
+, rebootstrap ? false
 }:
 
+assert rebootstrap -> throw "bootstrapping is irrelevant to stdenv/cross";
+
 let
   bootStages = import ../. {
     inherit lib localSystem overlays;

pkgs/stdenv/custom/default.nix

@@ -1,7 +1,10 @@
 { lib
 , localSystem, crossSystem, config, overlays, crossOverlays ? []
+, rebootstrap ? false
 }:
 
+assert rebootstrap -> throw "bootstrapping is not relevant to stdenv/custom";
+
 assert crossSystem == localSystem;
 
 let

pkgs/stdenv/darwin/default.nix

@@ -34,9 +34,11 @@
       cpio = fetch { file = "cpio"; sha256 = "sha256-SWkwvLaFyV44kLKL2nx720SvcL4ej/p2V/bX3uqAGO0="; };
       tarball = fetch { file = "bootstrap-tools.cpio.bz2"; sha256 = "sha256-kRC/bhCmlD4L7KAvJQgcukk7AinkMz4IwmG1rqlh5tA="; executable = false; };
     }
+, rebootstrap ? false
 }:
 
 assert crossSystem == localSystem;
+assert rebootstrap -> throw "darwin does not yet support automatic rebootstrapping";
 
 let
   inherit (localSystem) system;

pkgs/stdenv/default.nix

@@ -8,6 +8,13 @@
   lib
   # Args to pass on to the pkgset builder, too
 , localSystem, crossSystem, config, overlays, crossOverlays ? []
+
+# If true, the bootstrapFiles will be rebuilt, resulting in a stdenv
+# built by the same compiler as its bootstrap tools.  Turning this
+# feature on will considerably increase the rebuild cycle, but may
+# help with troubleshooting the bootstrap.
+, rebootstrap ? false
+
 } @ args:
 
 let

pkgs/stdenv/freebsd/default.nix

@@ -1,7 +1,10 @@
 { lib
 , localSystem, crossSystem, config, overlays, crossOverlays ? []
+, rebootstrap ? false
 }:
 
+assert rebootstrap -> throw "stdenv/freebsd does not yet support automatic rebootstrapping";
+
 assert crossSystem == localSystem;
 let inherit (localSystem) system;
     fetchURL = import <nix/fetchurl.nix>;

pkgs/stdenv/linux/default.nix

@@ -59,6 +59,16 @@
 { lib
 , localSystem, crossSystem, config, overlays, crossOverlays ? []
 
+# ARMv8.0 backward compatibility (`-moutline-intrinsics`) currently
+# requires regenerating bootstrapFiles; see:
+#
+#   https://github.com/NixOS/nixpkgs/issues/108111
+#   https://github.com/NixOS/nixpkgs/pull/108200
+#   https://bugzilla.redhat.com/show_bug.cgi?id=1830472
+, rebootstrap ? localSystem.isAarch64
+
+, minimal ? false
+
 , bootstrapFiles ?
   let table = {
     glibc = {
@@ -93,7 +103,7 @@
   files = archLookupTable.${localSystem.system} or (if getCompatibleTools != null then getCompatibleTools
     else (abort "unsupported platform for the pure Linux stdenv"));
   in files
-}:
+} @args:
 
 assert crossSystem == localSystem;
 
@@ -117,7 +127,7 @@ let
 
 
   # Download and unpack the bootstrap tools (coreutils, GCC, Glibc, ...).
-  bootstrapTools = import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) {
+  unpackBootstrapTools = bootstrapFiles: import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) {
     inherit system bootstrapFiles;
     extraAttrs = lib.optionalAttrs
       config.contentAddressedByDefault
@@ -135,10 +145,9 @@ let
   # the bootstrap.  In all stages, we build an stdenv and the package
   # set that can be built with that stdenv.
   stageFun = prevStage:
-    { name, overrides ? (self: super: {}), extraNativeBuildInputs ? [] }:
+    { name, overrides ? (self: super: {}), extraNativeBuildInputs ? [], bootstrapTools ? prevStage.bootstrapTools }:
 
     let
-
       thisStdenv = import ../generic {
         name = "${name}-stdenv-linux";
         buildPlatform = localSystem;
@@ -177,33 +186,62 @@ let
           stdenvNoCC = prevStage.ccWrapperStdenv;
         };
 
-        overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; };
+        overrides = self: super: (overrides self super) // {
+          fetchurl = thisStdenv.fetchurlBoot;
+          inherit bootstrapTools;
+        };
       };
-
     in {
       inherit config overlays;
       stdenv = thisStdenv;
     };
 
 in
 
-[
+# when rebootstrapping, we first prepend a copy of the stdenv stages
+# that use the fetched bootstrapFiles:
+lib.optionals rebootstrap (import ./. (args // {
+  rebootstrap = false;
+  minimal = true;
+  overlays = overlays ++ [ (final: prev: {
+    gettext = null;
+    help2man = null;
+    texinfo = null;
+    python3Minimal = null;
+    python3 = null;
+    libxcrypt = null;
+    coreutils = prev.coreutilsMinimal;
+  }) ];
+})) ++
 
-  ({}: {
+[
+  (prevStage: {
     __raw = true;
-
     gcc-unwrapped = null;
     binutils = null;
     coreutils = null;
     gnugrep = null;
+    bootstrapTools =
+      unpackBootstrapTools
+        (if   rebootstrap
+         then (prevStage.freshBootstrapTools.override {
+           # Copy busybox from the original bootstrapFiles; it is
+           # safe to do this because it is statically linked.  This
+           # avoids having to rebuild a musl-gcc compiler (glibc
+           # doesn't know how to do static linking).
+           busyboxMinimal = prevStage.stdenv.bootstrapFiles.busybox;
+           # don't rebuild a custom compiler
+           bootGCC = prevStage.gcc.cc;
+           nukeReferences = prevStage.nukeReferences.override { perl = prevStage.stdenv.bootstrapTools; };
+         }).bootstrapFiles else bootstrapFiles);
   })
 
   # Build a dummy stdenv with no GCC or working fetchurl.  This is
   # because we need a stdenv to build the GCC wrapper and fetchurl.
   #
   # resulting stage0 stdenv:
   # - coreutils, binutils, glibc, gcc: from bootstrapFiles
-  (prevStage: stageFun prevStage {
+  ({ bootstrapTools, ...}@prevStage: stageFun prevStage {
     name = "bootstrap-stage0";
 
     overrides = self: super: {
@@ -244,6 +282,8 @@ in
       };
       coreutils = bootstrapTools;
       gnugrep = bootstrapTools;
+    } // lib.optionalAttrs minimal {
+      nukeReferences = prevStage.nukeReferences.override { perl = prevStage.bootstrapTools; };
     };
   })
 
@@ -280,7 +320,7 @@ in
       # This is not an issue for the final stdenv, because this perl
       # won't be included in the final stdenv and won't be exported to
       # top-level pkgs as an override either.
-      perl = super.perl.override { enableThreading = false; enableCrypt = false; };
+      perl = if minimal then prevStage.bootstrapTools else super.perl.override { enableThreading = false; enableCrypt = false; };
     };
   })
 
@@ -332,7 +372,8 @@ in
         # and that can fail to load.  Therefore we upgrade `ld` to use newer libc;
         # apparently the interpreter needs to match libc, too.
         bintools = self.stdenvNoCC.mkDerivation {
-          inherit (prevStage.bintools.bintools) name;
+          pname = prevStage.bintools.bintools.pname + "-patchelfed";
+          inherit (prevStage.bintools.bintools) version;
           enableParallelBuilding = true;
           dontUnpack = true;
           dontBuild = true;
@@ -477,7 +518,7 @@ in
   #        and the bootstrapTools-built, statically-linked
   #        lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc
   #        (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d).
-  (prevStage: {
+  ({ bootstrapTools, ...}@prevStage: {
     inherit config overlays;
     stdenv = import ../generic rec {
       name = "stdenv-linux";
@@ -504,7 +545,7 @@ in
       inherit (prevStage.stdenv) fetchurlBoot;
 
       extraAttrs = {
-        inherit bootstrapTools;
+        inherit bootstrapTools bootstrapFiles;
         shellPackage = prevStage.bash;
       };
 
@@ -544,7 +585,12 @@ in
           inherit (self) stdenv runCommandLocal patchelf libunistring;
         };
 
+      } // lib.optionalAttrs (!minimal) {
         gnumake = super.gnumake.override { inBootstrap = false; };
+      } // lib.optionalAttrs minimal {
+        fetchurl = prevStage.stdenv.fetchurlBoot;
+        cpio = prevStage.bootstrapTools;
+        nukeReferences = prevStage.nukeReferences.override { perl = prevStage.bootstrapTools; };
       } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) {
         # Need to get rid of these when cross-compiling.
         inherit (prevStage) binutils binutils-unwrapped;

pkgs/stdenv/linux/make-bootstrap-tools.nix

@@ -1,22 +1,6 @@
-{ pkgs ? import ../../.. {} }:
-
-let
-  libc = pkgs.stdenv.cc.libc;
-in with pkgs; rec {
-
-
-  coreutilsMinimal = coreutils.override (args: {
-    # We want coreutils without ACL/attr support.
-    aclSupport = false;
-    attrSupport = false;
-    # Our tooling currently can't handle scripts in bin/, only ELFs and symlinks.
-    singleBinary = "symlinks";
-  });
-
-  tarMinimal = gnutar.override { acl = null; };
-
-  busyboxMinimal = busybox.override {
-    useMusl = !stdenv.targetPlatform.isRiscV;
+{ pkgs ? import ../../.. {}
+, busyboxMinimal ? "${pkgs.busybox.override {
+    useMusl = !pkgs.stdenv.targetPlatform.isRiscV;
     enableStatic = true;
     enableMinimal = true;
     extraConfig = ''
@@ -28,9 +12,21 @@ in with pkgs; rec {
       CONFIG_TAR y
       CONFIG_UNXZ y
     '';
-  };
+  }}/bin/busybox"
+, bootGCC ? pkgs.gcc.cc.override { enableLTO = false; }
+, nukeReferences ? pkgs.buildPackages.nukeReferences
+, cpio ? pkgs.buildPackages.cpio
+, rebootstrap ? false
+}:
+
+let
+  libc = pkgs.stdenv.cc.libc;
+in with pkgs; rec {
+
+  inherit (pkgs) coreutilsMinimal;
+
+  tarMinimal = gnutar.override { acl = null; };
 
-  bootGCC = gcc.cc.override { enableLTO = false; };
   bootBinutils = binutils.bintools.override {
     withAllTargets = false;
     # Don't need two linkers, disable whatever's not primary/default.
@@ -50,7 +46,7 @@ in with pkgs; rec {
         schedulingPriority = 200;
       };
 
-      nativeBuildInputs = [ buildPackages.nukeReferences buildPackages.cpio ];
+      nativeBuildInputs = [ nukeReferences cpio ];
 
       buildCommand = ''
         set -x
@@ -196,7 +192,7 @@ in with pkgs; rec {
 
         mkdir $out/on-server
         XZ_OPT="-9 -e" tar cvJf $out/on-server/bootstrap-tools.tar.xz --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack .
-        cp ${busyboxMinimal}/bin/busybox $out/on-server
+        cp ${busyboxMinimal} $out/on-server/busybox
         chmod u+w $out/on-server/busybox
         nuke-refs $out/on-server/busybox
       ''; # */

pkgs/stdenv/native/default.nix

@@ -1,8 +1,10 @@
 { lib
 , localSystem, crossSystem, config, overlays, crossOverlays ? []
+, rebootstrap
 }:
 
 assert crossSystem == localSystem;
+assert rebootstrap -> throw "stdenv/native does not yet support automatic rebootstrapping";
 
 let
   inherit (localSystem) system;

pkgs/top-level/all-packages.nix

@@ -6139,6 +6139,13 @@ with pkgs;
   coreutils =  callPackage ../tools/misc/coreutils { };
   coreutils-full = coreutils.override { minimal = false; };
   coreutils-prefixed = coreutils.override { withPrefix = true; singleBinary = false; };
+  coreutilsMinimal = callPackage ../tools/misc/coreutils {  # this is for the bootstrapFiles
+    # We want coreutils without ACL/attr support.
+    aclSupport = false;
+    attrSupport = false;
+    # Our tooling currently can't handle scripts in bin/, only ELFs and symlinks.
+    singleBinary = "symlinks";
+  };
 
   corkscrew = callPackage ../tools/networking/corkscrew { };
 
@@ -14052,7 +14059,6 @@ with pkgs;
   inherit (let
       num =
         if (with stdenv.targetPlatform; isVc4 || libc == "relibc") then 6
-        else if (stdenv.targetPlatform.isAarch64 && stdenv.isLinux) then 9
         else 11;
       numS = toString num;
     in {