nixos/initrd-ssh: switch from Dropbear to OpenSSH

Dropbear lags behind OpenSSH significantly in both support for modern
key formats like `ssh-ed25519`, let alone the recently-introduced
U2F/FIDO2-based `sk-ssh-ed25519@openssh.com` (as I found when I switched
my `authorizedKeys` over to it and promptly locked myself out of my
server's initrd SSH, breaking reboots), as well as security features
like multiprocess isolation. Using the same SSH daemon for stage-1 and
the main system ensures key formats will always remain compatible, as
well as more conveniently allowing the sharing of configuration and
host keys.

The main reason to use Dropbear over OpenSSH would be initrd space
concerns, but NixOS initrds are already large (17 MiB currently on my
server), and the size difference between the two isn't huge (the test's
initrd goes from 9.7 MiB to 12 MiB with this change). If the size is
still a problem, then it would be easy to shrink sshd down to a few
hundred kilobytes by using an initrd-specific build that uses musl and
disables things like Kerberos support.

This passes the test and works on my server, but more rigorous testing
and review from people who use initrd SSH would be appreciated!

nixos/doc/manual/release-notes/rl-2009.xml

@@ -108,6 +108,23 @@
       <link linkend="opt-security.duosec.integrationKey">security.duosec.integrationKey</link>.
     </para>
    </listitem>
+   <listitem>
+    <para>
+     The initrd SSH support now uses OpenSSH rather than Dropbear to
+     allow the use of Ed25519 keys and other OpenSSH-specific
+     functionality. Host keys must now be in the OpenSSH format, and at
+     least one pre-generated key must be specified.
+    </para>
+    <para>
+     If you used the <option>boot.initrd.network.ssh.host*Key</option>
+     options, you'll get an error explaining how to convert your host
+     keys and migrate to the new
+     <option>boot.initrd.network.ssh.hostKeys</option> option.
+     Otherwise, if you don't have any host keys set, you'll need to
+     generate some; see the <option>hostKeys</option> option
+     documentation for instructions.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 

nixos/modules/system/boot/initrd-ssh.nix

@@ -10,123 +10,193 @@ in
 
 {
 
-  options = {
-
-    boot.initrd.network.ssh.enable = mkOption {
+  options.boot.initrd.network.ssh = {
+    enable = mkOption {
       type = types.bool;
       default = false;
       description = ''
         Start SSH service during initrd boot. It can be used to debug failing
         boot on a remote server, enter pasphrase for an encrypted partition etc.
         Service is killed when stage-1 boot is finished.
+
+        The sshd configuration is largely inherited from
+        <option>services.openssh</option>.
       '';
     };
 
-    boot.initrd.network.ssh.port = mkOption {
+    port = mkOption {
       type = types.int;
       default = 22;
       description = ''
         Port on which SSH initrd service should listen.
       '';
     };
 
-    boot.initrd.network.ssh.shell = mkOption {
+    shell = mkOption {
       type = types.str;
       default = "/bin/ash";
       description = ''
         Login shell of the remote user. Can be used to limit actions user can do.
       '';
     };
 
-    boot.initrd.network.ssh.hostRSAKey = mkOption {
-      type = types.nullOr types.path;
-      default = null;
-      description = ''
-        RSA SSH private key file in the Dropbear format.
-
-        WARNING: Unless your bootloader supports initrd secrets, this key is
-        contained insecurely in the global Nix store. Do NOT use your regular
-        SSH host private keys for this purpose or you'll expose them to
-        regular users!
-      '';
-    };
-
-    boot.initrd.network.ssh.hostDSSKey = mkOption {
-      type = types.nullOr types.path;
-      default = null;
+    hostKeys = mkOption {
+      type = types.listOf (types.either types.str types.path);
+      default = [];
+      example = [
+        "/etc/secrets/initrd/ssh_host_rsa_key"
+        "/etc/secrets/initrd/ssh_host_ed25519_key"
+      ];
       description = ''
-        DSS SSH private key file in the Dropbear format.
-
-        WARNING: Unless your bootloader supports initrd secrets, this key is
-        contained insecurely in the global Nix store. Do NOT use your regular
-        SSH host private keys for this purpose or you'll expose them to
-        regular users!
+        Specify SSH host keys to import into the initrd.
+
+        To generate keys, use
+        <citerefentry><refentrytitle>ssh-keygen</refentrytitle><manvolnum>1</manvolnum></citerefentry>:
+
+        <screen>
+        <prompt># </prompt>ssh-keygen -t rsa -N "" -f /etc/secrets/initrd/ssh_host_rsa_key
+        <prompt># </prompt>ssh-keygen -t ed25519 -N "" -f /etc/secrets/initrd/ssh_host_ed_25519_key
+        </screen>
+
+        <warning>
+          <para>
+            Unless your bootloader supports initrd secrets, these keys
+            are stored insecurely in the global Nix store. Do NOT use
+            your regular SSH host private keys for this purpose or
+            you'll expose them to regular users!
+          </para>
+          <para>
+            Additionally, even if your initrd supports secrets, if
+            you're using initrd SSH to unlock an encrypted disk then
+            using your regular host keys exposes the private keys on
+            your unencrypted boot partition.
+          </para>
+        </warning>
       '';
     };
 
-    boot.initrd.network.ssh.hostECDSAKey = mkOption {
-      type = types.nullOr types.path;
-      default = null;
-      description = ''
-        ECDSA SSH private key file in the Dropbear format.
-
-        WARNING: Unless your bootloader supports initrd secrets, this key is
-        contained insecurely in the global Nix store. Do NOT use your regular
-        SSH host private keys for this purpose or you'll expose them to
-        regular users!
-      '';
-    };
-
-    boot.initrd.network.ssh.authorizedKeys = mkOption {
+    authorizedKeys = mkOption {
       type = types.listOf types.str;
       default = config.users.users.root.openssh.authorizedKeys.keys;
+      defaultText = "config.users.users.root.openssh.authorizedKeys.keys";
       description = ''
         Authorized keys for the root user on initrd.
-        Note that Dropbear doesn't support OpenSSH's Ed25519 key type.
       '';
     };
-
   };
 
-  config = mkIf (config.boot.initrd.network.enable && cfg.enable) {
+  imports =
+    map (opt: mkRemovedOptionModule ([ "boot" "initrd" "network" "ssh" ] ++ [ opt ]) ''
+      The initrd SSH functionality now uses OpenSSH rather than Dropbear.
+
+      If you want to keep your existing initrd SSH host keys, convert them with
+        $ dropbearconvert dropbear openssh dropbear_host_$type_key ssh_host_$type_key
+      and then set options.boot.initrd.network.ssh.hostKeys.
+    '') [ "hostRSAKey" "hostDSSKey" "hostECDSAKey" ];
+
+  config = let
+    # Nix complains if you include a store hash in initrd path names, so
+    # as an awful hack we drop the first character of the hash.
+    initrdKeyPath = path: if isString path
+      then path
+      else let name = builtins.baseNameOf path; in
+        builtins.unsafeDiscardStringContext ("/etc/ssh/" +
+          substring 1 (stringLength name) name);
+
+    sshdCfg = config.services.openssh;
+
+    sshdConfig = ''
+      Port ${toString cfg.port}
+
+      PasswordAuthentication no
+      ChallengeResponseAuthentication no
+
+      ${flip concatMapStrings cfg.hostKeys (path: ''
+        HostKey ${initrdKeyPath path}
+      '')}
+
+      KexAlgorithms ${concatStringsSep "," sshdCfg.kexAlgorithms}
+      Ciphers ${concatStringsSep "," sshdCfg.ciphers}
+      MACs ${concatStringsSep "," sshdCfg.macs}
+
+      LogLevel ${sshdCfg.logLevel}
+
+      ${if sshdCfg.useDns then ''
+        UseDNS yes
+      '' else ''
+        UseDNS no
+      ''}
+    '';
+  in mkIf (config.boot.initrd.network.enable && cfg.enable) {
     assertions = [
-      { assertion = cfg.authorizedKeys != [];
+      {
+        assertion = cfg.authorizedKeys != [];
         message = "You should specify at least one authorized key for initrd SSH";
       }
+
+      {
+        assertion = cfg.hostKeys != [];
+        message = ''
+          You must now pre-generate the host keys for initrd SSH.
+          See the boot.inird.network.ssh.hostKeys documentation
+          for instructions.
+        '';
+      }
     ];
 
     boot.initrd.extraUtilsCommands = ''
-      copy_bin_and_libs ${pkgs.dropbear}/bin/dropbear
+      copy_bin_and_libs ${pkgs.openssh}/bin/sshd
       cp -pv ${pkgs.glibc.out}/lib/libnss_files.so.* $out/lib
     '';
 
     boot.initrd.extraUtilsCommandsTest = ''
-      $out/bin/dropbear -V
+      # sshd requires a host key to check config, so we pass in the test's
+      echo -n ${escapeShellArg sshdConfig} |
+        $out/bin/sshd -t -f /dev/stdin \
+        -h ${../../../tests/initrd-network-ssh/ssh_host_ed25519_key}
     '';
 
     boot.initrd.network.postCommands = ''
       echo '${cfg.shell}' > /etc/shells
       echo 'root:x:0:0:root:/root:${cfg.shell}' > /etc/passwd
+      echo 'sshd:x:1:1:sshd:/var/empty:/bin/nologin' >> /etc/passwd
       echo 'passwd: files' > /etc/nsswitch.conf
 
-      mkdir -p /var/log
+      mkdir -p /var/log /var/empty
       touch /var/log/lastlog
 
-      mkdir -p /etc/dropbear
+      mkdir -p /etc/ssh
+      echo -n ${escapeShellArg sshdConfig} > /etc/ssh/sshd_config
+
+      echo "export PATH=$PATH" >> /etc/profile
+      echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /etc/profile
 
       mkdir -p /root/.ssh
       ${concatStrings (map (key: ''
         echo ${escapeShellArg key} >> /root/.ssh/authorized_keys
       '') cfg.authorizedKeys)}
 
-      dropbear -s -j -k -E -p ${toString cfg.port} ${optionalString (cfg.hostRSAKey == null && cfg.hostDSSKey == null && cfg.hostECDSAKey == null) "-R"}
+      ${flip concatMapStrings cfg.hostKeys (path: ''
+        # keys from Nix store are world-readable, which sshd doesn't like
+        chmod 0600 "${initrdKeyPath path}"
+      '')}
+
+      /bin/sshd -e
     '';
 
-    boot.initrd.secrets =
-     (optionalAttrs (cfg.hostRSAKey != null) { "/etc/dropbear/dropbear_rsa_host_key" = cfg.hostRSAKey; }) //
-     (optionalAttrs (cfg.hostDSSKey != null) { "/etc/dropbear/dropbear_dss_host_key" = cfg.hostDSSKey; }) //
-     (optionalAttrs (cfg.hostECDSAKey != null) { "/etc/dropbear/dropbear_ecdsa_host_key" = cfg.hostECDSAKey; });
+    boot.initrd.postMountCommands = ''
+      # Stop sshd cleanly before stage 2.
+      #
+      # If you want to keep it around to debug post-mount SSH issues,
+      # run `touch /.keep_sshd` (either from an SSH session or in
+      # another initrd hook like preDeviceCommands).
+      if ! [ -e /.keep_sshd ]; then
+        pkill -x sshd
+      fi
+    '';
 
+    boot.initrd.secrets = listToAttrs
+      (map (path: nameValuePair (initrdKeyPath path) path) cfg.hostKeys);
   };
 
 }

nixos/modules/system/boot/stage-1.nix

@@ -142,7 +142,10 @@ let
              let source' = if source == null then dest else source; in
                ''
                   mkdir -p $(dirname "$out/secrets/${dest}")
-                  cp -a ${source'} "$out/secrets/${dest}"
+                  # Some programs (e.g. ssh) doesn't like secrets to be
+                  # symlinks, so we use `cp -L` here to match the
+                  # behaviour when secrets are natively supported.
+                  cp -Lr ${source'} "$out/secrets/${dest}"
                 ''
           ) config.boot.initrd.secrets))
        }

nixos/tests/initrd-network-ssh/default.nix

@@ -3,7 +3,7 @@ import ../make-test-python.nix ({ lib, ... }:
 {
   name = "initrd-network-ssh";
   meta = with lib.maintainers; {
-    maintainers = [ willibutz ];
+    maintainers = [ willibutz emily ];
   };
 
   nodes = with lib; {
@@ -17,9 +17,9 @@ import ../make-test-python.nix ({ lib, ... }:
           enable = true;
           ssh = {
             enable = true;
-            authorizedKeys = [ "${readFile ./openssh.pub}" ];
+            authorizedKeys = [ (readFile ./id_ed25519.pub) ];
             port = 22;
-            hostRSAKey = ./dropbear.priv;
+            hostKeys = [ ./ssh_host_ed25519_key ];
           };
         };
         boot.initrd.preLVMCommands = ''
@@ -42,11 +42,11 @@ import ../make-test-python.nix ({ lib, ... }:
               "${toString (head (splitString " " (
                 toString (elemAt (splitString "\n" config.networking.extraHosts) 2)
               )))} "
-              "${readFile ./dropbear.pub}"
+              "${readFile ./ssh_host_ed25519_key.pub}"
             ];
           };
           sshKey = {
-            source = ./openssh.priv; # dont use this anywhere else
+            source = ./id_ed25519;
             mode = "0600";
           };
         };
@@ -56,7 +56,17 @@ import ../make-test-python.nix ({ lib, ... }:
   testScript = ''
     start_all()
     client.wait_for_unit("network.target")
-    client.wait_until_succeeds("ping -c 1 server")
+
+
+    def ssh_is_up(_) -> bool:
+        status, _ = client.execute("nc -z server 22")
+        return status == 0
+
+
+    with client.nested("waiting for SSH server to come up"):
+        retry(ssh_is_up)
+
+
     client.succeed(
         "ssh -i /etc/sshKey -o UserKnownHostsFile=/etc/knownHosts server 'touch /fnord'"
     )

nixos/tests/initrd-network-ssh/generate-keys.nix

@@ -1,12 +1,10 @@
 with import ../../.. {};
 
 runCommand "gen-keys" {
-    buildInputs = [ dropbear openssh ];
+    buildInputs = [ openssh ];
   }
   ''
     mkdir $out
-    dropbearkey -t rsa -f $out/dropbear.priv -s 4096 | sed -n 2p > $out/dropbear.pub
-    ssh-keygen -q -t rsa -b 4096 -N "" -f client
-    mv client $out/openssh.priv
-    mv client.pub $out/openssh.pub
+    ssh-keygen -q -t ed25519 -N "" -f $out/ssh_host_ed25519_key
+    ssh-keygen -q -t ed25519 -N "" -f $out/id_ed25519
   ''

nixos/tests/initrd-network-ssh/id_ed25519

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACAVcX+32Yqig25RxRA8bel/f604wV0p/63um+Oku/3vfwAAAJi/AJZMvwCW
+TAAAAAtzc2gtZWQyNTUxOQAAACAVcX+32Yqig25RxRA8bel/f604wV0p/63um+Oku/3vfw
+AAAEAPLjQusjrB90Lk3996G3AbtTeK+XweNgxaegYnml/A/RVxf7fZiqKDblHFEDxt6X9/
+rTjBXSn/re6b46S7/e9/AAAAEG5peGJsZEBsb2NhbGhvc3QBAgMEBQ==
+-----END OPENSSH PRIVATE KEY-----

nixos/tests/initrd-network-ssh/id_ed25519.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBVxf7fZiqKDblHFEDxt6X9/rTjBXSn/re6b46S7/e9/ nixbld@localhost