composefs: When using signatures, delay application until first boot

We can't safely apply the fs-verity with signature until we have
booted with the new initrd, because the public key that matches the
signature is loaded from it. So, instead we save the .sig file next
to the compoosefs, and on the first boot we detect that it is there, and
the composefs file isn't fs-verity, so we apply it.

Things get a bit more complex due to having to temporarily make
/sysroot read-write for the fsverity operation too.

src/libostree/ostree-sysroot-deploy.c

@@ -649,7 +649,6 @@ checkout_deployment_tree (OstreeSysroot *sysroot, OstreeRepo *repo, OstreeDeploy
 #ifdef HAVE_COMPOSEFS
   if (repo->composefs_wanted != OT_TRISTATE_NO)
     {
-      g_autoptr (GBytes) sig = NULL;
       gboolean apply_composefs_signature;
       g_autofree guchar *fsverity_digest = NULL;
       g_auto (GLnxTmpfile) tmpf = {
@@ -699,22 +698,26 @@ checkout_deployment_tree (OstreeSysroot *sysroot, OstreeRepo *repo, OstreeDeploy
       if (!glnx_fchmod (tmpf.fd, 0644, error))
         return FALSE;
 
-      if (metadata_composefs_sig)
+      if (metadata_composefs_sig && apply_composefs_signature)
         {
+          /* We can't apply the signature during deploy, because the corresponding public key for
+             this commit is not loaded into the keyring. So, we delay fs-verity application to the
+             first boot. */
+
           g_autofree char *composefs_sig_path
               = g_strdup_printf ("%s/.ostree.cfs.sig", checkout_target_name);
+          g_autoptr (GBytes) sig = g_variant_get_data_as_bytes (metadata_composefs_sig);
 
-          sig = g_variant_get_data_as_bytes (metadata_composefs_sig);
-
-          /* Write signature to file so it can be applied later if needed */
           if (!glnx_file_replace_contents_at (osdeploy_dfd, composefs_sig_path,
                                               g_bytes_get_data (sig, NULL), g_bytes_get_size (sig),
                                               0, cancellable, error))
             return FALSE;
         }
-
-      if (!_ostree_tmpf_fsverity (repo, &tmpf, apply_composefs_signature ? sig : NULL, error))
-        return FALSE;
+      else
+        {
+          if (!_ostree_tmpf_fsverity (repo, &tmpf, NULL, error))
+            return FALSE;
+        }
 
       if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, osdeploy_dfd, composefs_cfs_path,
                                  error))

src/switchroot/ostree-mount-util.h

@@ -24,12 +24,18 @@
 #include <err.h>
 #include <fcntl.h>
 #include <stdbool.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/ioctl.h>
 #include <sys/statvfs.h>
 #include <unistd.h>
 
+#ifdef HAVE_LINUX_FSVERITY_H
+#include <linux/fsverity.h>
+#endif
+
 #define INITRAMFS_MOUNT_VAR "/run/ostree/initramfs-mount-var"
 #define _OSTREE_SYSROOT_READONLY_STAMP "/run/ostree-sysroot-ro.stamp"
 #define _OSTREE_COMPOSEFS_ROOT_STAMP "/run/ostree-composefs-root.stamp"
@@ -123,4 +129,65 @@ touch_run_ostree (void)
   (void)close (fd);
 }
 
+static inline unsigned char *
+read_file (const char *path, size_t *out_len)
+{
+  int fd;
+
+  fd = open (path, O_RDONLY);
+  if (fd < 0)
+    {
+      if (errno == ENOENT)
+        return NULL;
+      err (EXIT_FAILURE, "failed to open %s", path);
+    }
+
+  struct stat stbuf;
+  if (fstat (fd, &stbuf))
+    err (EXIT_FAILURE, "fstat(%s) failed", path);
+
+  size_t file_size = stbuf.st_size;
+  unsigned char *buf = malloc (file_size);
+  if (buf == NULL)
+    err (EXIT_FAILURE, "Out of memory");
+
+  size_t file_read = 0;
+  while (file_read < file_size)
+    {
+      ssize_t bytes_read;
+      do
+        bytes_read = read (fd, buf + file_read, file_size - file_read);
+      while (bytes_read == -1 && errno == EINTR);
+      if (bytes_read == -1)
+        err (EXIT_FAILURE, "read_file(%s) failed", path);
+      if (bytes_read == 0)
+        break;
+
+      file_read += bytes_read;
+    }
+
+  close (fd);
+
+  *out_len = file_read;
+  return buf;
+}
+
+static inline void
+fsverity_sign (int fd, unsigned char *signature, size_t signature_len)
+{
+#ifdef HAVE_LINUX_FSVERITY_H
+  struct fsverity_enable_arg arg = {
+    0,
+  };
+  arg.version = 1;
+  arg.hash_algorithm = FS_VERITY_HASH_ALG_SHA256;
+  arg.block_size = 4096;
+  arg.sig_size = signature_len;
+  arg.sig_ptr = (uint64_t)signature;
+
+  if (ioctl (fd, FS_IOC_ENABLE_VERITY, &arg) < 0)
+    err (EXIT_FAILURE, "failed to fs-verity sign file");
+#endif
+}
+
 #endif /* __OSTREE_MOUNT_UTIL_H_ */

src/switchroot/ostree-prepare-root.c

@@ -66,13 +66,18 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/ioctl.h>
 #include <sys/mount.h>
 #include <sys/param.h>
 #include <sys/stat.h>
 #include <sys/syscall.h>
 #include <sys/types.h>
 #include <unistd.h>
 
+/* We can't include both linux/fs.h and sys/mount.h, so define these directly */
+#define FS_VERITY_FL 0x00100000 /* Verity protected inode */
+#define FS_IOC_GETFLAGS _IOR ('f', 1, long)
+
 #if defined(HAVE_LIBSYSTEMD) && !defined(OSTREE_PREPARE_ROOT_STATIC)
 #define USE_LIBSYSTEMD
 #endif
@@ -307,6 +312,47 @@ main (int argc, char *argv[])
         objdirs,
         1,
       };
+      int cfs_fd;
+      unsigned cfs_flags;
+
+      cfs_fd = open (".ostree.cfs", O_RDONLY);
+      if (cfs_fd < 0)
+        {
+          if (errno == ENOENT)
+            goto nocfs;
+
+          err (EXIT_FAILURE, "failed to open .ostree.cfs");
+        }
+
+      /* Check if file is already fsverity */
+      if (ioctl (cfs_fd, FS_IOC_GETFLAGS, &cfs_flags) < 0)
+        err (EXIT_FAILURE, "failed to get .ostree.cfs flags");
+
+      /* It is not, apply signature (if it exists) */
+      if ((cfs_flags & FS_VERITY_FL) == 0)
+        {
+          unsigned char *signature;
+          size_t signature_len;
+
+          signature = read_file (".ostree.cfs.sig", &signature_len);
+          if (signature != NULL)
+            {
+              /* If we're read-only we temporarily make it read-write to sign the image */
+              if (!sysroot_currently_writable
+                  && mount (root_mountpoint, root_mountpoint, NULL, MS_REMOUNT | MS_SILENT, NULL)
+                         < 0)
+                err (EXIT_FAILURE, "failed to remount rootfs writable (for signing)");
+
+              fsverity_sign (cfs_fd, signature, signature_len);
+              free (signature);
+
+              if (!sysroot_currently_writable
+                  && mount (root_mountpoint, root_mountpoint, NULL,
+                            MS_REMOUNT | MS_RDONLY | MS_SILENT, NULL)
+                         < 0)
+                err (EXIT_FAILURE, "failed to remount rootfs back read-only (after signing)");
+            }
+        }
 
       cfs_options.flags = LCFS_MOUNT_FLAGS_READONLY;
 
@@ -324,12 +370,7 @@ main (int argc, char *argv[])
           cfs_options.expected_digest = composefs_digest;
         }
 
-      if (lcfs_mount_image (".ostree.cfs", "/sysroot.tmp", &cfs_options) < 0)
-        {
-          if (composefs_mode > OSTREE_COMPOSEFS_MODE_MAYBE)
-            err (EXIT_FAILURE, "Failed to mount composefs");
-        }
-      else
+      if (lcfs_mount_fd (cfs_fd, TMP_SYSROOT, &cfs_options) == 0)
         {
           int fd = open (_OSTREE_COMPOSEFS_ROOT_STAMP, O_WRONLY | O_CREAT | O_CLOEXEC, 0644);
           if (fd < 0)
@@ -338,13 +379,20 @@ main (int argc, char *argv[])
 
           using_composefs = 1;
         }
+
+      close (cfs_fd);
+
+    nocfs:
 #else
       err (EXIT_FAILURE, "Composefs not supported");
 #endif
     }
 
   if (!using_composefs)
     {
+      if (composefs_mode > OSTREE_COMPOSEFS_MODE_MAYBE)
+        err (EXIT_FAILURE, "Failed to mount composefs");
+
       /* The deploy root starts out bind mounted to sysroot.tmp */
       if (mount (deploy_path, TMP_SYSROOT, NULL, MS_BIND | MS_SILENT, NULL) < 0)
         err (EXIT_FAILURE, "failed to make initial bind mount %s", deploy_path);