Merge pull request #2640 from alexlarsson/composefs

Add initial composefs integration

.github/workflows/tests.yml

@@ -73,7 +73,7 @@ jobs:
       - name: Build
         run: |
           env NOCONFIGURE=1 ./autogen.sh &&
-          ./configure --with-curl --with-selinux --with-dracut=yesbutnoconf &&
+          ./configure --with-curl --with-selinux --with-dracut=yesbutnoconf --with-composefs &&
           make -j 4 && make install DESTDIR=$(pwd)/install && tar -c -C install --zstd -f inst.tar.zst .
       - name: Upload binary
         uses: actions/upload-artifact@v2
@@ -193,6 +193,8 @@ jobs:
             pre-checkout-setup: |
               apt-get update
               apt-get install -y git
+            configure-options: >-
+              --with-composefs
 
           # A build using libsoup3. After bookworm is released, this can
           # be switched to Debian Stable.

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "bsdiff"]
 	path = bsdiff
 	url = https://github.com/mendsley/bsdiff
+[submodule "composefs"]
+	path = composefs
+	url = https://github.com/containers/composefs.git

Makefile-libostree.am

@@ -87,6 +87,7 @@ libostree_1_la_SOURCES = \
 	src/libostree/ostree-repo.c \
 	src/libostree/ostree-repo-checkout.c \
 	src/libostree/ostree-repo-commit.c \
+	src/libostree/ostree-repo-composefs.c \
 	src/libostree/ostree-repo-pull.c \
 	src/libostree/ostree-repo-pull-private.h \
 	src/libostree/ostree-repo-pull-verify.c \
@@ -184,7 +185,7 @@ EXTRA_DIST += \
 	$(top_srcdir)/src/libostree/libostree-released.sym \
 	$(NULL)
 
-libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(builddir)/src/libostree \
+libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$(srcdir)/composefs -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(builddir)/src/libostree \
 	$(OT_INTERNAL_GIO_UNIX_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS) $(OT_DEP_LZMA_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) $(OT_DEP_CRYPTO_CFLAGS) \
 	-fvisibility=hidden '-D_OSTREE_PUBLIC=__attribute__((visibility("default"))) extern' \
 	-DPKGLIBEXECDIR=\"$(pkglibexecdir)\"
@@ -266,6 +267,10 @@ libostree_1_la_CFLAGS += $(OT_DEP_LIBSODIUM_CFLAGS)
 libostree_1_la_LIBADD += $(OT_DEP_LIBSODIUM_LIBS)
 endif # USE_LIBSODIUM
 
+if USE_COMPOSEFS
+libostree_1_la_LIBADD += libcomposefs.la
+endif # USE_COMPOSEFS
+
 # XXX: work around clang being passed -fstack-clash-protection which it doesn't understand
 # See: https://bugzilla.redhat.com/show_bug.cgi?id=1672012
 INTROSPECTION_SCANNER_ENV = CC=gcc

Makefile-switchroot.am

@@ -27,7 +27,9 @@ ostree_prepare_root_SOURCES = \
     src/switchroot/ostree-mount-util.h \
     src/switchroot/ostree-prepare-root.c \
     $(NULL)
+ostree_prepare_root_CFLAGS =
 ostree_prepare_root_CPPFLAGS = $(AM_CPPFLAGS)
+ostree_prepare_root_LDADD =
 
 if BUILDOPT_USE_STATIC_COMPILER
 # ostree-prepare-root can be used as init in a system without a populated /lib.
@@ -46,7 +48,7 @@ ostree-prepare-root : $(ostree_prepare_root_SOURCES)
 	$(STATIC_COMPILER) -o $@ -static $(top_srcdir)/src/switchroot/ostree-prepare-root.c $(ostree_prepare_root_CPPFLAGS) $(AM_CFLAGS) $(DEFAULT_INCLUDES) -DOSTREE_PREPARE_ROOT_STATIC=1
 else
 ostree_boot_PROGRAMS += ostree-prepare-root
-ostree_prepare_root_CFLAGS = $(AM_CFLAGS) -Isrc/switchroot
+ostree_prepare_root_CFLAGS += $(AM_CFLAGS) -Isrc/switchroot -I$(srcdir)/composefs
 endif
 
 ostree_remount_SOURCES = \
@@ -56,9 +58,13 @@ ostree_remount_SOURCES = \
 ostree_remount_CPPFLAGS = $(AM_CPPFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -Isrc/switchroot -I$(srcdir)/libglnx
 ostree_remount_LDADD = $(AM_LDFLAGS) $(OT_INTERNAL_GIO_UNIX_LIBS) libglnx.la
 
+if USE_COMPOSEFS
+ostree_prepare_root_LDADD += libcomposefs.la
+endif
+
 if BUILDOPT_SYSTEMD
 ostree_prepare_root_CPPFLAGS += -DHAVE_SYSTEMD=1
-ostree_prepare_root_LDADD = $(AM_LDFLAGS) $(LIBSYSTEMD_LIBS)
+ostree_prepare_root_LDADD += $(AM_LDFLAGS) $(LIBSYSTEMD_LIBS)
 endif
 
 # This is the "new mode" of using a generator for /var; see

Makefile.am

@@ -117,6 +117,15 @@ include bsdiff/Makefile-bsdiff.am.inc
 EXTRA_DIST += bsdiff/Makefile-bsdiff.am
 noinst_LTLIBRARIES += libbsdiff.la
 
+COMPOSEFSDIR=$(srcdir)/composefs/libcomposefs
+LCFS_DEP_CRYPTO_CFLAGS=$(OT_DEP_CRYPTO_CFLAGS)
+LCFS_DEP_CRYPTO_LIBS=$(OT_DEP_CRYPTO_LIBS)
+include composefs/libcomposefs/Makefile-lib.am.inc
+EXTRA_DIST += composefs/libcomposefs/Makefile-lib.am
+if USE_COMPOSEFS
+noinst_LTLIBRARIES += libcomposefs.la
+endif
+
 include Makefile-otutil.am
 include Makefile-libostree.am
 include Makefile-ostree.am

autogen.sh

@@ -35,6 +35,7 @@ fi
 # changing this, please also change Makefile.am.
 sed -e 's,$(libglnx_srcpath),libglnx,g' < libglnx/Makefile-libglnx.am >libglnx/Makefile-libglnx.am.inc
 sed -e 's,$(libbsdiff_srcpath),bsdiff,g' < bsdiff/Makefile-bsdiff.am >bsdiff/Makefile-bsdiff.am.inc
+sed -e 's,$(COMPOSEFSDIR),composefs/libcomposefs,g' < composefs/libcomposefs/Makefile-lib.am >composefs/libcomposefs/Makefile-lib.am.inc
 
 # FIXME - figure out how to get aclocal to find this by default
 ln -sf ../libglnx/libglnx.m4 buildutil/libglnx.m4

configure.ac

@@ -280,6 +280,35 @@ AS_IF([test x$have_gpgme = xyes],
 )
 AM_CONDITIONAL(USE_GPGME, test "x$have_gpgme" = xyes)
 
+# These are needed by libcomposefs
+AC_MSG_CHECKING([for new mount API (fsconfig)])
+AC_COMPILE_IFELSE(
+	[AC_LANG_SOURCE([[
+			#include <sys/mount.h>
+			int cmd = FSCONFIG_CMD_CREATE;
+	]])],
+	[AC_MSG_RESULT(yes)
+		 AC_DEFINE([HAVE_FSCONFIG_CMD_CREATE_SYS_MOUNT_H], 1, [Define if FSCONFIG_CMD_CREATE is available in sys/mount.h])],
+	[AC_MSG_RESULT(no)])
+AC_COMPILE_IFELSE(
+	[AC_LANG_SOURCE([[
+			/* also make sure it doesn't conflict with <sys/mount.h> since it is always used.  */
+			#include <sys/mount.h>
+			#include <linux/mount.h>
+			int cmd = FSCONFIG_CMD_CREATE;
+	]])],
+	[AC_MSG_RESULT(yes)
+		 AC_DEFINE([HAVE_FSCONFIG_CMD_CREATE_LINUX_MOUNT_H], 1, [Define if FSCONFIG_CMD_CREATE is available in linux/mount.h])],
+	[AC_MSG_RESULT(no)])
+
+AC_ARG_WITH(composefs,
+	    AS_HELP_STRING([--with-composefs], [Support composefs]),
+	    :, with_composefs=no)
+
+if test x$with_composefs != xno; then OSTREE_FEATURES="$OSTREE_FEATURES composefs";
+   AC_DEFINE([HAVE_COMPOSEFS], 1, [Define if we have libcomposefs])
+fi
+AM_CONDITIONAL(USE_COMPOSEFS, test $with_composefs != no)
 
 LIBSODIUM_DEPENDENCY="1.0.14"
 AC_ARG_WITH(ed25519_libsodium,
@@ -675,7 +704,8 @@ echo "
     gjs-based tests:                              $have_gjs
     dracut:                                       $with_dracut
     mkinitcpio:                                   $with_mkinitcpio
-    Static compiler for ostree-prepare-root:      $with_static_compiler"
+    Static compiler for ostree-prepare-root:      $with_static_compiler
+    Composefs:                                    $with_composefs"
 AS_IF([test x$with_builtin_grub2_mkconfig = xyes], [
     echo "    builtin grub2-mkconfig (instead of system):   $with_builtin_grub2_mkconfig"
 ], [

src/libostree/ostree-repo-commit.c

@@ -184,7 +184,7 @@ _ostree_repo_commit_tmpf_final (OstreeRepo *self, const char *checksum, OstreeOb
   if (!_ostree_repo_ensure_loose_objdir_at (dest_dfd, tmpbuf, cancellable, error))
     return FALSE;
 
-  if (!_ostree_tmpf_fsverity (self, tmpf, error))
+  if (!_ostree_tmpf_fsverity (self, tmpf, NULL, error))
     return FALSE;
 
   if (!glnx_link_tmpfile_at (tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, dest_dfd, tmpbuf,
@@ -398,14 +398,9 @@ compare_ascii_checksums_for_sorting (gconstpointer a_pp, gconstpointer b_pp)
 /*
  * Create sizes metadata GVariant and add it to the metadata variant given.
  */
-static GVariant *
-add_size_index_to_metadata (OstreeRepo *self, GVariant *original_metadata)
+static void
+add_size_index_to_metadata (OstreeRepo *self, GVariantBuilder *builder)
 {
-  g_autoptr (GVariantBuilder) builder = NULL;
-
-  /* original_metadata may be NULL */
-  builder = ot_util_variant_builder_from_variant (original_metadata, G_VARIANT_TYPE ("a{sv}"));
-
   if (self->object_sizes && g_hash_table_size (self->object_sizes) > 0)
     {
       GVariantBuilder index_builder;
@@ -443,8 +438,6 @@ add_size_index_to_metadata (OstreeRepo *self, GVariant *original_metadata)
       /* Clear the object sizes hash table for a subsequent commit. */
       g_hash_table_remove_all (self->object_sizes);
     }
-
-  return g_variant_ref_sink (g_variant_builder_end (builder));
 }
 
 static gboolean
@@ -2912,6 +2905,23 @@ ostree_repo_write_commit (OstreeRepo *self, const char *parent, const char *subj
                                              out_commit, cancellable, error);
 }
 
+static GVariant *
+add_auto_metadata (OstreeRepo *self, GVariant *original_metadata, OstreeRepoFile *repo_root,
+                   GCancellable *cancellable, GError **error)
+{
+  g_autoptr (GVariantBuilder) builder = NULL;
+
+  /* original_metadata may be NULL */
+  builder = ot_util_variant_builder_from_variant (original_metadata, G_VARIANT_TYPE ("a{sv}"));
+
+  add_size_index_to_metadata (self, builder);
+
+  if (!ostree_repo_commit_add_composefs_metadata (self, builder, repo_root, cancellable, error))
+    return NULL;
+
+  return g_variant_ref_sink (g_variant_builder_end (builder));
+}
+
 /**
  * ostree_repo_write_commit_with_time:
  * @self: Repo
@@ -2938,7 +2948,10 @@ ostree_repo_write_commit_with_time (OstreeRepo *self, const char *parent, const
   OstreeRepoFile *repo_root = OSTREE_REPO_FILE (root);
 
   /* Add sizes information to our metadata object */
-  g_autoptr (GVariant) new_metadata = add_size_index_to_metadata (self, metadata);
+  g_autoptr (GVariant) new_metadata
+      = add_auto_metadata (self, metadata, repo_root, cancellable, error);
+  if (new_metadata == NULL)
+    return FALSE;
 
   g_autoptr (GVariant) commit = g_variant_new (
       "(@a{sv}@ay@a(say)sst@ay@ay)", new_metadata ? new_metadata : create_empty_gvariant_dict (),