Merge pull request #1732 from flatcar/krnowak/crypt

Migrate from glibc libcrypt to sys-libs/libxcrypt

.github/workflows/portage-stable-packages-list

@@ -572,6 +572,7 @@ sys-libs/libselinux
 sys-libs/libsepol
 sys-libs/libunwind
 sys-libs/liburing
+sys-libs/libxcrypt
 sys-libs/ncurses
 sys-libs/readline
 sys-libs/talloc

bootstrap_sdk

@@ -144,9 +144,10 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 #  outdated "seed tarball" libraries which have been updated to newer versions in stage 1.
 
 stage_repo() {
-    local repo="$1"
-    local path="$2"
-    local dest="$3"
+    local repo=${1}
+    local path=${2}
+    local dest=${3}
+    local update_seed_file=${4}
     local gitname="$repo"
 
     if [ "$gitname" = "gentoo" ] ; then
@@ -173,7 +174,7 @@ stage_repo() {
             name=${hook##*/}
             name=${name%"-${gitname}.sh"}
             info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
-            "${hook}" "${dest}/${repo}" "${!repo_var}"
+            "${hook}" "${dest}/${repo}" "${!repo_var}" "${update_seed_file}"
         done
     )
 }
@@ -189,10 +190,14 @@ build_stage1() {
     rm -rf "$stage1_repos"
     mkdir "$stage1_repos"
 
+    # If the file exists and is not empty, seed will be updated.
+    # Stage1 hooks may decide that the seed SDK needs updating.
+    local update_seed_file="${TEMPDIR}/update_seed"
+
     # prepare ebuild repos for stage 1, either from the local SDK (default)
     #  or from custom paths specified via command line flags
-    stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos"
-    stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos"
+    stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos" "${update_seed_file}"
+    stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" "${update_seed_file}"
 
     # Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
     #  This requires us to create a custom catalyst config to point it to the
@@ -210,11 +215,12 @@ build_stage1() {
         "$TEMPDIR/stage1.spec"
 
     # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
-    if [ -n "${FLAGS_stage1_portage_path}" -o -n "${FLAGS_stage1_overlay_path}" ] ; then
+    if [[ -n ${FLAGS_stage1_portage_path} ]] || [[ -n ${FLAGS_stage1_overlay_path} ]] || [[ -s ${update_seed_file} ]]; then
         sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
         echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
             >>"$TEMPDIR/stage1.spec"
     fi
+    rm -f "${update_seed_file}"
 
     # Finally, build stage 1
     build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"

changelog/changes/2024-03-08-libcrypt-migration.md

@@ -0,0 +1 @@
+- libcrypt is now provided by the libxcrypt library instead of glibc. Glibc libcrypt was deprecated long time ago.

sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-glibc-crypt-portage-stable.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -x
+set -euo pipefail
+
+stage1_repo=${1}
+new_repo=${2}
+update_seed_file=${3}
+
+cat=sys-libs
+pkg=libxcrypt
+
+if [[ -d "${stage1_repo}/${cat}/${pkg}" ]]; then
+    # libxcrypt is already a part of portage-stable, nothing to do
+    exit 0
+fi
+
+mkdir -p "${stage1_repo}/${cat}"
+cp -a "${new_repo}/${cat}/${pkg}" "${stage1_repo}/${cat}/${pkg}"
+echo x >"${update_seed_file}"

sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0002-glibc-crypt-coreos-overlay.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -x
+set -euo pipefail
+
+stage1_repo=${1}
+new_repo=${2}
+update_seed_file=${3}
+
+base_profile_dir='profiles/coreos/base'
+
+declare -A fixups_old=(
+    ['package.mask']='>=virtual/libcrypt-2'
+    ['package.unmask']='=virtual/libcrypt-1-r1'
+    ['package.use.force']='sys-libs/glibc crypt'
+    ['package.use.mask']='sys-libs/glibc -crypt'
+)
+
+declare -A fixups_new=(
+    ['package.mask']='>=virtual/libcrypt-2'
+    ['package.unmask']='<virtual/libcrypt-2'
+    ['package.use.force']='sys-libs/glibc crypt'
+    ['package.use.mask']='sys-libs/glibc -crypt'
+)
+
+for var_name in fixups_old fixups_new; do
+    declare -n fixups="${var_name}"
+
+    skip=''
+    for f in "${!fixups[@]}"; do
+        l=${fixups["${f}"]}
+        ff="${stage1_repo}/${base_profile_dir}/${f}"
+        if ! grep --quiet --fixed-strings --line-regexp --regexp="${l}" -- "${ff}"; then
+            # fixup not applicable, try next one
+            skip=x
+            break
+        fi
+    done
+
+    if [[ -n ${skip} ]]; then
+        unset -n fixups
+        continue
+    fi
+
+    for f in "${!fixups[@]}"; do
+        l=${fixups["${f}"]}
+        ff="${stage1_repo}/${base_profile_dir}/${f}"
+        ffb="${ff}.bak"
+        mv "${ff}" "${ffb}"
+        grep --invert-match --fixed-strings --line-regexp --regexp="${l}" -- "${ffb}" >"${ff}"
+    done
+    echo x >"${update_seed_file}"
+    exit 0
+done

sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.mask

@@ -11,12 +11,6 @@
 # certificate store provided in NSS rather than the Gentoo/Debian package.
 >=app-misc/ca-certificates-20000000
 
-# Overwrite portage-stable mask. We are delaying the transition to
-# libxcrypt, because we need to figure out how to solve the dep loop
-# that results from the migration (python -> virtual/libcrypt ->
-# libxcrypt -> glibc -> python).
->=virtual/libcrypt-2
-
 # Python 3.12 is in portage-stable (currently testing), so avoid picking it
 # up. Update this to mask later versions when we switch to 3.11.
 >=dev-lang/python-3.12

sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.unmask

@@ -1,9 +1,3 @@
-# Overwrite portage-stable mask. We are delaying the transition to
-# libxcrypt, because we need to figure out how to solve the dep loop
-# that results from the migration (python -> virtual/libcrypt ->
-# libxcrypt -> glibc -> python).
-<virtual/libcrypt-2
-
 # Overwrite portage-stable mask. We haven't seen LLVM related problems
 # with rust so far, so keep using 1.70.0.
 ~dev-lang/rust-1.78.0

sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.force

@@ -14,10 +14,6 @@ app-alternatives/sh bash
 app-alternatives/tar gnu
 app-alternatives/yacc bison
 
-# Force libcrypt so it's included in stage-1 of the SDK build.
-# +crypt was introduced in glibc-2.30 and above.
-sys-libs/glibc crypt
-
 # Do not force this flag, we don't need XATTR_PAX
 sys-apps/portage -xattr
 

sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask

@@ -17,13 +17,6 @@ sys-libs/ncurses cxx
 # which is defined in portage-stable.
 app-editors/nano unicode
 
-# Overwrite portage-stable mask. We are delaying the transition to
-# libxcrypt, because we need to figure out how to solve the dep loop
-# that results from the migration (python -> virtual/libcrypt ->
-# libxcrypt -> glibc -> python), and also we need to update gcc to
-# version 10 or later.
-sys-libs/glibc -crypt
-
 # We don't use pip.
 dev-lang/python ensurepip
 

sdk_container/src/third_party/portage-stable/sys-libs/libxcrypt/Manifest

@@ -0,0 +1 @@
+DIST libxcrypt-4.4.36-autotools.tar.xz 624660 BLAKE2B 8dc3d0f354baf8c64dc011e95e7df10d48b0dfe428503936ffd55edf2745de04003c7efe231ed5d9a14cea7f682ba377b7e00f0463b4060c50c9c29f555b790f SHA512 fb8391ecb89622eb0d74d13c5fc1369718e83c47671449044ca0c2f78a236d7b06177a60bf8cda47694caa840c68eaaf0b23690e8975fa5d64b734c8eb246d10

sdk_container/src/third_party/portage-stable/sys-libs/libxcrypt/files/libxcrypt-4.4.19-multibuild.patch

@@ -0,0 +1,14 @@
+diff --git a/Makefile.am b/Makefile.am
+index d0cca1d..4a5d4a1 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -86,9 +86,7 @@ noinst_HEADERS = \
+ 	test/des-cases.h \
+ 	test/ka-table.inc
+
+-if ENABLE_XCRYPT_COMPAT_FILES
+ nodist_include_HEADERS += xcrypt.h
+-endif
+
+ noinst_PROGRAMS = \
+ 	lib/gen-des-tables