build: refactor appimage build config

See e90f843, #798

- deterministically install missing dependencies
- find missing dependencies via shell:appimage_dependencies_linux{32,64}
  grunt tasks
- move missing dependency resolver script from build-docker.sh to
  get-dependencies.sh
- remove unnecessary ARCH parameter

build/resources/appimage/build-docker.sh

@@ -4,13 +4,8 @@ set -e
 APPDIR="${1}"
 INSTALLDIR="${2}"
 VERSION="${3}"
-ARCH="${4}"
 LIBDIR="${APPDIR}/${INSTALLDIR}/lib"
 
-declare -A excludepackages=(
-  [gtk3]=true
-)
-
 
 # ----
 
@@ -24,60 +19,40 @@ err() {
   exit 1
 }
 
-[[ $# == 4 ]] || err "Invalid arguments"
+[[ $# -lt 3 ]] && err "Invalid arguments"
+shift 3
+
 [[ -f /.dockerenv ]] || err "This script is supposed to be run from build.sh inside a docker container"
 
-for pkg in $(repoquery --queryformat="%{name}" --requires --recursive --resolve "${!excludepackages[@]}" | sort -u); do
-  excludepackages["${pkg}"]=true
-done
 
-declare -A excludelibraries
-for lib in $(sed -e '/#.*/d; /^[[:space:]]*|[[:space:]]*$/d; /^$/d' /usr/local/share/appimage/excludelist); do
-  excludelibraries["${lib}"]=true
+declare -A DEPS
+for dep in "$@"; do
+  DEPS["$(cut -d= -f2- <<< "${dep}")"]=$(cut -d= -f1 <<< "${dep}")
 done
 
 
-find_dependencies() {
-  log "Finding missing dependencies for: $@"
-  declare -A libs
-  for file in "$@"; do
-    [[ -f "${file}" && -x "${file}" ]] || err "File does not exist or is not executable: ${file}"
-    for lib in $(LD_LIBRARY_PATH=$(dirname "${file}") ldd "${file}" | awk '/ => not found/ {print $1}'); do
-      # don't check a lib more than once
-      [[ -n "${excludelibraries["${lib}"]}" ]] && continue
-      excludelibraries["${lib}"]=true
-      # find the lib's package
-      local package=$(repoquery --queryformat="%{name}" --file "${lib}" | head -n1)
-      [[ -z "${package}" ]] && err "Missing package for: ${lib}"
-      [[ -n "${excludepackages["${package}"]}" ]] && continue
-      libs["${lib}"]="${package}"
-    done
-  done
+install_dependencies() {
+  log "Installing missing dependencies"
+
+  log "Installing packages: ${DEPS[@]}"
+  yum install -y --setopt=tsflags= "${DEPS[@]}"
 
-  [[ ${#libs[@]} == 0 ]] && return
-
-  log "Installing packages: ${libs[@]}"
-  yum install -y --setopt=tsflags= "${libs[@]}"
-  log "Copying libraries"
-  for lib in "${!libs[@]}"; do
-    for path in $(repoquery --list --archlist "${ARCH}" "${libs["${lib}"]}"); do
-      if [[ "$(basename -- "${path}")" == "${lib}" ]]; then
-        ( set -x; install -m755 -t "${LIBDIR}" "${path}" )
-      fi
-      if echo "${path}" | grep -Ei '^/usr/share/(doc|licenses)/.*(copying|licen[cs]e|readme|terms).*'; then
-        ( set -x; install -Dm644 -t "${APPDIR}${path}" "${path}" )
-      fi
+  log "Copying libraries and license files"
+  for lib in "${!DEPS[@]}"; do
+    ( set -x; install -m755 -t "${LIBDIR}" "${lib}" )
+    for path in $(repoquery --list "${DEPS["${lib}"]}" \
+      | grep -Ei '^/usr/share/(doc|licenses)/.*(copying|licen[cs]e|readme|terms).*'
+    ); do
+      ( set -x; install -Dm644 -t "${APPDIR}${path}" "${path}" )
     done
   done
-
-  find_dependencies $(echo "${!libs[@]}" | tr ' ' '\n' | while read -r lib; do echo "${LIBDIR}/${lib}"; done)
 }
 
 build_appimage() {
   log "Building appimage"
   [ "${SOURCE_DATE_EPOCH}" ] && mtime="@${SOURCE_DATE_EPOCH}" || mtime=now
   find "${APPDIR}" -exec touch --no-dereference "--date=${mtime}" '{}' '+'
-  VERSION="${VERSION}" ARCH="${ARCH}" /usr/local/bin/appimagetool \
+  VERSION="${VERSION}" ARCH="$(uname -m)" /usr/local/bin/appimagetool \
     --verbose \
     --comp gzip \
     --no-appstream \
@@ -86,7 +61,7 @@ build_appimage() {
 }
 
 build() {
-  find_dependencies $(find "${APPDIR}/${INSTALLDIR}" -type f -exec sh -c 'readelf -h {} >/dev/null 2>&1' \; -print)
+  install_dependencies
   build_appimage
 }
 

build/resources/appimage/build.sh

@@ -6,11 +6,10 @@ set -e
 
 NAME="${1}"
 VERSION="${2}"
-ARCH="${3}"
-DOCKER_IMAGE="${4}"
-DOCKER_DIGEST="${5}"
-INPUT="${6}"
-OUTPUT="${7}"
+DOCKER_IMAGE="${3}"
+DOCKER_DIGEST="${4}"
+INPUT="${5}"
+OUTPUT="${6}"
 
 declare -A DEPS=(
   [docker]=docker
@@ -33,12 +32,15 @@ err() {
   exit 1
 }
 
-[[ $# == 7 ]] || err "Invalid arguments"
+[[ $# -lt 6 ]] && err "Invalid arguments"
+shift 6
 
 for dep in "${!DEPS[@]}"; do
   command -v "${dep}" 2>&1 >/dev/null || err "Missing dependency: ${DEPS["${dep}"]}"
 done
 
+APPIMAGEDEPS=("${@}")
+
 
 # ----
 
@@ -156,7 +158,7 @@ trap "chown -R $(id -u):$(id -g) '${target}'" EXIT
   "${appdir}" \
   "${installdir}" \
   "${VERSION}" \
-  "${ARCH}"
+  "${APPIMAGEDEPS[@]}"
 EOF
   install -Dm755 "${tempdir}/output" "${OUTPUT}"
 }

build/resources/appimage/get-dependencies.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+set -e
+
+DOCKER_IMAGE="${1}"
+DOCKER_DIGEST="${2}"
+INPUT="${3}"
+
+declare -A DEPS=(
+  [docker]=docker
+  [jq]=jq
+)
+
+
+# ----
+
+
+SELF=$(basename "$(readlink -f "${0}")")
+log() {
+  echo "[${SELF}] $@"
+}
+err() {
+  log >&2 "$@"
+  exit 1
+}
+
+[[ $# == 3 ]] || err "Invalid arguments"
+
+for dep in "${!DEPS[@]}"; do
+  command -v "${dep}" 2>&1 >/dev/null || err "Missing dependency: ${DEPS["${dep}"]}"
+done
+
+
+# ----
+
+
+DOCKER_SCRIPT=$(cat <<'EOF'
+set -e
+
+declare -A excludepackages=(
+  [gtk3]=true
+)
+
+for pkg in $(repoquery --queryformat="%{name}" --requires --recursive --resolve "${!excludepackages[@]}" | sort -u); do
+  excludepackages["${pkg}"]=true
+done
+
+declare -A excludelibraries
+for lib in $(sed -e '/#.*/d; /^[[:space:]]*|[[:space:]]*$/d; /^$/d' /usr/local/share/appimage/excludelist); do
+  excludelibraries["${lib}"]=true
+done
+
+find_dependencies() {
+  declare -A libs
+  for file in "$@"; do
+    [[ -f "${file}" && -x "${file}" ]] || { echo >&2 "File does not exist or is not executable: ${file}"; exit 1; }
+
+    for lib in $(LD_LIBRARY_PATH=$(dirname "${file}") ldd "${file}" | awk '/ => not found/ {print $1}'); do
+      # don't check a lib more than once
+      [[ -n "${excludelibraries["${lib}"]}" ]] && continue
+      excludelibraries["${lib}"]=true
+
+      # find the lib's package (for some reason, --file doesn't support selecting the package arch)
+      local package=$(repoquery --queryformat="%{name}" --file "${lib}" | head -n1)
+
+      [[ -z "${package}" ]] && { echo >&2 "Missing package for: ${lib}"; exit 1; }
+      [[ -n "${excludepackages["${package}"]}" ]] && continue
+
+      # resolve full package name with epoch, name, version, release and arch
+      libs["${lib}"]=$(repoquery --envra "${package}.$(uname -m)")
+    done
+  done
+
+  [[ ${#libs[@]} == 0 ]] && return
+
+  yum install -q -y "${libs[@]}"
+
+  for lib in "${!libs[@]}"; do
+    for path in $(repoquery --list "${libs["${lib}"]}"); do
+      if [[ "$(basename -- "${path}")" == "${lib}" ]]; then
+        echo "${libs["${lib}"]}=${path}"
+        find_dependencies "${path}"
+      fi
+    done
+  done
+}
+
+find_dependencies $(find . -type f -exec sh -c 'readelf -h {} >/dev/null 2>&1' \; -print)
+EOF
+)
+
+docker run \
+  --interactive \
+  --rm \
+  --env SOURCE_DATE_EPOCH \
+  --mount "type=bind,source=${INPUT},target=/app" \
+  --workdir /app \
+  "${DOCKER_IMAGE}@${DOCKER_DIGEST}" \
+  /usr/bin/bash <<< "${DOCKER_SCRIPT}" \
+  | jq -CRn '[(inputs | split("\n")) | .[]]'

build/tasks/configs/appimage.js

@@ -4,15 +4,19 @@ module.exports = {
 	linux32: {
 		image: "ghcr.io/streamlink/appimage-buildenv-i686",
 		digest: "sha256:cebe6266fee12cf5e64dbf7324bdf358cd1b97ff647d16e818ea66342d8c9ea4",
-		arch: "i686",
 		input: "<%= dir.releases %>/<%= package.name %>/linux32",
-		output: "<%= dir.dist %>/<%= package.name %>-<%= version %>-i686.AppImage"
+		output: "<%= dir.dist %>/<%= package.name %>-<%= version %>-i686.AppImage",
+		dependencies: [
+			"0:libatomic-4.8.5-44.el7.i686=/usr/lib/libatomic.so.1"
+		]
 	},
 	linux64: {
 		image: "ghcr.io/streamlink/appimage-buildenv-x86_64",
 		digest: "sha256:bd6d8c4a945e108e4d9198a551e2c7aaeb44995ddb1780bc2aa3f25636e47606",
-		arch: "x86_64",
 		input: "<%= dir.releases %>/<%= package.name %>/linux64",
-		output: "<%= dir.dist %>/<%= package.name %>-<%= version %>-x86_64.AppImage"
+		output: "<%= dir.dist %>/<%= package.name %>-<%= version %>-x86_64.AppImage",
+		dependencies: [
+			"0:libatomic-4.8.5-44.el7.x86_64=/usr/lib64/libatomic.so.1"
+		]
 	}
 };

build/tasks/configs/shell.js

@@ -83,23 +83,40 @@ module.exports = {
 			"bash '<%= dir.resources %>/appimage/build.sh'",
 			"'<%= package.name %>'",
 			"'<%= version %>'",
-			"'<%= appimage.linux32.arch %>'",
 			"'<%= appimage.linux32.image %>'",
 			"'<%= appimage.linux32.digest %>'",
 			"'<%= appimage.linux32.input %>'",
-			"'<%= appimage.linux32.output %>'"
+			"'<%= appimage.linux32.output %>'",
+			"<%= appimage.linux32.dependencies.map(d=>`'${d}'`).join(' ') %>"
 		].join( " " )
 	},
 	appimage_linux64: {
 		command: [
 			"bash '<%= dir.resources %>/appimage/build.sh'",
 			"'<%= package.name %>'",
 			"'<%= version %>'",
-			"'<%= appimage.linux64.arch %>'",
 			"'<%= appimage.linux64.image %>'",
 			"'<%= appimage.linux64.digest %>'",
 			"'<%= appimage.linux64.input %>'",
-			"'<%= appimage.linux64.output %>'"
+			"'<%= appimage.linux64.output %>'",
+			"<%= appimage.linux64.dependencies.map(d=>`'${d}'`).join(' ') %>"
+		].join( " " )
+	},
+
+	appimage_dependencies_linux32: {
+		command: [
+			"bash '<%= dir.resources %>/appimage/get-dependencies.sh'",
+			"'<%= appimage.linux32.image %>'",
+			"'<%= appimage.linux32.digest %>'",
+			"'<%= appimage.linux32.input %>'"
+		].join( " " )
+	},
+	appimage_dependencies_linux64: {
+		command: [
+			"bash '<%= dir.resources %>/appimage/get-dependencies.sh'",
+			"'<%= appimage.linux64.image %>'",
+			"'<%= appimage.linux64.digest %>'",
+			"'<%= appimage.linux64.input %>'"
 		].join( " " )
 	}
 };