bulk sources downloader

scripts/genbuildplan.py

@@ -351,12 +351,21 @@ def processPackages(args, packages):
 parser.add_argument("--ignore-invalid", action="store_true", default=False, \
                     help="Ignore invalid packages.")
 
-group =  parser.add_mutually_exclusive_group()
-group.add_argument("--show-wants", action="store_true", \
+header_group = parser.add_mutually_exclusive_group()
+header_group.add_argument("--show-header", action="store_true", default=True, \
+                    help="Show stats header on output. This is the default.")
+header_group.add_argument("--hide-header", action="store_false", dest="show_header", \
+                    help="Hide stats header on output")
+
+wants_group = parser.add_mutually_exclusive_group()
+wants_group.add_argument("--show-wants", action="store_true", \
                     help="Output \"wants\" dependencies for each step.")
-group.add_argument("--hide-wants", action="store_false", dest="show_wants", default=True, \
+wants_group.add_argument("--hide-wants", action="store_false", dest="show_wants", default=True, \
                     help="Disable --show-wants.  This is the default.")
 
+parser.add_argument("--list-packages", action="store_true", default=False, \
+                    help="Only list package atoms in build plan.")
+
 parser.add_argument("--with-json", metavar="FILE", \
                     help="File into which JSON formatted plan will be written.")
 
@@ -371,10 +380,11 @@ def processPackages(args, packages):
 # Identify list of packages to build/install
 steps = [step for step in get_build_steps(args, REQUIRED_PKGS)]
 
-eprint(f"Packages loaded : {loaded}")
-eprint(f"Build trigger(s): {len(args.build)} [{' '.join(args.build)}]")
-eprint(f"Package steps   : {len(steps)}")
-eprint("")
+if args.show_header:
+    eprint(f"Packages loaded : {loaded}")
+    eprint(f"Build trigger(s): {len(args.build)} [{' '.join(args.build)}]")
+    eprint(f"Package steps   : {len(steps)}")
+    eprint("")
 
 # Write the JSON build plan (with dependencies)
 if args.with_json:
@@ -396,6 +406,9 @@ def processPackages(args, packages):
         node = (REQUIRED_PKGS[step[1]])
         wants = [edge.fqname for edge in node.edges]
         print(f"{step[0]:<7} {step[1].replace(':target',''):<25} (wants: {', '.join(wants).replace(':target','')})")
+elif args.list_packages:
+    for step in steps:
+        print(f"{step[1].replace(':target','')}")
 else:
     for step in steps:
         print(f"{step[0]:<7} {step[1].replace(':target','')}")

tools/download-sources

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv)
+
+# Download all packages listed in the build plan to SOURCES_DIR
+
+unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACKAGE_LIST
+
+. config/options ""
+
+THREADCOUNT="${THREADCOUNT:-$(nproc)}"
+
+# function to enable use of parallel
+download_package_tarball() {
+  local package_name="${1}"
+
+  "${SCRIPTS}/get" "${package_name}"
+}
+
+# Packages listed in the build plan
+PACKAGE_LIST=$( \
+  "${SCRIPTS}/pkgjson" | \
+  "${SCRIPTS}/genbuildplan.py" --hide-header --list-packages --build ${@:-image} ${GENFLAGS} || \
+  die "FAILURE: Unable to generate plan"
+)
+
+# drop :target, :host, :init, and now repeated package entries
+UNIQUE_PACKAGES=$(
+  for package in ${PACKAGE_LIST}; do
+    echo "${package}" | cut -d ":" -f 1
+  done | awk '!seen[$0]++'
+)
+
+# Package downloading
+if command -v parallel > /dev/null; then
+  export -f download_package_tarball
+  export SCRIPTS
+
+  parallel --jobs "${THREADCOUNT}" download_package_tarball <<< "${UNIQUE_PACKAGES}"
+else
+  for package in ${UNIQUE_PACKAGES}; do
+    download_package_tarball "${package}"
+  done
+fi