From 077c797625724850e50b968aa631cc139e91021f Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sun, 27 Feb 2022 08:35:12 +0000 Subject: [PATCH 1/6] genbuildplan.py: add --list-packages Signed-off-by: Ian Leonard --- scripts/genbuildplan.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/genbuildplan.py b/scripts/genbuildplan.py index f5651f92c13..07db04264c9 100755 --- a/scripts/genbuildplan.py +++ b/scripts/genbuildplan.py @@ -357,6 +357,9 @@ def processPackages(args, packages): group.add_argument("--hide-wants", action="store_false", dest="show_wants", default=True, \ help="Disable --show-wants. This is the default.") +parser.add_argument("--list-packages", action="store_true", default=False, \ + help="Only list package atoms in build plan.") + parser.add_argument("--with-json", metavar="FILE", \ help="File into which JSON formatted plan will be written.") @@ -396,6 +399,9 @@ def processPackages(args, packages): node = (REQUIRED_PKGS[step[1]]) wants = [edge.fqname for edge in node.edges] print(f"{step[0]:<7} {step[1].replace(':target',''):<25} (wants: {', '.join(wants).replace(':target','')})") +elif args.list_packages: + for step in steps: + print(f"{step[1].replace(':target','')}") else: for step in steps: print(f"{step[0]:<7} {step[1].replace(':target','')}") From 918f14325a3bf7d88be609ee66ce187ddf78d07a Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Tue, 31 Oct 2023 20:06:32 -0400 Subject: [PATCH 2/6] genbuildplan.py: add --hide-header Signed-off-by: Ian Leonard --- scripts/genbuildplan.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/genbuildplan.py b/scripts/genbuildplan.py index 07db04264c9..0b3e8b1969f 100755 --- a/scripts/genbuildplan.py +++ b/scripts/genbuildplan.py @@ -351,12 +351,15 @@ def processPackages(args, packages): parser.add_argument("--ignore-invalid", action="store_true", default=False, \ help="Ignore invalid packages.") -group = parser.add_mutually_exclusive_group() +group = parser.add_mutually_exclusive_group() group.add_argument("--show-wants", action="store_true", \ help="Output \"wants\" dependencies for each step.") group.add_argument("--hide-wants", action="store_false", dest="show_wants", default=True, \ help="Disable --show-wants. This is the default.") +parser.add_argument("--hide-header", action="store_false", dest="show_header", \ + help="Hide stats header on output") + parser.add_argument("--list-packages", action="store_true", default=False, \ help="Only list package atoms in build plan.") @@ -374,10 +377,11 @@ def processPackages(args, packages): # Identify list of packages to build/install steps = [step for step in get_build_steps(args, REQUIRED_PKGS)] -eprint(f"Packages loaded : {loaded}") -eprint(f"Build trigger(s): {len(args.build)} [{' '.join(args.build)}]") -eprint(f"Package steps : {len(steps)}") -eprint("") +if args.show_header: + eprint(f"Packages loaded : {loaded}") + eprint(f"Build trigger(s): {len(args.build)} [{' '.join(args.build)}]") + eprint(f"Package steps : {len(steps)}") + eprint("") # Write the JSON build plan (with dependencies) if args.with_json: From a3745771c7862ee5af39c9335a6c2d368b787511 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sun, 29 Sep 2024 22:26:47 -0400 Subject: [PATCH 3/6] tools/genmirrorlist.py: initial commit Signed-off-by: Ian Leonard --- tools/genmirrorlist.py | 224 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100755 tools/genmirrorlist.py diff --git a/tools/genmirrorlist.py b/tools/genmirrorlist.py new file mode 100755 index 00000000000..c32fab9ff7b --- /dev/null +++ b/tools/genmirrorlist.py @@ -0,0 +1,224 @@ +#! /usr/bin/env python3 + +#SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2023-present Ian Leonard (antonlacon@gmail.com) + +''' +This generates a list of URLs of every file used by the selected image build. + +The script may be invoked like so: + +./genmirrorlist.py --all + +This considers every project/device/arch/uboot_system in builds_all. + +./genmirrorlist.py --builddirs + +This filters --all's build list to only ones with a build directory. + +PROJECT=W DEVICE=X ARCH=Y UBOOT_SYSTEM=Z ./genmirrorlist.py + +This runs against the project/device/arch specificed on the CLI. + +For working with multiple git branches (different versions), there are +additional options: + +--export filename to write a list of packages and their source tarballs to +file. + +./genmirror.py --all --export file1.txt +''' + + +import argparse +import os +import subprocess +import shutil +import sys + + +DISTRO_NAME = 'LibreELEC' + + +# project, device, arch, uboot_system +builds_all = [ + ['Allwinner', 'A64', 'aarch64', 'oranagepi-win'], + ['Allwinner', 'A64', 'aarch64', 'pine64'], + ['Allwinner', 'A64', 'aarch64', 'pine64-lts'], + ['Allwinner', 'A64', 'aarch64', 'pine64-plus'], + ['Allwinner', 'H3', 'arm', 'banana-m2p'], + ['Allwinner', 'H3', 'arm', 'beelink-x2'], + ['Allwinner', 'H3', 'arm', 'libretech-h3'], + ['Allwinner', 'H3', 'arm', 'nanopi-m1'], + ['Allwinner', 'H3', 'arm', 'orangepi-2'], + ['Allwinner', 'H3', 'arm', 'orangepi-pc'], + ['Allwinner', 'H3', 'arm', 'orangepi-pc-plus'], + ['Allwinner', 'H3', 'arm', 'orangepi-pc-plus2e'], + ['Allwinner', 'H5', 'aarch64', 'tritium-h5'], + ['Allwinner', 'H6', 'aarch64', 'beelink-gs1'], + ['Allwinner', 'H6', 'aarch64', 'orangepi-3'], + ['Allwinner', 'H6', 'aarch64', 'orangepi-3-lts'], + ['Allwinner', 'H6', 'aarch64', 'oranagepi-lite2'], + ['Allwinner', 'H6', 'aarch64', 'orangepi-one-plus'], + ['Allwinner', 'H6', 'aarch64', 'pine-h64'], + ['Allwinner', 'H6', 'aarch64', 'pine-h64-model-b'], + ['Allwinner', 'H6', 'aarch64', 'tanix-tx6'], + ['Allwinner', 'R40', 'arm', 'banana-m2u'], + ['Amlogic', 'AMLGX', 'aarch64', 'bananapi-m2-pro'], + ['Amlogic', 'AMLGX', 'aarch64', 'bananapi-m2s'], + ['Amlogic', 'AMLGX', 'aarch64', 'bananapi-m5'], + ['Amlogic', 'AMLGX', 'aarch64', 'box'], + ['Amlogic', 'AMLGX', 'aarch64', 'khadas-vim'], + ['Amlogic', 'AMLGX', 'aarch64', 'khadas-vim2'], + ['Amlogic', 'AMLGX', 'aarch64', 'khadas-vim3'], + ['Amlogic', 'AMLGX', 'aarch64', 'khadas-vim3l'], + ['Amlogic', 'AMLGX', 'aarch64', 'lafrite'], + ['Amlogic', 'AMLGX', 'aarch64', 'lepotato'], + ['Amlogic', 'AMLGX', 'aarch64', 'nanopi-k2'], + ['Amlogic', 'AMLGX', 'aarch64', 'odroid-c2'], + ['Amlogic', 'AMLGX', 'aarch64', 'odroid-c4'], + ['Amlogic', 'AMLGX', 'aarch64', 'odroid-hc4'], + ['Amlogic', 'AMLGX', 'aarch64', 'odroid-n2'], + ['Amlogic', 'AMLGX', 'aarch64', 'radxa-zero'], + ['Amlogic', 'AMLGX', 'aarch64', 'radxa-zero2'], + ['Amlogic', 'AMLGX', 'aarch64', 'wetek-core2'], + ['Amlogic', 'AMLGX', 'arm', 'wetek-hub'], + ['Amlogic', 'AMLGX', 'aarch64', 'wetek-play2'], + ['Generic', 'Generic', 'x86_64', None], + ['Generic', 'Generic-legacy', 'x86_64', None], + ['NXP', 'iMX6', 'arm', 'cubox'], + ['NXP', 'iMX6', 'arm', 'udoo'], + ['NXP', 'iMX6', 'arm', 'wandboard'], + ['NXP', 'iMX8', 'aarch64', 'mq-evk'], + ['NXP', 'iMX8', 'aarch64', 'pico-mq'], + ['Qualcomm', 'Dragonboard', 'aarch64', '410c'], + ['RPi', 'RPi2', 'arm', None], + ['RPi', 'RPi4', 'aarch64', None], + ['RPi', 'RPi5', 'aarch64', None], + ['Samsung', 'Exynos', 'arm', 'odroid-xu3'], + ['Samsung', 'Exynos', 'arm', 'odroid-xu4'], + ['Rockchip', 'RK3288', 'arm', 'miqi'], + ['Rockchip', 'RK3288', 'arm', 'tinker'], + ['Rockchip', 'RK3328', 'aarch64', 'a1'], + ['Rockchip', 'RK3328', 'aarch64', 'roc-cc'], + ['Rockchip', 'RK3328', 'aarch64', 'rock64'], + ['Rockchip', 'RK3399', 'aarch64', 'hugsun-x99'], + ['Rockchip', 'RK3399', 'aarch64', 'khadas-edge'], + ['Rockchip', 'RK3399', 'aarch64', 'nanopc-t4'], + ['Rockchip', 'RK3399', 'aarch64', 'nanopi-m4'], + ['Rockchip', 'RK3399', 'aarch64', 'orangepi'], + ['Rockchip', 'RK3399', 'aarch64', 'roc-pc'], + ['Rockchip', 'RK3399', 'aarch64', 'roc-pc-plus'], + ['Rockchip', 'RK3399', 'aarch64', 'rock-pi-4'], + ['Rockchip', 'RK3399', 'aarch64', 'rock-pi-4-plus'], + ['Rockchip', 'RK3399', 'aarch64', 'rock-pi-n10'], + ['Rockchip', 'RK3399', 'aarch64', 'rock960'], + ['Rockchip', 'RK3399', 'aarch64', 'rockpro64'], + ['Rockchip', 'RK3399', 'aarch64', 'sapphire'], +] +# no test builds +# ['RPi', 'RPi', 'arm', None], + + +def execute(command): + '''Run shell commands.''' + try: + cmd_status = subprocess.run(command, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + print(f'Command failed: {command}') + print(f'Executed command: {e.cmd}') + print(f'START COMMAND OUTPUT:\n{e.stdout.decode()}\nEND COMMAND OUTPUT') + return None + return cmd_status.stdout.decode() + + +def parse_distro_info(): + '''Read distro settings from file.''' + le_version = None + os_version = None + with open(f'{os.getcwd()}/distributions/{DISTRO_NAME}/version', mode='r', encoding='utf-8') as data: + content = data.read() + for line in content.splitlines(): + line = line.strip() + if line[0:17] == 'LIBREELEC_VERSION': + le_version = line.split('=')[1].strip('\"') + elif line[0:10] == 'OS_VERSION': + os_version = line.split('=')[1].strip('\"') + if le_version and os_version: + break + return le_version, os_version + + +# build list of packages with desired versions to keep +def get_git_packagelist(): + '''Create list of packages, their source package filenames, and URL to download for every setup in builds''' + if args.all or args.builddirs: + builds = builds_all + else: + project = os.getenv('PROJECT') + device = os.getenv('DEVICE') + arch = os.getenv('ARCH') + uboot_system = os.getenv('UBOOT_SYSTEM') if os.getenv('UBOOT_SYSTEM') else None + if project and device and arch: + builds = [[project, device, arch, uboot_system]] + else: + print('Error: Unkown build. Set PROJECT, DEVICE, ARCH and, if needed, UBOOT_SYSTEM or invoke with --all') + sys.exit(1) + pkg_list = [] + + if args.builddirs: + le_version, os_version = parse_distro_info() + for build in builds: + # skip entries from builds_all if not build directory present for it + if args.builddirs and not os.path.isdir(f'{os.getcwd()}/build.{DISTRO_NAME}-{build[1]}.{build[2]}-{os_version}-{le_version}'): + continue + # build list of packages that go into each build + cmd_build = f'PROJECT={build[0]} DEVICE={build[1]} ARCH={build[2]}' + if build[3]: + cmd_build = f'{cmd_build} UBOOT_SYSTEM={build[3]}' + cmd_buildplan = f'{cmd_build} scripts/pkgjson | scripts/genbuildplan.py --hide-header --list-packages --build image' + cmd_result = execute(f'{cmd_buildplan}') + if cmd_result: + for item in cmd_result.splitlines(): + # get package filename + pkg_details = execute(f'{cmd_build} tools/pkginfo --strip {item}').strip() + for line in pkg_details.splitlines(): + if line.startswith('PKG_URL'): + pkg_url = line.split('=')[-1].strip('"') + break + # add package and filename to master list if not present + if pkg_url and [item, pkg_url] not in pkg_list: + pkg_list.append([item, pkg_url]) + return pkg_list + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=""" + Prints a list of package names and URLs. Considers one, some, or all project builds in determining + which packages are relevant to mirror. Specify PROJECT=W DEVICE=X ARCH=Y UBOOT_SYSTEM=Z like other + build commands to mirror to a single project build's files. Specify other options listed below to + consider more than one project build. + """) + parser.add_argument('-a', '--all', action='store_true', \ + help='Consider all project/devices when determining what files to keep in sources directory.') + parser.add_argument('-b', '--builddirs', action='store_true', \ + help='Filter build list used by --all to only include builds with a present project/device/arch build directory.') + parser.add_argument('-e', '--export', action='store', nargs='?', \ + help='Export list of package source files to keep to file.') + args = parser.parse_args() + + + pkg_list = get_git_packagelist() + if args.export: + export_path = args.export if args.export.startswith('/') else os.path.join(os.getcwd(), args.export) + if not os.path.isfile(export_path): + with open(export_path, mode='w', encoding='utf-8') as export_file: + for package in pkg_list: + export_file.write(f'{package[0]} {package[1]}\n') + print(f'Exported list of files to: {export_path}') + else: + print(f'Error: Export file already exists: {export_path}') + sys.exit(1) + else: + for package in pkg_list: + print(f'{package[0]} {package[1]}') From 6cef1cfef6f932b77312faaa9b8ca909777bd6f5 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Mon, 30 Sep 2024 00:18:51 -0400 Subject: [PATCH 4/6] tools/build-mirror: initial commit Signed-off-by: Ian Leonard --- tools/build-mirror | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 tools/build-mirror diff --git a/tools/build-mirror b/tools/build-mirror new file mode 100755 index 00000000000..bb3c6590d5b --- /dev/null +++ b/tools/build-mirror @@ -0,0 +1,76 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2024 Ian Leonard (antonlacon@gmail.com) + +set -e + +# helper functions +# die (message, code) abort with optional message and error code +die() { + if [ -n "${1}" ]; then + echo -e "${1}" >&2 + fi + exit "${2:-1}" +} + +help() { + echo "Usage: ${0} [-i file] [-o directory]" + echo "Set PROJECT, DEVICE and ARCH as required." + echo " -h this help" + echo " -i input file - list of files to download" + echo " -o output directory - location to download files" + echo " -v verbose progress output" +} + +# command line opts +while getopts hi:o:v OPT; do + case "${OPT}" in + h) + help + exit 0 + ;; + i) + INPUT_FILE="${OPTARG}" + ;; + o) + OUTPUT_DIR="${OPTARG}" + ;; + v) + VERBOSE="true" + ;; + \?) + # error and output help on unknown + help + die + ;; + esac +done + +shift $((${OPTIND} - 1)) + +# sanity checking +if [ -n "${INPUT_FILE}" ] && [ ! -f "${INPUT_FILE}" ]; then + die "Error: Unable to find input file: ${INPUT_FILE}" +fi +if [ -z "${OUTPUT_DIR}" ]; then + die "Error: Output directory must be specified." +fi + +# import file or read from stdin +while read -r line; do + PKG_NAME=$(echo "${line}" | cut -d" " -f1 | cut -d":" -f1) + PKG_URL=$(echo "${line}" | cut -d" " -f2) + PKG_FILENAME=$(basename "${PKG_URL}") + if [ ! -d "${OUTPUT_DIR}/${PKG_NAME}" ]; then + mkdir -p "${OUTPUT_DIR}/${PKG_NAME}" + fi + if [ ! -f "${OUTPUT_DIR}/${PKG_NAME}/${PKG_FILENAME}" ]; then + if [ "${VERBOSE}" = "true" ]; then + echo "Downloading: ${PKG_NAME}" + else + VERBOSE_OPTS="--silent --show-error" + fi + curl --progress-bar ${VERBOSE_OPTS} --connect-timeout 30 --retry 3 --continue-at - --location --max-redirs 5 --output-dir "${OUTPUT_DIR}/${PKG_NAME}/" --output "${PKG_FILENAME}" "${PKG_URL}" + fi +done < "${INPUT_FILE:-/dev/stdin}" From a2fa0cd74876f9ddfe217a87d214acc5f0659c16 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Fri, 4 Oct 2024 04:10:09 -0400 Subject: [PATCH 5/6] tools/genmirrorlist.py: fail early if desired export file already exists Signed-off-by: Ian Leonard --- tools/genmirrorlist.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/genmirrorlist.py b/tools/genmirrorlist.py index c32fab9ff7b..bbba69f9f79 100755 --- a/tools/genmirrorlist.py +++ b/tools/genmirrorlist.py @@ -180,7 +180,8 @@ def get_git_packagelist(): cmd_result = execute(f'{cmd_buildplan}') if cmd_result: for item in cmd_result.splitlines(): - # get package filename + pkg_url = None + # get package details pkg_details = execute(f'{cmd_build} tools/pkginfo --strip {item}').strip() for line in pkg_details.splitlines(): if line.startswith('PKG_URL'): @@ -208,17 +209,22 @@ def get_git_packagelist(): args = parser.parse_args() - pkg_list = get_git_packagelist() + # sanity check if args.export: export_path = args.export if args.export.startswith('/') else os.path.join(os.getcwd(), args.export) + if os.path.isfile(export_path): + print(f'Error: Export file already exists: {export_path}') + sys.exit(1) + + + # get package details to build a source repository mirror + pkg_list = get_git_packagelist() + if args.export: if not os.path.isfile(export_path): with open(export_path, mode='w', encoding='utf-8') as export_file: for package in pkg_list: export_file.write(f'{package[0]} {package[1]}\n') print(f'Exported list of files to: {export_path}') - else: - print(f'Error: Export file already exists: {export_path}') - sys.exit(1) else: for package in pkg_list: print(f'{package[0]} {package[1]}') From b8a73af372ffbe94fe9e2e0425beaf31e3d899b7 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sat, 19 Oct 2024 03:08:54 -0400 Subject: [PATCH 6/6] tools/build-mirror: update script help message and curl command Signed-off-by: Ian Leonard --- tools/build-mirror | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/build-mirror b/tools/build-mirror index bb3c6590d5b..75006d128cb 100755 --- a/tools/build-mirror +++ b/tools/build-mirror @@ -16,7 +16,6 @@ die() { help() { echo "Usage: ${0} [-i file] [-o directory]" - echo "Set PROJECT, DEVICE and ARCH as required." echo " -h this help" echo " -i input file - list of files to download" echo " -o output directory - location to download files" @@ -71,6 +70,6 @@ while read -r line; do else VERBOSE_OPTS="--silent --show-error" fi - curl --progress-bar ${VERBOSE_OPTS} --connect-timeout 30 --retry 3 --continue-at - --location --max-redirs 5 --output-dir "${OUTPUT_DIR}/${PKG_NAME}/" --output "${PKG_FILENAME}" "${PKG_URL}" + curl --progress-bar ${VERBOSE_OPTS} --fail --connect-timeout 30 --retry 3 --continue-at - --location --max-redirs 5 --output-dir "${OUTPUT_DIR}/${PKG_NAME}/" --output "${PKG_FILENAME}" "${PKG_URL}" fi done < "${INPUT_FILE:-/dev/stdin}"