From 0cd203fe5ae59e825d8d6bf25aa8ff69902e787b Mon Sep 17 00:00:00 2001 From: John Andersen Date: Tue, 18 Apr 2023 12:05:31 -0700 Subject: [PATCH] scripts: images containers manifest: Build json from dirs Related: https://github.com/intel/dffml/pull/1273 Signed-off-by: John Andersen --- scripts/images_containers_manifest.py | 195 ++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 scripts/images_containers_manifest.py diff --git a/scripts/images_containers_manifest.py b/scripts/images_containers_manifest.py new file mode 100755 index 0000000000..e8362eca9e --- /dev/null +++ b/scripts/images_containers_manifest.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# This file generates a manifest for building container images +# Usage: JSON_INDENT=" " nodemon -e py,Dockerfile,HEAD --exec 'clear; python scripts/images_containers_manifest.py; test 1' +import os +import sys +import json +import pathlib +import itertools +import traceback +import subprocess +import urllib.request + +# For running under GitHub actions within a container +if "GITHUB_WORKSPACE" in os.environ: + subprocess.check_call(["git", "config", "--global", "--add", "safe.directory", os.environ["GITHUB_WORKSPACE"]]) + +try: + os.environ.update({ + "COMMIT": subprocess.check_output(["git", "log", "-n", "1", "--format=%H"]).decode().strip(), + }) +except: + traceback.print_exc(file=sys.stderr) + +try: + os.environ.update({ + "ROOT_PATH": str(pathlib.Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode().strip()).relative_to(os.getcwd())), + "SCHEMA": "https://github.com/intel/dffml/raw/c82f7ddd29a00d24217c50370907c281c4b5b54d/schema/github/actions/build/images/containers/0.0.0.schema.json", + "OWNER_REPOSITORY": "/".join(subprocess.check_output(["git", "remote", "get-url", "origin"]).decode().strip().replace(".git", "").split("/")[-2:]), + "BRANCH": subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode().strip(), + "PREFIX": os.environ.get("PREFIX", json.dumps([ + ".", + "scripts", + "dffml/skel/operations", + ])), + "NO_DELTA_PREFIX": os.environ.get("NO_DELTA_PREFIX", json.dumps([ + ".", + "scripts", + "dffml/skel/operations", + ])), + }) +except: + traceback.print_exc(file=sys.stderr) + +def path_to_image_name(path, root_path): + # Stem as image name + if path.stem != "Dockerfile": + return path.stem + # Non-top level no stem as image name (filename is Dockerfile) + hyphen_dir_path = str(path.parent.relative_to(root_path)).replace(os.sep, "-").replace(".", "") + if hyphen_dir_path != ".": + return hyphen_dir_path + # Top level dir Dockerfile use top level dirname + return str(root_path.resolve().name) + +# Pull request file change delta filter using GitHub API +prefixes = json.loads(os.environ["PREFIX"]) +no_delta_prefixes = json.loads(os.environ["NO_DELTA_PREFIX"]) +owner, repository = os.environ["OWNER_REPOSITORY"].split("/", maxsplit=1) +base = None +env_vars = ["BASE", "BASE_REF"] +for env_var in env_vars: + if env_var in os.environ and os.environ[env_var].strip(): + # Set if present and not blank + base = os.environ[env_var] + +# Empty manifest (list of manifests for each build file) in case not triggered +# from on file change (workflow changed or dispatched). +manifest = [] +# Path to root of repo +root_path = pathlib.Path(os.environ["ROOT_PATH"]) +# Grab commit from git +commit = os.environ["COMMIT"] +if base is None: + print(f"::notice file={__file__},line=1,endLine=1,title=nobase::None of {env_vars!r} found in os.environ", file=sys.stderr) + +else: + compare_url = os.environ["COMPARE_URL"] + compare_url = compare_url.replace("{base}", base) + compare_url = compare_url.replace("{head}", os.environ["HEAD"]) + with urllib.request.urlopen( + urllib.request.Request( + compare_url, + headers={ + "Authorization": "bearer " + os.environ["GH_ACCESS_TOKEN"], + }, + ) + ) as response: + response_json = json.load(response) + # Print for debug + print(json.dumps({ + "@context": { + "@vocab": "github_delta_response_json", + }, + "include": manifest, + }, sort_keys=True, indent=4), file=sys.stderr) + # Build the most recent commit + commit = response_json["commits"][-1]["sha"] + manifest = list(itertools.chain(*( + [ + [ + { + "image_name": path_to_image_name(path, root_path), + "dockerfile": str(path.relative_to(root_path)), + "owner": owner, + "repository": repository, + "branch": os.environ["BRANCH"], + "commit": commit, + } + for path in [ + (print(compare_file) or pathlib.Path(compare_file["filename"])) + for compare_file in response_json["files"] + if ( + any([ + compare_file["filename"].startswith(prefix_path) + for prefix_path in json.loads(os.environ["PREFIX"]) + ]) and compare_file["filename"].endswith("Dockerfile") + ) + ] + ] + ] + [ + [ + json.loads(path.read_text()) + for path in [ + (print(compare_file) or pathlib.Path(compare_file["filename"])) + for compare_file in response_json["files"] + if ( + any([ + compare_file["filename"].startswith(prefix_path) + for prefix_path in json.loads(os.environ["PREFIX"]) + ]) and compare_file["filename"].endswith("manifest.json") + ) + ] + ] + ] + ))) + +# Build everything if we aren't sure why we got here +if not manifest: + manifest = list(itertools.chain(*( + [ + [ + { + "image_name": path_to_image_name(path, root_path), + "dockerfile": str(path.relative_to(root_path)), + "owner": owner, + "repository": repository, + "branch": os.environ["BRANCH"], + "commit": commit, + } + for path in prefix_path.glob("*Dockerfile") + ] + for prefix_path in map(pathlib.Path, prefixes) + if any( + str(prefix_path.relative_to(root_path)) in no_delta_prefix + for no_delta_prefix in no_delta_prefixes + ) + ] + [ + [ + json.loads(path.read_text()) + for path in prefix_path.glob("*manifest.json") + ] + for prefix_path in map(pathlib.Path, prefixes) + if any( + str(prefix_path.relative_to(root_path)) in no_delta_prefix + for no_delta_prefix in no_delta_prefixes + ) + ] + ))) + +# Add proxies or other runtime args/env vars +for i in manifest: + build_args = {} + if "build_args" in i: + build_args = dict(json.loads(i["build_args"])) + for env_var in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + ]: + if not env_var in os.environ: + continue + build_args[env_var] = os.environ[env_var] + i["build_args"] = json.dumps(list(build_args.items())) + +manifest = { + "@context": { + "@vocab": os.environ["SCHEMA"], + }, + "include": manifest, +} +print(json.dumps(manifest, sort_keys=True, indent=os.environ.get("JSON_INDENT", None))) + +if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "a") as fileobj: + fileobj.write(f"manifest={json.dumps(manifest)}\n")