From f51b10245b2bf2e20b05c16da80b77d30bdcc735 Mon Sep 17 00:00:00 2001 From: Jonathan Metzman Date: Mon, 30 Jan 2023 21:59:08 -0500 Subject: [PATCH 1/6] Try speeding up trial builds Use regular base-images as cache as well to speed up first build. --- .../functions/build_and_push_test_images.py | 2 +- infra/build/functions/build_lib.py | 22 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/infra/build/functions/build_and_push_test_images.py b/infra/build/functions/build_and_push_test_images.py index fee95e9d7970..e17a9701afb2 100755 --- a/infra/build/functions/build_and_push_test_images.py +++ b/infra/build/functions/build_and_push_test_images.py @@ -99,7 +99,7 @@ def gcb_build_and_push_images(test_image_suffix): directory = os.path.join('infra', 'base-images', base_image) step = build_lib.get_docker_build_step([main_tag, test_tag], directory, - buildkit_cache_image=test_tag, + buildkit_cache_images=[main_tag, test_tag], src_root='.') steps.append(step) diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index c22d8dc3158c..385e570bbe27 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -376,6 +376,10 @@ def get_docker_build_step(image_names, architecture='x86_64'): """Returns the docker build step.""" assert len(image_names) >= 1 + + if buildkit_cache_image is None: + buildkit_cache_image = [] + directory = os.path.join(src_root, directory) if architecture != _ARM64: @@ -398,17 +402,21 @@ def get_docker_build_step(image_names, 'args': args, 'dir': directory, } + # Handle buildkit args # Note that we mutate "args" after making it a value in step. - - if buildkit_cache_image is not None: + buildkit_cache_args = [] + if build_cache_images: env = ['DOCKER_BUILDKIT=1'] step['env'] = env - assert buildkit_cache_image in args - additional_args = [ - '--build-arg', 'BUILDKIT_INLINE_CACHE=1', '--cache-from', + buildkit_cache_args = ['--build-arg', 'BUILDKIT_INLINE_CACHE=1'] + + for buildkit_cache_image in buildkit_cache_images: + buildkit_cache_args.extend([ + '--cache-from', buildkit_cache_image - ] - args.extend(additional_args) + ]) + args.extend(buildkit_cache_args) + args.append('.') return step From c46ccd9ef65d53f056003019da0e82903d0e597a Mon Sep 17 00:00:00 2001 From: Jonathan Metzman Date: Mon, 30 Jan 2023 22:02:41 -0500 Subject: [PATCH 2/6] tmp --- .../functions/build_and_push_test_images.py | 9 +++++---- infra/build/functions/build_lib.py | 20 +++++-------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/infra/build/functions/build_and_push_test_images.py b/infra/build/functions/build_and_push_test_images.py index e17a9701afb2..7977cb04f1c0 100755 --- a/infra/build/functions/build_and_push_test_images.py +++ b/infra/build/functions/build_and_push_test_images.py @@ -97,10 +97,11 @@ def gcb_build_and_push_images(test_image_suffix): main_tag, test_tag = get_image_tags(base_image, test_image_suffix) test_tags.append(test_tag) directory = os.path.join('infra', 'base-images', base_image) - step = build_lib.get_docker_build_step([main_tag, test_tag], - directory, - buildkit_cache_images=[main_tag, test_tag], - src_root='.') + step = build_lib.get_docker_build_step( + [main_tag, test_tag], + directory, + buildkit_cache_images=[main_tag, test_tag], + src_root='.') steps.append(step) overrides = {'images': test_tags} diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index 385e570bbe27..bfdb3f833d8c 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -371,15 +371,11 @@ def _make_image_name_architecture_specific(image_name, architecture): def get_docker_build_step(image_names, directory, - buildkit_cache_image=None, + use_buildkit_cache=False, src_root='oss-fuzz', architecture='x86_64'): """Returns the docker build step.""" assert len(image_names) >= 1 - - if buildkit_cache_image is None: - buildkit_cache_image = [] - directory = os.path.join(src_root, directory) if architecture != _ARM64: @@ -404,18 +400,12 @@ def get_docker_build_step(image_names, } # Handle buildkit args # Note that we mutate "args" after making it a value in step. - buildkit_cache_args = [] - if build_cache_images: + if use_buildkit_cache: env = ['DOCKER_BUILDKIT=1'] step['env'] = env - buildkit_cache_args = ['--build-arg', 'BUILDKIT_INLINE_CACHE=1'] - - for buildkit_cache_image in buildkit_cache_images: - buildkit_cache_args.extend([ - '--cache-from', - buildkit_cache_image - ]) - args.extend(buildkit_cache_args) + args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1']) + for image in image_names: + args.extend(['--cache-from', image]) args.append('.') From 2823783dbba481c9ffe6af3eddd441186fbe3412 Mon Sep 17 00:00:00 2001 From: Jonathan Metzman Date: Mon, 30 Jan 2023 22:03:22 -0500 Subject: [PATCH 3/6] fix --- infra/build/functions/build_and_push_test_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/build/functions/build_and_push_test_images.py b/infra/build/functions/build_and_push_test_images.py index 7977cb04f1c0..280f45e15c00 100755 --- a/infra/build/functions/build_and_push_test_images.py +++ b/infra/build/functions/build_and_push_test_images.py @@ -100,7 +100,7 @@ def gcb_build_and_push_images(test_image_suffix): step = build_lib.get_docker_build_step( [main_tag, test_tag], directory, - buildkit_cache_images=[main_tag, test_tag], + use_buildkit_cache=True, src_root='.') steps.append(step) From 231e2217c7e0e0e7aa813b42c0cf1bf387cda698 Mon Sep 17 00:00:00 2001 From: Jonathan Metzman Date: Mon, 30 Jan 2023 22:03:42 -0500 Subject: [PATCH 4/6] fmt --- infra/build/functions/build_and_push_test_images.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/infra/build/functions/build_and_push_test_images.py b/infra/build/functions/build_and_push_test_images.py index 280f45e15c00..64da29f1d811 100755 --- a/infra/build/functions/build_and_push_test_images.py +++ b/infra/build/functions/build_and_push_test_images.py @@ -97,11 +97,10 @@ def gcb_build_and_push_images(test_image_suffix): main_tag, test_tag = get_image_tags(base_image, test_image_suffix) test_tags.append(test_tag) directory = os.path.join('infra', 'base-images', base_image) - step = build_lib.get_docker_build_step( - [main_tag, test_tag], - directory, - use_buildkit_cache=True, - src_root='.') + step = build_lib.get_docker_build_step([main_tag, test_tag], + directory, + use_buildkit_cache=True, + src_root='.') steps.append(step) overrides = {'images': test_tags} From d1076d46c0fce8f2360831ce68517b62b7ffc44d Mon Sep 17 00:00:00 2001 From: Jonathan Metzman Date: Mon, 6 Feb 2023 11:56:10 -0500 Subject: [PATCH 5/6] fix --- infra/base-images/base-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-image/Dockerfile b/infra/base-images/base-image/Dockerfile index 3e0ad0fd02c6..dfffcb82aeaa 100644 --- a/infra/base-images/base-image/Dockerfile +++ b/infra/base-images/base-image/Dockerfile @@ -16,7 +16,7 @@ # Base image for all other images. -ARG parent_image=ubuntu:20.04 +ARG parent_image=ubuntu:20.04@sha256:1f1a2d56de1d604801a9671f301190704c25d604a416f59e03c04f5c6ffee0d6 FROM $parent_image From a724275063066f4d7bc94d5f99518d1c76e65ef4 Mon Sep 17 00:00:00 2001 From: Jonathan Metzman Date: Mon, 6 Feb 2023 12:28:56 -0500 Subject: [PATCH 6/6] fix --- infra/base-images/base-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-image/Dockerfile b/infra/base-images/base-image/Dockerfile index dfffcb82aeaa..c14adff5e411 100644 --- a/infra/base-images/base-image/Dockerfile +++ b/infra/base-images/base-image/Dockerfile @@ -16,7 +16,7 @@ # Base image for all other images. -ARG parent_image=ubuntu:20.04@sha256:1f1a2d56de1d604801a9671f301190704c25d604a416f59e03c04f5c6ffee0d6 +ARG parent_image=ubuntu:20.04@sha256:4a45212e9518f35983a976eead0de5eecc555a2f047134e9dd2cfc589076a00d FROM $parent_image