feat: implement boot asset cache

Fixes #4

This caches the result of the boot asset build (a call to Talos
`imager`) so that we don't rebuild the asset twice.

OCI Registry is used as a cache. An internal registry can be used for
caching, we don't need to expose it to the world.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

.github/workflows/ci.yaml

@@ -1,10 +1,10 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2023-10-11T10:56:42Z by kres latest.
+# Generated on 2023-10-27T14:12:36Z by kres latest.
 
 name: default
 concurrency:
-  group: ${{ github.event.label == null && github.head_ref || github.run_id }}
+  group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 "on":
   push:
@@ -20,12 +20,16 @@ concurrency:
 jobs:
   default:
     permissions:
+      actions: read
       contents: write
       packages: write
+      pull-requests: read
     runs-on:
       - self-hosted
       - generic
     if: (!startsWith(github.head_ref, 'renovate/') && !startsWith(github.head_ref, 'dependabot/'))
+    outputs:
+      labels: ${{ steps.workflow-run-info.outputs.pullRequestLabels }}
     services:
       buildkitd:
         image: moby/buildkit:v0.12.2
@@ -89,9 +93,14 @@ jobs:
       - name: integration
         env:
           REGISTRY: registry.dev.siderolabs.io
-          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs
+          TEST_FLAGS: -test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache
         run: |
           make integration
+      - name: Retrieve workflow info
+        id: workflow-run-info
+        uses: potiuk/get-workflow-origin@v1_5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Generate Checksums
         if: startsWith(github.ref, 'refs/tags/')
         run: |

.golangci.yml

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2023-08-11T18:26:20Z by kres d37bb8d.
+# Generated on 2023-10-27T14:12:36Z by kres latest.
 
 # options for analysis running
 run:
@@ -150,6 +150,9 @@ linters:
     - wrapcheck
     - depguard # Disabled because starting with golangci-lint 1.53.0 it doesn't allow denylist alone anymore
     - tagalign
+    - inamedparam
+    - testifylint # complains about our assert recorder and has a number of false positives for assert.Greater(t, thing, 1)
+    - protogetter # complains about us using Value field on typed spec, instead of GetValue which has a different signature
     # abandoned linters for which golangci shows the warning that the repo is archived by the owner
     - interfacer
     - maligned

.kres.yaml

@@ -108,7 +108,7 @@ spec:
     enabled: true
     environment:
       REGISTRY: registry.dev.siderolabs.io
-      TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs"
+      TEST_FLAGS: "-test.schematic-service-repository=registry.dev.siderolabs.io/image-factory/schematic -test.installer-external-repository=registry.dev.siderolabs.io/siderolabs -test.installer-internal-repository=registry.dev.siderolabs.io/siderolabs -test.cache-repository=registry.dev.siderolabs.io/image-factory/cache"
 ---
 kind: common.Build
 spec:

Makefile

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2023-10-03T19:35:51Z by kres latest.
+# Generated on 2023-10-27T14:12:36Z by kres latest.
 
 # common variables
 
@@ -19,10 +19,10 @@ GRPC_GO_VERSION ?= 1.3.0
 GRPC_GATEWAY_VERSION ?= 2.18.0
 VTPROTOBUF_VERSION ?= 0.5.0
 DEEPCOPY_VERSION ?= v0.5.5
-GOLANGCILINT_VERSION ?= v1.54.2
+GOLANGCILINT_VERSION ?= v1.55.1
 GOFUMPT_VERSION ?= v0.5.0
-GO_VERSION ?= 1.21.1
-GOIMPORTS_VERSION ?= v0.13.0
+GO_VERSION ?= 1.21.3
+GOIMPORTS_VERSION ?= v0.14.0
 GO_BUILDFLAGS ?=
 GO_LDFLAGS ?=
 CGO_ENABLED ?= 0

README.md

@@ -168,7 +168,7 @@ cosign verify --offline --insecure-ignore-tlog --insecure-ignore-sct --key signi
 Run integration tests in local mode, with registry mirrors:
 
 ```bash
-make integration TEST_FLAGS="-test.image-registry=127.0.0.1:5004 -test.schematic-service-repository=127.0.0.1:5005/image-factory/schematic -test.installer-external-repository=127.0.0.1:5005/test -test.installer-internal-repository=127.0.0.1:5005/test" REGISTRY=127.0.0.1:5005
+make integration TEST_FLAGS="-test.image-registry=127.0.0.1:5004 -test.schematic-service-repository=127.0.0.1:5005/image-factory/schematic -test.installer-external-repository=127.0.0.1:5005/test -test.installer-internal-repository=127.0.0.1:5005/test -test.cache-repository=127.0.0.1:5005/cache" REGISTRY=127.0.0.1:5005
 ```
 
 In order to run the Image Factory, generate a ECDSA key pair:
@@ -185,5 +185,6 @@ Run the Image Factory passing the flags:
 -schematic-service-repository 127.0.0.1:5005/image-factory/schematic # private registry for schematics
 -installer-internal-repository 127.0.0.1:5005/siderolabs # internal registry to push installer images to
 -installer-external-repository 127.0.0.1:5005/siderolabs # external registry to redirect users to pull installer
+-cache-repository 127.0.0.1:5005/cache # private registry for cached assets
 -cache-signing-key-path ./cache-signing-key.key # path to the ECDSA private key (to sign cached assets)
 ```

cmd/image-factory/cmd/options.go

@@ -47,6 +47,10 @@ type Options struct { //nolint:govet
 	//
 	// Best choice is to use ECDSA key.
 	CacheSigningKeyPath string
+
+	// OCI registry to use to store cached boot assets.
+	// Only used internally by the image factory.
+	CacheRepository string
 }
 
 // DefaultOptions are the default options.
@@ -69,4 +73,6 @@ var DefaultOptions = Options{
 	InstallerExternalRepository: "ghcr.io/siderolabs",
 
 	TalosVersionRecheckInterval: 15 * time.Minute,
+
+	CacheRepository: "ghcr.io/siderolabs/image-factory/cache",
 }

cmd/image-factory/cmd/service.go

@@ -52,15 +52,18 @@ func RunFactory(ctx context.Context, logger *zap.Logger, opts Options) error {
 		return err
 	}
 
-	assetBuilder := asset.NewBuilder(logger, artifactsManager, opts.AssetBuildMaxConcurrency)
-
-	var frontendOptions frontendhttp.Options
-
 	cacheSigningKey, err := loadPrivateKey(opts.CacheSigningKeyPath)
 	if err != nil {
 		return fmt.Errorf("failed to load cache signing key: %w", err)
 	}
 
+	assetBuilder, err := buildAssetBuilder(logger, artifactsManager, cacheSigningKey, opts)
+	if err != nil {
+		return err
+	}
+
+	var frontendOptions frontendhttp.Options
+
 	frontendOptions.CacheSigningKey = cacheSigningKey
 
 	frontendOptions.ExternalURL, err = url.Parse(opts.ExternalURL)
@@ -172,6 +175,24 @@ func buildArtifactsManager(ctx context.Context, logger *zap.Logger, opts Options
 	return artifactsManager, nil
 }
 
+func buildAssetBuilder(logger *zap.Logger, artifactsManager *artifacts.Manager, cacheSigningKey crypto.PrivateKey, opts Options) (*asset.Builder, error) {
+	builderOptions := asset.Options{
+		AllowedConcurrency: opts.AssetBuildMaxConcurrency,
+		CacheSigningKey:    cacheSigningKey,
+	}
+
+	builderOptions.RemoteOptions = append(builderOptions.RemoteOptions, remoteOptions()...)
+
+	var err error
+
+	builderOptions.CacheRepository, err = name.NewRepository(opts.CacheRepository)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse cache repository: %w", err)
+	}
+
+	return asset.NewBuilder(logger, artifactsManager, builderOptions)
+}
+
 func buildSchematicFactory(logger *zap.Logger, opts Options) (*schematic.Factory, error) {
 	repo, err := name.NewRepository(opts.SchematicServiceRepository)
 	if err != nil {

cmd/image-factory/flags.go

@@ -42,6 +42,8 @@ func initFlags() cmd.Options {
 
 	flag.StringVar(&opts.CacheSigningKeyPath, "cache-signing-key-path", cmd.DefaultOptions.CacheSigningKeyPath, "path to the default cache signing key (PEM-encoded, ECDSA private key)")
 
+	flag.StringVar(&opts.CacheRepository, "cache-repository", cmd.DefaultOptions.CacheRepository, "cache repository for boot assets")
+
 	flag.Parse()
 
 	return opts

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/siderolabs/talos v1.6.0-alpha.1.0.20231018180437-c23bc2f4a77c
 	github.com/siderolabs/talos/pkg/machinery v1.6.0-alpha.1.0.20231018180437-c23bc2f4a77c
 	github.com/sigstore/cosign/v2 v2.2.0
+	github.com/sigstore/sigstore v1.7.2
 	github.com/stretchr/testify v1.8.4
 	github.com/u-root/u-root v0.11.0
 	github.com/ulikunitz/xz v0.5.11
@@ -258,7 +259,6 @@ require (
 	github.com/siderolabs/siderolink v0.3.1 // indirect
 	github.com/sigstore/fulcio v1.4.0 // indirect
 	github.com/sigstore/rekor v1.2.2 // indirect
-	github.com/sigstore/sigstore v1.7.2 // indirect
 	github.com/sigstore/timestamp-authority v1.1.2 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect

internal/artifacts/artifacts.go

@@ -51,3 +51,5 @@ const (
 	ImagerImage            = "siderolabs/imager"
 	ExtensionManifestImage = "siderolabs/extensions"
 )
+
+const tmpSuffix = "-tmp"

internal/artifacts/fetch.go

@@ -141,32 +141,32 @@ func (m *Manager) fetchImager(tag string) error {
 	destinationPath := filepath.Join(m.storagePath, tag)
 
 	if err := m.fetchImageByTag(ImagerImage, tag, ArchAmd64, imageExportHandler(func(logger *zap.Logger, r io.Reader) error {
-		return untar(logger, r, destinationPath+"-tmp")
+		return untar(logger, r, destinationPath+tmpSuffix)
 	})); err != nil {
 		return err
 	}
 
-	return os.Rename(destinationPath+"-tmp", destinationPath)
+	return os.Rename(destinationPath+tmpSuffix, destinationPath)
 }
 
 // fetchExtensionImage fetches a specified extension image and exports it to the storage as OCI.
 func (m *Manager) fetchExtensionImage(arch Arch, ref ExtensionRef, destPath string) error {
 	imageRef := m.imageRegistry.Repo(ref.TaggedReference.RepositoryStr()).Digest(ref.Digest)
 
-	if err := m.fetchImageByDigest(imageRef, arch, imageOCIHandler(destPath+"-tmp")); err != nil {
+	if err := m.fetchImageByDigest(imageRef, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil {
 		return err
 	}
 
-	return os.Rename(destPath+"-tmp", destPath)
+	return os.Rename(destPath+tmpSuffix, destPath)
 }
 
 // fetchInstallerImage fetches a Talos installer image and exports it to the storage.
 func (m *Manager) fetchInstallerImage(arch Arch, versionTag string, destPath string) error {
-	if err := m.fetchImageByTag(InstallerImage, versionTag, arch, imageOCIHandler(destPath+"-tmp")); err != nil {
+	if err := m.fetchImageByTag(InstallerImage, versionTag, arch, imageOCIHandler(destPath+tmpSuffix)); err != nil {
 		return err
 	}
 
-	return os.Rename(destPath+"-tmp", destPath)
+	return os.Rename(destPath+tmpSuffix, destPath)
 }
 
 func untar(logger *zap.Logger, r io.Reader, destination string) error {