generator: recover from kustomize build panics

controllers/kustomization_generator.go

@@ -247,7 +247,7 @@ var kustomizeBuildMutex sync.Mutex
 //  - load files from outside the kustomization dir path
 //    (but not outside root)
 //  - disable plugins except for the builtin ones
-func secureBuildKustomization(root, dirPath string) (resmap.ResMap, error) {
+func secureBuildKustomization(root, dirPath string) (_ resmap.ResMap, err error) {
 	// Create secure FS for root
 	fs, err := securefs.MakeFsOnDiskSecureBuild(root)
 	if err != nil {
@@ -259,6 +259,15 @@ func secureBuildKustomization(root, dirPath string) (resmap.ResMap, error) {
 	kustomizeBuildMutex.Lock()
 	defer kustomizeBuildMutex.Unlock()
 
+	// Kustomize tends to panic in unpredicted ways due to (accidental)
+	// invalid object data; recover when this happens to ensure continuity of
+	// operations
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("recovered from kustomize build panic: %v", r)
+		}
+	}()
+
 	buildOptions := &krusty.Options{
 		LoadRestrictions: kustypes.LoadRestrictionsNone,
 		PluginConfig:     kustypes.DisabledPluginConfig(),

controllers/kustomization_generator_test.go

@@ -30,3 +30,16 @@ func Test_secureBuildKustomization(t *testing.T) {
 		g.Expect(err).ToNot(HaveOccurred())
 	})
 }
+
+func Test_secureBuildKustomization_panic(t *testing.T) {
+	t.Run("build panic", func(t *testing.T) {
+		g := NewWithT(t)
+
+		_, err := secureBuildKustomization("testdata/panic", "testdata/panic")
+		g.Expect(err).To(HaveOccurred())
+		g.Expect(err.Error()).To(ContainSubstring("recovered from kustomize build panic"))
+		// Run again to ensure the lock is released
+		_, err = secureBuildKustomization("testdata/panic", "testdata/panic")
+		g.Expect(err).To(HaveOccurred())
+	})
+}

controllers/testdata/panic/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: test
+namePrefix: test
+resources:
+  - secret.age.yaml

controllers/testdata/panic/secret.age.yaml

@@ -0,0 +1,27 @@
+apiVersion: ENC[AES256_GCM,data:RwzrBF8wy16SpfbQoeADeKyz,iv:DuJce2Ebx1Y49DaLCOJ74OOkgiv21roxhz/sZqKCSSs=,tag:Gg9XHapZI5q+rvtgeY6nrg==,type:str]
+kind: ENC[AES256_GCM,data:RwzrBF8wy16SpfbQoeADeKyz,iv:DuJce2Ebx1Y49DaLCOJ74OOkgiv21roxhz/sZqKCSSs=,tag:Gg9XHapZI5q+rvtgeY6nrg==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:RwzrBF8wy16SpfbQoeADeKyz,iv:DuJce2Ebx1Y49DaLCOJ74OOkgiv21roxhz/sZqKCSSs=,tag:Gg9XHapZI5q+rvtgeY6nrg==,type:str]
+    namespace: ENC[AES256_GCM,data:RwzrBF8wy16SpfbQoeADeKyz,iv:DuJce2Ebx1Y49DaLCOJ74OOkgiv21roxhz/sZqKCSSs=,tag:Gg9XHapZI5q+rvtgeY6nrg==,type:str]
+stringData:
+    secret: ENC[AES256_GCM,data:RwzrBF8wy16SpfbQoeADeKyz,iv:DuJce2Ebx1Y49DaLCOJ74OOkgiv21roxhz/sZqKCSSs=,tag:Gg9XHapZI5q+rvtgeY6nrg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1l44xcng8dqj32nlv6d930qvvrny05hglzcv9qpc7kxjc6902ma4qufys29
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNeGduOFZjRWw2WTFQdWdu
+            OS83OEZaN1E1aU1zSThhMlNEZzd0aEYvdURFCnE3bmJ5c3J2cDNEbXhselFPVC9v
+            NFhMRjZjOHZOdEpoYjdiS0ZPd2pvN1kKLS0tIDZUVEFoblpDNWhnaWxYRTBjaktk
+            bHRXV0o1K2ZDNm5Mem5SdzNBMTNuNFUKylE2cRLqydjj6e4+4Giwn4y8mIPej+CM
+            Bab3UWiK1da2rFNTOEnoHl6QDAVxNrWdrrIa5k22SzApT88VtJ4xuQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2021-04-06T09:07:05Z"
+    mac: ENC[AES256_GCM,data:oaM8qFtEP8dOCd/Tr5yb08uetsnDtZO8o1rCayN53ncQ1HUAdhRBrFdmbYx1YTh1mwQVVN6sGYqFZU1LBMVv5pTqvpwd41biJZEg8NznXQWx0GA2Z6HOrblGhFZKrqky3P5xN+6j63zkJizXWgBMKzRvBnsVKxjZGr/lk1vVVv4=,iv:p4y9Fo3SArkEMuoK2d9sQYgNdc0iw/StFhg/5LnhcXM=,tag:61JGbnEw35tv6WnGj46JOw==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.7.0