fix: should ignore create errors when namespace terminating

Makefile

@@ -100,7 +100,7 @@ envtest: setup-envtest
 
 .PHONY: test
 test: test-tools
-	go test -v -count 1 -race ./api/... ./pkg/...
+	go test -v -count 1 -race ./api/... ./internal/... ./pkg/...
 	go install ./...
 	go vet ./...
 	test -z $$(gofmt -s -l . | tee /dev/stderr)

controllers/namespace_controller.go

@@ -6,6 +6,7 @@ import (
 	"path"
 
 	accuratev2alpha1 "github.com/cybozu-go/accurate/api/accurate/v2alpha1"
+	utilerrors "github.com/cybozu-go/accurate/internal/util/errors"
 	"github.com/cybozu-go/accurate/pkg/constants"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -229,7 +230,7 @@ func (r *NamespaceReconciler) propagateCreate(ctx context.Context, res *unstruct
 	}
 
 	if err := r.Create(ctx, cloneResource(res, ns)); err != nil {
-		return err
+		return utilerrors.Ignore(err, utilerrors.IsNamespaceTerminating)
 	}
 
 	logger := log.FromContext(ctx)
@@ -249,7 +250,7 @@ func (r *NamespaceReconciler) propagateUpdate(ctx context.Context, res *unstruct
 			return err
 		}
 		if err := r.Create(ctx, cloneResource(res, ns)); err != nil {
-			return err
+			return utilerrors.Ignore(err, utilerrors.IsNamespaceTerminating)
 		}
 		logger.Info("created a resource", "namespace", ns, "name", res.GetName(), "gvk", gvk.String())
 		return nil

controllers/propagate.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	utilerrors "github.com/cybozu-go/accurate/internal/util/errors"
 	"github.com/cybozu-go/accurate/pkg/config"
 	"github.com/cybozu-go/accurate/pkg/constants"
 	"github.com/cybozu-go/accurate/pkg/feature"
@@ -200,6 +201,9 @@ func (r *PropagateController) handleDelete(ctx context.Context, req ctrl.Request
 			switch obj.GetAnnotations()[constants.AnnPropagate] {
 			case constants.PropagateCreate, constants.PropagateUpdate:
 				if err := r.Create(ctx, cloneResource(obj, req.Namespace)); err != nil {
+					if utilerrors.IsNamespaceTerminating(err) {
+						return nil
+					}
 					return fmt.Errorf("failed to re-create %s/%s: %w", req.Namespace, req.Name, err)
 				}
 				logger.Info("re-created", "from", fmt.Sprintf("%s/%s", p, req.Name))
@@ -254,6 +258,9 @@ func (r *PropagateController) propagateCreate(ctx context.Context, obj *unstruct
 		}
 
 		if err := r.Create(ctx, cloneResource(obj, child.Name)); err != nil {
+			if utilerrors.IsNamespaceTerminating(err) {
+				return nil
+			}
 			return fmt.Errorf("failed to create %s/%s: %w", child.Name, name, err)
 		}
 
@@ -301,6 +308,9 @@ func (r *PropagateController) propagateUpdate(ctx context.Context, obj, parent *
 
 			clone := cloneResource(obj, child.Name)
 			if err := r.Create(ctx, clone); err != nil {
+				if utilerrors.IsNamespaceTerminating(err) {
+					return nil
+				}
 				return fmt.Errorf("failed to create %s/%s: %w", child.Name, name, err)
 			}
 

controllers/subnamespace_controller.go

@@ -7,6 +7,7 @@ import (
 
 	accuratev2alpha1 "github.com/cybozu-go/accurate/api/accurate/v2alpha1"
 	accuratev2alpha1ac "github.com/cybozu-go/accurate/internal/applyconfigurations/accurate/v2alpha1"
+	utilerrors "github.com/cybozu-go/accurate/internal/util/errors"
 	"github.com/cybozu-go/accurate/pkg/constants"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -113,7 +114,7 @@ func (r *SubNamespaceReconciler) reconcileNS(ctx context.Context, sn *accuratev2
 			constants.LabelParent:    sn.Namespace,
 		}
 		if err := r.Create(ctx, ns); err != nil {
-			return err
+			return utilerrors.Ignore(err, utilerrors.IsNamespaceTerminating)
 		}
 		logger.Info("created a sub namespace", "name", sn.Name)
 	}

internal/util/errors/errors.go

@@ -0,0 +1,26 @@
+package errors
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+)
+
+// ErrorIs returns true if an error satisfies a particular condition.
+type ErrorIs func(err error) bool
+
+// Ignore ignores errors that satisfy any of the supplied ErrorIs functions
+// by returning nil. Errors that do not satisfy any of the supplied functions
+// are returned unmodified.
+func Ignore(err error, is ...ErrorIs) error {
+	for _, f := range is {
+		if f(err) {
+			return nil
+		}
+	}
+	return err
+}
+
+// IsNamespaceTerminating returns true if the error is a namespace is terminating error.
+func IsNamespaceTerminating(err error) bool {
+	return apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause)
+}

internal/util/errors/errors_test.go

@@ -0,0 +1,42 @@
+package errors
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestIgnore(t *testing.T) {
+	err := errors.New("error")
+
+	type args struct {
+		err error
+		is  []ErrorIs
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{name: "ignored", args: args{err: err, is: []ErrorIs{IsErrorAlways}}},
+		{name: "not ignored", args: args{err: err, is: []ErrorIs{IsErrorNever}}, wantErr: true},
+		{name: "ignored by first", args: args{err: err, is: []ErrorIs{IsErrorAlways, IsErrorNever}}},
+		{name: "ignored by second", args: args{err: err, is: []ErrorIs{IsErrorNever, IsErrorAlways}}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := Ignore(tt.args.err, tt.args.is...); (err != nil) != tt.wantErr {
+				t.Errorf("IgnoreAny() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+//goland:noinspection GoUnusedParameter
+func IsErrorAlways(err error) bool {
+	return true
+}
+
+//goland:noinspection GoUnusedParameter
+func IsErrorNever(err error) bool {
+	return false
+}