✨ Revert "Revert "✨ Reaper optimization."" (#696)

Reverts #687
konveyor · Jul 5, 2024 · 220be5d · 220be5d
1 parent 24bb742
commit 220be5d
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 77 deletions.
diff --git a/k8s/api/tackle/v1alpha2/zz_generated.deepcopy.go b/k8s/api/tackle/v1alpha2/zz_generated.deepcopy.go
diff --git a/reaper/bucket.go b/reaper/bucket.go
@@ -26,13 +26,24 @@ func (r *BucketReaper) Run() {
 		Log.Error(err, "")
 		return
 	}
-	for _, bucket := range list {
-		busy, err := r.busy(&bucket)
+	if len(list) == 0 {
+		return
+	}
+	ids := make(map[uint]byte)
+	finder := RefFinder{DB: r.DB}
+	for _, m := range []any{
+		&model.Application{},
+		&model.TaskGroup{},
+		&model.Task{},
+	} {
+		err := finder.Find(m, "bucket", ids)
 		if err != nil {
 			Log.Error(err, "")
 			continue
 		}
-		if busy {
+	}
+	for _, bucket := range list {
+		if _, found := ids[bucket.ID]; found {
 			if bucket.Expiration != nil {
 				bucket.Expiration = nil
 				err = r.DB.Save(&bucket).Error
@@ -59,27 +70,6 @@ func (r *BucketReaper) Run() {
 	}
 }
 
-// busy determines if anything references the bucket.
-func (r *BucketReaper) busy(bucket *model.Bucket) (busy bool, err error) {
-	nRef := int64(0)
-	var n int64
-	ref := RefCounter{DB: r.DB}
-	for _, m := range []any{
-		&model.Application{},
-		&model.TaskGroup{},
-		&model.Task{},
-	} {
-		n, err = ref.Count(m, "bucket", bucket.ID)
-		if err != nil {
-			Log.Error(err, "")
-			continue
-		}
-		nRef += n
-	}
-	busy = nRef > 0
-	return
-}
-
 // Delete bucket.
 func (r *BucketReaper) delete(bucket *model.Bucket) (err error) {
 	err = nas.RmDir(bucket.Path)

diff --git a/reaper/file.go b/reaper/file.go
@@ -25,13 +25,25 @@ func (r *FileReaper) Run() {
 		Log.Error(err, "")
 		return
 	}
-	for _, file := range list {
-		busy, err := r.busy(&file)
+	if len(list) == 0 {
+		return
+	}
+	ids := make(map[uint]byte)
+	finder := RefFinder{DB: r.DB}
+	for _, m := range []any{
+		&model.Task{},
+		&model.TaskReport{},
+		&model.Rule{},
+		&model.Target{},
+	} {
+		err := finder.Find(m, "file", ids)
 		if err != nil {
 			Log.Error(err, "")
 			continue
 		}
-		if busy {
+	}
+	for _, file := range list {
+		if _, found := ids[file.ID]; found {
 			if file.Expiration != nil {
 				file.Expiration = nil
 				err = r.DB.Save(&file).Error
@@ -58,29 +70,6 @@ func (r *FileReaper) Run() {
 	}
 }
 
-// busy determines if anything references the file.
-func (r *FileReaper) busy(file *model.File) (busy bool, err error) {
-	nRef := int64(0)
-	var n int64
-	ref := RefCounter{DB: r.DB}
-	for _, m := range []any{
-		&model.Task{},
-		&model.TaskReport{},
-		&model.RuleSet{},
-		&model.Rule{},
-		&model.Target{},
-	} {
-		n, err = ref.Count(m, "file", file.ID)
-		if err != nil {
-			Log.Error(err, "")
-			continue
-		}
-		nRef += n
-	}
-	busy = nRef > 0
-	return
-}
-
 // Delete file.
 func (r *FileReaper) delete(file *model.File) (err error) {
 	err = os.Remove(file.Path)

diff --git a/reaper/ref.go b/reaper/ref.go
@@ -8,40 +8,32 @@ import (
 	"gorm.io/gorm"
 )
 
-// RefCounter provides model inspection for files
-// tagged with: ref:<kind>.
-type RefCounter struct {
+// RefFinder provides model inspection for files
+// tagged with:
+//
+//	ref:<kind>
+//	[]ref:<kind>
+type RefFinder struct {
 	// DB
 	DB *gorm.DB
 }
 
-// Count find & count references.
-func (r *RefCounter) Count(m any, kind string, pk uint) (nRef int64, err error) {
-	db := r.DB.Model(m)
-	fields := 0
-	j := 0
+// Find returns a map of all references for the model and kind.
+func (r *RefFinder) Find(m any, kind string, ids map[uint]byte) (err error) {
+	var nfields []string
+	var jfields []string
 	add := func(ft reflect.StructField) {
 		tag, found := ft.Tag.Lookup("ref")
 		if found && tag == kind {
-			db = db.Or(ft.Name, pk)
-			fields++
+			nfields = append(
+				nfields,
+				ft.Name)
 			return
 		}
 		if found && tag == "[]"+kind {
-			db = db.Joins(
-				fmt.Sprintf(
-					",json_each(%s) j%d",
-					ft.Name,
-					j))
-			db = db.Or(
-				fmt.Sprintf(
-					"json_extract(j%d.value,?)=?",
-					j),
-				"$.id",
-				pk)
-			fields++
-			j++
-			return
+			jfields = append(
+				jfields,
+				ft.Name)
 		}
 	}
 	var find func(any)
@@ -76,21 +68,58 @@ func (r *RefCounter) Count(m any, kind string, pk uint) (nRef int64, err error)
 				add(ft)
 			case reflect.Slice:
 				add(ft)
+			default:
 			}
 		}
 	}
 	find(m)
-	if fields == 0 {
+	if len(nfields)+len(jfields) == 0 {
 		return
 	}
-	err = db.Count(&nRef).Error
+	db := r.DB.Model(m)
+	if Log.V(1).Enabled() {
+		db = db.Debug()
+	}
+	var fields []string
+	var list []map[string]any
+	for i := range nfields {
+		fields = append(fields, nfields[i])
+	}
+	for i := range jfields {
+		fields = append(
+			fields,
+			fmt.Sprintf(
+				"json_extract(j%d.value,'$.id')",
+				i))
+		db = db.Joins(
+			fmt.Sprintf(
+				",json_each(%s) j%d",
+				jfields[i],
+				i))
+	}
+	db = db.Select(fields)
+	err = db.Find(&list).Error
 	if err != nil {
 		err = liberr.Wrap(
 			err,
 			"object",
 			reflect.TypeOf(m).Name(),
 		)
 	}
+	for _, ref := range list {
+		for _, v := range ref {
+			switch n := v.(type) {
+			case uint:
+				ids[n] = 0
+			case *uint:
+				if n != nil {
+					ids[*n] = 0
+				}
+			case int64:
+				ids[uint(n)] = 0
+			}
+		}
+	}
 
 	return
 }