feat: optimized post reconciliation process for enhanced performance …

…and resource utilization (halo-dev#5250) #### What type of PR is this? /kind feature /milestone 2.12.x /area core #### What this PR does / why we need it: 我们为文章自定义模型的数据调协过程引入了重要的优化。在以前，当数据量大（例如，50,000篇文章）的情况下，每次系统重启都会触发耗时且资源密集的所有数据的协调过程，即使大部分数据并不需要调协。这导致了不必要的数据库查询和高资源消耗。为了解决这个问题，我们在文章自定义模型的 status 中添加了一个新的 `Long observedVersion` 属性。每次协调后，此属性将更新为 `metadata.version`，还调整了 `syncAllOnStart` 条件，只有当 `status.observedVersion < metadata.version` 时才会调协数据。这个改变确保了只有在启动时需要的数据会被协调，从而减少了资源使用和不必要的协调过程。因此，Halo 的数据承载能力得到了显著提高。 **how to test it?** 使用此 PR 测试：启动时文章只有首次会执行 reconcile，再次重启时则不会再执行，如果直接修改数据去除掉 `status.observedVersion` 来模拟迁移或漏 reconcile 的过程则启动时该数据会被再次执行 reconcile #### Which issue(s) this PR fixes: Fixes halo-dev#5147 #### Does this PR introduce a user-facing change? ```release-note 优化文章数据的调协过程以降低 Halo 启动时文章的调协耗时同时提高性能和资源利用率 ```
ruibaby · Jan 26, 2024 · 17a0fb9 · 17a0fb9
1 parent 22104fe
commit 17a0fb9
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 11 deletions.
diff --git a/api/src/main/java/run/halo/app/core/extension/content/Post.java b/api/src/main/java/run/halo/app/core/extension/content/Post.java
@@ -16,7 +16,6 @@
 import run.halo.app.extension.GVK;
 import run.halo.app.extension.GroupVersionKind;
 import run.halo.app.extension.MetadataOperator;
-import run.halo.app.extension.MetadataUtil;
 import run.halo.app.infra.ConditionList;
 
 /**
@@ -35,6 +34,8 @@ public class Post extends AbstractExtension {
 
     public static final String KIND = "Post";
 
+    public static final String REQUIRE_SYNC_ON_STARTUP_INDEX_NAME = "requireSyncOnStartup";
+
     public static final GroupVersionKind GVK = GroupVersionKind.fromExtension(Post.class);
 
     public static final String CATEGORIES_ANNO = "content.halo.run/categories";
@@ -158,6 +159,8 @@ public static class PostStatus {
 
         private Instant lastModifyTime;
 
+        private Long observedVersion;
+
         @JsonIgnore
         public ConditionList getConditionsOrDefault() {
             if (this.conditions == null) {
@@ -268,9 +271,4 @@ public CompactPost build() {
             }
         }
     }
-
-    public static void changePublishedState(Post post, boolean value) {
-        Map<String, String> labels = MetadataUtil.nullSafeLabels(post);
-        labels.put(PUBLISHED_LABEL, String.valueOf(value));
-    }
 }
diff --git a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java
@@ -4,6 +4,7 @@
 import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
 import static run.halo.app.extension.ExtensionUtil.addFinalizers;
 import static run.halo.app.extension.ExtensionUtil.removeFinalizers;
+import static run.halo.app.extension.index.query.QueryFactory.equal;
 
 import com.google.common.hash.Hashing;
 import java.time.Instant;
@@ -36,6 +37,7 @@
 import run.halo.app.event.post.PostUnpublishedEvent;
 import run.halo.app.event.post.PostUpdatedEvent;
 import run.halo.app.event.post.PostVisibleChangedEvent;
+import run.halo.app.extension.DefaultExtensionMatcher;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.ExtensionOperator;
 import run.halo.app.extension.ListOptions;
@@ -211,7 +213,11 @@ public Result reconcile(Request request) {
                 status.setInProgress(
                     !StringUtils.equals(headSnapshot, post.getSpec().getReleaseSnapshot()));
 
+                // version + 1 is required to truly equal version
+                // as a version will be incremented after the update
+                status.setObservedVersion(post.getMetadata().getVersion() + 1);
                 client.update(post);
+
                 // fire event after updating post
                 events.forEach(eventPublisher::publishEvent);
             });
@@ -222,8 +228,12 @@ public Result reconcile(Request request) {
     public Controller setupWith(ControllerBuilder builder) {
         return builder
             .extension(new Post())
-            // TODO Make it configurable
-            .workerCount(1)
+            .onAddMatcher(DefaultExtensionMatcher.builder(client, Post.GVK)
+                .fieldSelector(FieldSelector.of(
+                    equal(Post.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME, BooleanUtils.TRUE))
+                )
+                .build()
+            )
             .build();
     }
 

diff --git a/application/src/main/java/run/halo/app/infra/SchemeInitializer.java b/application/src/main/java/run/halo/app/infra/SchemeInitializer.java
@@ -5,6 +5,7 @@
 import static run.halo.app.extension.index.IndexAttributeFactory.simpleAttribute;
 
 import java.util.Set;
+import org.apache.commons.lang3.BooleanUtils;
 import org.springframework.boot.context.event.ApplicationContextInitializedEvent;
 import org.springframework.context.ApplicationListener;
 import org.springframework.lang.NonNull;
@@ -146,6 +147,18 @@ public void onApplicationEvent(@NonNull ApplicationContextInitializedEvent event
                 .setName("status.excerpt")
                 .setIndexFunc(
                     simpleAttribute(Post.class, post -> post.getStatusOrDefault().getExcerpt())));
+
+            indexSpecs.add(new IndexSpec()
+                .setName(Post.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME)
+                .setIndexFunc(simpleAttribute(Post.class, post -> {
+                    var version = post.getMetadata().getVersion();
+                    var observedVersion = post.getStatusOrDefault().getObservedVersion();
+                    if (observedVersion == null || observedVersion < version) {
+                        return BooleanUtils.TRUE;
+                    }
+                    // do not care about the false case so return null to avoid indexing
+                    return null;
+                })));
         });
         schemeManager.register(Category.class, indexSpecs -> {
             indexSpecs.add(new IndexSpec()

diff --git a/application/src/test/java/run/halo/app/content/TestPost.java b/application/src/test/java/run/halo/app/content/TestPost.java
@@ -19,6 +19,7 @@ public static Post postV1() {
         post.setApiVersion(getApiVersion(Post.class));
         Metadata metadata = new Metadata();
         metadata.setName("post-A");
+        metadata.setVersion(1L);
         post.setMetadata(metadata);
 
         Post.PostSpec postSpec = new Post.PostSpec();
@@ -38,6 +39,7 @@ public static Snapshot snapshotV1() {
         snapshot.setApiVersion(getApiVersion(Snapshot.class));
         Metadata metadata = new Metadata();
         metadata.setName("snapshot-A");
+        metadata.setVersion(1L);
         metadata.setCreationTimestamp(Instant.now());
         snapshot.setMetadata(metadata);
         MetadataUtil.nullSafeAnnotations(snapshot).put(Snapshot.KEEP_RAW_ANNO, "true");

diff --git a/application/src/test/java/run/halo/app/content/comment/CommentServiceImplTest.java b/application/src/test/java/run/halo/app/content/comment/CommentServiceImplTest.java
@@ -358,7 +358,8 @@ private String expectListResultJson() {
                             "apiVersion": "content.halo.run/v1alpha1",
                             "kind": "Post",
                             "metadata": {
-                                "name": "fake-post"
+                                "name": "fake-post",
+                                "version": 1
                             }
                         },
                         "stats": {
@@ -403,7 +404,8 @@ private String expectListResultJson() {
                             "apiVersion": "content.halo.run/v1alpha1",
                             "kind": "Post",
                             "metadata": {
-                                "name": "fake-post"
+                                "name": "fake-post",
+                                "version": 1
                             }
                         },
                         "stats": {
@@ -447,7 +449,8 @@ private String expectListResultJson() {
                             "apiVersion": "content.halo.run/v1alpha1",
                             "kind": "Post",
                             "metadata": {
-                                "name": "fake-post"
+                                "name": "fake-post",
+                                "version": 1
                             }
                         },
                         "stats": {