feat(split): implement child_handle_split_error (#642)

XiaoMi · Oct 15, 2020 · 2161300 · 2161300
1 parent 91f9f4e
commit 2161300
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 28 deletions.
diff --git a/src/replica/replica_config.cpp b/src/replica/replica_config.cpp
@@ -625,6 +625,13 @@ bool replica::is_same_ballot_status_change_allowed(partition_status::type olds,
 bool replica::update_local_configuration(const replica_configuration &config,
                                          bool same_ballot /* = false*/)
 {
+    FAIL_POINT_INJECT_F("replica_update_local_configuration", [=](dsn::string_view) -> bool {
+        auto old_status = status();
+        _config = config;
+        ddebug_replica("update status from {} to {}", enum_to_string(old_status), status());
+        return true;
+    });
+
     dassert(config.ballot > get_ballot() || (same_ballot && config.ballot == get_ballot()),
             "invalid ballot, %" PRId64 " VS %" PRId64 "",
             config.ballot,

diff --git a/src/replica/split/replica_split_manager.cpp b/src/replica/split/replica_split_manager.cpp
@@ -805,10 +805,11 @@ void replica_split_manager::child_handle_split_error(
     const std::string &error_msg) // on child partition
 {
     if (status() != partition_status::PS_ERROR) {
-        dwarn_replica("partition split failed because {}", error_msg);
-        // TODO(heyuchen):
-        // convert child partition_status from PS_PARTITION_SPLIT to PS_ERROR in further pull
-        // request
+        derror_replica("child partition split failed because {}, parent = {}",
+                       error_msg,
+                       _replica->_split_states.parent_gpid);
+        // TODO(heyuchen): add perf-counter (split_failed_count)
+        _replica->update_local_configuration_with_no_ballot_change(partition_status::PS_ERROR);
     }
 }
 

diff --git a/src/replica/split/test/replica_split_test.cpp b/src/replica/split/test/replica_split_test.cpp
@@ -397,33 +397,36 @@ TEST_F(replica_split_test, copy_prepare_list_succeed)
     cleanup_child_split_context();
 }
 
-// TODO(heyuchen): refactor child_learn_states unit tests
-TEST_F(replica_split_test, learn_states_succeed)
+// child_learn_states tests
+TEST_F(replica_split_test, child_learn_states_tests)
 {
-    generate_child(true, false);
-    // mock_child_split_context(true, false);
-
-    // fail::setup();
-    fail::cfg("replica_child_apply_private_logs", "return()");
-    fail::cfg("replica_child_catch_up_states", "return()");
-    test_child_learn_states();
-    // fail::teardown();
-    ASSERT_EQ(_child_replica->status(), partition_status::PS_PARTITION_SPLIT);
-
-    cleanup_prepare_list(_child_replica);
-    cleanup_child_split_context();
-}
+    generate_child();
 
-TEST_F(replica_split_test, learn_states_with_replay_private_log_error)
-{
-    generate_child(true, false);
+    // Test cases:
+    // - mock replay private log error
+    // - child learn states succeed
+    struct child_learn_state_test
+    {
+        bool mock_replay_log_error;
+        partition_status::type expected_child_status;
+    } tests[] = {{true, partition_status::PS_ERROR}, {false, partition_status::PS_PARTITION_SPLIT}};
+    for (auto test : tests) {
+        fail::setup();
+        fail::cfg("replica_child_catch_up_states", "return()");
+        fail::cfg("replica_stub_split_replica_exec", "return()");
+        if (test.mock_replay_log_error) {
+            fail::cfg("replica_child_apply_private_logs", "return(ERR_INVALID_STATE)");
+        } else {
+            fail::cfg("replica_child_apply_private_logs", "return()");
+        }
+        mock_child_split_context(true, false);
+        test_child_learn_states();
+        ASSERT_EQ(_child_replica->status(), test.expected_child_status);
 
-    fail::cfg("replica_child_apply_private_logs", "return(error)");
-    fail::cfg("replica_child_catch_up_states", "return()");
-    test_child_learn_states();
-    // TODO(heyuchen): child should be equal to error(after implement child_handle_split_error)
-    cleanup_prepare_list(_child_replica);
-    cleanup_child_split_context();
+        cleanup_prepare_list(_child_replica);
+        cleanup_child_split_context();
+        fail::teardown();
+    }
 }
 
 // child_apply_private_logs test