From 857e2c2d1b21951aec937eeca4c1e094af7cfa70 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Sun, 25 Sep 2022 21:11:51 +0800 Subject: [PATCH 1/2] refactor(test): remove useless upgrade_test --- src/test/upgrade_test/CMakeLists.txt | 51 -- src/test/upgrade_test/config.ini | 118 ----- src/test/upgrade_test/data_verifier.cpp | 435 ------------------ src/test/upgrade_test/data_verifier.h | 23 - src/test/upgrade_test/job.cpp | 40 -- src/test/upgrade_test/job.h | 65 --- src/test/upgrade_test/main.cpp | 44 -- src/test/upgrade_test/process_upgrader.cpp | 206 --------- src/test/upgrade_test/process_upgrader.h | 23 - src/test/upgrade_test/upgrade_testor.cpp | 230 --------- src/test/upgrade_test/upgrade_testor.h | 86 ---- src/test/upgrade_test/upgrader_handler.h | 77 ---- .../upgrade_test/upgrader_handler_shell.cpp | 240 ---------- .../upgrade_test/upgrader_handler_shell.h | 69 --- src/test/upgrade_test/upgrader_registry.cpp | 29 -- src/test/upgrade_test/upgrader_registry.h | 22 - 16 files changed, 1758 deletions(-) delete mode 100644 src/test/upgrade_test/CMakeLists.txt delete mode 100644 src/test/upgrade_test/config.ini delete mode 100644 src/test/upgrade_test/data_verifier.cpp delete mode 100644 src/test/upgrade_test/data_verifier.h delete mode 100644 src/test/upgrade_test/job.cpp delete mode 100644 src/test/upgrade_test/job.h delete mode 100644 src/test/upgrade_test/main.cpp delete mode 100644 src/test/upgrade_test/process_upgrader.cpp delete mode 100644 src/test/upgrade_test/process_upgrader.h delete mode 100644 src/test/upgrade_test/upgrade_testor.cpp delete mode 100644 src/test/upgrade_test/upgrade_testor.h delete mode 100644 src/test/upgrade_test/upgrader_handler.h delete mode 100644 src/test/upgrade_test/upgrader_handler_shell.cpp delete mode 100644 src/test/upgrade_test/upgrader_handler_shell.h delete mode 100644 src/test/upgrade_test/upgrader_registry.cpp delete mode 100644 src/test/upgrade_test/upgrader_registry.h diff --git a/src/test/upgrade_test/CMakeLists.txt b/src/test/upgrade_test/CMakeLists.txt deleted file mode 100644 index ee3a6eff81..0000000000 --- a/src/test/upgrade_test/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set(MY_PROJ_NAME "pegasus_upgrade_test") -project(${MY_PROJ_NAME} C CXX) - -# Source files under CURRENT project directory will be automatically included. -# You can manually set MY_PROJ_SRC to include source files under other directories. -set(MY_PROJ_SRC "") - -# Search mode for source files under CURRENT project directory? -# "GLOB_RECURSE" for recursive search -# "GLOB" for non-recursive search -set(MY_SRC_SEARCH_MODE "GLOB") - -set(MY_PROJ_LIBS - pegasus_base - pegasus_client_static - dsn_client - dsn_replication_common - dsn_runtime - dsn_utils - sasl2 - gssapi_krb5 - krb5 - ) -set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini") - -set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) - -if (UNIX) - SET(CMAKE_INSTALL_RPATH ".") - SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) -endif() - -dsn_add_executable() -dsn_install_executable() diff --git a/src/test/upgrade_test/config.ini b/src/test/upgrade_test/config.ini deleted file mode 100644 index ff5b99ee92..0000000000 --- a/src/test/upgrade_test/config.ini +++ /dev/null @@ -1,118 +0,0 @@ -; Licensed to the Apache Software Foundation (ASF) under one -; or more contributor license agreements. See the NOTICE file -; distributed with this work for additional information -; regarding copyright ownership. The ASF licenses this file -; to you under the Apache License, Version 2.0 (the -; "License"); you may not use this file except in compliance -; with the License. You may obtain a copy of the License at -; -; http://www.apache.org/licenses/LICENSE-2.0 -; -; Unless required by applicable law or agreed to in writing, -; software distributed under the License is distributed on an -; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -; KIND, either express or implied. See the License for the -; specific language governing permissions and limitations -; under the License. -[apps..default] -run = true -count = 1 -;network.client.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536 -;network.client.RPC_CHANNEL_UDP = dsn::tools::sim_network_provider, 65536 -;network.server.0.RPC_CHANNEL_TCP = NET_HDR_DSN, dsn::tools::sim_network_provider, 65536 - -[apps.mimic] -name = mimic -type = dsn.app.mimic -arguments = -pools = THREAD_POOL_DEFAULT, THREAD_POOL_META_SERVER -run = true -count = 1 - -[core] -;tool = simulator -tool = nativerun -;toollets = tracer -;toollets = tracer, profiler, fault_injector -pause_on_start = false - -logging_start_level = LOG_LEVEL_DEBUG -logging_factory_name = dsn::tools::simple_logger -;logging_factory_name = dsn::tools::screen_logger -logging_flush_on_exit = true - -enable_default_app_mimic = true - -data_dir = ./data - -[tools.simple_logger] -short_header = false -fast_flush = true -max_number_of_log_files_on_disk = 100000 -stderr_start_level = LOG_LEVEL_FATAL - -[tools.simulator] -random_seed = 0 - -[network] -; how many network threads for network library(used by asio) -io_service_worker_count = 4 - -; specification for each thread pool -[threadpool..default] -worker_count = 4 - -[threadpool.THREAD_POOL_DEFAULT] -name = default -partitioned = false -worker_priority = THREAD_xPRIORITY_NORMAL -worker_count = 8 - -[task..default] -is_trace = false -is_profile = false -allow_inline = false -rpc_call_channel = RPC_CHANNEL_TCP -rpc_call_header_format = NET_HDR_DSN -rpc_timeout_milliseconds = 5000 - -[pegasus.upgradetest] -upgrade_handler = shell -pegasus_cluster_name = onebox - -; total number of meta/replica/zookeeper jobs -total_meta_count = @META_COUNT@ -total_replica_count = @REPLICA_COUNT@ -total_zookeeper_count = @ZK_COUNT@ - -; unit seconds, sleep time for recover -sleep_time_before_recover_seconds = 60 -; unit seconds, upgrade inerval, must > 20s -kill_interval_seconds = 300 -; the max time for all the partitions of the table become to healthy status, unit seconds -max_seconds_for_all_partitions_to_recover = 600 - -; the max number that can be upgraded of replica/meta/zookeeper -upgrade_meta_max_count = 0 -upgrade_replica_max_count = 5 -upgrade_zookeeper_max_count = 0 -; log file that record the action of upgrade -upgrade_log_file = ./upgrade_history.txt - -verify_app_name = @APP_NAME@ -; unit milliseconds, timeout for set/get operation, must >= (sleep_time_for_recover * 1000 + 3000) -set_and_get_timeout_milliseconds = 9000 -; number of the set client -set_thread_count = @SET_THREAD_COUNT@ -; number of the get client -get_thread_count = @GET_THREAD_COUNT@ - -[upgrade.handler.shell] -; using run.sh script to upgrade/downgrade the jobs, just for testing onebox -; path to the folder that contain run.sh script -onebox_run_path = @ONEBOX_RUN_PATH@ -old_version_path = @OLD_VERSION_PATH@ -new_version_path = @NEW_VERSION_PATH@ - -[pegasus.clusters] -onebox = @LOCAL_HOSTNAME@:30601,@LOCAL_HOSTNAME@:30602,@LOCAL_HOSTNAME@:30603 diff --git a/src/test/upgrade_test/data_verifier.cpp b/src/test/upgrade_test/data_verifier.cpp deleted file mode 100644 index 7d0bb7f18e..0000000000 --- a/src/test/upgrade_test/data_verifier.cpp +++ /dev/null @@ -1,435 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "pegasus/client.h" -#include "data_verifier.h" - -using namespace std; -using namespace ::pegasus; - -static pegasus_client *client = nullptr; -static string app_name; -static string pegasus_cluster_name; -static uint32_t set_and_get_timeout_milliseconds; -static int set_thread_count = 0; - -static std::atomic_llong set_next(0); -static int get_thread_count = 0; -static std::vector set_thread_setting_id; - -static const char *set_next_key = "set_next"; -static const char *check_max_key = "check_max"; -static const char *hash_key_prefix = "upgrade_test_hash_key_"; -static const char *sort_key_prefix = "upgrade_test_sort_key_"; -static const char *value_prefix = "upgrade_test_value_"; -static const long stat_batch = 100000; -static const long stat_min_pos = 0; -static const long stat_p90_pos = stat_batch - stat_batch / 10 - 1; -static const long stat_p99_pos = stat_batch - stat_batch / 100 - 1; -static const long stat_p999_pos = stat_batch - stat_batch / 1000 - 1; -static const long stat_p9999_pos = stat_batch - stat_batch / 10000 - 1; -static const long stat_max_pos = stat_batch - 1; - -// return time in us. -long get_time() -{ - struct timeval tv; - gettimeofday(&tv, nullptr); - return tv.tv_sec * 1000000 + tv.tv_usec; -} - -long long get_min_thread_setting_id() -{ - long long id = set_thread_setting_id[0]; - for (int i = 1; i < set_thread_count; ++i) { - if (set_thread_setting_id[i] < id) - id = set_thread_setting_id[i]; - } - return id; -} - -void do_set(int thread_id) -{ - char buf[1024]; - std::string hash_key; - std::string sort_key; - std::string value; - long long id = 0; - int try_count = 0; - long stat_count = 0; - std::vector stat_time; - stat_time.resize(stat_batch); - long last_time = get_time(); - while (true) { - if (try_count == 0) { - id = set_next++; - set_thread_setting_id[thread_id] = id; - sprintf(buf, "%s%lld", hash_key_prefix, id); - hash_key.assign(buf); - sprintf(buf, "%s%lld", sort_key_prefix, id); - sort_key.assign(buf); - sprintf(buf, "%s%lld", value_prefix, id); - value.assign(buf); - } - pegasus_client::internal_info info; - int ret = - client->set(hash_key, sort_key, value, set_and_get_timeout_milliseconds, 0, &info); - if (ret == PERR_OK) { - long cur_time = get_time(); - // ddebug("SetThread[%d]: set succeed: id=%lld, try=%d, time=%ld (gpid=%d.%d, - // " - // "decree=%lld, server=%s)", - // thread_id, - // id, - // try_count, - // (cur_time - last_time), - // info.app_id, - // info.partition_index, - // info.decree, - // info.server.c_str()); - stat_time[stat_count++] = cur_time - last_time; - if (stat_count == stat_batch) { - std::sort(stat_time.begin(), stat_time.end()); - long total_time = 0; - for (auto t : stat_time) - total_time += t; - ddebug("SetThread[%d]: set statistics: count=%lld, min=%lld, P90=%lld, P99=%lld, " - "P999=%lld, P9999=%lld, max=%lld, avg=%lld", - thread_id, - stat_count, - stat_time[stat_min_pos], - stat_time[stat_p90_pos], - stat_time[stat_p99_pos], - stat_time[stat_p999_pos], - stat_time[stat_p9999_pos], - stat_time[stat_max_pos], - total_time / stat_batch); - stat_count = 0; - } - last_time = cur_time; - try_count = 0; - } else { - derror("SetThread[%d]: set failed: id=%lld, try=%d, ret=%d, error=%s (gpid=%d.%d, " - "decree=%lld, server=%s)", - thread_id, - id, - try_count, - ret, - client->get_error_string(ret), - info.app_id, - info.partition_index, - info.decree, - info.server.c_str()); - try_count++; - if (try_count > 3) { - sleep(1); - } - } - } -} - -// for each round: -// - loop from range [start_id, end_id] -void do_get_range(int thread_id, int round_id, long long start_id, long long end_id) -{ - ddebug( - "GetThread[%d]: round(%d): start get range [%u,%u]", thread_id, round_id, start_id, end_id); - char buf[1024]; - std::string hash_key; - std::string sort_key; - std::string value; - long long id = start_id; - int try_count = 0; - long stat_count = 0; - std::vector stat_time; - stat_time.resize(stat_batch); - long last_time = get_time(); - while (id <= end_id) { - if (try_count == 0) { - sprintf(buf, "%s%lld", hash_key_prefix, id); - hash_key.assign(buf); - sprintf(buf, "%s%lld", sort_key_prefix, id); - sort_key.assign(buf); - sprintf(buf, "%s%lld", value_prefix, id); - value.assign(buf); - } - pegasus_client::internal_info info; - std::string get_value; - int ret = - client->get(hash_key, sort_key, get_value, set_and_get_timeout_milliseconds, &info); - if (ret == PERR_OK || ret == PERR_NOT_FOUND) { - long cur_time = get_time(); - if (ret == PERR_NOT_FOUND) { - dfatal("GetThread[%d]: round(%d): get not found: id=%lld, try=%d, time=%ld " - "(gpid=%d.%d, server=%s), and exit", - thread_id, - round_id, - id, - try_count, - (cur_time - last_time), - info.app_id, - info.partition_index, - info.server.c_str()); - exit(-1); - } else if (value != get_value) { - dfatal("GetThread[%d]: round(%d): get mismatched: id=%lld, try=%d, time=%ld, " - "expect_value=%s, real_value=%s (gpid=%d.%d, server=%s), and exit", - thread_id, - round_id, - id, - try_count, - (cur_time - last_time), - value.c_str(), - get_value.c_str(), - info.app_id, - info.partition_index, - info.server.c_str()); - exit(-1); - } else { - dinfo("GetThread[%d]: round(%d): get succeed: id=%lld, try=%d, time=%ld " - "(gpid=%d.%d, server=%s)", - thread_id, - round_id, - id, - try_count, - (cur_time - last_time), - info.app_id, - info.partition_index, - info.server.c_str()); - stat_time[stat_count++] = cur_time - last_time; - if (stat_count == stat_batch) { - std::sort(stat_time.begin(), stat_time.end()); - long total_time = 0; - for (auto t : stat_time) - total_time += t; - ddebug("GetThread[%d]: get statistics: count=%lld, min=%lld, P90=%lld, " - "P99=%lld, P999=%lld, P9999=%lld, max=%lld, avg=%lld", - thread_id, - stat_count, - stat_time[stat_min_pos], - stat_time[stat_p90_pos], - stat_time[stat_p99_pos], - stat_time[stat_p999_pos], - stat_time[stat_p9999_pos], - stat_time[stat_max_pos], - total_time / stat_batch); - stat_count = 0; - } - } - last_time = cur_time; - try_count = 0; - id++; - } else { - derror("GetThread[%d]: round(%d): get failed: id=%lld, try=%d, ret=%d, error=%s " - "(gpid=%d.%d, server=%s)", - thread_id, - round_id, - id, - try_count, - ret, - client->get_error_string(ret), - info.app_id, - info.partition_index, - info.server.c_str()); - try_count++; - if (try_count > 3) { - sleep(1); - } - } - } - ddebug("GetThread[%d]: round(%d): finish get range [%u,%u]", - thread_id, - round_id, - start_id, - end_id); -} - -void do_check(int thread_count) -{ - int round_id = 1; - while (true) { - long long range_end = get_min_thread_setting_id() - 1; - if (range_end < thread_count) { - sleep(1); - continue; - } - ddebug("CheckThread: round(%d): start check round, range_end=%lld", round_id, range_end); - long start_time = get_time(); - std::vector worker_threads; - long long piece_count = range_end / thread_count; - for (int i = 0; i < thread_count; ++i) { - long long start_id = piece_count * i; - long long end_id = (i == thread_count - 1) ? range_end : (piece_count * (i + 1) - 1); - worker_threads.emplace_back(do_get_range, i, round_id, start_id, end_id); - } - for (auto &t : worker_threads) { - t.join(); - } - long finish_time = get_time(); - ddebug("CheckThread: round(%d): finish check round, range_end=%lld, total_time=%ld seconds", - round_id, - range_end, - (finish_time - start_time) / 1000000); - - // update check_max - while (true) { - char buf[1024]; - sprintf(buf, "%lld", range_end); - int ret = client->set(check_max_key, "", buf, set_and_get_timeout_milliseconds); - if (ret == PERR_OK) { - ddebug("CheckThread: round(%d): update \"%s\" succeed: check_max=%lld", - round_id, - check_max_key, - range_end); - break; - } else { - derror("CheckThread: round(%d): update \"%s\" failed: check_max=%lld, ret=%d, " - "error=%s", - round_id, - check_max_key, - range_end, - ret, - client->get_error_string(ret)); - } - } - - round_id++; - } -} - -void do_mark() -{ - char buf[1024]; - long last_time = get_time(); - long long old_id = 0; - std::string value; - while (true) { - sleep(1); - long long new_id = get_min_thread_setting_id(); - dassert(new_id >= old_id, "%" PRId64 " VS %" PRId64 "", new_id, old_id); - if (new_id == old_id) { - continue; - } - sprintf(buf, "%lld", new_id); - value.assign(buf); - int ret = client->set(set_next_key, "", value, set_and_get_timeout_milliseconds); - if (ret == PERR_OK) { - long cur_time = get_time(); - ddebug("MarkThread: update \"%s\" succeed: set_next=%lld, time=%ld", - set_next_key, - new_id, - (cur_time - last_time)); - old_id = new_id; - } else { - derror("MarkThread: update \"%s\" failed: set_next=%lld, ret=%d, error=%s", - set_next_key, - new_id, - ret, - client->get_error_string(ret)); - } - } -} - -void verifier_initialize(const char *config_file) -{ - const char *section = "pegasus.upgradetest"; - if (!pegasus_client_factory::initialize(config_file)) { - exit(-1); - } - - app_name = dsn_config_get_value_string( - section, "verify_app_name", "temp", "verify app name"); // default using temp - pegasus_cluster_name = - dsn_config_get_value_string(section, "pegasus_cluster_name", "", "pegasus cluster name"); - if (pegasus_cluster_name.empty()) { - derror("Should config the cluster name for verifier"); - exit(-1); - } - client = pegasus_client_factory::get_client(pegasus_cluster_name.c_str(), app_name.c_str()); - if (client == nullptr) { - derror("Initialize the _client failed"); - exit(-1); - } - - set_and_get_timeout_milliseconds = (uint32_t)dsn_config_get_value_uint64( - section, "set_and_get_timeout_milliseconds", 3000, "set and get timeout milliseconds"); - set_thread_count = - (uint32_t)dsn_config_get_value_uint64(section, "set_thread_count", 5, "set thread count"); - get_thread_count = (uint32_t)dsn_config_get_value_uint64( - section, "get_thread_count", set_thread_count * 4, "get thread count"); -} - -void verifier_start() -{ - // check the set_next - while (true) { - std::string set_next_value; - int ret = client->get(set_next_key, "", set_next_value, set_and_get_timeout_milliseconds); - if (ret == PERR_OK) { - long long i = atoll(set_next_value.c_str()); - if (i == 0 && !set_next_value.empty()) { - derror("MainThread: read \"%s\" failed: value_str=%s", - set_next_key, - set_next_value.c_str()); - exit(-1); - } - ddebug("MainThread: read \"%s\" succeed: value=%lld", set_next_key, i); - set_next.store(i); - break; - } else if (ret == PERR_NOT_FOUND) { - ddebug("MainThread: read \"%s\" not found, init set_next to 0", set_next_key); - set_next.store(0); - break; - } else { - derror("MainThread: read \"%s\" failed: error=%s", - set_next_key, - client->get_error_string(ret)); - } - } - set_thread_setting_id.resize(set_thread_count); - - std::vector set_threads; - for (int i = 0; i < set_thread_count; ++i) { - set_threads.emplace_back(do_set, i); - } - std::thread mark_thread(do_mark); - - // start several threads to read data from pegasus cluster and check data correctness, - // block until the check failed - do_check(get_thread_count); - - mark_thread.join(); - for (auto &t : set_threads) { - t.join(); - } -} diff --git a/src/test/upgrade_test/data_verifier.h b/src/test/upgrade_test/data_verifier.h deleted file mode 100644 index 7ec482d0bd..0000000000 --- a/src/test/upgrade_test/data_verifier.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -void verifier_initialize(const char *config_file); -void verifier_start(); diff --git a/src/test/upgrade_test/job.cpp b/src/test/upgrade_test/job.cpp deleted file mode 100644 index 27f3343867..0000000000 --- a/src/test/upgrade_test/job.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "job.h" - -namespace pegasus { -namespace test { - -void job::append_addrs(const std::string &addr) -{ - addrs.emplace_back(addr); - status.insert(make_pair(addr, true)); -} - -std::string job::get_addr_by_index(int index) -{ - if (index < addrs.size()) - return addrs[index]; - return std::string(); -} - -void job::set_name(const std::string &_name) { name = _name; } -} -} // end namespace diff --git a/src/test/upgrade_test/job.h b/src/test/upgrade_test/job.h deleted file mode 100644 index e0cfa9b57c..0000000000 --- a/src/test/upgrade_test/job.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include -#include - -namespace pegasus { -namespace test { - -enum job_type -{ - META = 0, - REPLICA = 1, - ZOOKEEPER = 2, - JOB_LENGTH = 3 -}; - -inline const char *job_type_str(enum job_type type) -{ - switch (type) { - case META: - return "meta"; - case REPLICA: - return "replica"; - case ZOOKEEPER: - return "zookeeper"; - default: - return "invalid"; - } -} - -struct job -{ - std::vector addrs; - std::unordered_map status; - std::string name; - - job() {} - job(const std::string &_name) : name(_name) {} - - void append_addrs(const std::string &addr); - std::string get_addr_by_index(int index); - void set_name(const std::string &_name); -}; -} -} // end namespace diff --git a/src/test/upgrade_test/main.cpp b/src/test/upgrade_test/main.cpp deleted file mode 100644 index c2b7787a9c..0000000000 --- a/src/test/upgrade_test/main.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include - -#include "data_verifier.h" -#include "process_upgrader.h" - -int main(int argc, const char **argv) -{ - if (argc < 3) { - printf( - "invalid arguments: pegasus_upgrade_test configfile worker_type(verifier|upgrader)\n"); - return -1; - } else if (strcmp(argv[2], "verifier") == 0) { - verifier_initialize(argv[1]); - verifier_start(); - } else if (strcmp(argv[2], "upgrader") == 0) { - upgrader_initialize(argv[1]); - upgrader_start(); - } else { - printf("invalid worker_type: %s\n", argv[2]); - return -1; - } - - return 0; -} diff --git a/src/test/upgrade_test/process_upgrader.cpp b/src/test/upgrade_test/process_upgrader.cpp deleted file mode 100644 index 4c84518a42..0000000000 --- a/src/test/upgrade_test/process_upgrader.cpp +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dsn/dist/fmt_logging.h" -#include "dsn/dist/replication/replication_ddl_client.h" - -#include - -#include "base/pegasus_const.h" -#include "upgrader_registry.h" -#include "upgrade_testor.h" -#include "process_upgrader.h" - -using namespace std; -using namespace ::pegasus; -using namespace ::pegasus::test; -using ::dsn::partition_configuration; -using ::dsn::replication::replication_ddl_client; - -static shared_ptr ddl_client; -static string app_name; -static string pegasus_cluster_name; -static vector meta_list; -static shared_ptr upgradetestor; - -static int upgrade_interval_seconds = 30; -static uint32_t max_seconds_for_partitions_recover = 600; - -dsn::error_code -get_partition_info(bool debug_unhealthy, int &healthy_partition_cnt, int &unhealthy_partition_cnt) -{ - healthy_partition_cnt = 0, unhealthy_partition_cnt = 0; - int32_t app_id; - int32_t partition_count; - std::vector partitions; - dsn::error_code err = ddl_client->list_app(app_name, app_id, partition_count, partitions); - if (err == ::dsn::ERR_OK) { - dinfo("access meta and query partition status success"); - for (int i = 0; i < partitions.size(); i++) { - const dsn::partition_configuration &p = partitions[i]; - int replica_count = 0; - if (!p.primary.is_invalid()) { - replica_count++; - } - replica_count += p.secondaries.size(); - if (replica_count == p.max_replica_count) { - healthy_partition_cnt++; - } else { - std::stringstream info; - info << "gpid=" << p.pid.get_app_id() << "." << p.pid.get_partition_index() << ", "; - info << "primay=" << p.primary.to_std_string() << ", "; - info << "secondaries=["; - for (int idx = 0; idx < p.secondaries.size(); idx++) { - if (idx != 0) - info << "," << p.secondaries[idx].to_std_string(); - else - info << p.secondaries[idx].to_std_string(); - } - info << "], "; - info << "last_committed_decree=" << p.last_committed_decree; - if (debug_unhealthy) { - ddebug("found unhealthy partition, %s", info.str().c_str()); - } else { - dinfo("found unhealthy partition, %s", info.str().c_str()); - } - } - } - unhealthy_partition_cnt = partition_count - healthy_partition_cnt; - } else { - dinfo("access meta and query partition status fail"); - healthy_partition_cnt = 0; - unhealthy_partition_cnt = 0; - } - return err; -} - -// false == partition unhealth, true == health -bool check_cluster_status() -{ - int healthy_partition_cnt = 0; - int unhealthy_partition_cnt = 0; - int try_count = 1; - while (try_count <= max_seconds_for_partitions_recover) { - dsn::error_code err = get_partition_info(try_count == max_seconds_for_partitions_recover, - healthy_partition_cnt, - unhealthy_partition_cnt); - if (err == dsn::ERR_OK) { - if (unhealthy_partition_cnt > 0) { - dinfo("query partition status success, but still have unhealthy partition, " - "healthy_partition_count = %d, unhealthy_partition_count = %d", - healthy_partition_cnt, - unhealthy_partition_cnt); - sleep(1); - } else - return true; - } else { - ddebug("query partition status fail, try times = %d", try_count); - sleep(1); - } - try_count += 1; - } - - return false; -} - -void upgrader_initialize(const char *config_file) -{ - register_upgrade_handlers(); - const char *section = "pegasus.upgradetest"; - // initialize the _client. - if (!pegasus_client_factory::initialize(config_file)) { - exit(-1); - } - - app_name = dsn_config_get_value_string( - section, "verify_app_name", "temp", "verify app name"); // default using temp - pegasus_cluster_name = - dsn_config_get_value_string(section, "pegasus_cluster_name", "", "pegasus cluster name"); - if (pegasus_cluster_name.empty()) { - derror("Should config the cluster name for upgradetest"); - exit(-1); - } - - // load meta_list - meta_list.clear(); - dsn::replication::replica_helper::load_meta_servers( - meta_list, PEGASUS_CLUSTER_SECTION_NAME.c_str(), pegasus_cluster_name.c_str()); - if (meta_list.empty()) { - derror("Should config the meta address for upgrader"); - exit(-1); - } - - ddl_client.reset(new replication_ddl_client(meta_list)); - if (ddl_client == nullptr) { - derror("Initialize the _ddl_client failed"); - exit(-1); - } - - upgrade_interval_seconds = - (uint32_t)dsn_config_get_value_uint64(section, "upgrade_interval_seconds", 30, ""); - max_seconds_for_partitions_recover = (uint32_t)dsn_config_get_value_uint64( - section, "max_seconds_for_all_partitions_to_recover", 600, ""); - - upgradetestor.reset(new upgrade_testor()); - if (upgradetestor == nullptr) { - derror("upgradetestor initialize fail"); - exit(-1); - } -} - -bool verifier_process_alive() -{ - const char *command = "ps aux | grep pegasus | grep verifier | wc -l"; - std::stringstream output; - int process_count; - - dcheck_eq(dsn::utils::pipe_execute(command, output), 0); - output >> process_count; - - // one for the verifier, one for command - return process_count > 1; -} - -void upgrader_start() -{ - ddebug("begin the upgrade-thread"); - while (true) { - if (!check_cluster_status()) { - upgrade_testor::stop_verifier_and_exit("check_cluster_status() fail, and exit"); - } - if (!verifier_process_alive()) { - upgrade_testor::stop_verifier_and_exit("the verifier process is dead"); - } - upgradetestor->run(); - ddebug("sleep %d seconds before checking", upgrade_interval_seconds); - sleep(upgrade_interval_seconds); - } -} diff --git a/src/test/upgrade_test/process_upgrader.h b/src/test/upgrade_test/process_upgrader.h deleted file mode 100644 index a49cbf7705..0000000000 --- a/src/test/upgrade_test/process_upgrader.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -void upgrader_initialize(const char *config_file); -void upgrader_start(); diff --git a/src/test/upgrade_test/upgrade_testor.cpp b/src/test/upgrade_test/upgrade_testor.cpp deleted file mode 100644 index 335929b932..0000000000 --- a/src/test/upgrade_test/upgrade_testor.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include -#include - -#include "upgrade_testor.h" - -namespace pegasus { -namespace test { - -// 在[a,b]之间生成cnt个不同的随机数到res -void upgrade_testor::generate_random(std::vector &res, int cnt, int a, int b) -{ - res.clear(); - if (a > b) - std::swap(a, b); - cnt = std::min(cnt, b - a + 1); - std::unordered_set numbers; - int tvalue; - for (int i = 0; i < cnt; i++) { - tvalue = (rand() % (b - a + 1)) + a; - while (numbers.find(tvalue) != numbers.end()) { - tvalue = (rand() % (b - a + 1)) + a; - } - numbers.insert(tvalue); - res.emplace_back(tvalue); - } -} - -int upgrade_testor::generate_one_number(int a, int b) -{ - if (a > b) - std::swap(a, b); - return ((rand() % (b - a + 1)) + a); -} - -upgrade_testor::upgrade_testor() -{ - const char *section = "pegasus.upgradetest"; - upgrade_round = 0; - - // initialize upgrader_handler - std::string upgrader_name = - dsn_config_get_value_string(section, "upgrade_handler", "", "upgrade handler"); - dassert(upgrader_name.size() > 0, ""); - _upgrader_handler.reset(upgrader_handler::new_handler(upgrader_name.c_str())); - dassert(_upgrader_handler.get() != nullptr, "invalid upgrader_name(%s)", upgrader_name.c_str()); - - _job_types = {META, REPLICA, ZOOKEEPER}; - _job_index_to_upgrade.resize(JOB_LENGTH); - _sleep_time_before_recover_seconds = (uint32_t)dsn_config_get_value_uint64( - section, "sleep_time_before_recover_seconds", 30, "sleep time before recover seconds"); - - _total_meta_count = - (int32_t)dsn_config_get_value_uint64(section, "total_meta_count", 0, "total meta count"); - _total_replica_count = (int32_t)dsn_config_get_value_uint64( - section, "total_replica_count", 0, "total replica count"); - _total_zookeeper_count = (int32_t)dsn_config_get_value_uint64( - section, "total_zookeeper_count", 0, "total zookeeper count"); - - if (_total_meta_count == 0 && _total_replica_count == 0 && _total_zookeeper_count == 0) { - dassert(false, "total number of meta/replica/zookeeper is 0"); - } - - _upgrade_replica_max_count = (int32_t)dsn_config_get_value_uint64( - section, "upgrade_replica_max_count", _total_replica_count, "replica upgradeed max count"); - _upgrade_meta_max_count = (int32_t)dsn_config_get_value_uint64( - section, "upgrade_meta_max_count", _total_meta_count, "meta upgradeed max count"); - _upgrade_zk_max_count = (int32_t)dsn_config_get_value_uint64(section, - "upgrade_zookeeper_max_count", - _total_zookeeper_count, - "zookeeper upgradeed max count"); - srand((unsigned)time(nullptr)); -} - -upgrade_testor::~upgrade_testor() {} - -void upgrade_testor::stop_verifier_and_exit(const char *msg) -{ - system("ps aux | grep verifier | grep -v grep| awk '{print $2}' | xargs kill -9"); - dassert(false, "%s", msg); -} - -bool upgrade_testor::check_coredump() -{ - bool has_core = false; - - // make sure all generated core are logged - for (int i = 1; i <= _total_meta_count; ++i) { - if (_upgrader_handler->has_meta_dumped_core(i)) { - derror("meta server %d generate core dump", i); - has_core = true; - } - } - - for (int i = 1; i <= _total_replica_count; ++i) { - if (_upgrader_handler->has_replica_dumped_core(i)) { - derror("replica server %d generate core dump", i); - has_core = true; - } - } - - return has_core; -} - -void upgrade_testor::run() -{ - if (check_coredump()) { - stop_verifier_and_exit("detect core dump in pegasus cluster"); - } - - if (upgrade_round == 0) { - ddebug("Number of meta-server: %d", _total_meta_count); - ddebug("Number of replica-server: %d", _total_replica_count); - ddebug("Number of zookeeper: %d", _total_zookeeper_count); - } - upgrade_round += 1; - int meta_cnt = 0; - int replica_cnt = 0; - int zk_cnt = 0; - while (replica_cnt == 0) { - replica_cnt = generate_one_number(1, _upgrade_replica_max_count); - } - ddebug("************************"); - ddebug("Round [%d]", upgrade_round); - ddebug("start upgrade..."); - ddebug( - "upgrade meta number=%d, replica number=%d, zk number=%d", meta_cnt, replica_cnt, zk_cnt); - - if (!upgrade(replica_cnt)) { - stop_verifier_and_exit("upgrade jobs failed"); - } - - auto sleep_time_random_seconds = generate_one_number(1, _sleep_time_before_recover_seconds); - ddebug("sleep %d seconds before downgrade", sleep_time_random_seconds); - sleep(sleep_time_random_seconds); - - ddebug("start downgrade..."); - if (!downgrade()) { - stop_verifier_and_exit("downgrade jobs failed"); - } - ddebug("after downgrade..."); - ddebug("************************"); -} - -bool upgrade_testor::upgrade(int replica_cnt) -{ - std::vector upgrade_counts = {0, replica_cnt, 0}; - std::vector total_count = { - _total_meta_count, _total_replica_count, _total_zookeeper_count}; - std::vector random_idxs; - generate_random(random_idxs, 1 /*REPLICA - REPLICA + 1*/, REPLICA, REPLICA); // 生成type列表 - for (auto id : random_idxs) { - std::vector &job_index_to_upgrade = _job_index_to_upgrade[_job_types[id]]; - job_index_to_upgrade.clear(); - generate_random(job_index_to_upgrade, - upgrade_counts[id], - 1, - total_count[id]); // 生成该type需要upgrade的index列表 - for (auto index : job_index_to_upgrade) { - ddebug("start to upgrade %s@%d", job_type_str(_job_types[id]), index); - if (!upgrade_job_by_index(_job_types[id], index)) { - ddebug("upgrade %s@%d failed", job_type_str(_job_types[id]), index); - return false; - } - ddebug("upgrade %s@%d succeed", job_type_str(_job_types[id]), index); - } - } - return true; -} - -bool upgrade_testor::downgrade() -{ - std::vector random_idxs; - generate_random(random_idxs, JOB_LENGTH, META, ZOOKEEPER); - for (auto id : random_idxs) { - std::vector &job_index_to_upgrade = _job_index_to_upgrade[_job_types[id]]; - for (auto index : job_index_to_upgrade) { - ddebug("start to downgrade %s@%d", job_type_str(_job_types[id]), index); - if (!downgrade_job_by_index(_job_types[id], index)) { - ddebug("downgrade %s@%d failed", job_type_str(_job_types[id]), index); - return false; - } - ddebug("downgrade %s@%d succeed", job_type_str(_job_types[id]), index); - } - } - return true; -} - -bool upgrade_testor::upgrade_job_by_index(job_type type, int index) -{ - if (type == META) - return _upgrader_handler->upgrade_meta(index); - if (type == REPLICA) - return _upgrader_handler->upgrade_replica(index); - if (type == ZOOKEEPER) - return _upgrader_handler->upgrade_zookeeper(index); - return false; -} - -bool upgrade_testor::downgrade_job_by_index(job_type type, int index) -{ - if (type == META) - return _upgrader_handler->downgrade_meta(index); - if (type == REPLICA) - return _upgrader_handler->downgrade_replica(index); - if (type == ZOOKEEPER) - return _upgrader_handler->downgrade_zookeeper(index); - return false; -} -} -} // end namespace diff --git a/src/test/upgrade_test/upgrade_testor.h b/src/test/upgrade_test/upgrade_testor.h deleted file mode 100644 index c67f8fb886..0000000000 --- a/src/test/upgrade_test/upgrade_testor.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "job.h" -#include "upgrader_handler.h" - -namespace pegasus { -namespace test { - -class upgrade_testor -{ -public: - upgrade_testor(); - ~upgrade_testor(); - - // 1. randomly generate the upgrade plan: how many replica to upgrade - // 2. execute the upgrade plan - // 3. start the upgrade job after sleep for a while - void run(); - - // upgrade replica_cnt replica-job - bool upgrade(int replica_cnt); - - // downgrade all jobs that have been upgraded - bool downgrade(); - - static void stop_verifier_and_exit(const char *msg); - -private: - bool upgrade_job_by_index(job_type type, int index); - bool downgrade_job_by_index(job_type type, int index); - - // generate cnt number belong to [a, b], - // if cnt > (b - a + 1), then just return the numbers between a ~ b - void generate_random(std::vector &res, int cnt, int a, int b); - // generate one number belong to [a, b] - int generate_one_number(int a, int b); - - bool check_coredump(); - -private: - std::shared_ptr _upgrader_handler; - uint32_t _sleep_time_before_recover_seconds; - - int32_t _total_meta_count; - int32_t _total_replica_count; - int32_t _total_zookeeper_count; - - int32_t _upgrade_replica_max_count; - int32_t _upgrade_meta_max_count; - int32_t _upgrade_zk_max_count; - - std::vector _job_types; - - int64_t upgrade_round; - - // current upgrade plan: - // _job_index_to_upgrade[i][j] indicate that index j of job type i will be upgraded - std::vector> _job_index_to_upgrade; -}; -} -} // end namespace diff --git a/src/test/upgrade_test/upgrader_handler.h b/src/test/upgrade_test/upgrader_handler.h deleted file mode 100644 index 24d5239c02..0000000000 --- a/src/test/upgrade_test/upgrader_handler.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include - -#include - -namespace pegasus { -namespace test { - -// define the interface that how to upgrade and downgrade the jobs [meta, replica, zookeeper]. -class upgrader_handler -{ -public: - template - static void register_factory(const char *name) - { - dsn::utils::factory_store::register_factory( - name, create, dsn::PROVIDER_TYPE_MAIN); - } - static upgrader_handler *new_handler(const char *name) - { - return dsn::utils::factory_store::create(name, dsn::PROVIDER_TYPE_MAIN); - } - -public: - virtual ~upgrader_handler() {} - // index begin from 1, not zero - // upgrade one - virtual bool upgrade_meta(int index) = 0; - virtual bool upgrade_replica(int index) = 0; - virtual bool upgrade_zookeeper(int index) = 0; - // downgrade one - virtual bool downgrade_meta(int index) = 0; - virtual bool downgrade_replica(int index) = 0; - virtual bool downgrade_zookeeper(int index) = 0; - // upgrade all meta/replica/zookeeper - virtual bool upgrade_all_meta(std::unordered_set &) = 0; - virtual bool upgrade_all_replica(std::unordered_set &) = 0; - virtual bool upgrade_all_zookeeper(std::unordered_set &) = 0; - // downgrade all meta/replica/zookeeper - virtual bool downgrade_all_meta(std::unordered_set &) = 0; - virtual bool downgrade_all_replica(std::unordered_set &) = 0; - virtual bool downgrade_all_zookeeper(std::unordered_set &) = 0; - - virtual bool has_meta_dumped_core(int index) { return false; } - virtual bool has_replica_dumped_core(int index) { return false; } - virtual bool has_zookeeper_dumped_core(int index) { return false; } - -private: - template - static upgrader_handler *create() - { - return new T(); - } -}; -} -} // end namespace diff --git a/src/test/upgrade_test/upgrader_handler_shell.cpp b/src/test/upgrade_test/upgrader_handler_shell.cpp deleted file mode 100644 index 2a02529c90..0000000000 --- a/src/test/upgrade_test/upgrader_handler_shell.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "upgrader_handler_shell.h" - -#include -#include -#include -#include -#include -#include - -#include "dsn/c/api_utilities.h" -#include "dsn/dist/fmt_logging.h" -#include "dsn/utility/config_api.h" - -namespace pegasus { -namespace test { - -upgrader_handler_shell::upgrader_handler_shell() -{ - const char *section = "upgrade.handler.shell"; - _run_script_path = dsn_config_get_value_string( - section, "onebox_run_path", "~/pegasus/run.sh", "onebox run path"); - dassert(_run_script_path.size() > 0, ""); - _new_version_path = dsn_config_get_value_string( - section, "new_version_path", "", "new server binary and library path"); - dassert(_new_version_path.size() > 0, ""); - _old_version_path = dsn_config_get_value_string( - section, "old_version_path", "", "old server binary and library path"); - dassert(_old_version_path.size() > 0, ""); -} - -bool upgrader_handler_shell::has_meta_dumped_core(int index) -{ - char find_core[1024]; - snprintf(find_core, - 1024, - "ls %s/onebox/meta%d | grep core | wc -l", - _run_script_path.c_str(), - index); - - std::stringstream output; - int core_count; - dcheck_eq(dsn::utils::pipe_execute(find_core, output), 0); - output >> core_count; - - return core_count != 0; -} - -bool upgrader_handler_shell::has_replica_dumped_core(int index) -{ - char find_core[1024]; - snprintf(find_core, - 1024, - "ls %s/onebox/replica%d | grep core | wc -l", - _run_script_path.c_str(), - index); - - std::stringstream output; - int core_count; - dcheck_eq(dsn::utils::pipe_execute(find_core, output), 0); - output >> core_count; - - return core_count != 0; -} - -bool upgrader_handler_shell::upgrade_meta(int index) -{ - // not implement. - return true; -} - -bool upgrader_handler_shell::upgrade_replica(int index) -{ - std::list cmds = generate_cmd(index, "replica", "upgrade"); - int try_times = 0; - do { - for (auto cmd : cmds) { - int res = system(cmd.c_str()); - ddebug("upgrade replica command: %s", cmd.c_str()); - if (res != 0 && errno != 0) { - ddebug("upgrade meta encounter error(%s)", strerror(errno)); - return false; - } - sleep(5); - } - if (check("replica", index, "upgrade")) - return true; - } while (++try_times < 3); - return false; -} - -bool upgrader_handler_shell::upgrade_zookeeper(int index) -{ - // not implement - return true; -} - -bool upgrader_handler_shell::downgrade_meta(int index) -{ - // not implement. - return true; -} - -bool upgrader_handler_shell::downgrade_replica(int index) -{ - std::list cmds = generate_cmd(index, "replica", "downgrade"); - - int try_times = 0; - do { - for (auto cmd : cmds) { - int res = system(cmd.c_str()); - ddebug("downgrade replica command: %s", cmd.c_str()); - if (res != 0 && errno != 0) { - ddebug("upgrade meta encounter error(%s)", strerror(errno)); - return false; - } - sleep(5); - } - if (check("replica", index, "downgrade")) - return true; - } while (++try_times < 3); - return false; -} - -bool upgrader_handler_shell::downgrade_zookeeper(int index) -{ - // not implement. - return true; -} - -bool upgrader_handler_shell::upgrade_all_meta(std::unordered_set &indexs) -{ - // not implement - return false; -} - -bool upgrader_handler_shell::upgrade_all_replica(std::unordered_set &indexs) -{ - // not implement. - return false; -} - -bool upgrader_handler_shell::upgrade_all_zookeeper(std::unordered_set &indexs) -{ - // not implement. - return false; -} - -bool upgrader_handler_shell::downgrade_all_meta(std::unordered_set &indexs) -{ - // not implement. - return false; -} - -bool upgrader_handler_shell::downgrade_all_replica(std::unordered_set &indexs) -{ - // not implement. - return false; -} - -bool upgrader_handler_shell::downgrade_all_zookeeper(std::unordered_set &indexs) -{ - // not implement. - return false; -} - -std::list -upgrader_handler_shell::generate_cmd(int index, const std::string &job, const std::string &action) -{ - std::list lst; - - std::stringstream res; - res << "cd " << _run_script_path << "; "; - - res << "bash run.sh"; - res << " stop_onebox_instance "; - if (job == "replica") - res << "-r " << index << "; "; - else - res << "-m " << index << "; "; - lst.push_back(res.str()); - - res.str(""); - res << "cd " << _run_script_path << "/onebox/" << job << index << "; "; - std::string version_path = ""; - if (action == "upgrade") - version_path = _new_version_path; - else - version_path = _old_version_path; - res << "ln -s -f " << version_path << "/pegasus_server; "; - res << "export LD_LIBRARY_PATH=" << version_path << ":$LD_LIBRARY_PATH; "; - res << "../replica" << index << "/pegasus_server config.ini -app_list "; - if (job == "replica") - res << "replica" - << " &>result &"; - else - res << "meta" - << " &>result &"; - lst.push_back(res.str()); - - return lst; -} - -// type = upgrade / downgrade, but not used now -bool upgrader_handler_shell::check(const std::string &job, int index, const std::string &type) -{ - std::stringstream command; - command << "ps aux | grep pegasus | grep " << job << index << " | grep -v grep | wc -l"; - std::stringstream output; - int process_count = 0; - - int check_times = 5; - do { - sleep(1); - dcheck_eq(dsn::utils::pipe_execute(command.str().c_str(), output), 0); - output >> process_count; - } while (check_times-- > 0 and process_count == 1); - - return process_count == 1; -} -} -} // end namespace diff --git a/src/test/upgrade_test/upgrader_handler_shell.h b/src/test/upgrade_test/upgrader_handler_shell.h deleted file mode 100644 index 0dc3993cae..0000000000 --- a/src/test/upgrade_test/upgrader_handler_shell.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include "upgrader_handler.h" -#include -#include - -namespace pegasus { -namespace test { - -class upgrader_handler_shell : public upgrader_handler -{ -public: - upgrader_handler_shell(); - virtual ~upgrader_handler_shell() {} - // index begin from 1, not zero - // upgrade one - virtual bool upgrade_meta(int index) override; - virtual bool upgrade_replica(int index) override; - virtual bool upgrade_zookeeper(int index) override; - // downgrade one - virtual bool downgrade_meta(int index) override; - virtual bool downgrade_replica(int index) override; - virtual bool downgrade_zookeeper(int index) override; - // upgrade all meta/replica/zookeeper - virtual bool upgrade_all_meta(std::unordered_set &) override; - virtual bool upgrade_all_replica(std::unordered_set &) override; - virtual bool upgrade_all_zookeeper(std::unordered_set &) override; - // downgrade all meta/replica/zookeeper - virtual bool downgrade_all_meta(std::unordered_set &) override; - virtual bool downgrade_all_replica(std::unordered_set &) override; - virtual bool downgrade_all_zookeeper(std::unordered_set &) override; - - virtual bool has_meta_dumped_core(int index) override; - virtual bool has_replica_dumped_core(int index) override; - -private: - // action = upgrade | downgrade. - std::list - generate_cmd(int index, const std::string &job, const std::string &action); - // check whether the command execute success. - bool check(const std::string &job, int index, const std::string &type); - -private: - // using ${_run_script_path}/run.sh to upgrade/downgrade - std::string _run_script_path; - std::string _new_version_path; - std::string _old_version_path; -}; -} -} // end namespace diff --git a/src/test/upgrade_test/upgrader_registry.cpp b/src/test/upgrade_test/upgrader_registry.cpp deleted file mode 100644 index ee33fbd203..0000000000 --- a/src/test/upgrade_test/upgrader_registry.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "upgrader_registry.h" -#include "upgrader_handler.h" -#include "upgrader_handler_shell.h" - -using namespace pegasus::test; - -void register_upgrade_handlers() -{ - upgrader_handler::register_factory("shell"); -} diff --git a/src/test/upgrade_test/upgrader_registry.h b/src/test/upgrade_test/upgrader_registry.h deleted file mode 100644 index 707f3f5210..0000000000 --- a/src/test/upgrade_test/upgrader_registry.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -void register_upgrade_handlers(); From 523a600966dad40b497c5ddd8efc17c49065d6ea Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Sun, 25 Sep 2022 21:15:20 +0800 Subject: [PATCH 2/2] rm more --- run.sh | 246 --------------------------------------------- src/CMakeLists.txt | 1 - 2 files changed, 247 deletions(-) diff --git a/run.sh b/run.sh index 1bf6710db4..67e774cc94 100755 --- a/run.sh +++ b/run.sh @@ -1372,236 +1372,6 @@ function run_clear_kill_test() rm -rf kill_history.txt *.data config-*.ini &>/dev/null } -##################### -## start_upgrade_test -##################### -function usage_start_upgrade_test() -{ - echo "Options for subcommand 'start_upgrade_test':" - echo " -h|--help print the help info" - echo " -m|--meta_count " - echo " meta server count, default is 3" - echo " -r|--replica_count " - echo " replica server count, default is 5" - echo " -a|--app_name " - echo " app name, default is temp" - echo " -p|--partition_count " - echo " app partition count, default is 16" - echo " --upgrade_type " - echo " upgrade type: meta | replica | all, default is all" - echo " -o|--old_version_path " - echo " old server binary and library path" - echo " -n|--new_version_path " - echo " new server binary and library path" - echo " -s|--sleep_time " - echo " max sleep time before next update, default is 10" - echo " actual sleep time will be a random value in range of [1, sleep_time]" - echo " -w|--worker_count " - echo " worker count for concurrently setting value, default is 10" -} - -function run_start_upgrade_test() -{ - META_COUNT=3 - REPLICA_COUNT=5 - APP_NAME=temp - PARTITION_COUNT=16 - UPGRADE_TYPE=replica - OLD_VERSION_PATH= - NEW_VERSION_PATH= - SLEEP_TIME=10 - THREAD_COUNT=10 - while [[ $# > 0 ]]; do - key="$1" - case $key in - -h|--help) - usage_start_upgrade_test - exit 0 - ;; - -m|--meta_count) - META_COUNT="$2" - shift - ;; - -r|--replica_count) - REPLICA_COUNT="$2" - shift - ;; - -a|--app_name) - APP_NAME="$2" - shift - ;; - -p|--partition_count) - PARTITION_COUNT="$2" - shift - ;; - -t|--upgrade_type) - UPGRADE_TYPE="$2" - shift - ;; - -o|--old_version_path) - OLD_VERSION_PATH="${ROOT}/$2" - shift - ;; - -n|--new_version_path) - NEW_VERSION_PATH="${ROOT}/$2" - shift - ;; - -s|--sleep_time) - SLEEP_TIME="$2" - shift - ;; - -w|--worker_count) - THREAD_COUNT="$2" - shift - ;; - *) - echo "ERROR: unknown option \"$key\"" - echo - usage_start_upgrade_test - exit 1 - ;; - esac - shift - done - - run_start_onebox -m $META_COUNT -r $REPLICA_COUNT -a $APP_NAME -p $PARTITION_COUNT -s $OLD_VERSION_PATH - echo - - cd $ROOT - CONFIG=config-upgrade-test.ini - - sed "s/@LOCAL_HOSTNAME@/${LOCAL_HOSTNAME}/g;\ -s/@META_COUNT@/${META_COUNT}/g;\ -s/@REPLICA_COUNT@/${REPLICA_COUNT}/g;\ -s/@ZK_COUNT@/1/g;s/@APP_NAME@/${APP_NAME}/g;\ -s/@SET_THREAD_COUNT@/${THREAD_COUNT}/g;\ -s/@GET_THREAD_COUNT@/${THREAD_COUNT}/g;\ -s+@ONEBOX_RUN_PATH@+`pwd`+g; \ -s+@OLD_VERSION_PATH@+${OLD_VERSION_PATH}+g;\ -s+@NEW_VERSION_PATH@+${NEW_VERSION_PATH}+g " ${ROOT}/src/test/upgrade_test/config.ini >$CONFIG - - # start verifier - mkdir -p onebox/verifier && cd onebox/verifier - ln -s -f ${DSN_ROOT}/bin/pegasus_upgrade_test/pegasus_upgrade_test - ln -s -f ${ROOT}/$CONFIG config.ini - echo "./pegasus_upgrade_test config.ini verifier &>/dev/null &" - ./pegasus_upgrade_test config.ini verifier &>/dev/null & - sleep 0.2 - echo - cd ${ROOT} - - #start upgrader - mkdir -p onebox/upgrader && cd onebox/upgrader - ln -s -f ${DSN_ROOT}/bin/pegasus_upgrade_test/pegasus_upgrade_test - ln -s -f ${ROOT}/$CONFIG config.ini - echo "./pegasus_upgrade_test config.ini upgrader &>/dev/null &" - ./pegasus_upgrade_test config.ini upgrader &>/dev/null & - sleep 0.2 - echo - cd ${ROOT} - - run_list_upgrade_test -} - -##################### -## stop_upgrade_test -##################### -function usage_stop_upgrade_test() -{ - echo "Options for subcommand 'stop_upgrade_test':" - echo " -h|--help print the help info" -} - -function run_stop_upgrade_test() -{ - while [[ $# > 0 ]]; do - key="$1" - case $key in - -h|--help) - usage_stop_upgrade_test - exit 0 - ;; - *) - echo "ERROR: unknown option \"$key\"" - echo - usage_stop_upgrade_test - exit 1 - ;; - esac - shift - done - - ps -ef | grep ' \./pegasus_upgrade_test ' | awk '{print $2}' | xargs kill &>/dev/null || true - run_stop_onebox -} - -##################### -## list_upgrade_test -##################### -function usage_list_upgrade_test() -{ - echo "Options for subcommand 'list_upgrade_test':" - echo " -h|--help print the help info" -} - -function run_list_upgrade_test() -{ - while [[ $# > 0 ]]; do - key="$1" - case $key in - -h|--help) - usage_list_upgrade_test - exit 0 - ;; - *) - echo "ERROR: unknown option \"$key\"" - echo - usage_list_upgrade_test - exit 1 - ;; - esac - shift - done - echo "------------------------------" - run_list_onebox - ps -ef | grep ' \./pegasus_upgrade_test ' | grep -v grep - echo "------------------------------" - echo "Server dir: ./onebox" - echo "------------------------------" -} - -##################### -## clear_upgrade_test -##################### -function usage_clear_upgrade_test() -{ - echo "Options for subcommand 'clear_upgrade_test':" - echo " -h|--help print the help info" -} - -function run_clear_upgrade_test() -{ - while [[ $# > 0 ]]; do - key="$1" - case $key in - -h|--help) - usage_clear_upgrade_test - exit 0 - ;; - *) - echo "ERROR: unknown option \"$key\"" - echo - usage_clear_upgrade_test - exit 1 - ;; - esac - shift - done - run_stop_upgrade_test - run_clear_onebox - rm -rf upgrade_history.txt *.data config-*.ini &>/dev/null -} - ##################### ## bench ##################### @@ -2103,22 +1873,6 @@ case $cmd in shift run_clear_kill_test $* ;; - start_upgrade_test) - shift - run_start_upgrade_test $* - ;; - stop_upgrade_test) - shift - run_stop_upgrade_test $* - ;; - list_upgrade_test) - shift - run_list_upgrade_test $* - ;; - clear_upgrade_test) - shift - run_clear_upgrade_test $* - ;; bench) shift run_bench $* diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f2aa6bd8b2..499ae93682 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -47,6 +47,5 @@ add_subdirectory(geo) add_subdirectory(redis_protocol) add_subdirectory(test/function_test) add_subdirectory(test/kill_test) -add_subdirectory(test/upgrade_test) add_subdirectory(test/pressure_test) add_subdirectory(test/bench_test)