From daab5d349d98ee477881803f9dbaf4ae7edb9765 Mon Sep 17 00:00:00 2001 From: Michael Boquard Date: Fri, 23 Feb 2024 12:12:44 -0500 Subject: [PATCH 1/3] admin/debug/get_leaders_info: stream json results The current approach can lead to an oversized allocation. This is not a full solution as get_leaders needs to not be prone to reactor stalls and oversized allocs. Signed-off-by: Tyler Rockwood (cherry picked from commit 34d9ec9262ebadf7da887fcf37fce40afbce467c) --- src/v/redpanda/admin_server.cc | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/v/redpanda/admin_server.cc b/src/v/redpanda/admin_server.cc index ba6f86119b2b1..02a100357b531 100644 --- a/src/v/redpanda/admin_server.cc +++ b/src/v/redpanda/admin_server.cc @@ -4208,29 +4208,29 @@ void admin_server::register_debug_routes() { [this](std::unique_ptr) { vlog(logger.info, "Request to get leaders info"); using result_t = ss::httpd::debug_json::leader_info; - std::vector ans; - + // TODO(rockwood): get_leaders can lead to a reactor stall, fix to use + // an async version auto leaders_info = _metadata_cache.local().get_leaders(); - ans.reserve(leaders_info.size()); - for (const auto& leader_info : leaders_info) { - result_t info; - info.ns = leader_info.tp_ns.ns; - info.topic = leader_info.tp_ns.tp; - info.partition_id = leader_info.pid; - info.leader = leader_info.current_leader.has_value() - ? leader_info.current_leader.value() - : -1; - info.previous_leader = leader_info.previous_leader.has_value() - ? leader_info.previous_leader.value() - : -1; - info.last_stable_leader_term - = leader_info.last_stable_leader_term; - info.update_term = leader_info.update_term; - info.partition_revision = leader_info.partition_revision; - ans.push_back(std::move(info)); - } - - return ss::make_ready_future(ans); + return ss::make_ready_future( + ss::json::stream_range_as_array( + lw_shared_container(std::move(leaders_info)), + [](const auto& leader_info) { + result_t info; + info.ns = leader_info.tp_ns.ns; + info.topic = leader_info.tp_ns.tp; + info.partition_id = leader_info.pid; + info.leader = leader_info.current_leader.has_value() + ? leader_info.current_leader.value() + : -1; + info.previous_leader = leader_info.previous_leader.has_value() + ? leader_info.previous_leader.value() + : -1; + info.last_stable_leader_term + = leader_info.last_stable_leader_term; + info.update_term = leader_info.update_term; + info.partition_revision = leader_info.partition_revision; + return info; + })); }); register_route( From fd03a560d6f2cf951ce61cd6f87383701ade6395 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Thu, 8 Feb 2024 14:40:06 -0600 Subject: [PATCH 2/3] admin/partition: prevent oversized allocation Signed-off-by: Tyler Rockwood (cherry picked from commit 2818c0265670367186f2919ed8698492f495a290) --- src/v/redpanda/admin_server.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/v/redpanda/admin_server.cc b/src/v/redpanda/admin_server.cc index 02a100357b531..8d69d47bb0683 100644 --- a/src/v/redpanda/admin_server.cc +++ b/src/v/redpanda/admin_server.cc @@ -3330,9 +3330,8 @@ admin_server::get_topic_partitions_handler( fmt::format("Could not find topic: {}/{}", tp_ns.ns, tp_ns.tp)); } using partition_t = ss::httpd::partition_json::partition; - std::vector partitions; + fragmented_vector partitions; const auto& assignments = tp_md->get().get_assignments(); - partitions.reserve(assignments.size()); // Normal topic for (const auto& p_as : assignments) { partition_t p; @@ -3364,7 +3363,8 @@ admin_server::get_topic_partitions_handler( }); }); - co_return ss::json::json_return_type(partitions); + co_return ss::json::json_return_type(ss::json::stream_range_as_array( + lw_shared_container(std::move(partitions)), [](auto& p) { return p; })); } ss::future From 2dcc64d2001d337b9a4f560a8df8a25ccfdb6d34 Mon Sep 17 00:00:00 2001 From: Tyler Rockwood Date: Thu, 8 Feb 2024 15:27:01 -0600 Subject: [PATCH 3/3] admin/transaction: prevent oversized allocs Signed-off-by: Tyler Rockwood (cherry picked from commit 3b0ef74009e59aaf7e6e139ee59390b3b06bae7b) --- src/v/redpanda/admin_server.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/v/redpanda/admin_server.cc b/src/v/redpanda/admin_server.cc index 8d69d47bb0683..ef54c2300b338 100644 --- a/src/v/redpanda/admin_server.cc +++ b/src/v/redpanda/admin_server.cc @@ -3526,8 +3526,7 @@ admin_server::get_all_transactions_handler( } using tx_info = ss::httpd::transaction_json::transaction_summary; - std::vector ans; - ans.reserve(res.value().size()); + fragmented_vector ans; for (auto& tx : res.value()) { if (tx.status == cluster::tm_transaction::tx_status::tombstone) { @@ -3576,9 +3575,12 @@ admin_server::get_all_transactions_handler( } ans.push_back(std::move(new_tx)); + co_await ss::coroutine::maybe_yield(); } - co_return ss::json::json_return_type(ans); + co_return ss::json::json_return_type(ss::json::stream_range_as_array( + lw_shared_container(std::move(ans)), + [](auto& tx_info) { return tx_info; })); } ss::future