Skip to content

Commit

Permalink
test: Retry all RPC commands to fix MacOS CI jobs
Browse files Browse the repository at this point in the history 
* Follow up to XRPLF#5120 (23991c9), which added a retry for submit commands.
  It improved MacOS test reliability, but other tests are failing now.
* Retry all failed RPC connections / commands in unit tests.
  • Loading branch information
@ximinez
ximinez committed Oct 30, 2024
1 parent 0d887ad commit 08a76c8
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 22 deletions.
2 changes: 1 addition & 1 deletion include/xrpl/protocol/TxFlags.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ constexpr std::uint32_t const tfTrustLine = 0x00000004;
constexpr std::uint32_t const tfTransferable = 0x00000008;

// MPTokenIssuanceCreate flags:
// NOTE - there is intentionally no flag here for lsfMPTLocked, which this transaction cannot mutate.
// NOTE - there is intentionally no flag here for lsfMPTLocked, which this transaction cannot mutate.
constexpr std::uint32_t const tfMPTCanLock = lsfMPTCanLock;
constexpr std::uint32_t const tfMPTRequireAuth = lsfMPTRequireAuth;
constexpr std::uint32_t const tfMPTCanEscrow = lsfMPTCanEscrow;
Expand Down
41 changes: 40 additions & 1 deletion src/test/jtx/Env.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,9 @@ class Env
The command is examined and used to build
the correct JSON as per the arguments.
*/
using rpcCallback = std::function<bool(Json::Value const&)>;
const rpcCallback useDefaultCallback{};

template <class... Args>
Json::Value
rpc(unsigned apiVersion,
Expand All @@ -303,12 +306,23 @@ class Env
Json::Value
rpc(unsigned apiVersion, std::string const& cmd, Args&&... args);

template <class... Args>
Json::Value
rpc(rpcCallback cb,
std::unordered_map<std::string, std::string> const& headers,
std::string const& cmd,
Args&&... args);

template <class... Args>
Json::Value
rpc(std::unordered_map<std::string, std::string> const& headers,
std::string const& cmd,
Args&&... args);

template <class... Args>
Json::Value
rpc(rpcCallback cb, std::string const& cmd, Args&&... args);

template <class... Args>
Json::Value
rpc(std::string const& cmd, Args&&... args);
Expand Down Expand Up @@ -514,7 +528,7 @@ class Env
/** Gets the TER result and `didApply` flag from a RPC Json result object.
*/
static ParsedResult
parseResult(Json::Value const& jr);
parseResult(Json::Value const& jr, bool checkTER = true);

/** Submit an existing JTx.
This calls postconditions.
Expand Down Expand Up @@ -696,6 +710,7 @@ class Env

Json::Value
do_rpc(
rpcCallback cb,
unsigned apiVersion,
std::vector<std::string> const& args,
std::unordered_map<std::string, std::string> const& headers = {});
Expand Down Expand Up @@ -746,6 +761,7 @@ Env::rpc(
Args&&... args)
{
return do_rpc(
useDefaultCallback,
apiVersion,
std::vector<std::string>{cmd, std::forward<Args>(args)...},
headers);
Expand All @@ -765,16 +781,39 @@ Env::rpc(unsigned apiVersion, std::string const& cmd, Args&&... args)
template <class... Args>
Json::Value
Env::rpc(
rpcCallback cb,
std::unordered_map<std::string, std::string> const& headers,
std::string const& cmd,
Args&&... args)
{
return do_rpc(
cb,
RPC::apiCommandLineVersion,
std::vector<std::string>{cmd, std::forward<Args>(args)...},
headers);
}

template <class... Args>
Json::Value
Env::rpc(
std::unordered_map<std::string, std::string> const& headers,
std::string const& cmd,
Args&&... args)
{
return rpc(useDefaultCallback, headers, cmd, std::forward<Args>(args)...);
}

template <class... Args>
Json::Value
Env::rpc(rpcCallback cb, std::string const& cmd, Args&&... args)
{
return rpc(
cb,
std::unordered_map<std::string, std::string>(),
cmd,
std::forward<Args>(args)...);
}

template <class... Args>
Json::Value
Env::rpc(std::string const& cmd, Args&&... args)
Expand Down
65 changes: 45 additions & 20 deletions src/test/jtx/impl/Env.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ Env::trust(STAmount const& amount, Account const& account)
}

Env::ParsedResult
Env::parseResult(Json::Value const& jr)
Env::parseResult(Json::Value const& jr, bool checkTER)
{
auto error = [](ParsedResult& parsed, Json::Value const& object) {
// Use an error code that is not used anywhere in the transaction
Expand All @@ -296,14 +296,19 @@ Env::parseResult(Json::Value const& jr)
if (jr.isObject() && jr.isMember(jss::result))
{
auto const& result = jr[jss::result];
if (result.isMember(jss::engine_result_code))
if (checkTER && result.isMember(jss::engine_result_code))
{
parsed.ter = TER::fromInt(result[jss::engine_result_code].asInt());
parsed.rpcCode.emplace(rpcSUCCESS);
}
else if (!checkTER && !result.isMember(jss::error))
parsed.rpcCode.emplace(rpcSUCCESS);
else
error(parsed, result);
}
else if (
jr.isObject() && jr.isMember(jss::error) && jr[jss::error].isObject())
error(parsed, jr[jss::error]);
else
error(parsed, jr);

Expand All @@ -317,24 +322,20 @@ Env::submit(JTx const& jt)
auto const jr = [&]() {
if (jt.stx)
{
// We shouldn't need to retry, but it fixes the test on macOS for
// the moment.
int retries = 3;
do
{
txid_ = jt.stx->getTransactionID();
Serializer s;
jt.stx->add(s);
auto const jr = rpc("submit", strHex(s.slice()));

parsedResult = parseResult(jr);
txid_ = jt.stx->getTransactionID();
Serializer s;
jt.stx->add(s);
auto const cb = [&](Json::Value const& jr) {
parsedResult = parseResult(jr, true);
test.expect(parsedResult.ter, "ter uninitialized!");
ter_ = parsedResult.ter.value_or(telENV_RPC_FAILED);
if (ter_ != telENV_RPC_FAILED ||
return (
ter_ != telENV_RPC_FAILED ||
parsedResult.rpcCode != rpcINTERNAL ||
jt.ter == telENV_RPC_FAILED || --retries <= 0)
return jr;
} while (true);
jt.ter == telENV_RPC_FAILED);
};
// rpc() will call cb(), which does all the parsing
return rpc(cb, "submit", strHex(s.slice()));
}
else
{
Expand Down Expand Up @@ -379,7 +380,7 @@ Env::sign_and_submit(JTx const& jt, Json::Value params)
if (!txid_.parseHex(jr[jss::result][jss::tx_json][jss::hash].asString()))
txid_.zero();

ParsedResult const parsedResult = parseResult(jr);
ParsedResult const parsedResult = parseResult(jr, true);
test.expect(parsedResult.ter, "ter uninitialized!");
ter_ = parsedResult.ter.value_or(telENV_RPC_FAILED);

Expand Down Expand Up @@ -562,12 +563,36 @@ Env::ust(JTx const& jt)

Json::Value
Env::do_rpc(
rpcCallback cb,
unsigned apiVersion,
std::vector<std::string> const& args,
std::unordered_map<std::string, std::string> const& headers)
{
return rpcClient(args, app().config(), app().logs(), apiVersion, headers)
.second;
// We shouldn't need to retry, but it fixes the test on macOS for
// the moment.
if (!cb)
{
static auto const defaultCallback = [](Json::Value const& jr) {
auto const parsedResult = parseResult(jr, false);
return parsedResult.rpcCode != rpcINTERNAL;
};
cb = defaultCallback;
}
int retries = 3;
do
{
auto const ret =
rpcClient(args, app().config(), app().logs(), apiVersion, headers)
.second;
if (cb(ret) || --retries <= 0)
return ret;
std::stringstream ss;
for (auto const& arg : args)
{
ss << arg << ", ";
}
test.log << "RPC failed: " << ss.str() << " -> " << to_string(ret);
} while (true);
}

void
Expand Down

0 comments on commit 08a76c8

Please sign in to comment.