Cherry-pick XRPFees bugfix

CMakeLists.txt

@@ -267,6 +267,7 @@ if (tests)
     unittests/rpc/handlers/AMMInfoTests.cpp
     # Backend
     unittests/data/BackendFactoryTests.cpp
+    unittests/data/BackendInterfaceTests.cpp
     unittests/data/BackendCountersTests.cpp
     unittests/data/cassandra/BaseTests.cpp
     unittests/data/cassandra/BackendTests.cpp

conanfile.py

@@ -26,7 +26,7 @@ class Clio(ConanFile):
         'protobuf/3.21.12',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.0.0',
+        'xrpl/2.1.0',
         'libbacktrace/cci.20210118'
     ]
 

src/data/BackendInterface.cpp

@@ -344,14 +344,42 @@ BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context
     ripple::SerialIter it(bytes->data(), bytes->size());
     ripple::SLE const sle{it, key};
 
-    if (sle.getFieldIndex(ripple::sfBaseFee) != -1)
-        fees.base = sle.getFieldU64(ripple::sfBaseFee);
+    // XRPFees amendment introduced new fields for fees calculations.
+    // New fields are set and the old fields are removed via `set_fees` tx.
+    // Fallback to old fields if `set_fees` was not yet used to update the fields on this tx.
+    auto hasNewFields = false;
+    {
+        auto const baseFeeXRP = sle.at(~ripple::sfBaseFeeDrops);
+        auto const reserveBaseXRP = sle.at(~ripple::sfReserveBaseDrops);
+        auto const reserveIncrementXRP = sle.at(~ripple::sfReserveIncrementDrops);
 
-    if (sle.getFieldIndex(ripple::sfReserveBase) != -1)
-        fees.reserve = sle.getFieldU32(ripple::sfReserveBase);
+        if (baseFeeXRP)
+            fees.base = baseFeeXRP->xrp();
 
-    if (sle.getFieldIndex(ripple::sfReserveIncrement) != -1)
-        fees.increment = sle.getFieldU32(ripple::sfReserveIncrement);
+        if (reserveBaseXRP)
+            fees.reserve = reserveBaseXRP->xrp();
+
+        if (reserveIncrementXRP)
+            fees.increment = reserveIncrementXRP->xrp();
+
+        hasNewFields = baseFeeXRP || reserveBaseXRP || reserveIncrementXRP;
+    }
+
+    if (not hasNewFields) {
+        // Fallback to old fields
+        auto const baseFee = sle.at(~ripple::sfBaseFee);
+        auto const reserveBase = sle.at(~ripple::sfReserveBase);
+        auto const reserveIncrement = sle.at(~ripple::sfReserveIncrement);
+
+        if (baseFee)
+            fees.base = baseFee.value();
+
+        if (reserveBase)
+            fees.reserve = reserveBase.value();
+
+        if (reserveIncrement)
+            fees.increment = reserveIncrement.value();
+    }
 
     return fees;
 }

unittests/CMakeLists.txt

@@ -0,0 +1,177 @@
+add_executable(clio_tests)
+
+target_sources(
+  clio_tests
+  PRIVATE # Common
+          ConfigTests.cpp
+          data/BackendCountersTests.cpp
+          data/BackendFactoryTests.cpp
+          data/BackendInterfaceTests.cpp
+          data/cassandra/AsyncExecutorTests.cpp
+          # Webserver
+          data/cassandra/BackendTests.cpp
+          data/cassandra/BaseTests.cpp
+          data/cassandra/ExecutionStrategyTests.cpp
+          data/cassandra/RetryPolicyTests.cpp
+          data/cassandra/SettingsProviderTests.cpp
+          DOSGuardTests.cpp
+          etl/AmendmentBlockHandlerTests.cpp
+          etl/CacheLoaderSettingsTests.cpp
+          etl/CacheLoaderTests.cpp
+          etl/CursorFromAccountProviderTests.cpp
+          etl/CursorFromDiffProviderTests.cpp
+          etl/CursorFromFixDiffNumProviderTests.cpp
+          etl/ETLStateTests.cpp
+          etl/ExtractionDataPipeTests.cpp
+          etl/ExtractorTests.cpp
+          etl/ForwardingCacheTests.cpp
+          etl/ForwardingSourceTests.cpp
+          etl/GrpcSourceTests.cpp
+          etl/LedgerPublisherTests.cpp
+          etl/SourceTests.cpp
+          etl/SubscriptionSourceDependenciesTests.cpp
+          etl/SubscriptionSourceTests.cpp
+          etl/TransformerTests.cpp
+          # RPC
+          feed/BookChangesFeedTests.cpp
+          feed/ForwardFeedTests.cpp
+          feed/LedgerFeedTests.cpp
+          feed/ProposedTransactionFeedTests.cpp
+          feed/SingleFeedBaseTests.cpp
+          feed/SubscriptionManagerTests.cpp
+          feed/TrackableSignalTests.cpp
+          feed/TransactionFeedTests.cpp
+          JsonUtilTests.cpp
+          LoggerTests.cpp
+          Main.cpp
+          Playground.cpp
+          ProfilerTests.cpp
+          rpc/AmendmentsTests.cpp
+          rpc/APIVersionTests.cpp
+          rpc/BaseTests.cpp
+          rpc/CountersTests.cpp
+          rpc/ErrorTests.cpp
+          rpc/ForwardingProxyTests.cpp
+          rpc/handlers/AccountChannelsTests.cpp
+          rpc/handlers/AccountCurrenciesTests.cpp
+          rpc/handlers/AccountInfoTests.cpp
+          rpc/handlers/AccountLinesTests.cpp
+          rpc/handlers/AccountNFTsTests.cpp
+          rpc/handlers/AccountObjectsTests.cpp
+          rpc/handlers/AccountOffersTests.cpp
+          rpc/handlers/AccountTxTests.cpp
+          rpc/handlers/AMMInfoTests.cpp
+          # Backend
+          rpc/handlers/BookChangesTests.cpp
+          rpc/handlers/BookOffersTests.cpp
+          rpc/handlers/DefaultProcessorTests.cpp
+          rpc/handlers/DepositAuthorizedTests.cpp
+          rpc/handlers/GatewayBalancesTests.cpp
+          rpc/handlers/LedgerDataTests.cpp
+          rpc/handlers/LedgerEntryTests.cpp
+          rpc/handlers/LedgerRangeTests.cpp
+          rpc/handlers/LedgerTests.cpp
+          rpc/handlers/NFTBuyOffersTests.cpp
+          rpc/handlers/NFTHistoryTests.cpp
+          rpc/handlers/NFTInfoTests.cpp
+          rpc/handlers/NFTsByIssuerTest.cpp
+          rpc/handlers/NFTSellOffersTests.cpp
+          rpc/handlers/NoRippleCheckTests.cpp
+          rpc/handlers/PingTests.cpp
+          rpc/handlers/RandomTests.cpp
+          rpc/handlers/ServerInfoTests.cpp
+          rpc/handlers/SubscribeTests.cpp
+          rpc/handlers/TestHandlerTests.cpp
+          rpc/handlers/TransactionEntryTests.cpp
+          rpc/handlers/TxTests.cpp
+          rpc/handlers/UnsubscribeTests.cpp
+          rpc/handlers/VersionHandlerTests.cpp
+          rpc/JsonBoolTests.cpp
+          # RPC handlers
+          rpc/RPCHelpersTests.cpp
+          rpc/WorkQueueTests.cpp
+          util/AssertTests.cpp
+          util/async/AnyExecutionContextTests.cpp
+          util/async/AnyOperationTests.cpp
+          util/async/AnyStopTokenTests.cpp
+          util/async/AnyStrandTests.cpp
+          util/async/AsyncExecutionContextTests.cpp
+          # Requests framework
+          util/BatchingTests.cpp
+          util/LedgerUtilsTests.cpp
+          # Prometheus support
+          util/prometheus/BoolTests.cpp
+          util/prometheus/CounterTests.cpp
+          util/prometheus/GaugeTests.cpp
+          util/prometheus/HistogramTests.cpp
+          util/prometheus/HttpTests.cpp
+          util/prometheus/LabelTests.cpp
+          util/prometheus/MetricBuilderTests.cpp
+          util/prometheus/MetricsFamilyTests.cpp
+          util/prometheus/OStreamTests.cpp
+          util/requests/RequestBuilderTests.cpp
+          util/requests/SslContextTests.cpp
+          util/requests/WsConnectionTests.cpp
+          # ETL
+          util/RetryTests.cpp
+          # Async framework
+          util/StringUtils.cpp
+          util/TestGlobals.cpp
+          util/TestHttpServer.cpp
+          util/TestObject.cpp
+          util/TestWsServer.cpp
+          util/TxUtilTests.cpp
+          web/AdminVerificationTests.cpp
+          web/RPCServerHandlerTests.cpp
+          web/ServerTests.cpp
+          web/SweepHandlerTests.cpp
+          # Feed
+          web/WhitelistHandlerTests.cpp
+)
+
+include(deps/gtest)
+
+# See https://github.com/google/googletest/issues/3475
+gtest_discover_tests(clio_tests DISCOVERY_TIMEOUT 90)
+
+# Fix for dwarf5 bug on ci
+target_compile_options(clio_options INTERFACE -gdwarf-4)
+
+target_compile_definitions(clio_tests PUBLIC UNITTEST_BUILD)
+target_include_directories(clio_tests PRIVATE .)
+target_link_libraries(clio_tests PUBLIC clio gtest::gtest)
+set_target_properties(clio_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+
+# Generate `coverage_report` target if coverage is enabled
+if (coverage)
+  if (DEFINED CODE_COVERAGE_REPORT_FORMAT)
+    set(CODE_COVERAGE_FORMAT ${CODE_COVERAGE_REPORT_FORMAT})
+  else ()
+    set(CODE_COVERAGE_FORMAT html-details)
+  endif ()
+
+  if (DEFINED CODE_COVERAGE_TESTS_ARGS)
+    set(TESTS_ADDITIONAL_ARGS ${CODE_COVERAGE_TESTS_ARGS})
+    separate_arguments(TESTS_ADDITIONAL_ARGS)
+  else ()
+    set(TESTS_ADDITIONAL_ARGS "")
+  endif ()
+
+  set(GCOVR_ADDITIONAL_ARGS --exclude-throw-branches -s)
+
+  setup_target_for_coverage_gcovr(
+    NAME
+    coverage_report
+    FORMAT
+    ${CODE_COVERAGE_FORMAT}
+    EXECUTABLE
+    clio_tests
+    EXECUTABLE_ARGS
+    --gtest_brief=1
+    ${TESTS_ADDITIONAL_ARGS}
+    EXCLUDE
+    "unittests"
+    DEPENDENCIES
+    clio_tests
+  )
+endif ()

unittests/data/BackendInterfaceTests.cpp

@@ -0,0 +1,72 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/BackendInterface.hpp"
+#include "util/Fixtures.hpp"
+#include "util/TestObject.hpp"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+using namespace data;
+using namespace util::prometheus;
+using namespace testing;
+
+constexpr static auto MAXSEQ = 30;
+constexpr static auto MINSEQ = 10;
+
+struct BackendInterfaceTest : MockBackendTestNaggy, SyncAsioContextTest, WithPrometheus {};
+
+TEST_F(BackendInterfaceTest, FetchFeesSuccessPath)
+{
+    using namespace ripple;
+    backend->setRange(MINSEQ, MAXSEQ);
+
+    // New fee setting (after XRPFees amendment)
+    EXPECT_CALL(*backend, doFetchLedgerObject(keylet::fees().key, MAXSEQ, _))
+        .WillRepeatedly(Return(CreateFeeSettingBlob(XRPAmount(1), XRPAmount(2), XRPAmount(3), 0)));
+
+    runSpawn([this](auto yield) {
+        auto fees = backend->fetchFees(MAXSEQ, yield);
+
+        EXPECT_TRUE(fees.has_value());
+        EXPECT_EQ(fees->base, XRPAmount(1));
+        EXPECT_EQ(fees->increment, XRPAmount(2));
+        EXPECT_EQ(fees->reserve, XRPAmount(3));
+    });
+}
+
+TEST_F(BackendInterfaceTest, FetchFeesLegacySuccessPath)
+{
+    using namespace ripple;
+    backend->setRange(MINSEQ, MAXSEQ);
+
+    // Legacy fee setting (before XRPFees amendment)
+    EXPECT_CALL(*backend, doFetchLedgerObject(keylet::fees().key, MAXSEQ, _))
+        .WillRepeatedly(Return(CreateLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
+
+    runSpawn([this](auto yield) {
+        auto fees = backend->fetchFees(MAXSEQ, yield);
+
+        EXPECT_TRUE(fees.has_value());
+        EXPECT_EQ(fees->base, XRPAmount(1));
+        EXPECT_EQ(fees->increment, XRPAmount(2));
+        EXPECT_EQ(fees->reserve, XRPAmount(3));
+    });
+}

unittests/data/cassandra/BackendTests.cpp

@@ -65,11 +65,12 @@
 using namespace util;
 using namespace std;
 using namespace rpc;
+using namespace prometheus;
 namespace json = boost::json;
 
 using namespace data::cassandra;
 
-class BackendCassandraTest : public SyncAsioContextTest {
+class BackendCassandraTest : public SyncAsioContextTest, public WithPrometheus {
 protected:
     Config cfg{json::parse(fmt::format(
         R"JSON({{

unittests/etl/LedgerPublisherTests.cpp

@@ -128,7 +128,7 @@ TEST_F(ETLLedgerPublisherTest, PublishLedgerInfoInRange)
     // mock fetch fee
     EXPECT_CALL(*backend, doFetchLedgerObject).Times(1);
     ON_CALL(*backend, doFetchLedgerObject(ripple::keylet::fees().key, SEQ, _))
-        .WillByDefault(Return(CreateFeeSettingBlob(1, 2, 3, 4, 0)));
+        .WillByDefault(Return(CreateLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
 
     // mock fetch transactions
     EXPECT_CALL(*backend, fetchAllTransactionsInLedger).Times(1);
@@ -176,7 +176,7 @@ TEST_F(ETLLedgerPublisherTest, PublishLedgerInfoCloseTimeGreaterThanNow)
     // mock fetch fee
     EXPECT_CALL(*backend, doFetchLedgerObject).Times(1);
     ON_CALL(*backend, doFetchLedgerObject(ripple::keylet::fees().key, SEQ, _))
-        .WillByDefault(Return(CreateFeeSettingBlob(1, 2, 3, 4, 0)));
+        .WillByDefault(Return(CreateLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
 
     // mock fetch transactions
     EXPECT_CALL(*backend, fetchAllTransactionsInLedger).Times(1);
@@ -265,7 +265,7 @@ TEST_F(ETLLedgerPublisherTest, PublishMultipleTxInOrder)
     // mock fetch fee
     EXPECT_CALL(*backend, doFetchLedgerObject).Times(1);
     ON_CALL(*backend, doFetchLedgerObject(ripple::keylet::fees().key, SEQ, _))
-        .WillByDefault(Return(CreateFeeSettingBlob(1, 2, 3, 4, 0)));
+        .WillByDefault(Return(CreateLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
 
     // mock fetch transactions
     EXPECT_CALL(*backend, fetchAllTransactionsInLedger).Times(1);

unittests/feed/LedgerFeedTests.cpp

@@ -42,7 +42,7 @@ TEST_F(FeedLedgerTest, SubPub)
     auto const ledgerInfo = CreateLedgerInfo(LEDGERHASH, 30);
     EXPECT_CALL(*backend, fetchLedgerBySequence).WillOnce(testing::Return(ledgerInfo));
 
-    auto const feeBlob = CreateFeeSettingBlob(1, 2, 3, 4, 0);
+    auto const feeBlob = CreateLegacyFeeSettingBlob(1, 2, 3, 4, 0);
     EXPECT_CALL(*backend, doFetchLedgerObject).WillOnce(testing::Return(feeBlob));
     // check the function response
     // Information about the ledgers on hand and current fee schedule. This
@@ -103,7 +103,7 @@ TEST_F(FeedLedgerTest, AutoDisconnect)
     auto const ledgerinfo = CreateLedgerInfo(LEDGERHASH, 30);
     EXPECT_CALL(*backend, fetchLedgerBySequence).WillOnce(testing::Return(ledgerinfo));
 
-    auto const feeBlob = CreateFeeSettingBlob(1, 2, 3, 4, 0);
+    auto const feeBlob = CreateLegacyFeeSettingBlob(1, 2, 3, 4, 0);
     EXPECT_CALL(*backend, doFetchLedgerObject).WillOnce(testing::Return(feeBlob));
     constexpr static auto LedgerResponse =
         R"({

unittests/feed/SubscriptionManagerTests.cpp

@@ -280,7 +280,7 @@ TEST_F(SubscriptionManagerTest, LedgerTest)
     auto const ledgerinfo = CreateLedgerInfo(LEDGERHASH, 30);
     EXPECT_CALL(*backend, fetchLedgerBySequence).WillOnce(testing::Return(ledgerinfo));
 
-    auto const feeBlob = CreateFeeSettingBlob(1, 2, 3, 4, 0);
+    auto const feeBlob = CreateLegacyFeeSettingBlob(1, 2, 3, 4, 0);
     EXPECT_CALL(*backend, doFetchLedgerObject).WillOnce(testing::Return(feeBlob));
     // check the function response
     // Information about the ledgers on hand and current fee schedule. This