From fc3e60f17fdf64b05596d6edd5d18589c838c552 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 17 Dec 2024 15:35:17 +0000 Subject: [PATCH] test: Fix integration tests (#1788) Fixes #1784 --- src/util/newconfig/ConfigDefinition.hpp | 32 ++++- .../integration/data/BackendFactoryTests.cpp | 126 ++++++++---------- .../data/cassandra/BackendTests.cpp | 33 ++++- 3 files changed, 116 insertions(+), 75 deletions(-) diff --git a/src/util/newconfig/ConfigDefinition.hpp b/src/util/newconfig/ConfigDefinition.hpp index a95399d53..cd20921e3 100644 --- a/src/util/newconfig/ConfigDefinition.hpp +++ b/src/util/newconfig/ConfigDefinition.hpp @@ -312,25 +312,33 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{ {"database.cassandra.username", ConfigValue{ConfigType::String}.optional()}, {"database.cassandra.password", ConfigValue{ConfigType::String}.optional()}, {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, + {"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateIP)}}, {"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}}, {"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}}, + {"forwarding.cache_timeout", ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)}, {"forwarding.request_timeout", ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)}, + {"rpc.cache_timeout", ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)}, + {"num_markers", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateNumMarkers)}, - {"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}}}, + + {"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}.optional()}}, {"dos_guard.max_fetches", ConfigValue{ConfigType::Integer}.defaultValue(1000'000u).withConstraint(validateUint32)}, {"dos_guard.max_connections", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)}, {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)}, {"dos_guard.sweep_interval", ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(validatePositiveDouble)}, + {"workers", ConfigValue{ConfigType::Integer}.defaultValue(std::thread::hardware_concurrency()).withConstraint(validateUint32) }, + {"server.ip", ConfigValue{ConfigType::String}.withConstraint(validateIP)}, {"server.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)}, {"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint32)}, @@ -341,11 +349,16 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{ {"server.parallel_requests_limit", ConfigValue{ConfigType::Integer}.optional()}, {"server.ws_max_sending_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(1500)}, {"server.__ng_web_server", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, {"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, + {"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateIOThreads)}, + {"subscription_workers", ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint32)}, + {"graceful_period", ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)}, + {"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32).withConstraint(validateUint16)}, {"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48).withConstraint(validateUint16)}, {"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)}, @@ -353,28 +366,45 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{ }, {"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(validateUint16)}, {"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(validateLoadMode)}, + {"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateChannelName)}}, {"log_channels.[].log_level", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateLogLevelName)}}, + {"log_level", ConfigValue{ConfigType::String}.defaultValue("info").withConstraint(validateLogLevelName)}, + {"log_format", ConfigValue{ConfigType::String}.defaultValue( R"(%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%)" )}, + {"log_to_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"log_directory", ConfigValue{ConfigType::String}.optional()}, + {"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(validateUint32)}, + {"log_directory_max_size", ConfigValue{ConfigType::Integer}.defaultValue(50 * 1024).withConstraint(validateUint32) }, + {"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(validateUint32)}, + {"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("none").withConstraint(validateLogTag)}, + {"extractor_threads", ConfigValue{ConfigType::Integer}.defaultValue(1u).withConstraint(validateUint32)}, + {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)}, + {"start_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, + {"finish_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, + {"ssl_cert_file", ConfigValue{ConfigType::String}.optional()}, + {"ssl_key_file", ConfigValue{ConfigType::String}.optional()}, + {"api_version.default", ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_DEFAULT).withConstraint(validateApiVersion)}, {"api_version.min", diff --git a/tests/integration/data/BackendFactoryTests.cpp b/tests/integration/data/BackendFactoryTests.cpp index fc0b559dd..3b8e1347e 100644 --- a/tests/integration/data/BackendFactoryTests.cpp +++ b/tests/integration/data/BackendFactoryTests.cpp @@ -21,50 +21,71 @@ #include "data/cassandra/Handle.hpp" #include "util/AsioContextTestFixture.hpp" #include "util/MockPrometheus.hpp" +#include "util/newconfig/ConfigConstraints.hpp" #include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ConfigFileJson.hpp" #include "util/newconfig/ConfigValue.hpp" #include "util/newconfig/Types.hpp" #include +#include #include #include +#include +#include +#include #include #include +#include using namespace util::config; -namespace { -constexpr auto keyspace = "factory_test"; -} // namespace +struct BackendCassandraFactoryTest : SyncAsioContextTest, util::prometheus::WithPrometheus { + constexpr static auto keyspace = "factory_test"; -class BackendCassandraFactoryTest : public SyncAsioContextTest, public util::prometheus::WithPrometheus { -protected: - void - SetUp() override - { - SyncAsioContextTest::SetUp(); - } + ClioConfigDefinition cfg_{ + {"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")}, + {"database.cassandra.contact_points", + ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, + {"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()}, + {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)}, + {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}, + {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)}, + {"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)}, + {"database.cassandra.threads", + ConfigValue{ConfigType::Integer}.defaultValue(static_cast(std::thread::hardware_concurrency()))}, + {"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)}, + {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()}, + {"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)}, + {"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()}, + {"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()}, + {"database.cassandra.username", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.password", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, + + {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)} + }; void - TearDown() override + useConfig(std::string config) { - SyncAsioContextTest::TearDown(); + auto jsonConfig = boost::json::parse(config).as_object(); + auto const parseErrors = cfg_.parse(ConfigFileJson{jsonConfig}); + if (parseErrors) { + std::ranges::for_each(*parseErrors, [](auto const& error) { std::cout << error.error << std::endl; }); + FAIL() << "Failed to parse config"; + } } }; class BackendCassandraFactoryTestWithDB : public BackendCassandraFactoryTest { protected: - void - SetUp() override - { - BackendCassandraFactoryTest::SetUp(); - } - void TearDown() override { - BackendCassandraFactoryTest::TearDown(); // drop the keyspace for next test data::cassandra::Handle const handle{TestGlobals::instance().backendHost}; EXPECT_TRUE(handle.connect()); @@ -74,34 +95,28 @@ class BackendCassandraFactoryTestWithDB : public BackendCassandraFactoryTest { TEST_F(BackendCassandraFactoryTest, NoSuchBackend) { - ClioConfigDefinition const cfg{{"database.type", ConfigValue{ConfigType::String}.defaultValue("unknown")}}; - EXPECT_THROW(data::make_Backend(cfg), std::runtime_error); + useConfig(R"json( {"database": {"type": "unknown"}} )json"); + EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error); } TEST_F(BackendCassandraFactoryTest, CreateCassandraBackendDBDisconnect) { - ClioConfigDefinition const cfg{ - {"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")}, - {"database.cassandra.contact_points", ConfigValue{ConfigType::String}.defaultValue("127.0.0.2")}, - {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)}, - {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}, - {"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(2)} - }; - EXPECT_THROW(data::make_Backend(cfg), std::runtime_error); + useConfig(R"json( + {"database": { + "type": "cassandra", + "cassandra": { + "contact_points": "127.0.0.2" + } + }} + )json"); + + EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error); } TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend) { - ClioConfigDefinition const cfg{ - {"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")}, - {"database.cassandra.contact_points", - ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, - {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)}, - {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)} - }; - { - auto backend = data::make_Backend(cfg); + auto backend = data::make_Backend(cfg_); EXPECT_TRUE(backend); // empty db does not have ledger range @@ -115,7 +130,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend) } { - auto backend = data::make_Backend(cfg); + auto backend = data::make_Backend(cfg_); EXPECT_TRUE(backend); auto const range = backend->fetchLedgerRange(); @@ -126,38 +141,15 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend) TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithEmptyDB) { - ClioConfigDefinition const cfg{ - {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, - {"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")}, - {"database.cassandra.contact_points", - ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, - {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)}, - {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)} - }; - - EXPECT_THROW(data::make_Backend(cfg), std::runtime_error); + useConfig(R"json( {"read_only": true} )json"); + EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error); } TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithDBReady) { - ClioConfigDefinition const cfgReadOnly{ - {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, - {"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")}, - {"database.cassandra.contact_points", - ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, - {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)}, - {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)} - }; - - ClioConfigDefinition const cfgWrite{ - {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, - {"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")}, - {"database.cassandra.contact_points", - ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, - {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)}, - {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)} - }; + auto cfgReadOnly = cfg_; + ASSERT_FALSE(cfgReadOnly.parse(ConfigFileJson{boost::json::parse(R"json( {"read_only": true} )json").as_object()})); - EXPECT_TRUE(data::make_Backend(cfgWrite)); + EXPECT_TRUE(data::make_Backend(cfg_)); EXPECT_TRUE(data::make_Backend(cfgReadOnly)); } diff --git a/tests/integration/data/cassandra/BackendTests.cpp b/tests/integration/data/cassandra/BackendTests.cpp index 3047ceddb..9729e6232 100644 --- a/tests/integration/data/cassandra/BackendTests.cpp +++ b/tests/integration/data/cassandra/BackendTests.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -73,15 +74,33 @@ using namespace data::cassandra; class BackendCassandraTest : public SyncAsioContextTest, public WithPrometheus { protected: - ClioConfigDefinition cfg{ - {{"database.cassandra.contact_points", - ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, - {"database.cassandra.keyspace", - ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)}, - {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}} + ClioConfigDefinition cfg_{ + {"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")}, + {"database.cassandra.contact_points", + ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)}, + {"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()}, + {"database.cassandra.keyspace", + ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)}, + {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}, + {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)}, + {"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)}, + {"database.cassandra.threads", + ConfigValue{ConfigType::Integer}.defaultValue(static_cast(std::thread::hardware_concurrency()))}, + {"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)}, + {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()}, + {"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)}, + {"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()}, + {"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()}, + {"database.cassandra.username", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.password", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, + + {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)} }; - ObjectView obj = cfg.getObject("database.cassandra"); + ObjectView obj = cfg_.getObject("database.cassandra"); SettingsProvider settingsProvider{obj}; // recreated for each test