Skip to content

Commit

Permalink
test: Fix integration tests (#1788)
Browse files Browse the repository at this point in the history 
Fixes #1784
  • Loading branch information
@kuznetsss
kuznetsss authored Dec 17, 2024
1 parent 8dc7f16 commit fc3e60f
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 75 deletions.
32 changes: 31 additions & 1 deletion src/util/newconfig/ConfigDefinition.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,25 +312,33 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
{"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},

{"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},

{"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateIP)}},
{"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}},
{"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}},

{"forwarding.cache_timeout",
ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)},
{"forwarding.request_timeout",
ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)},

{"rpc.cache_timeout", ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)},

{"num_markers", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateNumMarkers)},
{"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}}},

{"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}.optional()}},
{"dos_guard.max_fetches", ConfigValue{ConfigType::Integer}.defaultValue(1000'000u).withConstraint(validateUint32)},
{"dos_guard.max_connections", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)},
{"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)},
{"dos_guard.sweep_interval",
ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(validatePositiveDouble)},

{"workers",
ConfigValue{ConfigType::Integer}.defaultValue(std::thread::hardware_concurrency()).withConstraint(validateUint32)
},

{"server.ip", ConfigValue{ConfigType::String}.withConstraint(validateIP)},
{"server.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)},
{"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint32)},
Expand All @@ -341,40 +349,62 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
{"server.parallel_requests_limit", ConfigValue{ConfigType::Integer}.optional()},
{"server.ws_max_sending_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(1500)},
{"server.__ng_web_server", ConfigValue{ConfigType::Boolean}.defaultValue(false)},

{"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
{"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)},

{"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateIOThreads)},

{"subscription_workers", ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint32)},

{"graceful_period", ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)},

{"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32).withConstraint(validateUint16)},
{"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48).withConstraint(validateUint16)},
{"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)},
{"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)
},
{"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(validateUint16)},
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(validateLoadMode)},

{"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateChannelName)}},
{"log_channels.[].log_level",
Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateLogLevelName)}},

{"log_level", ConfigValue{ConfigType::String}.defaultValue("info").withConstraint(validateLogLevelName)},

{"log_format",
ConfigValue{ConfigType::String}.defaultValue(
R"(%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%)"
)},

{"log_to_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)},

{"log_directory", ConfigValue{ConfigType::String}.optional()},

{"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(validateUint32)},

{"log_directory_max_size", ConfigValue{ConfigType::Integer}.defaultValue(50 * 1024).withConstraint(validateUint32)
},

{"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(validateUint32)},

{"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("none").withConstraint(validateLogTag)},

{"extractor_threads", ConfigValue{ConfigType::Integer}.defaultValue(1u).withConstraint(validateUint32)},

{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)},

{"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)},

{"start_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},

{"finish_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},

{"ssl_cert_file", ConfigValue{ConfigType::String}.optional()},

{"ssl_key_file", ConfigValue{ConfigType::String}.optional()},

{"api_version.default",
ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_DEFAULT).withConstraint(validateApiVersion)},
{"api_version.min",
Expand Down
126 changes: 59 additions & 67 deletions tests/integration/data/BackendFactoryTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,50 +21,71 @@
#include "data/cassandra/Handle.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/newconfig/ConfigConstraints.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include "util/newconfig/ConfigFileJson.hpp"
#include "util/newconfig/ConfigValue.hpp"
#include "util/newconfig/Types.hpp"

#include <TestGlobals.hpp>
#include <boost/json/parse.hpp>
#include <fmt/core.h>
#include <gtest/gtest.h>

#include <algorithm>
#include <cstdint>
#include <iostream>
#include <stdexcept>
#include <string>
#include <thread>

using namespace util::config;

namespace {
constexpr auto keyspace = "factory_test";
} // namespace
struct BackendCassandraFactoryTest : SyncAsioContextTest, util::prometheus::WithPrometheus {
constexpr static auto keyspace = "factory_test";

class BackendCassandraFactoryTest : public SyncAsioContextTest, public util::prometheus::WithPrometheus {
protected:
void
SetUp() override
{
SyncAsioContextTest::SetUp();
}
ClioConfigDefinition cfg_{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)},
{"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)},
{"database.cassandra.threads",
ConfigValue{ConfigType::Integer}.defaultValue(static_cast<uint32_t>(std::thread::hardware_concurrency()))},
{"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)},
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},

{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}
};

void
TearDown() override
useConfig(std::string config)
{
SyncAsioContextTest::TearDown();
auto jsonConfig = boost::json::parse(config).as_object();
auto const parseErrors = cfg_.parse(ConfigFileJson{jsonConfig});
if (parseErrors) {
std::ranges::for_each(*parseErrors, [](auto const& error) { std::cout << error.error << std::endl; });
FAIL() << "Failed to parse config";
}
}
};

class BackendCassandraFactoryTestWithDB : public BackendCassandraFactoryTest {
protected:
void
SetUp() override
{
BackendCassandraFactoryTest::SetUp();
}

void
TearDown() override
{
BackendCassandraFactoryTest::TearDown();
// drop the keyspace for next test
data::cassandra::Handle const handle{TestGlobals::instance().backendHost};
EXPECT_TRUE(handle.connect());
Expand All @@ -74,34 +95,28 @@ class BackendCassandraFactoryTestWithDB : public BackendCassandraFactoryTest {

TEST_F(BackendCassandraFactoryTest, NoSuchBackend)
{
ClioConfigDefinition const cfg{{"database.type", ConfigValue{ConfigType::String}.defaultValue("unknown")}};
EXPECT_THROW(data::make_Backend(cfg), std::runtime_error);
useConfig(R"json( {"database": {"type": "unknown"}} )json");
EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error);
}

TEST_F(BackendCassandraFactoryTest, CreateCassandraBackendDBDisconnect)
{
ClioConfigDefinition const cfg{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points", ConfigValue{ConfigType::String}.defaultValue("127.0.0.2")},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(2)}
};
EXPECT_THROW(data::make_Backend(cfg), std::runtime_error);
useConfig(R"json(
{"database": {
"type": "cassandra",
"cassandra": {
"contact_points": "127.0.0.2"
}
}}
)json");

EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error);
}

TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
{
ClioConfigDefinition const cfg{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};

{
auto backend = data::make_Backend(cfg);
auto backend = data::make_Backend(cfg_);
EXPECT_TRUE(backend);

// empty db does not have ledger range
Expand All @@ -115,7 +130,7 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)
}

{
auto backend = data::make_Backend(cfg);
auto backend = data::make_Backend(cfg_);
EXPECT_TRUE(backend);

auto const range = backend->fetchLedgerRange();
Expand All @@ -126,38 +141,15 @@ TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackend)

TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithEmptyDB)
{
ClioConfigDefinition const cfg{
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};

EXPECT_THROW(data::make_Backend(cfg), std::runtime_error);
useConfig(R"json( {"read_only": true} )json");
EXPECT_THROW(data::make_Backend(cfg_), std::runtime_error);
}

TEST_F(BackendCassandraFactoryTestWithDB, CreateCassandraBackendReadOnlyWithDBReady)
{
ClioConfigDefinition const cfgReadOnly{
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};

ClioConfigDefinition const cfgWrite{
{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue(keyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}
};
auto cfgReadOnly = cfg_;
ASSERT_FALSE(cfgReadOnly.parse(ConfigFileJson{boost::json::parse(R"json( {"read_only": true} )json").as_object()}));

EXPECT_TRUE(data::make_Backend(cfgWrite));
EXPECT_TRUE(data::make_Backend(cfg_));
EXPECT_TRUE(data::make_Backend(cfgReadOnly));
}
33 changes: 26 additions & 7 deletions tests/integration/data/cassandra/BackendTests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
#include <optional>
#include <random>
#include <string>
#include <thread>
#include <tuple>
#include <unordered_map>
#include <utility>
Expand All @@ -73,15 +74,33 @@ using namespace data::cassandra;

class BackendCassandraTest : public SyncAsioContextTest, public WithPrometheus {
protected:
ClioConfigDefinition cfg{
{{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.keyspace",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)}}
ClioConfigDefinition cfg_{
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
{"database.cassandra.contact_points",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
{"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.keyspace",
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)},
{"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)},
{"database.cassandra.threads",
ConfigValue{ConfigType::Integer}.defaultValue(static_cast<uint32_t>(std::thread::hardware_concurrency()))},
{"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)},
{"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()},
{"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)},
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.defaultValue(1).optional()},
{"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
{"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},

{"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}
};

ObjectView obj = cfg.getObject("database.cassandra");
ObjectView obj = cfg_.getObject("database.cassandra");
SettingsProvider settingsProvider{obj};

// recreated for each test
Expand Down

0 comments on commit fc3e60f

Please sign in to comment.