diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d3622f..188bbed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.23.0 - TBD + +### Enhancements +- Added new `Cmbp1Msg` +- Added new consolidated publisher values for `XNAS.BASIC` and `DBEQ.MAX` + +### Breaking changes +- Changed the layout of `CbboMsg` to better match `BboMsg` +- Renamed `Schema::Cbbo` to `Schema::Cmbp1` + ## 0.22.0 - 2024-08-27 ### Enhancements diff --git a/include/databento/enums.hpp b/include/databento/enums.hpp index a3150f7..3504c86 100644 --- a/include/databento/enums.hpp +++ b/include/databento/enums.hpp @@ -34,7 +34,7 @@ enum class Schema : std::uint16_t { Statistics = 10, Status = 11, Imbalance = 12, - Cbbo = 14, + Cmbp1 = 14, Cbbo1S = 15, Cbbo1M = 16, Tcbbo = 17, @@ -139,7 +139,7 @@ enum RType : std::uint8_t { System = 0x17, Statistics = 0x18, Mbo = 0xA0, - Cbbo = 0xB1, + Cmbp1 = 0xB1, Cbbo1S = 0xC0, Cbbo1M = 0xC1, Tcbbo = 0xC2, diff --git a/include/databento/publishers.hpp b/include/databento/publishers.hpp index ff4904f..7c9daf0 100644 --- a/include/databento/publishers.hpp +++ b/include/databento/publishers.hpp @@ -353,6 +353,10 @@ enum class Publisher : std::uint16_t { XcisBbotradesXcis = 91, // NYSE BBO and Trades XnysBbotradesXnys = 92, + // Nasdaq Basic - Consolidated + XnasBasicDbeq = 93, + // DBEQ Max - Consolidated + DbeqMaxDbeq = 94, }; // Get a Publisher's Venue. diff --git a/include/databento/record.hpp b/include/databento/record.hpp index f9f9232..0d4317b 100644 --- a/include/databento/record.hpp +++ b/include/databento/record.hpp @@ -254,36 +254,68 @@ using Bbo1MMsg = BboMsg; static_assert(alignof(BboMsg) == 8, "Must have 8-byte alignment"); static_assert(sizeof(BboMsg) == sizeof(Mbp1Msg), "BboMsg size must match Rust"); +struct Cmbp1Msg { + static bool HasRType(RType rtype) { + switch (rtype) { + case RType::Cmbp1: // fallthrough + case RType::Tcbbo: + return true; + default: + return false; + }; + } + + UnixNanos IndexTs() const { return ts_recv; } + + RecordHeader hd; + std::int64_t price; + std::uint32_t size; + char action; + Side side; + FlagSet flags; + char reserved1; + UnixNanos ts_recv; + TimeDeltaNanos ts_in_delta; + std::array reserved2; + std::array levels; +}; +using TcbboMsg = Cmbp1Msg; +static_assert(alignof(Cmbp1Msg) == 8, "Must have 8-byte alignment"); +static_assert(sizeof(Cmbp1Msg) == + sizeof(TradeMsg) + sizeof(ConsolidatedBidAskPair), + "Cmbp1Msg size must match Rust"); + struct CbboMsg { static bool HasRType(RType rtype) { switch (rtype) { - case RType::Cbbo: // fallthrough case RType::Cbbo1S: // fallthrough case RType::Cbbo1M: // fallthrough - case RType::Tcbbo: return true; default: return false; }; } + static_assert(alignof(Cmbp1Msg) == 8, "Must have 8-byte alignment"); + static_assert(sizeof(Cmbp1Msg) == + sizeof(TradeMsg) + sizeof(ConsolidatedBidAskPair), + "Cmbp1Msg size must match Rust"); UnixNanos IndexTs() const { return ts_recv; } RecordHeader hd; std::int64_t price; std::uint32_t size; - Action action; + char reserved1; Side side; FlagSet flags; - char reserved; + char reserved2; UnixNanos ts_recv; - TimeDeltaNanos ts_in_delta; - std::uint32_t sequence; + std::array reserved3; + std::array reserved4; std::array levels; }; using Cbbo1SMsg = CbboMsg; using Cbbo1MMsg = CbboMsg; -using TcbboMsg = CbboMsg; static_assert(alignof(CbboMsg) == 8, "Must have 8-byte alignment"); static_assert(sizeof(CbboMsg) == sizeof(TradeMsg) + sizeof(ConsolidatedBidAskPair), @@ -604,12 +636,20 @@ inline bool operator!=(const BboMsg& lhs, const BboMsg& rhs) { return !(lhs == rhs); } -inline bool operator==(const CbboMsg& lhs, const CbboMsg& rhs) { +inline bool operator==(const Cmbp1Msg& lhs, const Cmbp1Msg& rhs) { return lhs.hd == rhs.hd && lhs.price == rhs.price && lhs.size == rhs.size && lhs.action == rhs.action && lhs.side == rhs.side && lhs.flags == rhs.flags && lhs.ts_recv == rhs.ts_recv && - lhs.ts_in_delta == rhs.ts_in_delta && lhs.sequence == rhs.sequence && - lhs.levels == rhs.levels; + lhs.ts_in_delta == rhs.ts_in_delta && lhs.levels == rhs.levels; +} +inline bool operator!=(const Cmbp1Msg& lhs, const Cmbp1Msg& rhs) { + return !(lhs == rhs); +} + +inline bool operator==(const CbboMsg& lhs, const CbboMsg& rhs) { + return lhs.hd == rhs.hd && lhs.price == rhs.price && lhs.size == rhs.size && + lhs.side == rhs.side && lhs.flags == rhs.flags && + lhs.ts_recv == rhs.ts_recv && lhs.levels == rhs.levels; } inline bool operator!=(const CbboMsg& lhs, const CbboMsg& rhs) { return !(lhs == rhs); diff --git a/src/enums.cpp b/src/enums.cpp index 46ae00a..b19567b 100644 --- a/src/enums.cpp +++ b/src/enums.cpp @@ -62,8 +62,8 @@ const char* ToString(Schema schema) { case Schema::Imbalance: { return "imbalance"; } - case Schema::Cbbo: { - return "cbbo"; + case Schema::Cmbp1: { + return "cmbp-1"; } case Schema::Cbbo1S: { return "cbbo-1s"; @@ -310,8 +310,8 @@ const char* ToString(RType rtype) { case RType::Mbo: { return "Mbo"; } - case RType::Cbbo: { - return "Cbbo"; + case RType::Cmbp1: { + return "Cmbp1"; } case RType::Cbbo1S: { return "Cbbo1S"; @@ -933,8 +933,8 @@ Schema FromString(const std::string& str) { if (str == "imbalance") { return Schema::Imbalance; } - if (str == "cbbo") { - return Schema::Cbbo; + if (str == "cmbp-1") { + return Schema::Cmbp1; } if (str == "cbbo-1s") { return Schema::Cbbo1S; diff --git a/src/publishers.cpp b/src/publishers.cpp index f762160..0a85476 100644 --- a/src/publishers.cpp +++ b/src/publishers.cpp @@ -787,6 +787,12 @@ Venue PublisherVenue(Publisher publisher) { case Publisher::XnysBbotradesXnys: { return Venue::Xnys; } + case Publisher::XnasBasicDbeq: { + return Venue::Dbeq; + } + case Publisher::DbeqMaxDbeq: { + return Venue::Dbeq; + } default: { throw InvalidArgumentError{ "PublisherVenue", "publisher", @@ -1073,6 +1079,12 @@ Dataset PublisherDataset(Publisher publisher) { case Publisher::XnysBbotradesXnys: { return Dataset::XnysBbotrades; } + case Publisher::XnasBasicDbeq: { + return Dataset::XnasBasic; + } + case Publisher::DbeqMaxDbeq: { + return Dataset::DbeqMax; + } default: { throw InvalidArgumentError{ "PublisherDataset", "publisher", @@ -1360,6 +1372,12 @@ const char* ToString(Publisher publisher) { case Publisher::XnysBbotradesXnys: { return "XNYS.BBOTRADES.XNYS"; } + case Publisher::XnasBasicDbeq: { + return "XNAS.BASIC.DBEQ"; + } + case Publisher::DbeqMaxDbeq: { + return "DBEQ.MAX.DBEQ"; + } default: { return "Unknown"; } @@ -1649,6 +1667,12 @@ Publisher FromString(const std::string& str) { if (str == "XNYS.BBOTRADES.XNYS") { return Publisher::XnysBbotradesXnys; } + if (str == "XNAS.BASIC.DBEQ") { + return Publisher::XnasBasicDbeq; + } + if (str == "DBEQ.MAX.DBEQ") { + return Publisher::DbeqMaxDbeq; + } throw InvalidArgumentError{"FromString", "str", "unknown value '" + str + '\''}; } diff --git a/src/record.cpp b/src/record.cpp index 3d41b03..c7f8c3f 100644 --- a/src/record.cpp +++ b/src/record.cpp @@ -322,6 +322,23 @@ std::ostream& operator<<(std::ostream& stream, const BboMsg& bbo_msg) { .AddField("levels", std::get<0>(bbo_msg.levels)) .Finish(); } +std::string ToString(const Cmbp1Msg& cbbo_msg) { return MakeString(cbbo_msg); } +std::ostream& operator<<(std::ostream& stream, const Cmbp1Msg& cmbp1_msg) { + return StreamOpBuilder{stream} + .SetTypeName("Cmbp1Msg") + .SetSpacer("\n ") + .Build() + .AddField("hd", cmbp1_msg.hd) + .AddField("price", FixPx{cmbp1_msg.price}) + .AddField("size", cmbp1_msg.size) + .AddField("action", cmbp1_msg.action) + .AddField("side", cmbp1_msg.side) + .AddField("flags", cmbp1_msg.flags) + .AddField("ts_recv", cmbp1_msg.ts_recv) + .AddField("ts_in_delta", cmbp1_msg.ts_in_delta) + .AddField("levels", std::get<0>(cmbp1_msg.levels)) + .Finish(); +} std::string ToString(const CbboMsg& cbbo_msg) { return MakeString(cbbo_msg); } std::ostream& operator<<(std::ostream& stream, const CbboMsg& cbbo_msg) { return StreamOpBuilder{stream} @@ -331,12 +348,9 @@ std::ostream& operator<<(std::ostream& stream, const CbboMsg& cbbo_msg) { .AddField("hd", cbbo_msg.hd) .AddField("price", FixPx{cbbo_msg.price}) .AddField("size", cbbo_msg.size) - .AddField("action", cbbo_msg.action) .AddField("side", cbbo_msg.side) .AddField("flags", cbbo_msg.flags) .AddField("ts_recv", cbbo_msg.ts_recv) - .AddField("ts_in_delta", cbbo_msg.ts_in_delta) - .AddField("sequence", cbbo_msg.sequence) .AddField("levels", std::get<0>(cbbo_msg.levels)) .Finish(); } diff --git a/tests/data/test_data.cbbo.dbn b/tests/data/test_data.cbbo.dbn index a275985..4312c96 100644 Binary files a/tests/data/test_data.cbbo.dbn and b/tests/data/test_data.cbbo.dbn differ diff --git a/tests/data/test_data.cbbo.dbn.zst b/tests/data/test_data.cbbo.dbn.zst index 9c66be1..a2986dc 100644 Binary files a/tests/data/test_data.cbbo.dbn.zst and b/tests/data/test_data.cbbo.dbn.zst differ diff --git a/tests/data/test_data.cbbo.v1.dbn b/tests/data/test_data.cbbo.v1.dbn index 2c718d6..2a8c11a 100644 Binary files a/tests/data/test_data.cbbo.v1.dbn and b/tests/data/test_data.cbbo.v1.dbn differ diff --git a/tests/data/test_data.cbbo.v1.dbn.zst b/tests/data/test_data.cbbo.v1.dbn.zst index 29d1460..1ea8fa0 100644 Binary files a/tests/data/test_data.cbbo.v1.dbn.zst and b/tests/data/test_data.cbbo.v1.dbn.zst differ diff --git a/tests/data/test_data.cmbp-1.dbn b/tests/data/test_data.cmbp-1.dbn new file mode 100644 index 0000000..9cad038 Binary files /dev/null and b/tests/data/test_data.cmbp-1.dbn differ diff --git a/tests/data/test_data.cmbp-1.dbn.zst b/tests/data/test_data.cmbp-1.dbn.zst new file mode 100644 index 0000000..8fe2928 Binary files /dev/null and b/tests/data/test_data.cmbp-1.dbn.zst differ diff --git a/tests/data/test_data.cmbp-1.v1.dbn b/tests/data/test_data.cmbp-1.v1.dbn new file mode 100644 index 0000000..2955e6f Binary files /dev/null and b/tests/data/test_data.cmbp-1.v1.dbn differ diff --git a/tests/data/test_data.cmbp-1.v1.dbn.zst b/tests/data/test_data.cmbp-1.v1.dbn.zst new file mode 100644 index 0000000..3a3c776 Binary files /dev/null and b/tests/data/test_data.cmbp-1.v1.dbn.zst differ diff --git a/tests/src/dbn_decoder_tests.cpp b/tests/src/dbn_decoder_tests.cpp index 13d90ff..460ebcf 100644 --- a/tests/src/dbn_decoder_tests.cpp +++ b/tests/src/dbn_decoder_tests.cpp @@ -488,6 +488,82 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeMbp10) { EXPECT_EQ(ch_mbp2.levels[2].ask_ct, 25); } +TEST_P(DbnDecoderSchemaTests, TestDecodeCmbp1) { + const auto extension = GetParam().first; + const auto version = GetParam().second; + ReadFromFile("cmbp-1", extension, version); + + const Metadata ch_metadata = channel_target_->DecodeMetadata(); + const Metadata f_metadata = file_target_->DecodeMetadata(); + EXPECT_EQ(ch_metadata, f_metadata); + EXPECT_EQ(ch_metadata.version, version); + EXPECT_EQ(ch_metadata.dataset, dataset::kGlbxMdp3); + EXPECT_EQ(ch_metadata.schema, Schema::Cmbp1); + EXPECT_EQ(ch_metadata.start.time_since_epoch().count(), 1609160400000000000); + EXPECT_EQ(ch_metadata.end.time_since_epoch().count(), 1609200000000000000); + EXPECT_EQ(ch_metadata.limit, 2); + EXPECT_EQ(ch_metadata.stype_in, SType::RawSymbol); + EXPECT_EQ(ch_metadata.stype_out, SType::InstrumentId); + EXPECT_EQ(ch_metadata.symbols, std::vector{"ESH1"}); + EXPECT_TRUE(ch_metadata.partial.empty()); + EXPECT_TRUE(ch_metadata.not_found.empty()); + AssertMappings(ch_metadata.mappings); + + const auto ch_record1 = channel_target_->DecodeRecord(); + const auto f_record1 = file_target_->DecodeRecord(); + ASSERT_NE(ch_record1, nullptr); + ASSERT_NE(f_record1, nullptr); + ASSERT_TRUE(ch_record1->Holds()); + ASSERT_TRUE(f_record1->Holds()); + const auto& ch_cbbo1 = ch_record1->Get(); + const auto& f_mbp1 = f_record1->Get(); + EXPECT_EQ(ch_cbbo1, f_mbp1); + EXPECT_EQ(ch_cbbo1.hd.publisher_id, 1); + EXPECT_EQ(ch_cbbo1.hd.instrument_id, 5482); + EXPECT_EQ(ch_cbbo1.hd.ts_event.time_since_epoch().count(), + 1609160400006001487); + EXPECT_EQ(ch_cbbo1.price, 3720500000000); + EXPECT_EQ(ch_cbbo1.size, 1); + EXPECT_EQ(ch_cbbo1.action, Action::Add); + EXPECT_EQ(ch_cbbo1.side, Side::Ask); + EXPECT_EQ(ch_cbbo1.flags.Raw(), 128); + EXPECT_EQ(ch_cbbo1.ts_recv.time_since_epoch().count(), 1609160400006136329); + EXPECT_EQ(ch_cbbo1.ts_in_delta.count(), 17214); + EXPECT_EQ(ch_cbbo1.levels[0].bid_px, 3720250000000); + EXPECT_EQ(ch_cbbo1.levels[0].ask_px, 3720500000000); + EXPECT_EQ(ch_cbbo1.levels[0].bid_sz, 24); + EXPECT_EQ(ch_cbbo1.levels[0].ask_sz, 11); + EXPECT_EQ(ch_cbbo1.levels[0].bid_pb, 1); + EXPECT_EQ(ch_cbbo1.levels[0].ask_pb, 1); + + const auto ch_record2 = channel_target_->DecodeRecord(); + const auto f_record2 = file_target_->DecodeRecord(); + ASSERT_NE(ch_record2, nullptr); + ASSERT_NE(f_record2, nullptr); + ASSERT_TRUE(ch_record2->Holds()); + ASSERT_TRUE(f_record2->Holds()); + const auto& ch_cbbo2 = ch_record2->Get(); + const auto& f_mbp2 = f_record2->Get(); + EXPECT_EQ(ch_cbbo2, f_mbp2); + EXPECT_EQ(ch_cbbo2.hd.publisher_id, 1); + EXPECT_EQ(ch_cbbo2.hd.instrument_id, 5482); + EXPECT_EQ(ch_cbbo2.hd.ts_event.time_since_epoch().count(), + 1609160400006146661); + EXPECT_EQ(ch_cbbo2.price, 3720500000000); + EXPECT_EQ(ch_cbbo2.size, 1); + EXPECT_EQ(ch_cbbo2.action, Action::Add); + EXPECT_EQ(ch_cbbo2.side, Side::Ask); + EXPECT_EQ(ch_cbbo2.flags.Raw(), 128); + EXPECT_EQ(ch_cbbo2.ts_recv.time_since_epoch().count(), 1609160400006246513); + EXPECT_EQ(ch_cbbo2.ts_in_delta.count(), 18858); + EXPECT_EQ(ch_cbbo2.levels[0].bid_px, 3720250000000); + EXPECT_EQ(ch_cbbo2.levels[0].ask_px, 3720500000000); + EXPECT_EQ(ch_cbbo2.levels[0].bid_sz, 24); + EXPECT_EQ(ch_cbbo2.levels[0].ask_sz, 12); + EXPECT_EQ(ch_cbbo2.levels[0].bid_pb, 1); + EXPECT_EQ(ch_cbbo2.levels[0].ask_pb, 1); +} + TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) { const auto extension = GetParam().first; const auto version = GetParam().second; @@ -498,7 +574,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) { EXPECT_EQ(ch_metadata, f_metadata); EXPECT_EQ(ch_metadata.version, version); EXPECT_EQ(ch_metadata.dataset, dataset::kGlbxMdp3); - EXPECT_EQ(ch_metadata.schema, Schema::Cbbo); + EXPECT_EQ(ch_metadata.schema, Schema::Cbbo1S); EXPECT_EQ(ch_metadata.start.time_since_epoch().count(), 1609160400000000000); EXPECT_EQ(ch_metadata.end.time_since_epoch().count(), 1609200000000000000); EXPECT_EQ(ch_metadata.limit, 2); @@ -524,12 +600,9 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) { 1609160400006001487); EXPECT_EQ(ch_cbbo1.price, 3720500000000); EXPECT_EQ(ch_cbbo1.size, 1); - EXPECT_EQ(ch_cbbo1.action, Action::Add); EXPECT_EQ(ch_cbbo1.side, Side::Ask); EXPECT_EQ(ch_cbbo1.flags.Raw(), 128); EXPECT_EQ(ch_cbbo1.ts_recv.time_since_epoch().count(), 1609160400006136329); - EXPECT_EQ(ch_cbbo1.ts_in_delta.count(), 17214); - EXPECT_EQ(ch_cbbo1.sequence, 1170362); EXPECT_EQ(ch_cbbo1.levels[0].bid_px, 3720250000000); EXPECT_EQ(ch_cbbo1.levels[0].ask_px, 3720500000000); EXPECT_EQ(ch_cbbo1.levels[0].bid_sz, 24); @@ -552,12 +625,9 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) { 1609160400006146661); EXPECT_EQ(ch_cbbo2.price, 3720500000000); EXPECT_EQ(ch_cbbo2.size, 1); - EXPECT_EQ(ch_cbbo2.action, Action::Add); EXPECT_EQ(ch_cbbo2.side, Side::Ask); EXPECT_EQ(ch_cbbo2.flags.Raw(), 128); EXPECT_EQ(ch_cbbo2.ts_recv.time_since_epoch().count(), 1609160400006246513); - EXPECT_EQ(ch_cbbo2.ts_in_delta.count(), 18858); - EXPECT_EQ(ch_cbbo2.sequence, 1170364); EXPECT_EQ(ch_cbbo2.levels[0].bid_px, 3720250000000); EXPECT_EQ(ch_cbbo2.levels[0].ask_px, 3720500000000); EXPECT_EQ(ch_cbbo2.levels[0].bid_sz, 24);