ADD: DBN updates to support XNAS.BASIC

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.23.0 - TBD
+
+### Enhancements
+- Added new `Cmbp1Msg`
+- Added new consolidated publisher values for `XNAS.BASIC` and `DBEQ.MAX`
+
+### Breaking changes
+- Changed the layout of `CbboMsg` to better match `BboMsg`
+- Renamed `Schema::Cbbo` to `Schema::Cmbp1`
+
 ## 0.22.0 - 2024-08-27
 
 ### Enhancements

include/databento/enums.hpp

@@ -34,7 +34,7 @@ enum class Schema : std::uint16_t {
   Statistics = 10,
   Status = 11,
   Imbalance = 12,
-  Cbbo = 14,
+  Cmbp1 = 14,
   Cbbo1S = 15,
   Cbbo1M = 16,
   Tcbbo = 17,
@@ -139,7 +139,7 @@ enum RType : std::uint8_t {
   System = 0x17,
   Statistics = 0x18,
   Mbo = 0xA0,
-  Cbbo = 0xB1,
+  Cmbp1 = 0xB1,
   Cbbo1S = 0xC0,
   Cbbo1M = 0xC1,
   Tcbbo = 0xC2,

include/databento/publishers.hpp

@@ -353,6 +353,10 @@ enum class Publisher : std::uint16_t {
   XcisBbotradesXcis = 91,
   // NYSE BBO and Trades
   XnysBbotradesXnys = 92,
+  // Nasdaq Basic - Consolidated
+  XnasBasicDbeq = 93,
+  // DBEQ Max - Consolidated
+  DbeqMaxDbeq = 94,
 };
 
 // Get a Publisher's Venue.

include/databento/record.hpp

@@ -254,36 +254,68 @@ using Bbo1MMsg = BboMsg;
 static_assert(alignof(BboMsg) == 8, "Must have 8-byte alignment");
 static_assert(sizeof(BboMsg) == sizeof(Mbp1Msg), "BboMsg size must match Rust");
 
+struct Cmbp1Msg {
+  static bool HasRType(RType rtype) {
+    switch (rtype) {
+      case RType::Cmbp1:  // fallthrough
+      case RType::Tcbbo:
+        return true;
+      default:
+        return false;
+    };
+  }
+
+  UnixNanos IndexTs() const { return ts_recv; }
+
+  RecordHeader hd;
+  std::int64_t price;
+  std::uint32_t size;
+  char action;
+  Side side;
+  FlagSet flags;
+  char reserved1;
+  UnixNanos ts_recv;
+  TimeDeltaNanos ts_in_delta;
+  std::array<char, 4> reserved2;
+  std::array<ConsolidatedBidAskPair, 1> levels;
+};
+using TcbboMsg = Cmbp1Msg;
+static_assert(alignof(Cmbp1Msg) == 8, "Must have 8-byte alignment");
+static_assert(sizeof(Cmbp1Msg) ==
+                  sizeof(TradeMsg) + sizeof(ConsolidatedBidAskPair),
+              "Cmbp1Msg size must match Rust");
+
 struct CbboMsg {
   static bool HasRType(RType rtype) {
     switch (rtype) {
-      case RType::Cbbo:    // fallthrough
       case RType::Cbbo1S:  // fallthrough
       case RType::Cbbo1M:  // fallthrough
-      case RType::Tcbbo:
         return true;
       default:
         return false;
     };
   }
+  static_assert(alignof(Cmbp1Msg) == 8, "Must have 8-byte alignment");
+  static_assert(sizeof(Cmbp1Msg) ==
+                    sizeof(TradeMsg) + sizeof(ConsolidatedBidAskPair),
+                "Cmbp1Msg size must match Rust");
 
   UnixNanos IndexTs() const { return ts_recv; }
 
   RecordHeader hd;
   std::int64_t price;
   std::uint32_t size;
-  Action action;
+  char reserved1;
   Side side;
   FlagSet flags;
-  char reserved;
+  char reserved2;
   UnixNanos ts_recv;
-  TimeDeltaNanos ts_in_delta;
-  std::uint32_t sequence;
+  std::array<char, 4> reserved3;
+  std::array<char, 4> reserved4;
   std::array<ConsolidatedBidAskPair, 1> levels;
 };
 using Cbbo1SMsg = CbboMsg;
 using Cbbo1MMsg = CbboMsg;
-using TcbboMsg = CbboMsg;
 static_assert(alignof(CbboMsg) == 8, "Must have 8-byte alignment");
 static_assert(sizeof(CbboMsg) ==
                   sizeof(TradeMsg) + sizeof(ConsolidatedBidAskPair),
@@ -604,12 +636,20 @@ inline bool operator!=(const BboMsg& lhs, const BboMsg& rhs) {
   return !(lhs == rhs);
 }
 
-inline bool operator==(const CbboMsg& lhs, const CbboMsg& rhs) {
+inline bool operator==(const Cmbp1Msg& lhs, const Cmbp1Msg& rhs) {
   return lhs.hd == rhs.hd && lhs.price == rhs.price && lhs.size == rhs.size &&
          lhs.action == rhs.action && lhs.side == rhs.side &&
          lhs.flags == rhs.flags && lhs.ts_recv == rhs.ts_recv &&
-         lhs.ts_in_delta == rhs.ts_in_delta && lhs.sequence == rhs.sequence &&
-         lhs.levels == rhs.levels;
+         lhs.ts_in_delta == rhs.ts_in_delta && lhs.levels == rhs.levels;
+}
+inline bool operator!=(const Cmbp1Msg& lhs, const Cmbp1Msg& rhs) {
+  return !(lhs == rhs);
+}
+
+inline bool operator==(const CbboMsg& lhs, const CbboMsg& rhs) {
+  return lhs.hd == rhs.hd && lhs.price == rhs.price && lhs.size == rhs.size &&
+         lhs.side == rhs.side && lhs.flags == rhs.flags &&
+         lhs.ts_recv == rhs.ts_recv && lhs.levels == rhs.levels;
 }
 inline bool operator!=(const CbboMsg& lhs, const CbboMsg& rhs) {
   return !(lhs == rhs);

src/enums.cpp

@@ -62,8 +62,8 @@ const char* ToString(Schema schema) {
     case Schema::Imbalance: {
       return "imbalance";
     }
-    case Schema::Cbbo: {
-      return "cbbo";
+    case Schema::Cmbp1: {
+      return "cmbp-1";
     }
     case Schema::Cbbo1S: {
       return "cbbo-1s";
@@ -310,8 +310,8 @@ const char* ToString(RType rtype) {
     case RType::Mbo: {
       return "Mbo";
     }
-    case RType::Cbbo: {
-      return "Cbbo";
+    case RType::Cmbp1: {
+      return "Cmbp1";
     }
     case RType::Cbbo1S: {
       return "Cbbo1S";
@@ -933,8 +933,8 @@ Schema FromString(const std::string& str) {
   if (str == "imbalance") {
     return Schema::Imbalance;
   }
-  if (str == "cbbo") {
-    return Schema::Cbbo;
+  if (str == "cmbp-1") {
+    return Schema::Cmbp1;
   }
   if (str == "cbbo-1s") {
     return Schema::Cbbo1S;

src/publishers.cpp

@@ -787,6 +787,12 @@ Venue PublisherVenue(Publisher publisher) {
     case Publisher::XnysBbotradesXnys: {
       return Venue::Xnys;
     }
+    case Publisher::XnasBasicDbeq: {
+      return Venue::Dbeq;
+    }
+    case Publisher::DbeqMaxDbeq: {
+      return Venue::Dbeq;
+    }
     default: {
       throw InvalidArgumentError{
           "PublisherVenue", "publisher",
@@ -1073,6 +1079,12 @@ Dataset PublisherDataset(Publisher publisher) {
     case Publisher::XnysBbotradesXnys: {
       return Dataset::XnysBbotrades;
     }
+    case Publisher::XnasBasicDbeq: {
+      return Dataset::XnasBasic;
+    }
+    case Publisher::DbeqMaxDbeq: {
+      return Dataset::DbeqMax;
+    }
     default: {
       throw InvalidArgumentError{
           "PublisherDataset", "publisher",
@@ -1360,6 +1372,12 @@ const char* ToString(Publisher publisher) {
     case Publisher::XnysBbotradesXnys: {
       return "XNYS.BBOTRADES.XNYS";
     }
+    case Publisher::XnasBasicDbeq: {
+      return "XNAS.BASIC.DBEQ";
+    }
+    case Publisher::DbeqMaxDbeq: {
+      return "DBEQ.MAX.DBEQ";
+    }
     default: {
       return "Unknown";
     }
@@ -1649,6 +1667,12 @@ Publisher FromString(const std::string& str) {
   if (str == "XNYS.BBOTRADES.XNYS") {
     return Publisher::XnysBbotradesXnys;
   }
+  if (str == "XNAS.BASIC.DBEQ") {
+    return Publisher::XnasBasicDbeq;
+  }
+  if (str == "DBEQ.MAX.DBEQ") {
+    return Publisher::DbeqMaxDbeq;
+  }
   throw InvalidArgumentError{"FromString<Publisher>", "str",
                              "unknown value '" + str + '\''};
 }

src/record.cpp

@@ -322,6 +322,23 @@ std::ostream& operator<<(std::ostream& stream, const BboMsg& bbo_msg) {
       .AddField("levels", std::get<0>(bbo_msg.levels))
       .Finish();
 }
+std::string ToString(const Cmbp1Msg& cbbo_msg) { return MakeString(cbbo_msg); }
+std::ostream& operator<<(std::ostream& stream, const Cmbp1Msg& cmbp1_msg) {
+  return StreamOpBuilder{stream}
+      .SetTypeName("Cmbp1Msg")
+      .SetSpacer("\n    ")
+      .Build()
+      .AddField("hd", cmbp1_msg.hd)
+      .AddField("price", FixPx{cmbp1_msg.price})
+      .AddField("size", cmbp1_msg.size)
+      .AddField("action", cmbp1_msg.action)
+      .AddField("side", cmbp1_msg.side)
+      .AddField("flags", cmbp1_msg.flags)
+      .AddField("ts_recv", cmbp1_msg.ts_recv)
+      .AddField("ts_in_delta", cmbp1_msg.ts_in_delta)
+      .AddField("levels", std::get<0>(cmbp1_msg.levels))
+      .Finish();
+}
 std::string ToString(const CbboMsg& cbbo_msg) { return MakeString(cbbo_msg); }
 std::ostream& operator<<(std::ostream& stream, const CbboMsg& cbbo_msg) {
   return StreamOpBuilder{stream}
@@ -331,12 +348,9 @@ std::ostream& operator<<(std::ostream& stream, const CbboMsg& cbbo_msg) {
       .AddField("hd", cbbo_msg.hd)
       .AddField("price", FixPx{cbbo_msg.price})
       .AddField("size", cbbo_msg.size)
-      .AddField("action", cbbo_msg.action)
       .AddField("side", cbbo_msg.side)
       .AddField("flags", cbbo_msg.flags)
       .AddField("ts_recv", cbbo_msg.ts_recv)
-      .AddField("ts_in_delta", cbbo_msg.ts_in_delta)
-      .AddField("sequence", cbbo_msg.sequence)
       .AddField("levels", std::get<0>(cbbo_msg.levels))
       .Finish();
 }

tests/src/dbn_decoder_tests.cpp

@@ -488,6 +488,82 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeMbp10) {
   EXPECT_EQ(ch_mbp2.levels[2].ask_ct, 25);
 }
 
+TEST_P(DbnDecoderSchemaTests, TestDecodeCmbp1) {
+  const auto extension = GetParam().first;
+  const auto version = GetParam().second;
+  ReadFromFile("cmbp-1", extension, version);
+
+  const Metadata ch_metadata = channel_target_->DecodeMetadata();
+  const Metadata f_metadata = file_target_->DecodeMetadata();
+  EXPECT_EQ(ch_metadata, f_metadata);
+  EXPECT_EQ(ch_metadata.version, version);
+  EXPECT_EQ(ch_metadata.dataset, dataset::kGlbxMdp3);
+  EXPECT_EQ(ch_metadata.schema, Schema::Cmbp1);
+  EXPECT_EQ(ch_metadata.start.time_since_epoch().count(), 1609160400000000000);
+  EXPECT_EQ(ch_metadata.end.time_since_epoch().count(), 1609200000000000000);
+  EXPECT_EQ(ch_metadata.limit, 2);
+  EXPECT_EQ(ch_metadata.stype_in, SType::RawSymbol);
+  EXPECT_EQ(ch_metadata.stype_out, SType::InstrumentId);
+  EXPECT_EQ(ch_metadata.symbols, std::vector<std::string>{"ESH1"});
+  EXPECT_TRUE(ch_metadata.partial.empty());
+  EXPECT_TRUE(ch_metadata.not_found.empty());
+  AssertMappings(ch_metadata.mappings);
+
+  const auto ch_record1 = channel_target_->DecodeRecord();
+  const auto f_record1 = file_target_->DecodeRecord();
+  ASSERT_NE(ch_record1, nullptr);
+  ASSERT_NE(f_record1, nullptr);
+  ASSERT_TRUE(ch_record1->Holds<Cmbp1Msg>());
+  ASSERT_TRUE(f_record1->Holds<Cmbp1Msg>());
+  const auto& ch_cbbo1 = ch_record1->Get<Cmbp1Msg>();
+  const auto& f_mbp1 = f_record1->Get<Cmbp1Msg>();
+  EXPECT_EQ(ch_cbbo1, f_mbp1);
+  EXPECT_EQ(ch_cbbo1.hd.publisher_id, 1);
+  EXPECT_EQ(ch_cbbo1.hd.instrument_id, 5482);
+  EXPECT_EQ(ch_cbbo1.hd.ts_event.time_since_epoch().count(),
+            1609160400006001487);
+  EXPECT_EQ(ch_cbbo1.price, 3720500000000);
+  EXPECT_EQ(ch_cbbo1.size, 1);
+  EXPECT_EQ(ch_cbbo1.action, Action::Add);
+  EXPECT_EQ(ch_cbbo1.side, Side::Ask);
+  EXPECT_EQ(ch_cbbo1.flags.Raw(), 128);
+  EXPECT_EQ(ch_cbbo1.ts_recv.time_since_epoch().count(), 1609160400006136329);
+  EXPECT_EQ(ch_cbbo1.ts_in_delta.count(), 17214);
+  EXPECT_EQ(ch_cbbo1.levels[0].bid_px, 3720250000000);
+  EXPECT_EQ(ch_cbbo1.levels[0].ask_px, 3720500000000);
+  EXPECT_EQ(ch_cbbo1.levels[0].bid_sz, 24);
+  EXPECT_EQ(ch_cbbo1.levels[0].ask_sz, 11);
+  EXPECT_EQ(ch_cbbo1.levels[0].bid_pb, 1);
+  EXPECT_EQ(ch_cbbo1.levels[0].ask_pb, 1);
+
+  const auto ch_record2 = channel_target_->DecodeRecord();
+  const auto f_record2 = file_target_->DecodeRecord();
+  ASSERT_NE(ch_record2, nullptr);
+  ASSERT_NE(f_record2, nullptr);
+  ASSERT_TRUE(ch_record2->Holds<Cmbp1Msg>());
+  ASSERT_TRUE(f_record2->Holds<Cmbp1Msg>());
+  const auto& ch_cbbo2 = ch_record2->Get<Cmbp1Msg>();
+  const auto& f_mbp2 = f_record2->Get<Cmbp1Msg>();
+  EXPECT_EQ(ch_cbbo2, f_mbp2);
+  EXPECT_EQ(ch_cbbo2.hd.publisher_id, 1);
+  EXPECT_EQ(ch_cbbo2.hd.instrument_id, 5482);
+  EXPECT_EQ(ch_cbbo2.hd.ts_event.time_since_epoch().count(),
+            1609160400006146661);
+  EXPECT_EQ(ch_cbbo2.price, 3720500000000);
+  EXPECT_EQ(ch_cbbo2.size, 1);
+  EXPECT_EQ(ch_cbbo2.action, Action::Add);
+  EXPECT_EQ(ch_cbbo2.side, Side::Ask);
+  EXPECT_EQ(ch_cbbo2.flags.Raw(), 128);
+  EXPECT_EQ(ch_cbbo2.ts_recv.time_since_epoch().count(), 1609160400006246513);
+  EXPECT_EQ(ch_cbbo2.ts_in_delta.count(), 18858);
+  EXPECT_EQ(ch_cbbo2.levels[0].bid_px, 3720250000000);
+  EXPECT_EQ(ch_cbbo2.levels[0].ask_px, 3720500000000);
+  EXPECT_EQ(ch_cbbo2.levels[0].bid_sz, 24);
+  EXPECT_EQ(ch_cbbo2.levels[0].ask_sz, 12);
+  EXPECT_EQ(ch_cbbo2.levels[0].bid_pb, 1);
+  EXPECT_EQ(ch_cbbo2.levels[0].ask_pb, 1);
+}
+
 TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) {
   const auto extension = GetParam().first;
   const auto version = GetParam().second;
@@ -498,7 +574,7 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) {
   EXPECT_EQ(ch_metadata, f_metadata);
   EXPECT_EQ(ch_metadata.version, version);
   EXPECT_EQ(ch_metadata.dataset, dataset::kGlbxMdp3);
-  EXPECT_EQ(ch_metadata.schema, Schema::Cbbo);
+  EXPECT_EQ(ch_metadata.schema, Schema::Cbbo1S);
   EXPECT_EQ(ch_metadata.start.time_since_epoch().count(), 1609160400000000000);
   EXPECT_EQ(ch_metadata.end.time_since_epoch().count(), 1609200000000000000);
   EXPECT_EQ(ch_metadata.limit, 2);
@@ -524,12 +600,9 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) {
             1609160400006001487);
   EXPECT_EQ(ch_cbbo1.price, 3720500000000);
   EXPECT_EQ(ch_cbbo1.size, 1);
-  EXPECT_EQ(ch_cbbo1.action, Action::Add);
   EXPECT_EQ(ch_cbbo1.side, Side::Ask);
   EXPECT_EQ(ch_cbbo1.flags.Raw(), 128);
   EXPECT_EQ(ch_cbbo1.ts_recv.time_since_epoch().count(), 1609160400006136329);
-  EXPECT_EQ(ch_cbbo1.ts_in_delta.count(), 17214);
-  EXPECT_EQ(ch_cbbo1.sequence, 1170362);
   EXPECT_EQ(ch_cbbo1.levels[0].bid_px, 3720250000000);
   EXPECT_EQ(ch_cbbo1.levels[0].ask_px, 3720500000000);
   EXPECT_EQ(ch_cbbo1.levels[0].bid_sz, 24);
@@ -552,12 +625,9 @@ TEST_P(DbnDecoderSchemaTests, TestDecodeCbbo) {
             1609160400006146661);
   EXPECT_EQ(ch_cbbo2.price, 3720500000000);
   EXPECT_EQ(ch_cbbo2.size, 1);
-  EXPECT_EQ(ch_cbbo2.action, Action::Add);
   EXPECT_EQ(ch_cbbo2.side, Side::Ask);
   EXPECT_EQ(ch_cbbo2.flags.Raw(), 128);
   EXPECT_EQ(ch_cbbo2.ts_recv.time_since_epoch().count(), 1609160400006246513);
-  EXPECT_EQ(ch_cbbo2.ts_in_delta.count(), 18858);
-  EXPECT_EQ(ch_cbbo2.sequence, 1170364);
   EXPECT_EQ(ch_cbbo2.levels[0].bid_px, 3720250000000);
   EXPECT_EQ(ch_cbbo2.levels[0].ask_px, 3720500000000);
   EXPECT_EQ(ch_cbbo2.levels[0].bid_sz, 24);