Switch to new tx format storing real and base size per tx

src/blocksci/chain/raw_transaction.cpp

@@ -8,5 +8,5 @@
 #include "raw_transaction.hpp"
 
 namespace blocksci {
-    RawTransaction::RawTransaction(uint32_t sizeBytes_, uint32_t locktime_, uint16_t inputCount_, uint16_t outputCount_) : sizeBytes(sizeBytes_), locktime(locktime_), inputCount(inputCount_), outputCount(outputCount_), inOuts(static_cast<uint32_t>(inputCount_ + outputCount_)) {}
+    RawTransaction::RawTransaction(uint32_t realSize_, uint32_t baseSize_, uint32_t locktime_, uint16_t inputCount_, uint16_t outputCount_) : realSize(realSize_), baseSize(baseSize_), locktime(locktime_), inputCount(inputCount_), outputCount(outputCount_) {}
 }

src/blocksci/chain/raw_transaction.hpp

@@ -14,37 +14,46 @@
 
 namespace blocksci {
     struct RawTransaction {
-        uint32_t sizeBytes;
+        uint32_t realSize;
+        uint32_t baseSize;
         uint32_t locktime;
         uint16_t inputCount;
         uint16_t outputCount;
-        InPlaceArray<Inout> inOuts;
 
-        RawTransaction(uint32_t sizeBytes, uint32_t locktime, uint16_t inputCount, uint16_t outputCount);
+        RawTransaction(uint32_t realSize, uint32_t baseSize, uint32_t locktime, uint16_t inputCount, uint16_t outputCount);
 
         RawTransaction(const RawTransaction &) = delete;
         RawTransaction(RawTransaction &&) = delete;
         RawTransaction &operator=(const RawTransaction &) = delete;
         RawTransaction &operator=(RawTransaction &&) = delete;
 
         Inout &getOutput(uint16_t outputNum) {
-            return reinterpret_cast<Inout &>(inOuts[inputCount + outputNum]);
+            return getInouts()[inputCount + outputNum];
         }
 
         Inout &getInput(uint16_t inputNum) {
-            return reinterpret_cast<Inout &>(inOuts[inputNum]);
+            return getInouts()[inputNum];
         }
 
         const Inout &getOutput(uint16_t outputNum) const {
-            return reinterpret_cast<const Inout &>(inOuts[inputCount + outputNum]);
+            return getInouts()[inputCount + outputNum];
         }
 
         const Inout &getInput(uint16_t inputNum) const {
-            return reinterpret_cast<const Inout &>(inOuts[inputNum]);
+            return getInouts()[inputNum];
         }
 
-        size_t realSize() const {
-            return sizeof(RawTransaction) + inOuts.extraSize();
+        size_t serializedSize() const {
+            return sizeof(RawTransaction) + sizeof(Inout) * (inputCount + outputCount);
+        }
+
+    private:
+        const Inout *getInouts() const {
+            return reinterpret_cast<const Inout *>(this + 1);
+        }
+
+        Inout *getInouts() {
+            return reinterpret_cast<Inout *>(this + 1);
         }
     };
 }

src/blocksci/chain/transaction.hpp

@@ -60,8 +60,20 @@ namespace blocksci {
         std::vector<OutputPointer> getOutputPointers(const InputPointer &pointer) const;
         std::vector<InputPointer> getInputPointers(const OutputPointer &pointer) const;
 
+        uint32_t baseSize() const {
+            return data->baseSize;
+        }
+
+        uint32_t totalSize() const {
+            return data->realSize;
+        }
+
+        uint32_t virtualSize() const {
+            return (data->realSize + data->baseSize * 3 + 3) / 4;
+        }
+
         uint32_t sizeBytes() const {
-            return data->sizeBytes;
+            return virtualSize();
         }
 
         uint32_t locktime() const {

src/parser/block_processor.cpp

@@ -559,7 +559,7 @@ void BlockProcessor::addNewBlocks(const ParserConfiguration<ParseTag> &config, s
     };
 
     auto serializeAddressAdvanceFunc = [&](RawTransaction *tx) {
-        bool shouldSend = tx->sizeBytes < 800 && finished_transaction_queue.write_available() >= 1;
+        bool shouldSend = tx->realSize < 800 && finished_transaction_queue.write_available() >= 1;
         if (!shouldSend) delete tx;
         return shouldSend;
     };

src/parser/preproccessed_block.cpp

@@ -68,19 +68,21 @@ void RawTransaction::load(SafeMemReader &reader, uint32_t txNum_, blocksci::Bloc
     isSegwit = witnessActivated;
     blockHeight = blockHeight_;
     auto startOffset = reader.offset();
+    auto curOffset = reader.offset();
     version = reader.readNext<decltype(version)>();
     txHashStart = reader.unsafePos();
-
+    baseSize = static_cast<uint32_t>(reader.offset() - curOffset);
+    curOffset = reader.offset();
     auto inputCount = reader.readVariableLengthInteger();
     bool containsSegwit = false;
     if (inputCount == 0) {
         auto flag = reader.readNext<uint8_t>();
         assert(flag == 1);
         containsSegwit = true;
         txHashStart = reader.unsafePos();
+        curOffset = reader.offset();
         inputCount = reader.readVariableLengthInteger();
     }
-
     inputs.clear();
     inputs.reserve(inputCount);
     for (decltype(inputCount) i = 0; i < inputCount; i++) {
@@ -94,9 +96,8 @@ void RawTransaction::load(SafeMemReader &reader, uint32_t txNum_, blocksci::Bloc
     for (decltype(outputCount) i = 0; i < outputCount; i++) {
         outputs.emplace_back(reader);
     }
-
+    baseSize += static_cast<uint32_t>(reader.offset() - curOffset);
     txHashLength = static_cast<uint32_t>(reader.unsafePos() - txHashStart);
-
     if (containsSegwit) {
         for (decltype(inputCount) i = 0; i < inputCount; i++) {
             auto &input = inputs[i];
@@ -106,21 +107,26 @@ void RawTransaction::load(SafeMemReader &reader, uint32_t txNum_, blocksci::Bloc
             }
         }
     }
+    curOffset = reader.offset();
     locktime = reader.readNext<Locktime>();
-    sizeBytes = static_cast<uint32_t>(reader.offset() - startOffset);
+    baseSize += static_cast<uint32_t>(reader.offset() - curOffset);
+    realSize = static_cast<uint32_t>(reader.offset() - startOffset);
     hash.SetNull();
 }
 
 TransactionHeader::TransactionHeader(SafeMemReader &reader) {
     auto startOffset = reader.offset();
+    auto curOffset = reader.offset();
     version = reader.readNext<decltype(version)>();
-
-    inputCount = reader.readVariableLengthInteger();
+    baseSize = static_cast<uint32_t>(reader.offset() - curOffset);
+    curOffset = reader.offset();
+    auto inputCount = reader.readVariableLengthInteger();
     bool containsSegwit = false;
     if (inputCount == 0) {
         auto flag = reader.readNext<uint8_t>();
         assert(flag == 1);
         containsSegwit = true;
+        curOffset = reader.offset();
         inputCount = reader.readVariableLengthInteger();
     }
 
@@ -130,13 +136,14 @@ TransactionHeader::TransactionHeader(SafeMemReader &reader) {
         reader.advance(scriptLength + sizeof(SequenceNum));
     }
 
-    outputCount = reader.readVariableLengthInteger();
+    auto outputCount = reader.readVariableLengthInteger();
     for (decltype(outputCount) i = 0; i < outputCount; i++) {
         reader.advance(sizeof(Value));
         auto scriptLength = reader.readVariableLengthInteger();
         reader.advance(scriptLength);
     }
 
+    baseSize += static_cast<uint32_t>(reader.offset() - curOffset);
     if (containsSegwit) {
         for (decltype(inputCount) i = 0; i < inputCount; i++) {
             auto stackItemCount = reader.readVariableLengthInteger();
@@ -146,8 +153,10 @@ TransactionHeader::TransactionHeader(SafeMemReader &reader) {
             }
         }
     }
+    curOffset = reader.offset();
     locktime = reader.readNext<Locktime>();
-    sizeBytes = static_cast<uint32_t>(reader.offset() - startOffset);
+    baseSize += static_cast<uint32_t>(reader.offset() - curOffset);
+    realSize = static_cast<uint32_t>(reader.offset() - startOffset);
 }
 
 void RawTransaction::calculateHash() {
@@ -183,7 +192,7 @@ void RawTransaction::load(const getrawtransaction_t &txinfo, uint32_t txNum_, bl
     blockHeight = blockHeight_;
     version = txinfo.version;
     locktime = static_cast<uint32_t>(txinfo.locktime);
-    sizeBytes = static_cast<uint32_t>(txinfo.hex.size() / 2);
+    realSize = static_cast<uint32_t>(txinfo.hex.size() / 2);
     auto inputCount = txinfo.vin.size();
     inputs.clear();
     inputs.reserve(inputCount);
@@ -203,7 +212,7 @@ void RawTransaction::load(const getrawtransaction_t &txinfo, uint32_t txNum_, bl
 #endif
 
 blocksci::RawTransaction RawTransaction::getRawTransaction() const {
-    return {sizeBytes, locktime, static_cast<uint16_t>(inputs.size()), static_cast<uint16_t>(outputs.size())};
+    return {realSize, baseSize, locktime, static_cast<uint16_t>(inputs.size()), static_cast<uint16_t>(outputs.size())};
 }
 
 blocksci::OutputPointer RawInput::getOutputPointer() const {

src/parser/preproccessed_block.hpp

@@ -103,15 +103,17 @@ struct TransactionHeader {
     int32_t version;
     uint32_t inputCount;
     uint32_t outputCount;
-    uint32_t sizeBytes;
+    uint32_t baseSize;
+    uint32_t realSize;
     uint32_t locktime;
     TransactionHeader(SafeMemReader &reader);
 };
 
 struct RawTransaction {
     uint32_t txNum;
     blocksci::uint256 hash;
-    uint32_t sizeBytes;
+    uint32_t baseSize;
+    uint32_t realSize;
     uint32_t locktime;
     int32_t version;
     blocksci::BlockHeight blockHeight;
@@ -128,7 +130,8 @@ struct RawTransaction {
     RawTransaction() :
       txNum(0),
       hash(),
-      sizeBytes(0),
+      baseSize(0),
+      realSize(0),
       locktime(0),
       version(0),
       blockHeight(0) {}