deserialize all the things

dex/networks/ltc/block.go

@@ -15,12 +15,58 @@ const (
 	// mwebVer is the bit of the block header's version that indicates the
 	// presence of a MWEB.
 	mwebVer = 0x20000000 // 1 << 29
-
-	// EmptyTxHash is the txid of an empty wire.MsgTx that may be included in a
-	// block with a MW integration transaction (HogEx).
-	EmptyTxHash = "f702453dd03b0f055e5437d76128141803984fb10acb85fc3b2184fae2f3fa78"
 )
 
+func parseMWEB(blk io.Reader) error {
+	dec := newDecoder(blk)
+	// src/mweb/mweb_models.h - struct Block
+	// "A convenience wrapper around a possibly-null extension block.""
+	// OptionalPtr around a mw::Block. Read the option byte:
+	hasMWEB, err := dec.readByte()
+	if err != nil {
+		return fmt.Errorf("failed to check MWEB option byte: %w", err)
+	}
+	if hasMWEB == 0 {
+		return nil
+	}
+
+	// src/libmw/include/mw/models/block/Block.h - class Block
+	// (1) Header and (2) TxBody
+
+	// src/libmw/include/mw/models/block/Header.h - class Header
+	// height
+	if _, err = dec.readVLQ(); err != nil {
+		return fmt.Errorf("failed to decode MWEB height: %w", err)
+	}
+
+	// 3x Hash + 2x BlindingFactor
+	if err = dec.discardBytes(32*3 + 32*2); err != nil {
+		return fmt.Errorf("failed to decode MWEB junk: %w", err)
+	}
+
+	// Number of TXOs: outputMMRSize
+	if _, err = dec.readVLQ(); err != nil {
+		return fmt.Errorf("failed to decode TXO count: %w", err)
+	}
+
+	// Number of kernels: kernelMMRSize
+	if _, err = dec.readVLQ(); err != nil {
+		return fmt.Errorf("failed to decode kernel count: %w", err)
+	}
+
+	// TxBody
+	_, err = dec.readMWTXBody()
+	if err != nil {
+		return fmt.Errorf("failed to decode MWEB tx: %w", err)
+	}
+	// if len(kern0) > 0 {
+	// 	mwebTxID := chainhash.Hash(blake3.Sum256(kern0))
+	// 	fmt.Println(mwebTxID.String())
+	// }
+
+	return nil
+}
+
 // DeserializeBlock decodes the bytes of a serialized Litecoin block. This
 // function exists because MWEB changes both the block and transaction
 // serializations. Blocks may have a MW "extension block" for "peg-out"
@@ -30,9 +76,7 @@ const (
 // (also known as a HogEx transaction), all still in the regular LTC block. The
 // peg-in txns decode correctly, but the integration tx is a special transaction
 // with the witness tx flag with bit 3 set (8), which prevents correct
-// wire.MsgTx deserialization. These integration transactions are replaced with
-// a dummy MsgTx recognizable by EmptyTxHash. We make no effort to decode the EW
-// or any of the peg-out transactions, both of which have unique encodings.
+// wire.MsgTx deserialization.
 // Refs:
 // https://github.com/litecoin-project/lips/blob/master/lip-0002.mediawiki#PegOut_Transactions
 // https://github.com/litecoin-project/lips/blob/master/lip-0003.mediawiki#Specification
@@ -48,36 +92,32 @@ func DeserializeBlock(blk io.Reader) (*wire.MsgBlock, error) {
 		return nil, fmt.Errorf("failed to deserialize block header: %w", err)
 	}
 
-	// May contain a MWEB *after* the txns.
-	containsMWEB := hdr.Version&mwebVer != 0
-
 	// This block's transactions
 	txnCount, err := wire.ReadVarInt(blk, 0)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse transaction count: %w", err)
 	}
 
-	// We can only decode the canonical txns, not the integ/hogex txn at the end
-	// or the mw peg-in txs in the EB.
+	// We can only decode the canonical txns, not the mw peg-in txs in the EB.
+	var hasHogEx bool
 	txns := make([]*wire.MsgTx, 0, int(txnCount))
 	for i := 0; i < cap(txns); i++ {
-		msgTx := &wire.MsgTx{}
-		err = msgTx.Deserialize(blk)
+		msgTx, err := DeserializeTx(blk)
 		if err != nil {
-			if !containsMWEB {
-				return nil, fmt.Errorf("failed to deserialize transaction %d of %d in block %v: %w",
-					i+1, txnCount, hdr.BlockHash(), err)
-			}
-			break // the last tx should be an integration tx (hogex)
+			return nil, fmt.Errorf("failed to deserialize transaction %d of %d in block %v: %w",
+				i+1, txnCount, hdr.BlockHash(), err)
 		}
-		txns = append(txns, msgTx) // txns = append(txns, msgTx)
-	}
-	for i := len(txns); i < cap(txns); i++ { // should be just one, but be tolerant
-		txns = append(txns, &wire.MsgTx{}) // just a placeholder => EmptyTxHash
+		txns = append(txns, msgTx.MsgTx) // txns = append(txns, msgTx)
+		hasHogEx = msgTx.IsHogEx         // hogex is the last txn
 	}
 
 	// If containsMWEB, blk.Len (remaining) will be non-zero since we did not
-	// fully read the integ/hogex tx or any of the EB.
+	// fully read the EB. The block is still optional, apparently.
+	if hdr.Version&mwebVer != 0 && hasHogEx {
+		if err = parseMWEB(blk); err != nil {
+			return nil, err
+		}
+	}
 
 	return &wire.MsgBlock{
 		Header:       *hdr,

dex/networks/ltc/block_online_test.go

@@ -72,6 +72,7 @@ func TestOnlineDeserializeBlock(t *testing.T) {
 	}
 
 	// start 1000 blocks prior to mweb testnet blocks
+	// 2215586 for mainnet, 2214584 for testnet4
 	for iBlk := int64(2214584); ; iBlk++ {
 		hash, err := client.GetBlockHash(context.Background(), iBlk)
 		if err != nil {
@@ -83,7 +84,7 @@ func TestOnlineDeserializeBlock(t *testing.T) {
 		blkStr := getBlkStr(hash)
 		blk, err := DeserializeBlock(hex.NewDecoder(strings.NewReader(blkStr)))
 		if err != nil {
-			t.Fatalf("Unmarshal: %v", err)
+			t.Fatalf("Unmarshal (%d): %v", iBlk, err)
 		}
 		if iBlk%500 == 0 {
 			t.Log(iBlk)
@@ -94,9 +95,6 @@ func TestOnlineDeserializeBlock(t *testing.T) {
 		}
 		for i, tx := range blk.Transactions {
 			txid := tx.TxHash().String()
-			if i == len(blk.Transactions)-1 && txid == EmptyTxHash {
-				break // don't error over the hogex at the end of the block
-			}
 			if txid != gbv.Tx[i] {
 				t.Errorf("got txid %v, wanted %v", txid, gbv.Tx[i])
 			}

dex/networks/ltc/block_test.go

@@ -15,10 +15,11 @@ import (
 
 var (
 	// Testnet4 block 1821752 is pre-MWEB activation, 3 txns.
+	// But version 20000000.
 	//go:embed testnet4Block1821752.dat
 	block1821752 []byte
 
-	// Block 2215584 is the first with MW txns, a peg-in with witness verion 9
+	// Block 2215584 is the first with MW txns, a peg-in with witness version 9
 	// script, an integ tx with witness version 8 script, block version 20000000
 	// (with a MWEB), and 5 txns.
 	// 7e35fabe7b3c694ebeb0368a1a1c31e83962f3c5b4cc8dcede3ae94ed3deb306
@@ -36,6 +37,9 @@ var (
 	// e9fe2c6496aedefa8bf6529bdc5c1f9fd4af565ca4c98cab73e3a1f616fb3502
 	//go:embed testnet4Block2319633.dat
 	block2319633 []byte
+
+	//go:embed block2215586testnet4.dat
+	block2215586 []byte
 )
 
 func TestDeserializeBlockBytes(t *testing.T) {
@@ -58,21 +62,28 @@ func TestDeserializeBlockBytes(t *testing.T) {
 			block2215584,
 			"7e35fabe7b3c694ebeb0368a1a1c31e83962f3c5b4cc8dcede3ae94ed3deb306",
 			5,
-			EmptyTxHash, // hogex, really 4c86658e64861c2f2b7fbbf26bbf7a6640ae3824d24293a009ad5ea1e8ab4418
+			"4c86658e64861c2f2b7fbbf26bbf7a6640ae3824d24293a009ad5ea1e8ab4418",
+		},
+		{
+			"block 2215586 MWEB",
+			block2215586,
+			"3000cc2076a568a8eb5f56a06112a57264446e2c7d2cca28cdc85d91820dfa17",
+			37,
+			"3a7299f5e6ee9975bdcc2d754ff5de3312d92db177b55c68753a1cdf9ce63a7c",
 		},
 		{
 			"block 2321749 MWEB",
 			block2321749,
 			"57929846db4a92d937eb596354d10949e33c815ee45df0c9b3bbdfb283e15bcd",
 			4,
-			EmptyTxHash, // hogex, really 1bad5e78b145947d32eeeb1d24295891ba03359508d5f09921bada3be66bbe17
+			"1bad5e78b145947d32eeeb1d24295891ba03359508d5f09921bada3be66bbe17",
 		},
 		{
 			"block 2319633 MWEB",
 			block2319633,
 			"e9fe2c6496aedefa8bf6529bdc5c1f9fd4af565ca4c98cab73e3a1f616fb3502",
 			2,
-			EmptyTxHash, // hogex, really 3cd43df64e9382040eff0bf54ba1c2389d5111eb5ab0968ab7af67e3c30cac04
+			"3cd43df64e9382040eff0bf54ba1c2389d5111eb5ab0968ab7af67e3c30cac04",
 		},
 	}