Fix proof generation and verification code

proof_ipa.go

@@ -387,66 +387,99 @@ type stemInfo struct {
 // PreStateTreeFromProof builds a stateless prestate tree from the proof.
 func PreStateTreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) { // skipcq: GO-R1005
 	if len(proof.Keys) != len(proof.PreValues) {
-		return nil, fmt.Errorf("incompatible number of keys and values: %d != %d", len(proof.Keys), len(proof.PreValues))
+		return nil, fmt.Errorf("incompatible number of keys and pre-values: %d != %d", len(proof.Keys), len(proof.PreValues))
+	}
+	if len(proof.Keys) != len(proof.PostValues) {
+		return nil, fmt.Errorf("incompatible number of keys and post-values: %d != %d", len(proof.Keys), len(proof.PostValues))
 	}
 	stems := make([][]byte, 0, len(proof.Keys))
 	for _, k := range proof.Keys {
 		if len(stems) == 0 || !bytes.Equal(stems[len(stems)-1], k[:31]) {
 			stems = append(stems, k[:31])
 		}
 	}
-	stemIndex := 0
-
+	if len(stems) != len(proof.ExtStatus) {
+		return nil, fmt.Errorf("invalid number of stems and extension statuses: %d != %d", len(stems), len(proof.ExtStatus))
+	}
 	var (
 		info  = map[string]stemInfo{}
 		paths [][]byte
 		err   error
 		poas  = proof.PoaStems
 	)
 
-	// assign one or more stem to each stem info
+	// The proof of absence stems must be sorted. If that isn't the case, the proof is invalid.
+	if !sort.IsSorted(bytesSlice(proof.PoaStems)) {
+		return nil, fmt.Errorf("proof of absence stems are not sorted")
+	}
+
+	// We build a cache of paths that have a presence extension status.
+	pathsWithExtPresent := map[string]struct{}{}
+	i := 0
 	for _, es := range proof.ExtStatus {
-		depth := es >> 3
-		path := stems[stemIndex][:depth]
+		if es&3 == extStatusPresent {
+			pathsWithExtPresent[string(stems[i][:es>>3])] = struct{}{}
+		}
+		i++
+	}
+
+	// assign one or more stem to each stem info
+	for i, es := range proof.ExtStatus {
 		si := stemInfo{
-			depth:    depth,
+			depth:    es >> 3,
 			stemType: es & 3,
 		}
+		path := stems[i][:si.depth]
 		switch si.stemType {
 		case extStatusAbsentEmpty:
+			// All keys that are part of a proof of absence, must contain empty
+			// prestate values. If that isn't the case, the proof is invalid.
+			for j := range proof.Keys { // TODO: DoS risk, use map or binary search.
+				if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil {
+					return nil, fmt.Errorf("proof of absence (empty) stem %x has a value", si.stem)
+				}
+			}
 		case extStatusAbsentOther:
+			// All keys that are part of a proof of absence, must contain empty
+			// prestate values. If that isn't the case, the proof is invalid.
+			for j := range proof.Keys { // TODO: DoS risk, use map or binary search.
+				if bytes.HasPrefix(proof.Keys[j], stems[i]) && proof.PreValues[j] != nil {
+					return nil, fmt.Errorf("proof of absence (other) stem %x has a value", si.stem)
+				}
+			}
+
+			// For this absent path, we must first check if this path contains a proof of presence.
+			// If that is the case, we don't have to do anything since the corresponding leaf will be
+			// constructed by that extension status (already processed or to be processed).
+			// In other case, we should get the stem from the list of proof of absence stems.
+			if _, ok := pathsWithExtPresent[string(path)]; ok {
+				continue
+			}
+
+			// Note that this path doesn't have proof of presence (previous if check above), but
+			// it can have multiple proof of absence. If a previous proof of absence had already
+			// created the stemInfo for this path, we don't have to do anything.
+			if _, ok := info[string(path)]; ok {
+				continue
+			}
+
 			si.stem = poas[0]
 			poas = poas[1:]
-		default:
-			// the first stem could be missing (e.g. the second stem in the
-			// group is the one that is present. Compare each key to the first
-			// stem, along the length of the path only.
-			stemPath := stems[stemIndex][:len(path)]
+		case extStatusPresent:
 			si.values = map[byte][]byte{}
-			for i, k := range proof.Keys {
-				if bytes.Equal(k[:len(path)], stemPath) && proof.PreValues[i] != nil {
-					si.values[k[31]] = proof.PreValues[i]
+			si.stem = stems[i]
+			for j, k := range proof.Keys { // TODO: DoS risk, use map or binary search.
+				if bytes.Equal(k[:31], si.stem) {
+					si.values[k[31]] = proof.PreValues[j]
 					si.has_c1 = si.has_c1 || (k[31] < 128)
 					si.has_c2 = si.has_c2 || (k[31] >= 128)
-					// This key has values, its stem is the one that
-					// is present.
-					si.stem = k[:31]
 				}
 			}
+		default:
+			return nil, fmt.Errorf("invalid extension status: %d", si.stemType)
 		}
 		info[string(path)] = si
 		paths = append(paths, path)
-
-		// Skip over all the stems that share the same path
-		// to the extension tree. This happens e.g. if two
-		// stems have the same path, but one is a proof of
-		// absence and the other one is present.
-		stemIndex++
-		for ; stemIndex < len(stems); stemIndex++ {
-			if !bytes.Equal(stems[stemIndex][:depth], path) {
-				break
-			}
-		}
 	}
 
 	if len(poas) != 0 {
@@ -513,3 +546,9 @@ func PostStateTreeFromStateDiff(preroot VerkleNode, statediff StateDiff) (Verkle
 
 	return postroot, nil
 }
+
+type bytesSlice [][]byte
+
+func (x bytesSlice) Len() int           { return len(x) }
+func (x bytesSlice) Less(i, j int) bool { return bytes.Compare(x[i], x[j]) < 0 }
+func (x bytesSlice) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }

proof_test.go

@@ -550,8 +550,8 @@ func TestProofOfAbsenceNoneMultipleStems(t *testing.T) {
 		t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems))
 	}
 
-	if len(proof.ExtStatus) != 1 {
-		t.Fatalf("invalid number of none extension statuses: %d ≠ 1", len(proof.ExtStatus))
+	if len(proof.ExtStatus) != 2 {
+		t.Fatalf("invalid number of extension statuses: %d ≠ 2", len(proof.ExtStatus))
 	}
 }
 
@@ -1016,6 +1016,47 @@ func TestProofOfAbsenceBorderCase(t *testing.T) {
 	}
 }
 
+func TestProofOfAbsenceBorderCaseReversed(t *testing.T) {
+	root := New()
+
+	key1, _ := hex.DecodeString("0001000000000000000000000000000000000000000000000000000000000001")
+	key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001")
+
+	// Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001
+	if err := root.Insert(key1, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+
+	// Generate a proof for the following keys:
+	// - key1, which is present.
+	// - key2, which isn't present.
+	// Note that all three keys will land on the same leaf value.
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key1, key2}, nil)
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := PreStateTreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !droot.Commit().Equal(root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) {
+		t.Fatal("differing commitment for child #0")
+	}
+}
+
 func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) {
 	t.Parallel()
 
@@ -1088,3 +1129,135 @@ func TestGenerateProofWithOnlyAbsentKeys(t *testing.T) {
 		}
 	}
 }
+
+func TestProofOfPresenceWithEmptyValue(t *testing.T) {
+	root := New()
+
+	key1, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001")
+
+	// Insert an arbitrary value at key 0000000000000000000000000000000000000000000000000000000000000001
+	if err := root.Insert(key1, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+
+	key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000002")
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2}, nil)
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := PreStateTreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !droot.Commit().Equal(root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	if !droot.(*InternalNode).children[0].Commit().Equal(root.(*InternalNode).children[0].Commit()) {
+		t.Fatal("differing commitment for child #0")
+	}
+}
+
+func TestDoubleProofOfAbsence(t *testing.T) {
+	root := New()
+
+	// Insert some keys.
+	key11, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000001")
+	key12, _ := hex.DecodeString("0003000000000000000000000000000000000000000000000000000000000001")
+
+	if err := root.Insert(key11, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+	if err := root.Insert(key12, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+
+	// Try to prove to different stems that end up in the same LeafNode without any other proof of presence
+	// in that leaf node. i.e: two proof of absence in the same leaf node with no proof of presence.
+	key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000100")
+	key3, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000200")
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2, key3}, nil)
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := PreStateTreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !droot.Commit().Equal(root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	// Depite we have two proof of absences for different steams, we should only have one
+	// stem in `others`. i.e: we only need one for both steams.
+	if len(proof.PoaStems) != 1 {
+		t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems))
+	}
+
+	// We need one extension status for each stem.
+	if len(proof.ExtStatus) != 2 {
+		t.Fatalf("invalid number of extension status: %d", len(proof.PoaStems))
+	}
+}
+
+func TestProveAbsenceInEmptyHalf(t *testing.T) {
+	root := New()
+
+	key1, _ := hex.DecodeString("00000000000000000000000000000000000000000000000000000000000000FF")
+
+	if err := root.Insert(key1, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+	if err := root.Insert(key1, fourtyKeyTest, nil); err != nil {
+		t.Fatalf("could not insert key: %v", err)
+	}
+
+	key2, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000100")
+	key3, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000000")
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, nil, keylist{key2, key3}, nil)
+
+	serialized, statediff, err := SerializeProof(proof)
+	if err != nil {
+		t.Fatalf("could not serialize proof: %v", err)
+	}
+
+	dproof, err := DeserializeProof(serialized, statediff)
+	if err != nil {
+		t.Fatalf("error deserializing proof: %v", err)
+	}
+
+	droot, err := PreStateTreeFromProof(dproof, root.Commit())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !droot.Commit().Equal(root.Commit()) {
+		t.Fatal("differing root commitments")
+	}
+
+	if len(proof.PoaStems) != 0 {
+		t.Fatalf("invalid number of proof-of-absence stems: %d", len(proof.PoaStems))
+	}
+
+	if len(proof.ExtStatus) != 2 {
+		t.Fatalf("invalid number of extension status: %d", len(proof.ExtStatus))
+	}
+}