Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wip] SSZ-optimize proof (type 3) #447

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

[wip] SSZ-optimize proof (type 3) #447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f9a3cbb
ssz type 3
jsign Jul 26, 2024
722a15c
fix
jsign Jul 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 121 additions & 68 deletions proof_ipa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"errors"
"fmt"
"sort"
"unsafe"

ipa "github.com/crate-crypto/go-ipa"
"github.com/crate-crypto/go-ipa/common"
Expand Down Expand Up @@ -83,17 +82,33 @@ type Proof struct {
PostValues [][]byte
}

type SuffixStateDiff struct {
Suffix byte `json:"suffix"`
CurrentValue *[32]byte `json:"currentValue"`
NewValue *[32]byte `json:"newValue"`
type StemStateDiff struct {
Stem [StemSize]byte `json:"stem"`

Updates []UpdateDiff `json:"updates"`
Reads []ReadDiff `json:"reads"`
Inserts []InsertDiff `json:"inserts"`
Missing []MissingDiff `json:"missing"`
}

type SuffixStateDiffs []SuffixStateDiff
type UpdateDiff struct {
Suffix byte `json:"suffix"`
Current [32]byte `json:"current"`
New [32]byte `json:"new"`
}

type StemStateDiff struct {
Stem [StemSize]byte `json:"stem"`
SuffixDiffs SuffixStateDiffs `json:"suffixDiffs"`
type ReadDiff struct {
Suffix byte `json:"suffix"`
Current [32]byte `json:"current"`
}

type InsertDiff struct {
Suffix byte `json:"suffix"`
New [32]byte `json:"new"`
}

type MissingDiff struct {
Suffix byte `json:"suffix"`
}

type StateDiff []StemStateDiff
Expand All @@ -102,18 +117,28 @@ func (sd StateDiff) Copy() StateDiff {
ret := make(StateDiff, len(sd))
for i := range sd {
copy(ret[i].Stem[:], sd[i].Stem[:])
ret[i].SuffixDiffs = make([]SuffixStateDiff, len(sd[i].SuffixDiffs))
for j := range sd[i].SuffixDiffs {
ret[i].SuffixDiffs[j].Suffix = sd[i].SuffixDiffs[j].Suffix
if sd[i].SuffixDiffs[j].CurrentValue != nil {
ret[i].SuffixDiffs[j].CurrentValue = &[32]byte{}
copy((*ret[i].SuffixDiffs[j].CurrentValue)[:], (*sd[i].SuffixDiffs[j].CurrentValue)[:])
}
if sd[i].SuffixDiffs[j].NewValue != nil {
ret[i].SuffixDiffs[j].NewValue = &[32]byte{}
copy((*ret[i].SuffixDiffs[j].NewValue)[:], (*sd[i].SuffixDiffs[j].NewValue)[:])
}

ret[i].Updates = make([]UpdateDiff, len(sd[i].Updates))
for j := range sd[i].Updates {
ret[i].Updates[j].Suffix = sd[i].Updates[j].Suffix
ret[i].Updates[j].Current = sd[i].Updates[j].Current
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we stop having []byte, many of if len(xxx) == 0 cases and similar are gone. We can always copy-by-value.

ret[i].Updates[j].New = sd[i].Updates[j].New
}

ret[i].Reads = make([]ReadDiff, len(sd[i].Reads))
for j := range sd[i].Reads {
ret[i].Reads[j].Suffix = sd[i].Reads[j].Suffix
ret[i].Reads[j].Current = sd[i].Reads[j].Current
}

ret[i].Inserts = make([]InsertDiff, len(sd[i].Inserts))
for j := range sd[i].Inserts {
ret[i].Inserts[j].Suffix = sd[i].Inserts[j].Suffix
ret[i].Inserts[j].New = sd[i].Inserts[j].New
}

ret[i].Missing = make([]MissingDiff, len(sd[i].Missing))
copy(ret[i].Missing, sd[i].Missing)
}
return ret
}
Expand Down Expand Up @@ -221,6 +246,16 @@ func VerifyVerkleProof(proof *Proof, Cs []*Point, indices []uint8, ys []*Fr, tc
return ipa.CheckMultiProof(tr, tc.conf, proof.Multipoint, Cs, ys, indices)
}

func isInsertion(preLen, postLen int) bool {
return preLen == 0 && postLen != 0
}
func isRead(preLen, postLen int) bool {
return preLen != 0 && postLen == 0
}
func isUpdate(preLen, postLen int) bool {
return preLen != 0 && postLen != 0
}

// SerializeProof serializes the proof in the rust-verkle format:
// * len(Proof of absence stem) || Proof of absence stems
// * len(depths) || serialize(depth || ext statusi)
Expand Down Expand Up @@ -257,32 +292,35 @@ func SerializeProof(proof *Proof) (*VerkleProof, StateDiff, error) {
stemdiff = &statediff[len(statediff)-1]
copy(stemdiff.Stem[:], stem)
}
stemdiff.SuffixDiffs = append(stemdiff.SuffixDiffs, SuffixStateDiff{Suffix: key[StemSize]})
newsd := &stemdiff.SuffixDiffs[len(stemdiff.SuffixDiffs)-1]

var valueLen = len(proof.PreValues[i])
switch valueLen {
case 0:
// null value
case 32:
newsd.CurrentValue = (*[32]byte)(proof.PreValues[i])
preLen := len(proof.PreValues[i])
postLen := len(proof.PostValues[i])
switch {
case isInsertion(preLen, postLen):
var newValue [32]byte
copy(newValue[:], proof.PostValues[i])
Copy link
Collaborator Author

@jsign jsign Jul 26, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that we don't have to do the postLen == 32 and all the alignment thing.

If proof.PostValues[i] isn't 32 bytes, then copy will copy whatever amount of bytes there are (in the front of the slice). So, both in aligned and unaligned cases, the right thing will happen.

Same applies to other cases.

stemdiff.Inserts = append(stemdiff.Inserts, InsertDiff{
Suffix: key[StemSize],
New: newValue,
})
case isRead(preLen, postLen):
var currentValue [32]byte
copy(currentValue[:], proof.PreValues[i])
stemdiff.Reads = append(stemdiff.Reads, ReadDiff{
Suffix: key[StemSize],
Current: currentValue,
})
case isUpdate(preLen, postLen):
var currentValue [32]byte
copy(currentValue[:], proof.PreValues[i])
var newValue [32]byte
copy(newValue[:], proof.PostValues[i])
stemdiff.Updates = append(stemdiff.Updates, UpdateDiff{
Suffix: key[StemSize],
Current: currentValue,
New: newValue,
})
default:
var aligned [32]byte
copy(aligned[:valueLen], proof.PreValues[i])
newsd.CurrentValue = (*[32]byte)(unsafe.Pointer(&aligned[0]))
}

valueLen = len(proof.PostValues[i])
switch valueLen {
case 0:
// null value
case 32:
newsd.NewValue = (*[32]byte)(proof.PostValues[i])
default:
// TODO remove usage of unsafe
var aligned [32]byte
copy(aligned[:valueLen], proof.PostValues[i])
newsd.NewValue = (*[32]byte)(unsafe.Pointer(&aligned[0]))
stemdiff.Missing = append(stemdiff.Missing, MissingDiff{Suffix: key[StemSize]})
}
}

Expand Down Expand Up @@ -347,22 +385,37 @@ func DeserializeProof(vp *VerkleProof, statediff StateDiff) (*Proof, error) {

// turn statediff into keys and values
for _, stemdiff := range statediff {
for _, suffixdiff := range stemdiff.SuffixDiffs {
for _, upt := range stemdiff.Updates {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = suffixdiff.Suffix
k[StemSize] = upt.Suffix
keys = append(keys, k[:])
if suffixdiff.CurrentValue != nil {
prevalues = append(prevalues, suffixdiff.CurrentValue[:])
} else {
prevalues = append(prevalues, nil)
}

if suffixdiff.NewValue != nil {
postvalues = append(postvalues, suffixdiff.NewValue[:])
} else {
postvalues = append(postvalues, nil)
}
prevalues = append(prevalues, upt.Current[:])
postvalues = append(postvalues, upt.New[:])
}
for _, ins := range stemdiff.Inserts {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = ins.Suffix
keys = append(keys, k[:])
prevalues = append(prevalues, nil)
postvalues = append(postvalues, ins.New[:])
}
for _, rd := range stemdiff.Reads {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = rd.Suffix
keys = append(keys, k[:])
prevalues = append(prevalues, rd.Current[:])
postvalues = append(postvalues, nil)
}
for _, mi := range stemdiff.Missing {
var k [32]byte
copy(k[:StemSize], stemdiff.Stem[:])
k[StemSize] = mi.Suffix
keys = append(keys, k[:])
prevalues = append(prevalues, nil)
postvalues = append(postvalues, nil)
}
}

Expand Down Expand Up @@ -524,20 +577,20 @@ func PostStateTreeFromStateDiff(preroot VerkleNode, statediff StateDiff) (Verkle

for _, stemstatediff := range statediff {
var (
values = make([][]byte, NodeWidth)
overwrites bool
values = make([][]byte, NodeWidth)
overwrite bool
)

for _, suffixdiff := range stemstatediff.SuffixDiffs {
if /* len(suffixdiff.NewValue) > 0 - this only works for a slice */ suffixdiff.NewValue != nil {
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This previous if (that was kept in Type 2) was removed. This can also be removed in Type 2 as I mentioned in my review in that PR.

Both in Inserts and Updates, we can't have "nil" or empty values in New.

// if this value is non-nil, it means InsertValuesAtStem should be
// called, otherwise, skip updating the tree.
overwrites = true
values[suffixdiff.Suffix] = suffixdiff.NewValue[:]
}
for _, ins := range stemstatediff.Inserts {
values[ins.Suffix] = ins.New[:]
overwrite = true
}
for _, upd := range stemstatediff.Updates {
values[upd.Suffix] = upd.New[:]
overwrite = true
}

if overwrites {
if overwrite {
var stem [StemSize]byte
copy(stem[:StemSize], stemstatediff.Stem[:])
if err := postroot.(*InternalNode).InsertValuesAtStem(stem[:], values, nil); err != nil {
Expand Down
104 changes: 32 additions & 72 deletions proof_json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
"encoding/hex"
"encoding/json"
"fmt"
"sort"
)

// HexToPrefixedString turns a byte slice into its hex representation
Expand Down Expand Up @@ -183,92 +184,51 @@
return nil
}

// TODO use gencodec if that works
type stemStateDiffMarshaller struct {
Stem string `json:"stem"`
SuffixDiffs SuffixStateDiffs `json:"suffixDiffs"`
Stem string `json:"stem"`
Suffixes string `json:"suffixes"`
Current []string `json:"current"`
New []string `json:"new"`
}

func (ssd StemStateDiff) MarshalJSON() ([]byte, error) {
suffixes := make([]byte, 0, len(ssd.Updates)+len(ssd.Inserts))
for i := range ssd.Updates {
suffixes = append(suffixes, ssd.Updates[i].Suffix)
}
for i := range ssd.Inserts {
suffixes = append(suffixes, ssd.Inserts[i].Suffix)
}
sort.Slice(suffixes, func(i, j int) bool { return suffixes[i] < suffixes[j] })
Comment on lines +196 to +203
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think in Type 2 there might be a bug since it's only inserting Updates, but prob we need both Updates+Inserts as I did here?
I also sort them since I'd guess that's expected.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that's possible, I'm only focusing on the SSZ encoding at the moment. We'll pick the best method and fix it if need be.

return json.Marshal(&stemStateDiffMarshaller{
Stem: HexToPrefixedString(ssd.Stem[:]),
SuffixDiffs: ssd.SuffixDiffs,
Stem: HexToPrefixedString(ssd.Stem[:]),
Suffixes: HexToPrefixedString(suffixes),
// Current:
// TODO implement if it makes sense.
})
}

func (ssd *StemStateDiff) UnmarshalJSON(data []byte) error {
var aux stemStateDiffMarshaller
if err := json.Unmarshal(data, &aux); err != nil {
return fmt.Errorf("stemdiff unmarshal error: %w", err)
}

stem, err := PrefixedHexStringToBytes(aux.Stem)
if err != nil {
return fmt.Errorf("invalid hex string for stem: %w", err)
}
*ssd = StemStateDiff{
SuffixDiffs: aux.SuffixDiffs,
}
copy(ssd.Stem[:], stem)
// var aux stemStateDiffMarshaller
// if err := json.Unmarshal(data, &aux); err != nil {
// return fmt.Errorf("stemdiff unmarshal error: %w", err)
// }

// stem, err := PrefixedHexStringToBytes(aux.Stem)
// if err != nil {
// return fmt.Errorf("invalid hex string for stem: %w", err)
// }
// *ssd = StemStateDiff{
// SuffixDiffs: aux.SuffixDiffs,
// }
// copy(ssd.Stem[:], stem)
// TODO implement if it makes sense
return nil
}

type suffixStateDiffMarshaller struct {

Check failure on line 230 in proof_json.go

View workflow job for this annotation

GitHub Actions / lint 

type `suffixStateDiffMarshaller` is unused (unused)
Suffix byte `json:"suffix"`
CurrentValue *string `json:"currentValue"`
NewValue *string `json:"newValue"`
}

func (ssd SuffixStateDiff) MarshalJSON() ([]byte, error) {
var cvstr, nvstr *string
if ssd.CurrentValue != nil {
tempstr := HexToPrefixedString(ssd.CurrentValue[:])
cvstr = &tempstr
}
if ssd.NewValue != nil {
tempstr := HexToPrefixedString(ssd.NewValue[:])
nvstr = &tempstr
}
return json.Marshal(&suffixStateDiffMarshaller{
Suffix: ssd.Suffix,
CurrentValue: cvstr,
NewValue: nvstr,
})
}

func (ssd *SuffixStateDiff) UnmarshalJSON(data []byte) error {
aux := &suffixStateDiffMarshaller{}

if err := json.Unmarshal(data, &aux); err != nil {
return fmt.Errorf("suffix diff unmarshal error: %w", err)
}

if aux.CurrentValue != nil && len(*aux.CurrentValue) != 64 && len(*aux.CurrentValue) != 0 && len(*aux.CurrentValue) != 66 {
return fmt.Errorf("invalid hex string for current value: %s", *aux.CurrentValue)
}

*ssd = SuffixStateDiff{
Suffix: aux.Suffix,
}

if aux.CurrentValue != nil && len(*aux.CurrentValue) != 0 {
currentValueBytes, err := PrefixedHexStringToBytes(*aux.CurrentValue)
if err != nil {
return fmt.Errorf("error decoding hex string for current value: %v", err)
}

ssd.CurrentValue = &[32]byte{}
copy(ssd.CurrentValue[:], currentValueBytes)
}

if aux.NewValue != nil && len(*aux.NewValue) != 0 {
newValueBytes, err := PrefixedHexStringToBytes(*aux.NewValue)
if err != nil {
return fmt.Errorf("error decoding hex string for current value: %v", err)
}

ssd.NewValue = &[32]byte{}
copy(ssd.NewValue[:], newValueBytes)
}

return nil
}
Loading
Loading