From 95e294bcdfc6192454eee1908b9677c4bba9557c Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 14:51:04 +0100 Subject: [PATCH 01/13] Add partial-match travesal of large bytes --- node/basicnode/bytes.go | 13 ++- node/basicnode/bytes_stream.go | 81 ++++++++++++++++++ traversal/selector/builder/builder.go | 14 +++ traversal/selector/exploreAll.go | 5 ++ traversal/selector/exploreFields.go | 5 ++ traversal/selector/exploreIndex.go | 5 ++ traversal/selector/exploreInterpretAs.go | 5 ++ traversal/selector/exploreRange.go | 5 ++ traversal/selector/exploreRecursive.go | 5 ++ traversal/selector/exploreRecursiveEdge.go | 5 ++ traversal/selector/exploreUnion.go | 12 +++ traversal/selector/fieldKeys.go | 3 + traversal/selector/matcher.go | 99 +++++++++++++++++++++- traversal/selector/matcher_util.go | 15 ++++ traversal/selector/selector.go | 6 ++ traversal/walk.go | 6 +- 16 files changed, 278 insertions(+), 6 deletions(-) create mode 100644 node/basicnode/bytes_stream.go create mode 100644 traversal/selector/matcher_util.go diff --git a/node/basicnode/bytes.go b/node/basicnode/bytes.go index 7c45d208..566c32fe 100644 --- a/node/basicnode/bytes.go +++ b/node/basicnode/bytes.go @@ -106,7 +106,7 @@ func (nb *plainBytes__Builder) Reset() { // -- NodeAssembler --> type plainBytes__Assembler struct { - w *plainBytes + w datamodel.Node } func (plainBytes__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { @@ -131,17 +131,24 @@ func (plainBytes__Assembler) AssignString(string) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignString("") } func (na *plainBytes__Assembler) AssignBytes(v []byte) error { - *na.w = plainBytes(v) + na.w = datamodel.Node(plainBytes(v)) return nil } func (plainBytes__Assembler) AssignLink(datamodel.Link) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignLink(nil) } func (na *plainBytes__Assembler) AssignNode(v datamodel.Node) error { + if lb, ok := v.(datamodel.LargeBytesNode); ok { + lbn, err := lb.AsLargeBytes() + if err == nil { + na.w = streamBytes{lbn} + return nil + } + } if v2, err := v.AsBytes(); err != nil { return err } else { - *na.w = plainBytes(v2) + na.w = plainBytes(v2) return nil } } diff --git a/node/basicnode/bytes_stream.go b/node/basicnode/bytes_stream.go new file mode 100644 index 00000000..ad238bcf --- /dev/null +++ b/node/basicnode/bytes_stream.go @@ -0,0 +1,81 @@ +package basicnode + +import ( + "io" + + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/mixins" +) + +var ( + _ datamodel.Node = streamBytes{nil} + _ datamodel.NodePrototype = Prototype__Bytes{} + _ datamodel.NodeBuilder = &plainBytes__Builder{} + _ datamodel.NodeAssembler = &plainBytes__Assembler{} +) + +func NewBytesFromReader(rs io.ReadSeeker) datamodel.Node { + return streamBytes{rs} +} + +// streamBytes is a boxed reader that complies with datamodel.Node. +type streamBytes struct { + io.ReadSeeker +} + +// -- Node interface methods --> + +func (streamBytes) Kind() datamodel.Kind { + return datamodel.Kind_Bytes +} +func (streamBytes) LookupByString(string) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByString("") +} +func (streamBytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByNode(nil) +} +func (streamBytes) LookupByIndex(idx int64) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByIndex(0) +} +func (streamBytes) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupBySegment(seg) +} +func (streamBytes) MapIterator() datamodel.MapIterator { + return nil +} +func (streamBytes) ListIterator() datamodel.ListIterator { + return nil +} +func (streamBytes) Length() int64 { + return -1 +} +func (streamBytes) IsAbsent() bool { + return false +} +func (streamBytes) IsNull() bool { + return false +} +func (streamBytes) AsBool() (bool, error) { + return mixins.Bytes{TypeName: "bytes"}.AsBool() +} +func (streamBytes) AsInt() (int64, error) { + return mixins.Bytes{TypeName: "bytes"}.AsInt() +} +func (streamBytes) AsFloat() (float64, error) { + return mixins.Bytes{TypeName: "bytes"}.AsFloat() +} +func (streamBytes) AsString() (string, error) { + return mixins.Bytes{TypeName: "bytes"}.AsString() +} +func (n streamBytes) AsBytes() ([]byte, error) { + return io.ReadAll(n) +} +func (streamBytes) AsLink() (datamodel.Link, error) { + return mixins.Bytes{TypeName: "bytes"}.AsLink() +} +func (streamBytes) Prototype() datamodel.NodePrototype { + return Prototype__Bytes{} +} +func (n streamBytes) AsLargeBytes() (io.ReadSeeker, error) { + return n.ReadSeeker, nil +} diff --git a/traversal/selector/builder/builder.go b/traversal/selector/builder/builder.go index 80067758..6cc69e3c 100644 --- a/traversal/selector/builder/builder.go +++ b/traversal/selector/builder/builder.go @@ -32,6 +32,7 @@ type SelectorSpecBuilder interface { ExploreFields(ExploreFieldsSpecBuildingClosure) SelectorSpec ExploreInterpretAs(as string, next SelectorSpec) SelectorSpec Matcher() SelectorSpec + MatcherSubset(from, to int64) SelectorSpec } // ExploreFieldsSpecBuildingClosure is a function that provided to SelectorSpecBuilder's @@ -170,6 +171,19 @@ func (ssb *selectorSpecBuilder) Matcher() SelectorSpec { } } +func (ssb *selectorSpecBuilder) MatcherSubset(from, to int64) SelectorSpec { + return selectorSpec{ + fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(1, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_Subset).CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_From).AssignInt(from) + na.AssembleEntry(selector.SelectorKey_To).AssignInt(to) + }) + }) + }), + } +} + type exploreFieldsSpecBuilder struct { na fluent.MapAssembler } diff --git a/traversal/selector/exploreAll.go b/traversal/selector/exploreAll.go index a7749f8e..7d889d7e 100644 --- a/traversal/selector/exploreAll.go +++ b/traversal/selector/exploreAll.go @@ -27,6 +27,11 @@ func (s ExploreAll) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreAll) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreAll assembles a Selector from a ExploreAll selector node func (pc ParseContext) ParseExploreAll(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { diff --git a/traversal/selector/exploreFields.go b/traversal/selector/exploreFields.go index 5d2abdd2..b2796ff2 100644 --- a/traversal/selector/exploreFields.go +++ b/traversal/selector/exploreFields.go @@ -38,6 +38,11 @@ func (s ExploreFields) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreFields) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreFields assembles a Selector // from a ExploreFields selector node func (pc ParseContext) ParseExploreFields(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreIndex.go b/traversal/selector/exploreIndex.go index 0c8edf16..56a2badc 100644 --- a/traversal/selector/exploreIndex.go +++ b/traversal/selector/exploreIndex.go @@ -37,6 +37,11 @@ func (s ExploreIndex) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreIndex) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreIndex assembles a Selector // from a ExploreIndex selector node func (pc ParseContext) ParseExploreIndex(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreInterpretAs.go b/traversal/selector/exploreInterpretAs.go index 25cf045b..6f4aed67 100644 --- a/traversal/selector/exploreInterpretAs.go +++ b/traversal/selector/exploreInterpretAs.go @@ -27,6 +27,11 @@ func (s ExploreInterpretAs) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreInterpretAs) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // NamedReifier indicates how this selector expects to Reify the current datamodel.Node. func (s ExploreInterpretAs) NamedReifier() string { return s.adl diff --git a/traversal/selector/exploreRange.go b/traversal/selector/exploreRange.go index 73e356ba..e9e041bc 100644 --- a/traversal/selector/exploreRange.go +++ b/traversal/selector/exploreRange.go @@ -41,6 +41,11 @@ func (s ExploreRange) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreRange) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreRange assembles a Selector // from a ExploreRange selector node func (pc ParseContext) ParseExploreRange(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreRecursive.go b/traversal/selector/exploreRecursive.go index 17614025..28a9bedc 100644 --- a/traversal/selector/exploreRecursive.go +++ b/traversal/selector/exploreRecursive.go @@ -176,6 +176,11 @@ func (s ExploreRecursive) Decide(n datamodel.Node) bool { return s.current.Decide(n) } +// Match always returns false because this is not a matcher +func (s ExploreRecursive) Match(node datamodel.Node) (datamodel.Node, error) { + return s.current.Match(node) +} + type exploreRecursiveContext struct { edgesFound int } diff --git a/traversal/selector/exploreRecursiveEdge.go b/traversal/selector/exploreRecursiveEdge.go index 65438572..534c73b8 100644 --- a/traversal/selector/exploreRecursiveEdge.go +++ b/traversal/selector/exploreRecursiveEdge.go @@ -31,6 +31,11 @@ func (s ExploreRecursiveEdge) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreRecursiveEdge) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreRecursiveEdge assembles a Selector // from a exploreRecursiveEdge selector node func (pc ParseContext) ParseExploreRecursiveEdge(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreUnion.go b/traversal/selector/exploreUnion.go index 0af42cd6..21594954 100644 --- a/traversal/selector/exploreUnion.go +++ b/traversal/selector/exploreUnion.go @@ -74,6 +74,18 @@ func (s ExploreUnion) Decide(n datamodel.Node) bool { return false } +// Match returns true for a Union selector based on the matched union. +func (s ExploreUnion) Match(n datamodel.Node) (datamodel.Node, error) { + for _, m := range s.Members { + if mn, err := m.Match(n); mn != nil { + return mn, nil + } else if err != nil { + return nil, err + } + } + return nil, nil +} + // ParseExploreUnion assembles a Selector // from an ExploreUnion selector node func (pc ParseContext) ParseExploreUnion(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/fieldKeys.go b/traversal/selector/fieldKeys.go index 58024b61..39ec61e6 100644 --- a/traversal/selector/fieldKeys.go +++ b/traversal/selector/fieldKeys.go @@ -23,5 +23,8 @@ const ( SelectorKey_StopAt = "!" SelectorKey_Condition = "&" SelectorKey_As = "as" + SelectorKey_Subset = "subset" + SelectorKey_From = "[" + SelectorKey_To = "]" // not filling conditional keys since it's not complete ) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index b8f64f68..d9538dc6 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -2,8 +2,10 @@ package selector import ( "fmt" + "io" "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" ) // Matcher marks a node to be included in the "result" set. @@ -16,7 +18,61 @@ import ( // A selector tree with only "explore*"-type selectors and no Matcher selectors // is valid; it will just generate a "covered" set of nodes and no "result" set. // TODO: From spec: implement conditions and labels -type Matcher struct{} +type Matcher struct { + *Slice +} + +// Slice limits a result node to a subset of the node. +// The returned node will be limited based on slicing the specified range of the +// node into a new node, or making use of the `AsLargeBytes` io.ReadSeeker to +// restrict response with a SectionReader. +type Slice struct { + From int64 + To int64 +} + +func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { + if n.Kind() == datamodel.Kind_String { + str, err := n.AsString() + if err != nil { + return nil, err + } + to := s.To + if len(str) < int(to) { + to = int64(len(str)) + } + from := s.From + if len(str) < int(from) { + from = int64(len(str)) + } + return basicnode.NewString(str[from:to]), nil + } else if n.Kind() == datamodel.Kind_Bytes { + if lbn, ok := n.(datamodel.LargeBytesNode); ok { + rdr, err := lbn.AsLargeBytes() + if err != nil { + return nil, err + } + + sr := io.NewSectionReader(readerat{rdr}, int64(s.From), int64(s.To)) + return basicnode.NewBytesFromReader(sr), nil + } + bytes, err := n.AsBytes() + if err != nil { + return nil, err + } + to := s.To + if len(bytes) < int(to) { + to = int64(len(bytes)) + } + from := s.From + if len(bytes) < int(from) { + from = int64(len(bytes)) + } + + return basicnode.NewBytes(bytes[from:to]), nil + } + return nil, fmt.Errorf("selector slice rejected: subset match must be over string or bytes") +} // Interests are empty for a matcher (for now) because // It is always just there to match, not explore further @@ -35,6 +91,14 @@ func (s Matcher) Decide(n datamodel.Node) bool { return true } +// Match is always true for a match cause it's in the result set +func (s Matcher) Match(node datamodel.Node) (datamodel.Node, error) { + if s.Slice != nil { + return s.Slice.Slice(node) + } + return node, nil +} + // ParseMatcher assembles a Selector // from a matcher selector node // TODO: Parse labels and conditions @@ -42,5 +106,38 @@ func (pc ParseContext) ParseMatcher(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } + + // check if a slice is specified + if subset, err := n.LookupByString("subset"); err == nil { + if subset.Kind() != datamodel.Kind_Map { + return nil, fmt.Errorf("selector spec parse rejected: subset body must be a map") + } + from, err := subset.LookupByString("[") + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a from '[' key") + } + fromN, err := from.AsInt() + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is a number") + } + to, err := subset.LookupByString("]") + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a to ']' key") + } + toN, err := to.AsInt() + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'to' key that is a number") + } + if fromN > toN { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is less than or equal to the 'to' key") + } + if fromN < 0 || toN < 0 { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with keys not less than 0") + } + return Matcher{&Slice{ + From: fromN, + To: toN, + }}, nil + } return Matcher{}, nil } diff --git a/traversal/selector/matcher_util.go b/traversal/selector/matcher_util.go new file mode 100644 index 00000000..b67cad09 --- /dev/null +++ b/traversal/selector/matcher_util.go @@ -0,0 +1,15 @@ +package selector + +import "io" + +type readerat struct { + io.ReadSeeker +} + +func (r readerat) ReadAt(p []byte, off int64) (n int, err error) { + _, err = r.Seek(off, 0) + if err != nil { + return 0, err + } + return r.Read(p) +} diff --git a/traversal/selector/selector.go b/traversal/selector/selector.go index 4d0a7814..b58fcd86 100644 --- a/traversal/selector/selector.go +++ b/traversal/selector/selector.go @@ -106,6 +106,12 @@ type Selector interface { // Only "Matcher" clauses actually implement this in a way that ever returns "true". // See the Selector specs for discussion on "matched" vs "reached"/"visited" nodes. Decide(node datamodel.Node) bool + + // Match is an extension to Decide allowing the matcher to `decide` a transformation of + // the matched node. This is used for `Subset` match behavior. If the node is matched, + // the first argument will be the matched node. If it is not matched, the first argument + // will be null. If there is an error, the first argument will be null. + Match(node datamodel.Node) (datamodel.Node, error) } // REVIEW: do ParsedParent and ParseContext need to be exported? They're mostly used during the compilation process. diff --git a/traversal/walk.go b/traversal/walk.go index 80250170..0f659189 100644 --- a/traversal/walk.go +++ b/traversal/walk.go @@ -182,10 +182,12 @@ func (prog Progress) walkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitF } // Decide if this node is matched -- do callbacks as appropriate. - if s.Decide(n) { - if err := fn(prog, n, VisitReason_SelectionMatch); err != nil { + if match, err := s.Match(n); match != nil { + if err := fn(prog, match, VisitReason_SelectionMatch); err != nil { return err } + } else if err != nil { + return err } else { if err := fn(prog, n, VisitReason_SelectionCandidate); err != nil { return err From 3f71e8e6e3c932bb6ffb7f47cafbbbb0413b7f0f Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 15:28:23 +0100 Subject: [PATCH 02/13] bytes spec test --- node/basicnode/bytes_test.go | 12 ++++++++++++ node/tests/byteSpecs.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 node/basicnode/bytes_test.go create mode 100644 node/tests/byteSpecs.go diff --git a/node/basicnode/bytes_test.go b/node/basicnode/bytes_test.go new file mode 100644 index 00000000..ca408484 --- /dev/null +++ b/node/basicnode/bytes_test.go @@ -0,0 +1,12 @@ +package basicnode_test + +import ( + "testing" + + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/node/tests" +) + +func TestBytes(t *testing.T) { + tests.SpecTestBytes(t, basicnode.Prototype__Bytes{}) +} diff --git a/node/tests/byteSpecs.go b/node/tests/byteSpecs.go new file mode 100644 index 00000000..69bf494b --- /dev/null +++ b/node/tests/byteSpecs.go @@ -0,0 +1,35 @@ +package tests + +import ( + "io" + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/ipld/go-ipld-prime/datamodel" +) + +func SpecTestBytes(t *testing.T, np datamodel.NodePrototype) { + t.Run("byte node", func(t *testing.T) { + nb := np.NewBuilder() + err := nb.AssignBytes([]byte("asdf")) + qt.Check(t, err, qt.IsNil) + n := nb.Build() + + qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Bytes) + qt.Check(t, n.IsNull(), qt.IsFalse) + x, err := n.AsBytes() + qt.Check(t, err, qt.IsNil) + qt.Check(t, x, qt.DeepEquals, []byte("asdf")) + + lbn, ok := n.(datamodel.LargeBytesNode) + if ok { + str, err := lbn.AsLargeBytes() + qt.Check(t, err, qt.IsNil) + bytes, err := io.ReadAll(str) + qt.Check(t, err, qt.IsNil) + qt.Check(t, bytes, qt.DeepEquals, []byte("asdf")) + } + + }) +} From 981297b3c950ea89788478a0dbcf2c9b06914c0b Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 15:32:55 +0100 Subject: [PATCH 03/13] extend traversal to handle partial byte range match access behavior --- traversal/selector/spec_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/traversal/selector/spec_test.go b/traversal/selector/spec_test.go index 215bfd6b..7d309d05 100644 --- a/traversal/selector/spec_test.go +++ b/traversal/selector/spec_test.go @@ -2,6 +2,7 @@ package selector_test import ( "bytes" + "io" "os" "testing" @@ -14,7 +15,7 @@ import ( "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" - "github.com/ipld/go-ipld-prime/traversal/selector/parse" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) func TestSpecFixtures(t *testing.T) { @@ -81,6 +82,18 @@ func testOneSpecFixtureFile(t *testing.T, filename string) { })) qp.MapEntry(ma, "matched", qp.Bool(reason == traversal.VisitReason_SelectionMatch)) }) + if reason == traversal.VisitReason_SelectionMatch && n.Kind() == datamodel.Kind_Bytes { + if lbn, ok := n.(datamodel.LargeBytesNode); ok { + rdr, err := lbn.AsLargeBytes() + if err == nil { + io.Copy(io.Discard, rdr) + } + } + _, err := n.AsBytes() + if err != nil { + panic("insanity at a deeper level than this test's target") + } + } if err != nil { panic("insanity at a deeper level than this test's target") } From 5a41f7fcbac8f7e699b57ca22cce62bc2761d448 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 15:47:06 +0100 Subject: [PATCH 04/13] code review --- traversal/selector/matcher.go | 19 +++++++++++-------- traversal/selector/matcher_util.go | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index d9538dc6..38314db4 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -32,46 +32,49 @@ type Slice struct { } func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { - if n.Kind() == datamodel.Kind_String { + var from, to int64 + switch n.Kind() { + case datamodel.Kind_String: str, err := n.AsString() if err != nil { return nil, err } - to := s.To + to = s.To if len(str) < int(to) { to = int64(len(str)) } - from := s.From + from = s.From if len(str) < int(from) { from = int64(len(str)) } return basicnode.NewString(str[from:to]), nil - } else if n.Kind() == datamodel.Kind_Bytes { + case datamodel.Kind_Bytes: if lbn, ok := n.(datamodel.LargeBytesNode); ok { rdr, err := lbn.AsLargeBytes() if err != nil { return nil, err } - sr := io.NewSectionReader(readerat{rdr}, int64(s.From), int64(s.To)) + sr := io.NewSectionReader(readerat{rdr}, s.From, s.To) return basicnode.NewBytesFromReader(sr), nil } bytes, err := n.AsBytes() if err != nil { return nil, err } - to := s.To + to = s.To if len(bytes) < int(to) { to = int64(len(bytes)) } - from := s.From + from = s.From if len(bytes) < int(from) { from = int64(len(bytes)) } return basicnode.NewBytes(bytes[from:to]), nil + default: + return nil, fmt.Errorf("selector slice rejected: subset match must be over string or bytes") } - return nil, fmt.Errorf("selector slice rejected: subset match must be over string or bytes") } // Interests are empty for a matcher (for now) because diff --git a/traversal/selector/matcher_util.go b/traversal/selector/matcher_util.go index b67cad09..b60d11c8 100644 --- a/traversal/selector/matcher_util.go +++ b/traversal/selector/matcher_util.go @@ -7,6 +7,7 @@ type readerat struct { } func (r readerat) ReadAt(p []byte, off int64) (n int, err error) { + // TODO: consider keeping track of current offset. _, err = r.Seek(off, 0) if err != nil { return 0, err From e0f4c23806e3f65a03530a55db5c4f7040dce062 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 16:00:28 +0100 Subject: [PATCH 05/13] warning on readerat --- traversal/selector/matcher_util.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/traversal/selector/matcher_util.go b/traversal/selector/matcher_util.go index b60d11c8..237c5e71 100644 --- a/traversal/selector/matcher_util.go +++ b/traversal/selector/matcher_util.go @@ -6,6 +6,10 @@ type readerat struct { io.ReadSeeker } +// ReadAt provides the io.ReadAt method over a ReadSeeker. +// This implementation does not support concurrent calls to `ReadAt`, +// as specified by the ReaderAt interface, and so must only be used +// in non-concurrent use cases. func (r readerat) ReadAt(p []byte, off int64) (n int, err error) { // TODO: consider keeping track of current offset. _, err = r.Seek(off, 0) From 1a0310848f49d81c24dd293acee95ca254312599 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 19:06:38 +0100 Subject: [PATCH 06/13] fix section reader --- traversal/selector/matcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index 38314db4..4186221f 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -55,7 +55,7 @@ func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { return nil, err } - sr := io.NewSectionReader(readerat{rdr}, s.From, s.To) + sr := io.NewSectionReader(readerat{rdr}, s.From, s.To-s.From) return basicnode.NewBytesFromReader(sr), nil } bytes, err := n.AsBytes() From 40773754af1c432c5a67599cbab165b57c9253ba Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 14:51:04 +0100 Subject: [PATCH 07/13] Add partial-match travesal of large bytes --- node/basicnode/bytes.go | 13 ++- node/basicnode/bytes_stream.go | 81 ++++++++++++++++++ traversal/selector/builder/builder.go | 14 +++ traversal/selector/exploreAll.go | 5 ++ traversal/selector/exploreFields.go | 5 ++ traversal/selector/exploreIndex.go | 5 ++ traversal/selector/exploreInterpretAs.go | 5 ++ traversal/selector/exploreRange.go | 5 ++ traversal/selector/exploreRecursive.go | 5 ++ traversal/selector/exploreRecursiveEdge.go | 5 ++ traversal/selector/exploreUnion.go | 12 +++ traversal/selector/fieldKeys.go | 3 + traversal/selector/matcher.go | 99 +++++++++++++++++++++- traversal/selector/matcher_util.go | 15 ++++ traversal/selector/selector.go | 6 ++ traversal/walk.go | 22 ++--- 16 files changed, 286 insertions(+), 14 deletions(-) create mode 100644 node/basicnode/bytes_stream.go create mode 100644 traversal/selector/matcher_util.go diff --git a/node/basicnode/bytes.go b/node/basicnode/bytes.go index 7c45d208..566c32fe 100644 --- a/node/basicnode/bytes.go +++ b/node/basicnode/bytes.go @@ -106,7 +106,7 @@ func (nb *plainBytes__Builder) Reset() { // -- NodeAssembler --> type plainBytes__Assembler struct { - w *plainBytes + w datamodel.Node } func (plainBytes__Assembler) BeginMap(sizeHint int64) (datamodel.MapAssembler, error) { @@ -131,17 +131,24 @@ func (plainBytes__Assembler) AssignString(string) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignString("") } func (na *plainBytes__Assembler) AssignBytes(v []byte) error { - *na.w = plainBytes(v) + na.w = datamodel.Node(plainBytes(v)) return nil } func (plainBytes__Assembler) AssignLink(datamodel.Link) error { return mixins.BytesAssembler{TypeName: "bytes"}.AssignLink(nil) } func (na *plainBytes__Assembler) AssignNode(v datamodel.Node) error { + if lb, ok := v.(datamodel.LargeBytesNode); ok { + lbn, err := lb.AsLargeBytes() + if err == nil { + na.w = streamBytes{lbn} + return nil + } + } if v2, err := v.AsBytes(); err != nil { return err } else { - *na.w = plainBytes(v2) + na.w = plainBytes(v2) return nil } } diff --git a/node/basicnode/bytes_stream.go b/node/basicnode/bytes_stream.go new file mode 100644 index 00000000..ad238bcf --- /dev/null +++ b/node/basicnode/bytes_stream.go @@ -0,0 +1,81 @@ +package basicnode + +import ( + "io" + + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/mixins" +) + +var ( + _ datamodel.Node = streamBytes{nil} + _ datamodel.NodePrototype = Prototype__Bytes{} + _ datamodel.NodeBuilder = &plainBytes__Builder{} + _ datamodel.NodeAssembler = &plainBytes__Assembler{} +) + +func NewBytesFromReader(rs io.ReadSeeker) datamodel.Node { + return streamBytes{rs} +} + +// streamBytes is a boxed reader that complies with datamodel.Node. +type streamBytes struct { + io.ReadSeeker +} + +// -- Node interface methods --> + +func (streamBytes) Kind() datamodel.Kind { + return datamodel.Kind_Bytes +} +func (streamBytes) LookupByString(string) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByString("") +} +func (streamBytes) LookupByNode(key datamodel.Node) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByNode(nil) +} +func (streamBytes) LookupByIndex(idx int64) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupByIndex(0) +} +func (streamBytes) LookupBySegment(seg datamodel.PathSegment) (datamodel.Node, error) { + return mixins.Bytes{TypeName: "bytes"}.LookupBySegment(seg) +} +func (streamBytes) MapIterator() datamodel.MapIterator { + return nil +} +func (streamBytes) ListIterator() datamodel.ListIterator { + return nil +} +func (streamBytes) Length() int64 { + return -1 +} +func (streamBytes) IsAbsent() bool { + return false +} +func (streamBytes) IsNull() bool { + return false +} +func (streamBytes) AsBool() (bool, error) { + return mixins.Bytes{TypeName: "bytes"}.AsBool() +} +func (streamBytes) AsInt() (int64, error) { + return mixins.Bytes{TypeName: "bytes"}.AsInt() +} +func (streamBytes) AsFloat() (float64, error) { + return mixins.Bytes{TypeName: "bytes"}.AsFloat() +} +func (streamBytes) AsString() (string, error) { + return mixins.Bytes{TypeName: "bytes"}.AsString() +} +func (n streamBytes) AsBytes() ([]byte, error) { + return io.ReadAll(n) +} +func (streamBytes) AsLink() (datamodel.Link, error) { + return mixins.Bytes{TypeName: "bytes"}.AsLink() +} +func (streamBytes) Prototype() datamodel.NodePrototype { + return Prototype__Bytes{} +} +func (n streamBytes) AsLargeBytes() (io.ReadSeeker, error) { + return n.ReadSeeker, nil +} diff --git a/traversal/selector/builder/builder.go b/traversal/selector/builder/builder.go index 80067758..6cc69e3c 100644 --- a/traversal/selector/builder/builder.go +++ b/traversal/selector/builder/builder.go @@ -32,6 +32,7 @@ type SelectorSpecBuilder interface { ExploreFields(ExploreFieldsSpecBuildingClosure) SelectorSpec ExploreInterpretAs(as string, next SelectorSpec) SelectorSpec Matcher() SelectorSpec + MatcherSubset(from, to int64) SelectorSpec } // ExploreFieldsSpecBuildingClosure is a function that provided to SelectorSpecBuilder's @@ -170,6 +171,19 @@ func (ssb *selectorSpecBuilder) Matcher() SelectorSpec { } } +func (ssb *selectorSpecBuilder) MatcherSubset(from, to int64) SelectorSpec { + return selectorSpec{ + fluent.MustBuildMap(ssb.np, 1, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_Matcher).CreateMap(1, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_Subset).CreateMap(2, func(na fluent.MapAssembler) { + na.AssembleEntry(selector.SelectorKey_From).AssignInt(from) + na.AssembleEntry(selector.SelectorKey_To).AssignInt(to) + }) + }) + }), + } +} + type exploreFieldsSpecBuilder struct { na fluent.MapAssembler } diff --git a/traversal/selector/exploreAll.go b/traversal/selector/exploreAll.go index a7749f8e..7d889d7e 100644 --- a/traversal/selector/exploreAll.go +++ b/traversal/selector/exploreAll.go @@ -27,6 +27,11 @@ func (s ExploreAll) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreAll) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreAll assembles a Selector from a ExploreAll selector node func (pc ParseContext) ParseExploreAll(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { diff --git a/traversal/selector/exploreFields.go b/traversal/selector/exploreFields.go index 5d2abdd2..b2796ff2 100644 --- a/traversal/selector/exploreFields.go +++ b/traversal/selector/exploreFields.go @@ -38,6 +38,11 @@ func (s ExploreFields) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreFields) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreFields assembles a Selector // from a ExploreFields selector node func (pc ParseContext) ParseExploreFields(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreIndex.go b/traversal/selector/exploreIndex.go index 0c8edf16..56a2badc 100644 --- a/traversal/selector/exploreIndex.go +++ b/traversal/selector/exploreIndex.go @@ -37,6 +37,11 @@ func (s ExploreIndex) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreIndex) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreIndex assembles a Selector // from a ExploreIndex selector node func (pc ParseContext) ParseExploreIndex(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreInterpretAs.go b/traversal/selector/exploreInterpretAs.go index 25cf045b..6f4aed67 100644 --- a/traversal/selector/exploreInterpretAs.go +++ b/traversal/selector/exploreInterpretAs.go @@ -27,6 +27,11 @@ func (s ExploreInterpretAs) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreInterpretAs) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // NamedReifier indicates how this selector expects to Reify the current datamodel.Node. func (s ExploreInterpretAs) NamedReifier() string { return s.adl diff --git a/traversal/selector/exploreRange.go b/traversal/selector/exploreRange.go index 73e356ba..e9e041bc 100644 --- a/traversal/selector/exploreRange.go +++ b/traversal/selector/exploreRange.go @@ -41,6 +41,11 @@ func (s ExploreRange) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreRange) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreRange assembles a Selector // from a ExploreRange selector node func (pc ParseContext) ParseExploreRange(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreRecursive.go b/traversal/selector/exploreRecursive.go index 17614025..28a9bedc 100644 --- a/traversal/selector/exploreRecursive.go +++ b/traversal/selector/exploreRecursive.go @@ -176,6 +176,11 @@ func (s ExploreRecursive) Decide(n datamodel.Node) bool { return s.current.Decide(n) } +// Match always returns false because this is not a matcher +func (s ExploreRecursive) Match(node datamodel.Node) (datamodel.Node, error) { + return s.current.Match(node) +} + type exploreRecursiveContext struct { edgesFound int } diff --git a/traversal/selector/exploreRecursiveEdge.go b/traversal/selector/exploreRecursiveEdge.go index 65438572..534c73b8 100644 --- a/traversal/selector/exploreRecursiveEdge.go +++ b/traversal/selector/exploreRecursiveEdge.go @@ -31,6 +31,11 @@ func (s ExploreRecursiveEdge) Decide(n datamodel.Node) bool { return false } +// Match always returns false because this is not a matcher +func (s ExploreRecursiveEdge) Match(node datamodel.Node) (datamodel.Node, error) { + return nil, nil +} + // ParseExploreRecursiveEdge assembles a Selector // from a exploreRecursiveEdge selector node func (pc ParseContext) ParseExploreRecursiveEdge(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/exploreUnion.go b/traversal/selector/exploreUnion.go index 0af42cd6..21594954 100644 --- a/traversal/selector/exploreUnion.go +++ b/traversal/selector/exploreUnion.go @@ -74,6 +74,18 @@ func (s ExploreUnion) Decide(n datamodel.Node) bool { return false } +// Match returns true for a Union selector based on the matched union. +func (s ExploreUnion) Match(n datamodel.Node) (datamodel.Node, error) { + for _, m := range s.Members { + if mn, err := m.Match(n); mn != nil { + return mn, nil + } else if err != nil { + return nil, err + } + } + return nil, nil +} + // ParseExploreUnion assembles a Selector // from an ExploreUnion selector node func (pc ParseContext) ParseExploreUnion(n datamodel.Node) (Selector, error) { diff --git a/traversal/selector/fieldKeys.go b/traversal/selector/fieldKeys.go index 58024b61..39ec61e6 100644 --- a/traversal/selector/fieldKeys.go +++ b/traversal/selector/fieldKeys.go @@ -23,5 +23,8 @@ const ( SelectorKey_StopAt = "!" SelectorKey_Condition = "&" SelectorKey_As = "as" + SelectorKey_Subset = "subset" + SelectorKey_From = "[" + SelectorKey_To = "]" // not filling conditional keys since it's not complete ) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index b8f64f68..d9538dc6 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -2,8 +2,10 @@ package selector import ( "fmt" + "io" "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" ) // Matcher marks a node to be included in the "result" set. @@ -16,7 +18,61 @@ import ( // A selector tree with only "explore*"-type selectors and no Matcher selectors // is valid; it will just generate a "covered" set of nodes and no "result" set. // TODO: From spec: implement conditions and labels -type Matcher struct{} +type Matcher struct { + *Slice +} + +// Slice limits a result node to a subset of the node. +// The returned node will be limited based on slicing the specified range of the +// node into a new node, or making use of the `AsLargeBytes` io.ReadSeeker to +// restrict response with a SectionReader. +type Slice struct { + From int64 + To int64 +} + +func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { + if n.Kind() == datamodel.Kind_String { + str, err := n.AsString() + if err != nil { + return nil, err + } + to := s.To + if len(str) < int(to) { + to = int64(len(str)) + } + from := s.From + if len(str) < int(from) { + from = int64(len(str)) + } + return basicnode.NewString(str[from:to]), nil + } else if n.Kind() == datamodel.Kind_Bytes { + if lbn, ok := n.(datamodel.LargeBytesNode); ok { + rdr, err := lbn.AsLargeBytes() + if err != nil { + return nil, err + } + + sr := io.NewSectionReader(readerat{rdr}, int64(s.From), int64(s.To)) + return basicnode.NewBytesFromReader(sr), nil + } + bytes, err := n.AsBytes() + if err != nil { + return nil, err + } + to := s.To + if len(bytes) < int(to) { + to = int64(len(bytes)) + } + from := s.From + if len(bytes) < int(from) { + from = int64(len(bytes)) + } + + return basicnode.NewBytes(bytes[from:to]), nil + } + return nil, fmt.Errorf("selector slice rejected: subset match must be over string or bytes") +} // Interests are empty for a matcher (for now) because // It is always just there to match, not explore further @@ -35,6 +91,14 @@ func (s Matcher) Decide(n datamodel.Node) bool { return true } +// Match is always true for a match cause it's in the result set +func (s Matcher) Match(node datamodel.Node) (datamodel.Node, error) { + if s.Slice != nil { + return s.Slice.Slice(node) + } + return node, nil +} + // ParseMatcher assembles a Selector // from a matcher selector node // TODO: Parse labels and conditions @@ -42,5 +106,38 @@ func (pc ParseContext) ParseMatcher(n datamodel.Node) (Selector, error) { if n.Kind() != datamodel.Kind_Map { return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map") } + + // check if a slice is specified + if subset, err := n.LookupByString("subset"); err == nil { + if subset.Kind() != datamodel.Kind_Map { + return nil, fmt.Errorf("selector spec parse rejected: subset body must be a map") + } + from, err := subset.LookupByString("[") + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a from '[' key") + } + fromN, err := from.AsInt() + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is a number") + } + to, err := subset.LookupByString("]") + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a to ']' key") + } + toN, err := to.AsInt() + if err != nil { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'to' key that is a number") + } + if fromN > toN { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with a 'from' key that is less than or equal to the 'to' key") + } + if fromN < 0 || toN < 0 { + return nil, fmt.Errorf("selector spec parse rejected: selector body must be a map with keys not less than 0") + } + return Matcher{&Slice{ + From: fromN, + To: toN, + }}, nil + } return Matcher{}, nil } diff --git a/traversal/selector/matcher_util.go b/traversal/selector/matcher_util.go new file mode 100644 index 00000000..b67cad09 --- /dev/null +++ b/traversal/selector/matcher_util.go @@ -0,0 +1,15 @@ +package selector + +import "io" + +type readerat struct { + io.ReadSeeker +} + +func (r readerat) ReadAt(p []byte, off int64) (n int, err error) { + _, err = r.Seek(off, 0) + if err != nil { + return 0, err + } + return r.Read(p) +} diff --git a/traversal/selector/selector.go b/traversal/selector/selector.go index 4d0a7814..b58fcd86 100644 --- a/traversal/selector/selector.go +++ b/traversal/selector/selector.go @@ -106,6 +106,12 @@ type Selector interface { // Only "Matcher" clauses actually implement this in a way that ever returns "true". // See the Selector specs for discussion on "matched" vs "reached"/"visited" nodes. Decide(node datamodel.Node) bool + + // Match is an extension to Decide allowing the matcher to `decide` a transformation of + // the matched node. This is used for `Subset` match behavior. If the node is matched, + // the first argument will be the matched node. If it is not matched, the first argument + // will be null. If there is an error, the first argument will be null. + Match(node datamodel.Node) (datamodel.Node, error) } // REVIEW: do ParsedParent and ParseContext need to be exported? They're mostly used during the compilation process. diff --git a/traversal/walk.go b/traversal/walk.go index 1656de15..a794b301 100644 --- a/traversal/walk.go +++ b/traversal/walk.go @@ -182,16 +182,18 @@ func (prog Progress) walkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitF } if prog.Path.Len() >= prog.Cfg.StartAtPath.Len() || !prog.PastStartAtPath { - // Decide if this node is matched -- do callbacks as appropriate. - if s.Decide(n) { - if err := fn(prog, n, VisitReason_SelectionMatch); err != nil { - return err - } - } else { - if err := fn(prog, n, VisitReason_SelectionCandidate); err != nil { - return err - } - } + // Decide if this node is matched -- do callbacks as appropriate. + if match, err := s.Match(n); match != nil { + if err := fn(prog, match, VisitReason_SelectionMatch); err != nil { + return err + } + } else if err != nil { + return err + } else { + if err := fn(prog, n, VisitReason_SelectionCandidate); err != nil { + return err + } + } } // If we're handling scalars (e.g. not maps and lists) we can return now. nk := n.Kind() From 671b1bee1257c30d7791c3acd35db6e488476701 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 15:28:23 +0100 Subject: [PATCH 08/13] bytes spec test --- node/basicnode/bytes_test.go | 12 ++++++++++++ node/tests/byteSpecs.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 node/basicnode/bytes_test.go create mode 100644 node/tests/byteSpecs.go diff --git a/node/basicnode/bytes_test.go b/node/basicnode/bytes_test.go new file mode 100644 index 00000000..ca408484 --- /dev/null +++ b/node/basicnode/bytes_test.go @@ -0,0 +1,12 @@ +package basicnode_test + +import ( + "testing" + + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/node/tests" +) + +func TestBytes(t *testing.T) { + tests.SpecTestBytes(t, basicnode.Prototype__Bytes{}) +} diff --git a/node/tests/byteSpecs.go b/node/tests/byteSpecs.go new file mode 100644 index 00000000..69bf494b --- /dev/null +++ b/node/tests/byteSpecs.go @@ -0,0 +1,35 @@ +package tests + +import ( + "io" + "testing" + + qt "github.com/frankban/quicktest" + + "github.com/ipld/go-ipld-prime/datamodel" +) + +func SpecTestBytes(t *testing.T, np datamodel.NodePrototype) { + t.Run("byte node", func(t *testing.T) { + nb := np.NewBuilder() + err := nb.AssignBytes([]byte("asdf")) + qt.Check(t, err, qt.IsNil) + n := nb.Build() + + qt.Check(t, n.Kind(), qt.Equals, datamodel.Kind_Bytes) + qt.Check(t, n.IsNull(), qt.IsFalse) + x, err := n.AsBytes() + qt.Check(t, err, qt.IsNil) + qt.Check(t, x, qt.DeepEquals, []byte("asdf")) + + lbn, ok := n.(datamodel.LargeBytesNode) + if ok { + str, err := lbn.AsLargeBytes() + qt.Check(t, err, qt.IsNil) + bytes, err := io.ReadAll(str) + qt.Check(t, err, qt.IsNil) + qt.Check(t, bytes, qt.DeepEquals, []byte("asdf")) + } + + }) +} From 918b60840333d728beec996b93a39a0586337728 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 15:32:55 +0100 Subject: [PATCH 09/13] extend traversal to handle partial byte range match access behavior --- traversal/selector/spec_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/traversal/selector/spec_test.go b/traversal/selector/spec_test.go index 215bfd6b..7d309d05 100644 --- a/traversal/selector/spec_test.go +++ b/traversal/selector/spec_test.go @@ -2,6 +2,7 @@ package selector_test import ( "bytes" + "io" "os" "testing" @@ -14,7 +15,7 @@ import ( "github.com/ipld/go-ipld-prime/fluent/qp" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal" - "github.com/ipld/go-ipld-prime/traversal/selector/parse" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" ) func TestSpecFixtures(t *testing.T) { @@ -81,6 +82,18 @@ func testOneSpecFixtureFile(t *testing.T, filename string) { })) qp.MapEntry(ma, "matched", qp.Bool(reason == traversal.VisitReason_SelectionMatch)) }) + if reason == traversal.VisitReason_SelectionMatch && n.Kind() == datamodel.Kind_Bytes { + if lbn, ok := n.(datamodel.LargeBytesNode); ok { + rdr, err := lbn.AsLargeBytes() + if err == nil { + io.Copy(io.Discard, rdr) + } + } + _, err := n.AsBytes() + if err != nil { + panic("insanity at a deeper level than this test's target") + } + } if err != nil { panic("insanity at a deeper level than this test's target") } From 5c15a9efaeaab9ebda4c085b4088101bbd911f15 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 15:47:06 +0100 Subject: [PATCH 10/13] code review --- traversal/selector/matcher.go | 19 +++++++++++-------- traversal/selector/matcher_util.go | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index d9538dc6..38314db4 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -32,46 +32,49 @@ type Slice struct { } func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { - if n.Kind() == datamodel.Kind_String { + var from, to int64 + switch n.Kind() { + case datamodel.Kind_String: str, err := n.AsString() if err != nil { return nil, err } - to := s.To + to = s.To if len(str) < int(to) { to = int64(len(str)) } - from := s.From + from = s.From if len(str) < int(from) { from = int64(len(str)) } return basicnode.NewString(str[from:to]), nil - } else if n.Kind() == datamodel.Kind_Bytes { + case datamodel.Kind_Bytes: if lbn, ok := n.(datamodel.LargeBytesNode); ok { rdr, err := lbn.AsLargeBytes() if err != nil { return nil, err } - sr := io.NewSectionReader(readerat{rdr}, int64(s.From), int64(s.To)) + sr := io.NewSectionReader(readerat{rdr}, s.From, s.To) return basicnode.NewBytesFromReader(sr), nil } bytes, err := n.AsBytes() if err != nil { return nil, err } - to := s.To + to = s.To if len(bytes) < int(to) { to = int64(len(bytes)) } - from := s.From + from = s.From if len(bytes) < int(from) { from = int64(len(bytes)) } return basicnode.NewBytes(bytes[from:to]), nil + default: + return nil, fmt.Errorf("selector slice rejected: subset match must be over string or bytes") } - return nil, fmt.Errorf("selector slice rejected: subset match must be over string or bytes") } // Interests are empty for a matcher (for now) because diff --git a/traversal/selector/matcher_util.go b/traversal/selector/matcher_util.go index b67cad09..b60d11c8 100644 --- a/traversal/selector/matcher_util.go +++ b/traversal/selector/matcher_util.go @@ -7,6 +7,7 @@ type readerat struct { } func (r readerat) ReadAt(p []byte, off int64) (n int, err error) { + // TODO: consider keeping track of current offset. _, err = r.Seek(off, 0) if err != nil { return 0, err From 598b16a02fc0e1464279faaf40bbe020690669a0 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 16:00:28 +0100 Subject: [PATCH 11/13] warning on readerat --- traversal/selector/matcher_util.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/traversal/selector/matcher_util.go b/traversal/selector/matcher_util.go index b60d11c8..237c5e71 100644 --- a/traversal/selector/matcher_util.go +++ b/traversal/selector/matcher_util.go @@ -6,6 +6,10 @@ type readerat struct { io.ReadSeeker } +// ReadAt provides the io.ReadAt method over a ReadSeeker. +// This implementation does not support concurrent calls to `ReadAt`, +// as specified by the ReaderAt interface, and so must only be used +// in non-concurrent use cases. func (r readerat) ReadAt(p []byte, off int64) (n int, err error) { // TODO: consider keeping track of current offset. _, err = r.Seek(off, 0) From c3d07b4530078699dce5b6ac3e9532a40fd1c440 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 3 Mar 2022 19:06:38 +0100 Subject: [PATCH 12/13] fix section reader --- traversal/selector/matcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index 38314db4..4186221f 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -55,7 +55,7 @@ func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { return nil, err } - sr := io.NewSectionReader(readerat{rdr}, s.From, s.To) + sr := io.NewSectionReader(readerat{rdr}, s.From, s.To-s.From) return basicnode.NewBytesFromReader(sr), nil } bytes, err := n.AsBytes() From 777d78d9625238dca7e706845465f338a4a50580 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 7 Mar 2022 09:01:04 +0100 Subject: [PATCH 13/13] clearer comments --- traversal/selector/matcher.go | 4 ++-- traversal/walk.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/traversal/selector/matcher.go b/traversal/selector/matcher.go index 4186221f..ff2596c1 100644 --- a/traversal/selector/matcher.go +++ b/traversal/selector/matcher.go @@ -73,7 +73,7 @@ func (s Slice) Slice(n datamodel.Node) (datamodel.Node, error) { return basicnode.NewBytes(bytes[from:to]), nil default: - return nil, fmt.Errorf("selector slice rejected: subset match must be over string or bytes") + return nil, fmt.Errorf("selector slice rejected on %s: subset match must be over string or bytes", n.Kind()) } } @@ -89,7 +89,7 @@ func (s Matcher) Explore(n datamodel.Node, p datamodel.PathSegment) (Selector, e } // Decide is always true for a match cause it's in the result set -// TODO: Implement boolean logic for conditionals +// Deprecated: use Match instead func (s Matcher) Decide(n datamodel.Node) bool { return true } diff --git a/traversal/walk.go b/traversal/walk.go index 0f659189..c2d5b35e 100644 --- a/traversal/walk.go +++ b/traversal/walk.go @@ -174,6 +174,7 @@ func (prog Progress) walkAdv(n datamodel.Node, s selector.Selector, fn AdvVisitF if err != nil { return fmt.Errorf("failed to reify node as %q: %w", adl, err) } + // explore into the `InterpretAs` clause to the child selector. s, err = s.Explore(n, datamodel.PathSegment{}) if err != nil { return err