From f1308d31790cb85303867fb1cf8fa5e2afc6f932 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 6 Dec 2023 19:35:46 +0900 Subject: [PATCH 01/72] numberKind byte slice parser --- examples/gno.land/p/demo/json/gno.mod | 1 + examples/gno.land/p/demo/json/number.gno | 83 +++++++++++++++++++ examples/gno.land/p/demo/json/number_test.gno | 37 +++++++++ examples/gno.land/p/demo/json/parser.gno | 1 + 4 files changed, 122 insertions(+) create mode 100644 examples/gno.land/p/demo/json/gno.mod create mode 100644 examples/gno.land/p/demo/json/number.gno create mode 100644 examples/gno.land/p/demo/json/number_test.gno create mode 100644 examples/gno.land/p/demo/json/parser.gno diff --git a/examples/gno.land/p/demo/json/gno.mod b/examples/gno.land/p/demo/json/gno.mod new file mode 100644 index 00000000000..831fa56c0f9 --- /dev/null +++ b/examples/gno.land/p/demo/json/gno.mod @@ -0,0 +1 @@ +module gno.land/p/demo/json diff --git a/examples/gno.land/p/demo/json/number.gno b/examples/gno.land/p/demo/json/number.gno new file mode 100644 index 00000000000..2d819f1cf88 --- /dev/null +++ b/examples/gno.land/p/demo/json/number.gno @@ -0,0 +1,83 @@ +package json + +import ( + "encoding/binary" + "math" + "strconv" +) + +const ( + absMinInt64 = 1 << 63 + maxInt64 = 1<<63 - 1 + maxUint64 = 1<<64 - 1 +) + +func ParseNumberKind(bytes []byte) (value float64, isFloat, ok, overflow bool) { + if len(bytes) == 0 { + return 0, false, false, false + } + + neg := bytes[0] == '-' + if neg { + bytes = bytes[1:] + } + + var intPart, fracPart uint64 + var decimalFound bool + var fracDivisor uint64 = 1 + + for _, c := range bytes { + if c == '.' { + if decimalFound { + return 0, false, false, false + } + decimalFound = true + continue + } + + if c < '0' || c > '9' { + return 0, false, false, false + } + + digit := uint64(c - '0') + if !decimalFound { + if intPart > (maxUint64-uint64(digit))/10 { + return 0, false, false, true + } + intPart = intPart*10 + digit + } else { + if fracPart > (maxUint64-uint64(digit))/10 { + return 0, false, false, true + } + fracPart = fracPart*10 + digit + fracDivisor *= 10 + } + } + + if decimalFound { + isFloat = true + } + + totalValue := float64(intPart) + float64(fracPart)/float64(fracDivisor) + if neg { + totalValue = -totalValue + } + + // Overflow check for integer part + if !isFloat { + if neg { + if intPart > absMinInt64 { + return 0, false, false, true + } + } else { + if intPart > maxInt64 { + return 0, false, false, true + } + } + + // return totalValue instead of float64(intPart) + return totalValue, isFloat, true, false + } + + return totalValue, isFloat, true, false +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/number_test.gno b/examples/gno.land/p/demo/json/number_test.gno new file mode 100644 index 00000000000..275d3484eb1 --- /dev/null +++ b/examples/gno.land/p/demo/json/number_test.gno @@ -0,0 +1,37 @@ +package json + +import ( + "encoding/binary" + "math" + "testing" +) + +func TestParseNumberKind(t *testing.T) { + cases := []struct { + name string + input []byte + expected float64 + isFloat bool + ok bool + overflow bool + }{ + {"Positive Integer", []byte("12345"), 12345, false, true, false}, + {"Negative Integer", []byte("-12345"), -12345, false, true, false}, + {"Positive Float", []byte("123.45"), 123.45, true, true, false}, + {"Negative Float", []byte("-123.45"), -123.45, true, true, false}, + {"Integer Overflow", []byte("18446744073709551616"), 0, false, false, true}, + {"Invalid Format", []byte("123a45"), 0, false, false, false}, + {"Multiple Decimal Points", []byte("123.45.67"), 0, false, false, false}, + {"Empty Input", []byte(""), 0, false, false, false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + value, isFloat, ok, overflow := ParseNumberKind(c.input) + if value != c.expected || isFloat != c.isFloat || ok != c.ok || overflow != c.overflow { + t.Errorf("ParseNumberKind(%s) == (%f, %t, %t, %t), expected (%f, %t, %t, %t)", + c.input, value, isFloat, ok, overflow, c.expected, c.isFloat, c.ok, c.overflow) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno new file mode 100644 index 00000000000..a5b981cc690 --- /dev/null +++ b/examples/gno.land/p/demo/json/parser.gno @@ -0,0 +1 @@ +package json From c30634c69f3904e26b2cbdff0aa9e7e96ea186a1 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 6 Dec 2023 19:54:54 +0900 Subject: [PATCH 02/72] remove unused packages --- examples/gno.land/p/demo/json/number.gno | 120 +++++++++--------- examples/gno.land/p/demo/json/number_test.gno | 2 - 2 files changed, 57 insertions(+), 65 deletions(-) diff --git a/examples/gno.land/p/demo/json/number.gno b/examples/gno.land/p/demo/json/number.gno index 2d819f1cf88..cde03b2f2db 100644 --- a/examples/gno.land/p/demo/json/number.gno +++ b/examples/gno.land/p/demo/json/number.gno @@ -1,11 +1,5 @@ package json -import ( - "encoding/binary" - "math" - "strconv" -) - const ( absMinInt64 = 1 << 63 maxInt64 = 1<<63 - 1 @@ -13,71 +7,71 @@ const ( ) func ParseNumberKind(bytes []byte) (value float64, isFloat, ok, overflow bool) { - if len(bytes) == 0 { - return 0, false, false, false - } + if len(bytes) == 0 { + return 0, false, false, false + } - neg := bytes[0] == '-' - if neg { - bytes = bytes[1:] - } + neg := bytes[0] == '-' + if neg { + bytes = bytes[1:] + } - var intPart, fracPart uint64 - var decimalFound bool - var fracDivisor uint64 = 1 + var intPart, fracPart uint64 + var decimalFound bool + var fracDivisor uint64 = 1 - for _, c := range bytes { - if c == '.' { - if decimalFound { - return 0, false, false, false - } - decimalFound = true - continue - } + for _, c := range bytes { + if c == '.' { + if decimalFound { + return 0, false, false, false + } + decimalFound = true + continue + } - if c < '0' || c > '9' { - return 0, false, false, false - } + if c < '0' || c > '9' { + return 0, false, false, false + } - digit := uint64(c - '0') - if !decimalFound { - if intPart > (maxUint64-uint64(digit))/10 { - return 0, false, false, true - } - intPart = intPart*10 + digit - } else { - if fracPart > (maxUint64-uint64(digit))/10 { - return 0, false, false, true - } - fracPart = fracPart*10 + digit - fracDivisor *= 10 - } - } + digit := uint64(c - '0') + if !decimalFound { + if intPart > (maxUint64-uint64(digit))/10 { + return 0, false, false, true + } + intPart = intPart*10 + digit + } else { + if fracPart > (maxUint64-uint64(digit))/10 { + return 0, false, false, true + } + fracPart = fracPart*10 + digit + fracDivisor *= 10 + } + } - if decimalFound { - isFloat = true - } + if decimalFound { + isFloat = true + } - totalValue := float64(intPart) + float64(fracPart)/float64(fracDivisor) - if neg { - totalValue = -totalValue - } + totalValue := float64(intPart) + float64(fracPart)/float64(fracDivisor) + if neg { + totalValue = -totalValue + } - // Overflow check for integer part - if !isFloat { - if neg { - if intPart > absMinInt64 { - return 0, false, false, true - } - } else { - if intPart > maxInt64 { - return 0, false, false, true - } - } + // Overflow check for integer part + if !isFloat { + if neg { + if intPart > absMinInt64 { + return 0, false, false, true + } + } else { + if intPart > maxInt64 { + return 0, false, false, true + } + } // return totalValue instead of float64(intPart) - return totalValue, isFloat, true, false - } + return totalValue, isFloat, true, false + } - return totalValue, isFloat, true, false -} \ No newline at end of file + return totalValue, isFloat, true, false +} diff --git a/examples/gno.land/p/demo/json/number_test.gno b/examples/gno.land/p/demo/json/number_test.gno index 275d3484eb1..89350ee5cd0 100644 --- a/examples/gno.land/p/demo/json/number_test.gno +++ b/examples/gno.land/p/demo/json/number_test.gno @@ -1,8 +1,6 @@ package json import ( - "encoding/binary" - "math" "testing" ) From 7b0e6c2d4d4e70626767a42ea6e82855213a9bce Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 7 Dec 2023 12:12:51 +0900 Subject: [PATCH 03/72] add utf16 package --- docs/reference/go-gno-compatibility.md | 2 +- gnovm/stdlibs/unicode/utf16/export_test.gno | 11 + gnovm/stdlibs/unicode/utf16/utf16.gno | 125 +++++++++++ gnovm/stdlibs/unicode/utf16/utf16_test.gno | 232 ++++++++++++++++++++ 4 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 gnovm/stdlibs/unicode/utf16/export_test.gno create mode 100644 gnovm/stdlibs/unicode/utf16/utf16.gno create mode 100644 gnovm/stdlibs/unicode/utf16/utf16_test.gno diff --git a/docs/reference/go-gno-compatibility.md b/docs/reference/go-gno-compatibility.md index 42ebe3c8ff7..8fec59eed28 100644 --- a/docs/reference/go-gno-compatibility.md +++ b/docs/reference/go-gno-compatibility.md @@ -258,7 +258,7 @@ Legend: | time | `full`[^7] | | time/tzdata | `tbd` | | unicode | `full` | -| unicode/utf16 | `tbd` | +| unicode/utf16 | `full` | | unicode/utf8 | `full` | | unsafe | `nondet` | diff --git a/gnovm/stdlibs/unicode/utf16/export_test.gno b/gnovm/stdlibs/unicode/utf16/export_test.gno new file mode 100644 index 00000000000..e0c57f52aef --- /dev/null +++ b/gnovm/stdlibs/unicode/utf16/export_test.gno @@ -0,0 +1,11 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package utf16 + +// Extra names for constants so we can validate them during testing. +const ( + MaxRune = maxRune + ReplacementChar = replacementChar +) diff --git a/gnovm/stdlibs/unicode/utf16/utf16.gno b/gnovm/stdlibs/unicode/utf16/utf16.gno new file mode 100644 index 00000000000..89755a8b8a7 --- /dev/null +++ b/gnovm/stdlibs/unicode/utf16/utf16.gno @@ -0,0 +1,125 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package utf16 implements encoding and decoding of UTF-16 sequences. +package utf16 + +// The conditions replacementChar==unicode.ReplacementChar and +// maxRune==unicode.MaxRune are verified in the tests. +// Defining them locally avoids this package depending on package unicode. + +const ( + replacementChar = '\uFFFD' // Unicode replacement character + maxRune = '\U0010FFFF' // Maximum valid Unicode code point. +) + +const ( + // 0xd800-0xdc00 encodes the high 10 bits of a pair. + // 0xdc00-0xe000 encodes the low 10 bits of a pair. + // the value is those 20 bits plus 0x10000. + surr1 = 0xd800 + surr2 = 0xdc00 + surr3 = 0xe000 + + surrSelf = 0x10000 +) + +// IsSurrogate reports whether the specified Unicode code point +// can appear in a surrogate pair. +func IsSurrogate(r rune) bool { + return surr1 <= r && r < surr3 +} + +// DecodeRune returns the UTF-16 decoding of a surrogate pair. +// If the pair is not a valid UTF-16 surrogate pair, DecodeRune returns +// the Unicode replacement code point U+FFFD. +func DecodeRune(r1, r2 rune) rune { + if surr1 <= r1 && r1 < surr2 && surr2 <= r2 && r2 < surr3 { + return (r1-surr1)<<10 | (r2 - surr2) + surrSelf + } + return replacementChar +} + +// EncodeRune returns the UTF-16 surrogate pair r1, r2 for the given rune. +// If the rune is not a valid Unicode code point or does not need encoding, +// EncodeRune returns U+FFFD, U+FFFD. +func EncodeRune(r rune) (r1, r2 rune) { + if r < surrSelf || r > maxRune { + return replacementChar, replacementChar + } + r -= surrSelf + return surr1 + (r>>10)&0x3ff, surr2 + r&0x3ff +} + +// Encode returns the UTF-16 encoding of the Unicode code point sequence s. +func Encode(s []rune) []uint16 { + n := len(s) + for _, v := range s { + if v >= surrSelf { + n++ + } + } + + a := make([]uint16, n) + n = 0 + for _, v := range s { + switch { + case 0 <= v && v < surr1, surr3 <= v && v < surrSelf: + // normal rune + a[n] = uint16(v) + n++ + case surrSelf <= v && v <= maxRune: + // needs surrogate sequence + r1, r2 := EncodeRune(v) + a[n] = uint16(r1) + a[n+1] = uint16(r2) + n += 2 + default: + a[n] = uint16(replacementChar) + n++ + } + } + return a[:n] +} + +// AppendRune appends the UTF-16 encoding of the Unicode code point r +// to the end of p and returns the extended buffer. If the rune is not +// a valid Unicode code point, it appends the encoding of U+FFFD. +func AppendRune(a []uint16, r rune) []uint16 { + // This function is inlineable for fast handling of ASCII. + switch { + case 0 <= r && r < surr1, surr3 <= r && r < surrSelf: + // normal rune + return append(a, uint16(r)) + case surrSelf <= r && r <= maxRune: + // needs surrogate sequence + r1, r2 := EncodeRune(r) + return append(a, uint16(r1), uint16(r2)) + } + return append(a, replacementChar) +} + +// Decode returns the Unicode code point sequence represented +// by the UTF-16 encoding s. +func Decode(s []uint16) []rune { + a := make([]rune, len(s)) + n := 0 + for i := 0; i < len(s); i++ { + switch r := s[i]; { + case r < surr1, surr3 <= r: + // normal rune + a[n] = rune(r) + case surr1 <= r && r < surr2 && i+1 < len(s) && + surr2 <= s[i+1] && s[i+1] < surr3: + // valid surrogate sequence + a[n] = DecodeRune(rune(r), rune(s[i+1])) + i++ + default: + // invalid surrogate sequence + a[n] = replacementChar + } + n++ + } + return a[:n] +} \ No newline at end of file diff --git a/gnovm/stdlibs/unicode/utf16/utf16_test.gno b/gnovm/stdlibs/unicode/utf16/utf16_test.gno new file mode 100644 index 00000000000..6de30004053 --- /dev/null +++ b/gnovm/stdlibs/unicode/utf16/utf16_test.gno @@ -0,0 +1,232 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package utf16 + +import ( + // "reflect" + "testing" + "unicode" +) + +// Validate the constants redefined from unicode. +func TestConstants(t *testing.T) { + if MaxRune != unicode.MaxRune { + t.Errorf("utf16.maxRune is wrong: %x should be %x", MaxRune, unicode.MaxRune) + } + if ReplacementChar != unicode.ReplacementChar { + t.Errorf("utf16.replacementChar is wrong: %x should be %x", ReplacementChar, unicode.ReplacementChar) + } +} + +type encodeTest struct { + in []rune + out []uint16 +} + +var encodeTests = []encodeTest{ + {[]rune{1, 2, 3, 4}, []uint16{1, 2, 3, 4}}, + { + []rune{0xffff, 0x10000, 0x10001, 0x12345, 0x10ffff}, + []uint16{0xffff, 0xd800, 0xdc00, 0xd800, 0xdc01, 0xd808, 0xdf45, 0xdbff, 0xdfff}, + }, + { + []rune{'a', 'b', 0xd7ff, 0xd800, 0xdfff, 0xe000, 0x110000, -1}, + []uint16{'a', 'b', 0xd7ff, 0xfffd, 0xfffd, 0xe000, 0xfffd, 0xfffd}, + }, +} + +func slicesEqual(a, b []uint16) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} + +func TestEncode(t *testing.T) { + for _, tt := range encodeTests { + out := Encode(tt.in) + if !slicesEqual(out, tt.out) { + t.Errorf("Encode(%x) = %x; want %x", tt.in, out, tt.out) + } + } +} + +func TestEncodeRune(t *testing.T) { + for i, tt := range encodeTests { + j := 0 + for _, r := range tt.in { + r1, r2 := EncodeRune(r) + if r < 0x10000 || r > unicode.MaxRune { + if j >= len(tt.out) { + t.Errorf("#%d: ran out of tt.out", i) + break + } + if r1 != unicode.ReplacementChar || r2 != unicode.ReplacementChar { + t.Errorf("EncodeRune(%#x) = %#x, %#x; want 0xfffd, 0xfffd", r, r1, r2) + } + j++ + } else { + if j+1 >= len(tt.out) { + t.Errorf("#%d: ran out of tt.out", i) + break + } + if r1 != rune(tt.out[j]) || r2 != rune(tt.out[j+1]) { + t.Errorf("EncodeRune(%#x) = %#x, %#x; want %#x, %#x", r, r1, r2, tt.out[j], tt.out[j+1]) + } + j += 2 + dec := DecodeRune(r1, r2) + if dec != r { + t.Errorf("DecodeRune(%#x, %#x) = %#x; want %#x", r1, r2, dec, r) + } + } + } + if j != len(tt.out) { + t.Errorf("#%d: EncodeRune didn't generate enough output", i) + } + } +} + +type decodeTest struct { + in []uint16 + out []rune +} + +var decodeTests = []decodeTest{ + {[]uint16{1, 2, 3, 4}, []rune{1, 2, 3, 4}}, + { + []uint16{0xffff, 0xd800, 0xdc00, 0xd800, 0xdc01, 0xd808, 0xdf45, 0xdbff, 0xdfff}, + []rune{0xffff, 0x10000, 0x10001, 0x12345, 0x10ffff}, + }, + {[]uint16{0xd800, 'a'}, []rune{0xfffd, 'a'}}, + {[]uint16{0xdfff}, []rune{0xfffd}}, +} + + +func TestDecode(t *testing.T) { + for _, tt := range decodeTests { + out := Decode(tt.in) + if !runesEqual(out, tt.out) { + t.Errorf("Decode(%x) = %x; want %x", tt.in, out, tt.out) + } + } +} + +func runesEqual(a, b []rune) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} + +var decodeRuneTests = []struct { + r1, r2 rune + want rune +}{ + {0xd800, 0xdc00, 0x10000}, + {0xd800, 0xdc01, 0x10001}, + {0xd808, 0xdf45, 0x12345}, + {0xdbff, 0xdfff, 0x10ffff}, + {0xd800, 'a', 0xfffd}, // illegal, replacement rune substituted +} + +func TestDecodeRune(t *testing.T) { + for i, tt := range decodeRuneTests { + got := DecodeRune(tt.r1, tt.r2) + if got != tt.want { + t.Errorf("%d: DecodeRune(%q, %q) = %v; want %v", i, tt.r1, tt.r2, got, tt.want) + } + } +} + +var surrogateTests = []struct { + r rune + want bool +}{ + // from https://en.wikipedia.org/wiki/UTF-16 + {'\u007A', false}, // LATIN SMALL LETTER Z + {'\u6C34', false}, // CJK UNIFIED IDEOGRAPH-6C34 (water) + {'\uFEFF', false}, // Byte Order Mark + {'\U00010000', false}, // LINEAR B SYLLABLE B008 A (first non-BMP code point) + {'\U0001D11E', false}, // MUSICAL SYMBOL G CLEF + {'\U0010FFFD', false}, // PRIVATE USE CHARACTER-10FFFD (last Unicode code point) + + {rune(0xd7ff), false}, // surr1-1 + {rune(0xd800), true}, // surr1 + {rune(0xdc00), true}, // surr2 + {rune(0xe000), false}, // surr3 + {rune(0xdfff), true}, // surr3-1 +} + +func TestIsSurrogate(t *testing.T) { + for i, tt := range surrogateTests { + got := IsSurrogate(tt.r) + if got != tt.want { + t.Errorf("%d: IsSurrogate(%q) = %v; want %v", i, tt.r, got, tt.want) + } + } +} + +func BenchmarkDecodeValidASCII(b *testing.B) { + // "hello world" + data := []uint16{104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100} + for i := 0; i < b.N; i++ { + Decode(data) + } +} + +func BenchmarkDecodeValidJapaneseChars(b *testing.B) { + // "日本語日本語日本語" + data := []uint16{26085, 26412, 35486, 26085, 26412, 35486, 26085, 26412, 35486} + for i := 0; i < b.N; i++ { + Decode(data) + } +} + +func BenchmarkDecodeRune(b *testing.B) { + rs := make([]rune, 10) + // U+1D4D0 to U+1D4D4: MATHEMATICAL BOLD SCRIPT CAPITAL LETTERS + for i, u := range []rune{'𝓐', '𝓑', '𝓒', '𝓓', '𝓔'} { + rs[2*i], rs[2*i+1] = EncodeRune(u) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < 5; j++ { + DecodeRune(rs[2*j], rs[2*j+1]) + } + } +} + +func BenchmarkEncodeValidASCII(b *testing.B) { + data := []rune{'h', 'e', 'l', 'l', 'o'} + for i := 0; i < b.N; i++ { + Encode(data) + } +} + +func BenchmarkEncodeValidJapaneseChars(b *testing.B) { + data := []rune{'日', '本', '語'} + for i := 0; i < b.N; i++ { + Encode(data) + } +} + +func BenchmarkEncodeRune(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, u := range []rune{'𝓐', '𝓑', '𝓒', '𝓓', '𝓔'} { + EncodeRune(u) + } + } +} \ No newline at end of file From cf0655e312ec7bf296df29996ba63fd60f2348a7 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 7 Dec 2023 18:42:47 +0900 Subject: [PATCH 04/72] add escape handler --- examples/gno.land/p/demo/json/escape.gno | 175 ++++++++++++++++ examples/gno.land/p/demo/json/escape_test.gno | 186 ++++++++++++++++++ gnovm/stdlibs/unicode/utf16/utf16.gno | 2 +- gnovm/stdlibs/unicode/utf16/utf16_test.gno | 51 +++-- 4 files changed, 387 insertions(+), 27 deletions(-) create mode 100644 examples/gno.land/p/demo/json/escape.gno create mode 100644 examples/gno.land/p/demo/json/escape_test.gno diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno new file mode 100644 index 00000000000..b7ce647b9fd --- /dev/null +++ b/examples/gno.land/p/demo/json/escape.gno @@ -0,0 +1,175 @@ +package json + +import ( + "bytes" + "unicode/utf8" +) + +const ( + SupplementalPlanesOffset = 0x10000 + HighSurrogateOffset = 0xD800 + LowSurrogateOffset = 0xDC00 + + SurrogateEnd = 0xDFFF + BasicMultilingualPlaneOffset = 0xFFFF + + BadHex = -1 +) + +var escapeMap = map[byte]byte{ + '"': '"', + '\\': '\\', + '/': '/', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', +} + +// hexToInt converts a hex character to its integer value. +func hexToInt(c byte) int { + switch { + case c >= '0' && c <= '9': + return int(c - '0') + case c >= 'A' && c <= 'F': + return int(c - 'A' + 10) + case c >= 'a' && c <= 'f': + return int(c - 'a' + 10) + } + + return BadHex +} + +// isSurrogatePair returns true if the rune is a surrogate pair. +// +// A surrogate pairs are used in UTF-16 encoding to encode characters +// outside the Basic Multilingual Plane (BMP). +func isSurrogatePair(r rune) bool { + return HighSurrogateOffset <= r && r <= SurrogateEnd +} + +// combineSurrogates reconstruct the original unicode code points in the +// supplemental plane by combinin the high and low surrogate. +// +// The hight surrogate in the range from U+D800 to U+DBFF, +// and the low surrogate in the range from U+DC00 to U+DFFF. +// +// The formula to combine the surrogates is: +// (high - 0xD800) * 0x400 + (low - 0xDC00) + 0x10000 +func combineSurrogates(high, low rune) rune { + return ((high - HighSurrogateOffset) << 10) + (low - LowSurrogateOffset) + SupplementalPlanesOffset +} + +// deocdeSingleUnicodeEscape decodes a unicode escape sequence (e.g., \uXXXX) into a rune. +func decodeSingleUnicodeEscape(b []byte) (rune, bool) { + if len(b) < 6 { + return utf8.RuneError, false + } + + // convert hex to decimal + h1, h2, h3, h4 := hexToInt(b[2]), hexToInt(b[3]), hexToInt(b[4]), hexToInt(b[5]) + if h1 == BadHex || h2 == BadHex || h3 == BadHex || h4 == BadHex { + return utf8.RuneError, false + } + + return rune(h1<<12 + h2<<8 + h3<<4 + h4), true +} + +// decodeUnicodeEscape decodes a Unicode escape sequence from a byte slice. +func decodeUnicodeEscape(b []byte) (rune, int) { + r, ok := decodeSingleUnicodeEscape(b) + if !ok { + return utf8.RuneError, -1 + } + + // determine valid unicode escapes within the BMP + if r <= BasicMultilingualPlaneOffset && !isSurrogatePair(r) { + return r, 6 + } + + // Decode the following escape sequence to verify a UTF-16 susergate pair. + r2, ok := decodeSingleUnicodeEscape(b[6:]) + if !ok { + return utf8.RuneError, -1 + } + + if r2 < LowSurrogateOffset { + return utf8.RuneError, -1 + } + + return combineSurrogates(r, r2), 12 +} + +// processEscapedUTF8 processes the escape sequence in the given byte slice and +// and converts them to UTF-8 characters. The function returns the length of the processed input and output. +// +// The input 'in' must contain the escape sequence to be processed, +// and 'out' provides a space to store the converted characters. +// +// The function returns (input length, output length) if the escape sequence is correct. +// Unicode escape sequences (e.g. \uXXXX) are decoded to UTF-8, other default escape sequences are +// converted to their corresponding special characters (e.g. \n -> newline). +// +// If the escape sequence is invalid, or if 'in' does not completely enclose the escape sequence, +// function returns (-1, -1) to indicate an error. +func processEscapedUTF8(in, out []byte) (inLen int, outLen int) { + if len(in) < 2 || in[0] != '\\' { + return -1, -1 + } + + if val, ok := escapeMap[in[1]]; ok { + out[0] = val + return 2, 1 + } + + if in[1] == 'u' { + if r, size := decodeUnicodeEscape(in); size != -1 { + outLen = utf8.EncodeRune(out, r) + return size, outLen + } + } + + return -1, -1 +} + +func Unescape(input, output []byte) []byte { + firstBackslash := bytes.IndexByte(input, '\\') + if firstBackslash == -1 { + return input + } + + if cap(output) < len(input) { + output = make([]byte, len(input)) + } else { + output = output[:len(input)] + } + + copy(output, input[:firstBackslash]) + input = input[firstBackslash:] + buf := output[firstBackslash:] + + for len(input) > 0 { + inLen, bufLen := processEscapedUTF8(input, buf) + if inLen == -1 { + return nil + } + + input = input[inLen:] + buf = buf[bufLen:] + + // copy everything until the next backslash + nextBackslash := bytes.IndexByte(input, '\\') + if nextBackslash == -1 { + copy(buf, input) + buf = buf[len(input):] + break + } else { + copy(buf, input[:nextBackslash]) + buf = buf[nextBackslash:] + input = input[nextBackslash:] + } + } + + return output[:len(output)-len(buf)] +} diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno new file mode 100644 index 00000000000..1c9dea6e7be --- /dev/null +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -0,0 +1,186 @@ +package json + +import ( + "bytes" + "testing" + "unicode/utf8" +) + +func TestHexToInt(t *testing.T) { + tests := []struct { + name string + c byte + want int + }{ + {"Digit 0", '0', 0}, + {"Digit 9", '9', 9}, + {"Uppercase A", 'A', 10}, + {"Uppercase F", 'F', 15}, + {"Lowercase a", 'a', 10}, + {"Lowercase f", 'f', 15}, + {"Invalid character1", 'g', BadHex}, + {"Invalid character2", 'G', BadHex}, + {"Invalid character3", 'z', BadHex}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hexToInt(tt.c); got != tt.want { + t.Errorf("hexToInt() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsSurrogatePair(t *testing.T) { + testCases := []struct { + name string + r rune + expected bool + }{ + {"high surrogate start", 0xD800, true}, + {"high surrogate end", 0xDBFF, true}, + {"low surrogate start", 0xDC00, true}, + {"low surrogate end", 0xDFFF, true}, + {"Non-surrogate", 0x0000, false}, + {"Non-surrogate 2", 0xE000, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if got := isSurrogatePair(tc.r); got != tc.expected { + t.Errorf("isSurrogate() = %v, want %v", got, tc.expected) + } + }) + } +} + +func TestCombineSurrogates(t *testing.T) { + testCases := []struct { + high, low rune + expected rune + }{ + {0xD83D, 0xDC36, 0x1F436}, // 🐶 U+1F436 DOG FACE + {0xD83D, 0xDE00, 0x1F600}, // 😀 U+1F600 GRINNING FACE + {0xD83C, 0xDF03, 0x1F303}, // 🌃 U+1F303 NIGHT WITH STARS + } + + for _, tc := range testCases { + result := combineSurrogates(tc.high, tc.low) + if result != tc.expected { + t.Errorf("combineSurrogates(%U, %U) = %U; want %U", tc.high, tc.low, result, tc.expected) + } + } +} + +func TestDecodeSingleUnicodeEscape(t *testing.T) { + testCases := []struct { + input []byte + expected rune + isValid bool + len int + }{ + // valid unicode escape sequences + {[]byte(`\u0041`), 'A', true}, + {[]byte(`\u03B1`), 'α', true}, + {[]byte(`\u00E9`), 'é', true}, // valid non-English character + {[]byte(`\u0021`), '!', true}, // valid special character + {[]byte(`\uFF11`), '１', true}, + {[]byte(`\uD83D`), 0xD83D, true}, + {[]byte(`\uDE03`), 0xDE03, true}, + + // invalid unicode escape sequences + {[]byte(`\u004`), utf8.RuneError, false}, // too short + {[]byte(`\uXYZW`), utf8.RuneError, false}, // invalid hex + {[]byte(`\u00G1`), utf8.RuneError, false}, // non-hex character + } + + for _, tc := range testCases { + result, isValid := decodeSingleUnicodeEscape(tc.input) + if result != tc.expected || isValid != tc.isValid { + t.Errorf("decodeSingleUnicodeEscape(%s) = (%U, %v); want (%U, %v)", tc.input, result, isValid, tc.expected, tc.isValid) + } + } +} + +func TestDecodeUnicodeEscape(t *testing.T) { + testCases := []struct { + input string + expected rune + size int + }{ + {"\\u0041", 'A', 6}, + {"\\u03B1", 'α', 6}, + {"\\u1F600", 0x1F60, 6}, + {"\\uD830\\uDE03", 0x1C203, 12}, + {"\\uD800\\uDC00", 0x00010000, 12}, + + {"\\u004", utf8.RuneError, -1}, + {"\\uXYZW", utf8.RuneError, -1}, + {"\\uD83D\\u0041", utf8.RuneError, -1}, + } + + for _, tc := range testCases { + r, size := decodeUnicodeEscape([]byte(tc.input)) + if r != tc.expected || size != tc.size { + t.Errorf("decodeUnicodeEscape(%q) = (%U, %d); want (%U, %d)", tc.input, r, size, tc.expected, tc.size) + } + } +} + +func TestUnescapeToUTF8(t *testing.T) { + testCases := []struct { + input []byte + expectedIn int + expectedOut int + }{ + // valid escape sequences + {[]byte(`\n`), 2, 1}, + {[]byte(`\t`), 2, 1}, + {[]byte(`\u0041`), 6, 1}, + {[]byte(`\u03B1`), 6, 2}, + {[]byte(`\uD830\uDE03`), 12, 4}, + + // invalid escape sequences + {[]byte(`\`), -1, -1}, // incomplete escape sequence + {[]byte(`\x`), -1, -1}, // invalid escape character + {[]byte(`\u`), -1, -1}, // incomplete unicode escape sequence + {[]byte(`\u004`), -1, -1}, // invalid unicode escape sequence + {[]byte(`\uXYZW`), -1, -1}, // invalid unicode escape sequence + {[]byte(`\uD83D\u0041`), -1, -1}, // invalid unicode escape sequence + } + + for _, tc := range testCases { + input := make([]byte, len(tc.input)) + copy(input, tc.input) + output := make([]byte, utf8.UTFMax) + inLen, outLen := processEscapedUTF8(input, output) + + if inLen != tc.expectedIn || outLen != tc.expectedOut { + t.Errorf("processEscapedUTF8(%q) = (%d, %d); want (%d, %d)", tc.input, inLen, outLen, tc.expectedIn, tc.expectedOut) + } + } +} + +func TestUnescape(t *testing.T) { + testCases := []struct { + name string + input []byte + expected []byte + }{ + {"NoEscape", []byte("hello world"), []byte("hello world")}, + {"SingleEscape", []byte("hello\\nworld"), []byte("hello\nworld")}, + {"MultipleEscapes", []byte("line1\\nline2\\r\\nline3"), []byte("line1\nline2\r\nline3")}, + {"UnicodeEscape", []byte("snowman:\\u2603"), []byte("snowman:\u2603")}, + {"Complex", []byte("test\\n\\u2603\\r\\nend"), []byte("test\n\u2603\r\nend")}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output := Unescape(tc.input, make([]byte, len(tc.input)+10)) + if !bytes.Equal(output, tc.expected) { + t.Errorf("unescape(%q) = %q; want %q", tc.input, output, tc.expected) + } + }) + } +} diff --git a/gnovm/stdlibs/unicode/utf16/utf16.gno b/gnovm/stdlibs/unicode/utf16/utf16.gno index 89755a8b8a7..38d8be60602 100644 --- a/gnovm/stdlibs/unicode/utf16/utf16.gno +++ b/gnovm/stdlibs/unicode/utf16/utf16.gno @@ -122,4 +122,4 @@ func Decode(s []uint16) []rune { n++ } return a[:n] -} \ No newline at end of file +} diff --git a/gnovm/stdlibs/unicode/utf16/utf16_test.gno b/gnovm/stdlibs/unicode/utf16/utf16_test.gno index 6de30004053..5dc577af805 100644 --- a/gnovm/stdlibs/unicode/utf16/utf16_test.gno +++ b/gnovm/stdlibs/unicode/utf16/utf16_test.gno @@ -38,24 +38,24 @@ var encodeTests = []encodeTest{ } func slicesEqual(a, b []uint16) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true } func TestEncode(t *testing.T) { - for _, tt := range encodeTests { - out := Encode(tt.in) - if !slicesEqual(out, tt.out) { - t.Errorf("Encode(%x) = %x; want %x", tt.in, out, tt.out) - } - } + for _, tt := range encodeTests { + out := Encode(tt.in) + if !slicesEqual(out, tt.out) { + t.Errorf("Encode(%x) = %x; want %x", tt.in, out, tt.out) + } + } } func TestEncodeRune(t *testing.T) { @@ -108,7 +108,6 @@ var decodeTests = []decodeTest{ {[]uint16{0xdfff}, []rune{0xfffd}}, } - func TestDecode(t *testing.T) { for _, tt := range decodeTests { out := Decode(tt.in) @@ -119,15 +118,15 @@ func TestDecode(t *testing.T) { } func runesEqual(a, b []rune) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true } var decodeRuneTests = []struct { @@ -229,4 +228,4 @@ func BenchmarkEncodeRune(b *testing.B) { EncodeRune(u) } } -} \ No newline at end of file +} From dbeb921f5af08eb2e390b9bf2a2d6d436a6605dd Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 8 Dec 2023 13:16:20 +0900 Subject: [PATCH 05/72] parse float --- .../gno.land/p/demo/json/eisel_lemire.gno | 912 ++++++++++++++++++ .../p/demo/json/eisel_lemire_test.gno | 43 + examples/gno.land/p/demo/json/number.gno | 127 +-- examples/gno.land/p/demo/json/number_test.gno | 94 +- 4 files changed, 1089 insertions(+), 87 deletions(-) create mode 100644 examples/gno.land/p/demo/json/eisel_lemire.gno create mode 100644 examples/gno.land/p/demo/json/eisel_lemire_test.gno diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno new file mode 100644 index 00000000000..3c13c0e30e7 --- /dev/null +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -0,0 +1,912 @@ +package json + +// This file implements the Eisel-Lemire ParseFloat algorithm, published in +// 2020 and discussed extensively at +// https://nigeltao.github.io/blog/2020/eisel-lemire.html +// +// The original C++ implementation is at +// https://github.com/lemire/fast_double_parser/blob/644bef4306059d3be01a04e77d3cc84b379c596f/include/fast_double_parser.h#L840 +// +// This Go re-implementation closely follows the C re-implementation at +// https://github.com/google/wuffs/blob/ba3818cb6b473a2ed0b38ecfc07dbbd3a97e8ae7/internal/cgen/base/floatconv-submodule-code.c#L990 +// +// Additional testing (on over several million test strings) is done by +// https://github.com/nigeltao/parse-number-fxx-test-data/blob/5280dcfccf6d0b02a65ae282dad0b6d9de50e039/script/test-go-strconv.go + +import ( + "math" + "math/bits" +) + +// The Eisel-Lemire algorithm accomplishes its task through the following methods: +// +// 1. Extracting Mantissa and Exponent +// It involves separating the mantissa, or the significant part of the number, +// from the exponent when converting a string into a floating-point number. +// +// 2. Normalization and Range checking +// This step normalizes the mantissa and ensures the exponent falls within an acceptable range. +// +// 3. Conversion to a Floating-Point Number +// Finally, the algorithm uses the normalized mantissa and exponent to convert the data +// into an actual floating-point number. + +// eiselLemire64 parses a floating-point number from its mantissa and exponent representation. +// This implementation is based on the Eisel-Lemire ParseFloat algorithm, which is efficient +// and precise for converting strings to floating-point numbers. +// +// Arguments: +// man (uint64): The mantissa part of the floating-point number. +// exp10 (int): The exponent part, representing the power of 10. +// neg (bool): Indicates if the number is negative. +// +// Returns: +// f (float64): The parsed floating-point number. +// ok (bool): Indicates whether the parsing was successful. +// +// The function starts by handling special cases, such as zero mantissa. +// It then checks if the exponent is within the allowed range. +// After that, it normalizes the mantissa by left-shifting it to fill +// the leading zeros. This is followed by the main algorithm logic that +// converts the normalized mantissa and exponent into a 64-bit floating-point number. +// The function returns this number along with a boolean indicating the success of the operation. +func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { + // The terse comments in this function body refer to sections of the + // https://nigeltao.github.io/blog/2020/eisel-lemire.html blog post. + + // Exp10 Range. + if man == 0 { + if neg { + f = math.Float64frombits(0x80000000_00000000) // Negative zero. + } + return f, true + } + if exp10 < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < exp10 { + return 0, false + } + + // Normalization. + clz := bits.LeadingZeros64(man) + man <<= uint(clz) + const float64ExponentBias = 1023 + retExp2 := uint64(217706*exp10>>16+64+float64ExponentBias) - uint64(clz) + + // Multiplication. + xHi, xLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][1]) + + // Wider Approximation. + if xHi&0x1FF == 0x1FF && xLo+man < man { + yHi, yLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][0]) + mergedHi, mergedLo := xHi, xLo+yHi + if mergedLo < xLo { + mergedHi++ + } + if mergedHi&0x1FF == 0x1FF && mergedLo+1 == 0 && yLo+man < man { + return 0, false + } + xHi, xLo = mergedHi, mergedLo + } + + // Shifting to 54 Bits. + msb := xHi >> 63 + retMantissa := xHi >> (msb + 9) + retExp2 -= 1 ^ msb + + // Half-way Ambiguity. + if xLo == 0 && xHi&0x1FF == 0 && retMantissa&3 == 1 { + return 0, false + } + + // From 54 to 53 Bits. + retMantissa += retMantissa & 1 + retMantissa >>= 1 + if retMantissa>>53 > 0 { + retMantissa >>= 1 + retExp2 += 1 + } + // retExp2 is a uint64. Zero or underflow means that we're in subnormal + // float64 space. 0x7FF or above means that we're in Inf/NaN float64 space. + // + // The if block is equivalent to (but has fewer branches than): + // if retExp2 <= 0 || retExp2 >= 0x7FF { etc } + if retExp2-1 >= 0x7FF-1 { + return 0, false + } + retBits := retExp2<<52 | retMantissa&0x000FFFFF_FFFFFFFF + if neg { + retBits |= 0x80000000_00000000 + } + return math.Float64frombits(retBits), true +} + +func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { + // The terse comments in this function body refer to sections of the + // https://nigeltao.github.io/blog/2020/eisel-lemire.html blog post. + // + // That blog post discusses the float64 flavor (11 exponent bits with a + // -1023 bias, 52 mantissa bits) of the algorithm, but the same approach + // applies to the float32 flavor (8 exponent bits with a -127 bias, 23 + // mantissa bits). The computation here happens with 64-bit values (e.g. + // man, xHi, retMantissa) before finally converting to a 32-bit float. + + // Exp10 Range. + if man == 0 { + if neg { + f = math.Float32frombits(0x80000000) // Negative zero. + } + return f, true + } + if exp10 < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < exp10 { + return 0, false + } + + // Normalization. + clz := bits.LeadingZeros64(man) + man <<= uint(clz) + const float32ExponentBias = 127 + retExp2 := uint64(217706*exp10>>16+64+float32ExponentBias) - uint64(clz) + + // Multiplication. + xHi, xLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][1]) + + // Wider Approximation. + if xHi&0x3F_FFFFFFFF == 0x3F_FFFFFFFF && xLo+man < man { + yHi, yLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][0]) + mergedHi, mergedLo := xHi, xLo+yHi + if mergedLo < xLo { + mergedHi++ + } + if mergedHi&0x3F_FFFFFFFF == 0x3F_FFFFFFFF && mergedLo+1 == 0 && yLo+man < man { + return 0, false + } + xHi, xLo = mergedHi, mergedLo + } + + // Shifting to 54 Bits (and for float32, it's shifting to 25 bits). + msb := xHi >> 63 + retMantissa := xHi >> (msb + 38) + retExp2 -= 1 ^ msb + + // Half-way Ambiguity. + if xLo == 0 && xHi&0x3F_FFFFFFFF == 0 && retMantissa&3 == 1 { + return 0, false + } + + // From 54 to 53 Bits (and for float32, it's from 25 to 24 bits). + retMantissa += retMantissa & 1 + retMantissa >>= 1 + if retMantissa>>24 > 0 { + retMantissa >>= 1 + retExp2 += 1 + } + // retExp2 is a uint64. Zero or underflow means that we're in subnormal + // float32 space. 0xFF or above means that we're in Inf/NaN float32 space. + // + // The if block is equivalent to (but has fewer branches than): + // if retExp2 <= 0 || retExp2 >= 0xFF { etc } + if retExp2-1 >= 0xFF-1 { + return 0, false + } + retBits := retExp2<<23 | retMantissa&0x007FFFFF + if neg { + retBits |= 0x80000000 + } + return math.Float32frombits(uint32(retBits)), true +} + +// detailedPowersOfTen{Min,Max}Exp10 is the power of 10 represented by the +// first and last rows of detailedPowersOfTen. Both bounds are inclusive. +const ( + detailedPowersOfTenMinExp10 = -348 + detailedPowersOfTenMaxExp10 = +347 +) + +// detailedPowersOfTen contains 128-bit mantissa approximations (rounded down) +// to the powers of 10. For example: +// +// - 1e43 ≈ (0xE596B7B0_C643C719 * (2 ** 79)) +// - 1e43 = (0xE596B7B0_C643C719_6D9CCD05_D0000000 * (2 ** 15)) +// +// The mantissas are explicitly listed. The exponents are implied by a linear +// expression with slope 217706.0/65536.0 ≈ log(10)/log(2). +// +// The table was generated by +// https://github.com/google/wuffs/blob/ba3818cb6b473a2ed0b38ecfc07dbbd3a97e8ae7/script/print-mpb-powers-of-10.go +var detailedPowersOfTen = [...][2]uint64{ + {0x1732C869CD60E453, 0xFA8FD5A0081C0288}, // 1e-348 + {0x0E7FBD42205C8EB4, 0x9C99E58405118195}, // 1e-347 + {0x521FAC92A873B261, 0xC3C05EE50655E1FA}, // 1e-346 + {0xE6A797B752909EF9, 0xF4B0769E47EB5A78}, // 1e-345 + {0x9028BED2939A635C, 0x98EE4A22ECF3188B}, // 1e-344 + {0x7432EE873880FC33, 0xBF29DCABA82FDEAE}, // 1e-343 + {0x113FAA2906A13B3F, 0xEEF453D6923BD65A}, // 1e-342 + {0x4AC7CA59A424C507, 0x9558B4661B6565F8}, // 1e-341 + {0x5D79BCF00D2DF649, 0xBAAEE17FA23EBF76}, // 1e-340 + {0xF4D82C2C107973DC, 0xE95A99DF8ACE6F53}, // 1e-339 + {0x79071B9B8A4BE869, 0x91D8A02BB6C10594}, // 1e-338 + {0x9748E2826CDEE284, 0xB64EC836A47146F9}, // 1e-337 + {0xFD1B1B2308169B25, 0xE3E27A444D8D98B7}, // 1e-336 + {0xFE30F0F5E50E20F7, 0x8E6D8C6AB0787F72}, // 1e-335 + {0xBDBD2D335E51A935, 0xB208EF855C969F4F}, // 1e-334 + {0xAD2C788035E61382, 0xDE8B2B66B3BC4723}, // 1e-333 + {0x4C3BCB5021AFCC31, 0x8B16FB203055AC76}, // 1e-332 + {0xDF4ABE242A1BBF3D, 0xADDCB9E83C6B1793}, // 1e-331 + {0xD71D6DAD34A2AF0D, 0xD953E8624B85DD78}, // 1e-330 + {0x8672648C40E5AD68, 0x87D4713D6F33AA6B}, // 1e-329 + {0x680EFDAF511F18C2, 0xA9C98D8CCB009506}, // 1e-328 + {0x0212BD1B2566DEF2, 0xD43BF0EFFDC0BA48}, // 1e-327 + {0x014BB630F7604B57, 0x84A57695FE98746D}, // 1e-326 + {0x419EA3BD35385E2D, 0xA5CED43B7E3E9188}, // 1e-325 + {0x52064CAC828675B9, 0xCF42894A5DCE35EA}, // 1e-324 + {0x7343EFEBD1940993, 0x818995CE7AA0E1B2}, // 1e-323 + {0x1014EBE6C5F90BF8, 0xA1EBFB4219491A1F}, // 1e-322 + {0xD41A26E077774EF6, 0xCA66FA129F9B60A6}, // 1e-321 + {0x8920B098955522B4, 0xFD00B897478238D0}, // 1e-320 + {0x55B46E5F5D5535B0, 0x9E20735E8CB16382}, // 1e-319 + {0xEB2189F734AA831D, 0xC5A890362FDDBC62}, // 1e-318 + {0xA5E9EC7501D523E4, 0xF712B443BBD52B7B}, // 1e-317 + {0x47B233C92125366E, 0x9A6BB0AA55653B2D}, // 1e-316 + {0x999EC0BB696E840A, 0xC1069CD4EABE89F8}, // 1e-315 + {0xC00670EA43CA250D, 0xF148440A256E2C76}, // 1e-314 + {0x380406926A5E5728, 0x96CD2A865764DBCA}, // 1e-313 + {0xC605083704F5ECF2, 0xBC807527ED3E12BC}, // 1e-312 + {0xF7864A44C633682E, 0xEBA09271E88D976B}, // 1e-311 + {0x7AB3EE6AFBE0211D, 0x93445B8731587EA3}, // 1e-310 + {0x5960EA05BAD82964, 0xB8157268FDAE9E4C}, // 1e-309 + {0x6FB92487298E33BD, 0xE61ACF033D1A45DF}, // 1e-308 + {0xA5D3B6D479F8E056, 0x8FD0C16206306BAB}, // 1e-307 + {0x8F48A4899877186C, 0xB3C4F1BA87BC8696}, // 1e-306 + {0x331ACDABFE94DE87, 0xE0B62E2929ABA83C}, // 1e-305 + {0x9FF0C08B7F1D0B14, 0x8C71DCD9BA0B4925}, // 1e-304 + {0x07ECF0AE5EE44DD9, 0xAF8E5410288E1B6F}, // 1e-303 + {0xC9E82CD9F69D6150, 0xDB71E91432B1A24A}, // 1e-302 + {0xBE311C083A225CD2, 0x892731AC9FAF056E}, // 1e-301 + {0x6DBD630A48AAF406, 0xAB70FE17C79AC6CA}, // 1e-300 + {0x092CBBCCDAD5B108, 0xD64D3D9DB981787D}, // 1e-299 + {0x25BBF56008C58EA5, 0x85F0468293F0EB4E}, // 1e-298 + {0xAF2AF2B80AF6F24E, 0xA76C582338ED2621}, // 1e-297 + {0x1AF5AF660DB4AEE1, 0xD1476E2C07286FAA}, // 1e-296 + {0x50D98D9FC890ED4D, 0x82CCA4DB847945CA}, // 1e-295 + {0xE50FF107BAB528A0, 0xA37FCE126597973C}, // 1e-294 + {0x1E53ED49A96272C8, 0xCC5FC196FEFD7D0C}, // 1e-293 + {0x25E8E89C13BB0F7A, 0xFF77B1FCBEBCDC4F}, // 1e-292 + {0x77B191618C54E9AC, 0x9FAACF3DF73609B1}, // 1e-291 + {0xD59DF5B9EF6A2417, 0xC795830D75038C1D}, // 1e-290 + {0x4B0573286B44AD1D, 0xF97AE3D0D2446F25}, // 1e-289 + {0x4EE367F9430AEC32, 0x9BECCE62836AC577}, // 1e-288 + {0x229C41F793CDA73F, 0xC2E801FB244576D5}, // 1e-287 + {0x6B43527578C1110F, 0xF3A20279ED56D48A}, // 1e-286 + {0x830A13896B78AAA9, 0x9845418C345644D6}, // 1e-285 + {0x23CC986BC656D553, 0xBE5691EF416BD60C}, // 1e-284 + {0x2CBFBE86B7EC8AA8, 0xEDEC366B11C6CB8F}, // 1e-283 + {0x7BF7D71432F3D6A9, 0x94B3A202EB1C3F39}, // 1e-282 + {0xDAF5CCD93FB0CC53, 0xB9E08A83A5E34F07}, // 1e-281 + {0xD1B3400F8F9CFF68, 0xE858AD248F5C22C9}, // 1e-280 + {0x23100809B9C21FA1, 0x91376C36D99995BE}, // 1e-279 + {0xABD40A0C2832A78A, 0xB58547448FFFFB2D}, // 1e-278 + {0x16C90C8F323F516C, 0xE2E69915B3FFF9F9}, // 1e-277 + {0xAE3DA7D97F6792E3, 0x8DD01FAD907FFC3B}, // 1e-276 + {0x99CD11CFDF41779C, 0xB1442798F49FFB4A}, // 1e-275 + {0x40405643D711D583, 0xDD95317F31C7FA1D}, // 1e-274 + {0x482835EA666B2572, 0x8A7D3EEF7F1CFC52}, // 1e-273 + {0xDA3243650005EECF, 0xAD1C8EAB5EE43B66}, // 1e-272 + {0x90BED43E40076A82, 0xD863B256369D4A40}, // 1e-271 + {0x5A7744A6E804A291, 0x873E4F75E2224E68}, // 1e-270 + {0x711515D0A205CB36, 0xA90DE3535AAAE202}, // 1e-269 + {0x0D5A5B44CA873E03, 0xD3515C2831559A83}, // 1e-268 + {0xE858790AFE9486C2, 0x8412D9991ED58091}, // 1e-267 + {0x626E974DBE39A872, 0xA5178FFF668AE0B6}, // 1e-266 + {0xFB0A3D212DC8128F, 0xCE5D73FF402D98E3}, // 1e-265 + {0x7CE66634BC9D0B99, 0x80FA687F881C7F8E}, // 1e-264 + {0x1C1FFFC1EBC44E80, 0xA139029F6A239F72}, // 1e-263 + {0xA327FFB266B56220, 0xC987434744AC874E}, // 1e-262 + {0x4BF1FF9F0062BAA8, 0xFBE9141915D7A922}, // 1e-261 + {0x6F773FC3603DB4A9, 0x9D71AC8FADA6C9B5}, // 1e-260 + {0xCB550FB4384D21D3, 0xC4CE17B399107C22}, // 1e-259 + {0x7E2A53A146606A48, 0xF6019DA07F549B2B}, // 1e-258 + {0x2EDA7444CBFC426D, 0x99C102844F94E0FB}, // 1e-257 + {0xFA911155FEFB5308, 0xC0314325637A1939}, // 1e-256 + {0x793555AB7EBA27CA, 0xF03D93EEBC589F88}, // 1e-255 + {0x4BC1558B2F3458DE, 0x96267C7535B763B5}, // 1e-254 + {0x9EB1AAEDFB016F16, 0xBBB01B9283253CA2}, // 1e-253 + {0x465E15A979C1CADC, 0xEA9C227723EE8BCB}, // 1e-252 + {0x0BFACD89EC191EC9, 0x92A1958A7675175F}, // 1e-251 + {0xCEF980EC671F667B, 0xB749FAED14125D36}, // 1e-250 + {0x82B7E12780E7401A, 0xE51C79A85916F484}, // 1e-249 + {0xD1B2ECB8B0908810, 0x8F31CC0937AE58D2}, // 1e-248 + {0x861FA7E6DCB4AA15, 0xB2FE3F0B8599EF07}, // 1e-247 + {0x67A791E093E1D49A, 0xDFBDCECE67006AC9}, // 1e-246 + {0xE0C8BB2C5C6D24E0, 0x8BD6A141006042BD}, // 1e-245 + {0x58FAE9F773886E18, 0xAECC49914078536D}, // 1e-244 + {0xAF39A475506A899E, 0xDA7F5BF590966848}, // 1e-243 + {0x6D8406C952429603, 0x888F99797A5E012D}, // 1e-242 + {0xC8E5087BA6D33B83, 0xAAB37FD7D8F58178}, // 1e-241 + {0xFB1E4A9A90880A64, 0xD5605FCDCF32E1D6}, // 1e-240 + {0x5CF2EEA09A55067F, 0x855C3BE0A17FCD26}, // 1e-239 + {0xF42FAA48C0EA481E, 0xA6B34AD8C9DFC06F}, // 1e-238 + {0xF13B94DAF124DA26, 0xD0601D8EFC57B08B}, // 1e-237 + {0x76C53D08D6B70858, 0x823C12795DB6CE57}, // 1e-236 + {0x54768C4B0C64CA6E, 0xA2CB1717B52481ED}, // 1e-235 + {0xA9942F5DCF7DFD09, 0xCB7DDCDDA26DA268}, // 1e-234 + {0xD3F93B35435D7C4C, 0xFE5D54150B090B02}, // 1e-233 + {0xC47BC5014A1A6DAF, 0x9EFA548D26E5A6E1}, // 1e-232 + {0x359AB6419CA1091B, 0xC6B8E9B0709F109A}, // 1e-231 + {0xC30163D203C94B62, 0xF867241C8CC6D4C0}, // 1e-230 + {0x79E0DE63425DCF1D, 0x9B407691D7FC44F8}, // 1e-229 + {0x985915FC12F542E4, 0xC21094364DFB5636}, // 1e-228 + {0x3E6F5B7B17B2939D, 0xF294B943E17A2BC4}, // 1e-227 + {0xA705992CEECF9C42, 0x979CF3CA6CEC5B5A}, // 1e-226 + {0x50C6FF782A838353, 0xBD8430BD08277231}, // 1e-225 + {0xA4F8BF5635246428, 0xECE53CEC4A314EBD}, // 1e-224 + {0x871B7795E136BE99, 0x940F4613AE5ED136}, // 1e-223 + {0x28E2557B59846E3F, 0xB913179899F68584}, // 1e-222 + {0x331AEADA2FE589CF, 0xE757DD7EC07426E5}, // 1e-221 + {0x3FF0D2C85DEF7621, 0x9096EA6F3848984F}, // 1e-220 + {0x0FED077A756B53A9, 0xB4BCA50B065ABE63}, // 1e-219 + {0xD3E8495912C62894, 0xE1EBCE4DC7F16DFB}, // 1e-218 + {0x64712DD7ABBBD95C, 0x8D3360F09CF6E4BD}, // 1e-217 + {0xBD8D794D96AACFB3, 0xB080392CC4349DEC}, // 1e-216 + {0xECF0D7A0FC5583A0, 0xDCA04777F541C567}, // 1e-215 + {0xF41686C49DB57244, 0x89E42CAAF9491B60}, // 1e-214 + {0x311C2875C522CED5, 0xAC5D37D5B79B6239}, // 1e-213 + {0x7D633293366B828B, 0xD77485CB25823AC7}, // 1e-212 + {0xAE5DFF9C02033197, 0x86A8D39EF77164BC}, // 1e-211 + {0xD9F57F830283FDFC, 0xA8530886B54DBDEB}, // 1e-210 + {0xD072DF63C324FD7B, 0xD267CAA862A12D66}, // 1e-209 + {0x4247CB9E59F71E6D, 0x8380DEA93DA4BC60}, // 1e-208 + {0x52D9BE85F074E608, 0xA46116538D0DEB78}, // 1e-207 + {0x67902E276C921F8B, 0xCD795BE870516656}, // 1e-206 + {0x00BA1CD8A3DB53B6, 0x806BD9714632DFF6}, // 1e-205 + {0x80E8A40ECCD228A4, 0xA086CFCD97BF97F3}, // 1e-204 + {0x6122CD128006B2CD, 0xC8A883C0FDAF7DF0}, // 1e-203 + {0x796B805720085F81, 0xFAD2A4B13D1B5D6C}, // 1e-202 + {0xCBE3303674053BB0, 0x9CC3A6EEC6311A63}, // 1e-201 + {0xBEDBFC4411068A9C, 0xC3F490AA77BD60FC}, // 1e-200 + {0xEE92FB5515482D44, 0xF4F1B4D515ACB93B}, // 1e-199 + {0x751BDD152D4D1C4A, 0x991711052D8BF3C5}, // 1e-198 + {0xD262D45A78A0635D, 0xBF5CD54678EEF0B6}, // 1e-197 + {0x86FB897116C87C34, 0xEF340A98172AACE4}, // 1e-196 + {0xD45D35E6AE3D4DA0, 0x9580869F0E7AAC0E}, // 1e-195 + {0x8974836059CCA109, 0xBAE0A846D2195712}, // 1e-194 + {0x2BD1A438703FC94B, 0xE998D258869FACD7}, // 1e-193 + {0x7B6306A34627DDCF, 0x91FF83775423CC06}, // 1e-192 + {0x1A3BC84C17B1D542, 0xB67F6455292CBF08}, // 1e-191 + {0x20CABA5F1D9E4A93, 0xE41F3D6A7377EECA}, // 1e-190 + {0x547EB47B7282EE9C, 0x8E938662882AF53E}, // 1e-189 + {0xE99E619A4F23AA43, 0xB23867FB2A35B28D}, // 1e-188 + {0x6405FA00E2EC94D4, 0xDEC681F9F4C31F31}, // 1e-187 + {0xDE83BC408DD3DD04, 0x8B3C113C38F9F37E}, // 1e-186 + {0x9624AB50B148D445, 0xAE0B158B4738705E}, // 1e-185 + {0x3BADD624DD9B0957, 0xD98DDAEE19068C76}, // 1e-184 + {0xE54CA5D70A80E5D6, 0x87F8A8D4CFA417C9}, // 1e-183 + {0x5E9FCF4CCD211F4C, 0xA9F6D30A038D1DBC}, // 1e-182 + {0x7647C3200069671F, 0xD47487CC8470652B}, // 1e-181 + {0x29ECD9F40041E073, 0x84C8D4DFD2C63F3B}, // 1e-180 + {0xF468107100525890, 0xA5FB0A17C777CF09}, // 1e-179 + {0x7182148D4066EEB4, 0xCF79CC9DB955C2CC}, // 1e-178 + {0xC6F14CD848405530, 0x81AC1FE293D599BF}, // 1e-177 + {0xB8ADA00E5A506A7C, 0xA21727DB38CB002F}, // 1e-176 + {0xA6D90811F0E4851C, 0xCA9CF1D206FDC03B}, // 1e-175 + {0x908F4A166D1DA663, 0xFD442E4688BD304A}, // 1e-174 + {0x9A598E4E043287FE, 0x9E4A9CEC15763E2E}, // 1e-173 + {0x40EFF1E1853F29FD, 0xC5DD44271AD3CDBA}, // 1e-172 + {0xD12BEE59E68EF47C, 0xF7549530E188C128}, // 1e-171 + {0x82BB74F8301958CE, 0x9A94DD3E8CF578B9}, // 1e-170 + {0xE36A52363C1FAF01, 0xC13A148E3032D6E7}, // 1e-169 + {0xDC44E6C3CB279AC1, 0xF18899B1BC3F8CA1}, // 1e-168 + {0x29AB103A5EF8C0B9, 0x96F5600F15A7B7E5}, // 1e-167 + {0x7415D448F6B6F0E7, 0xBCB2B812DB11A5DE}, // 1e-166 + {0x111B495B3464AD21, 0xEBDF661791D60F56}, // 1e-165 + {0xCAB10DD900BEEC34, 0x936B9FCEBB25C995}, // 1e-164 + {0x3D5D514F40EEA742, 0xB84687C269EF3BFB}, // 1e-163 + {0x0CB4A5A3112A5112, 0xE65829B3046B0AFA}, // 1e-162 + {0x47F0E785EABA72AB, 0x8FF71A0FE2C2E6DC}, // 1e-161 + {0x59ED216765690F56, 0xB3F4E093DB73A093}, // 1e-160 + {0x306869C13EC3532C, 0xE0F218B8D25088B8}, // 1e-159 + {0x1E414218C73A13FB, 0x8C974F7383725573}, // 1e-158 + {0xE5D1929EF90898FA, 0xAFBD2350644EEACF}, // 1e-157 + {0xDF45F746B74ABF39, 0xDBAC6C247D62A583}, // 1e-156 + {0x6B8BBA8C328EB783, 0x894BC396CE5DA772}, // 1e-155 + {0x066EA92F3F326564, 0xAB9EB47C81F5114F}, // 1e-154 + {0xC80A537B0EFEFEBD, 0xD686619BA27255A2}, // 1e-153 + {0xBD06742CE95F5F36, 0x8613FD0145877585}, // 1e-152 + {0x2C48113823B73704, 0xA798FC4196E952E7}, // 1e-151 + {0xF75A15862CA504C5, 0xD17F3B51FCA3A7A0}, // 1e-150 + {0x9A984D73DBE722FB, 0x82EF85133DE648C4}, // 1e-149 + {0xC13E60D0D2E0EBBA, 0xA3AB66580D5FDAF5}, // 1e-148 + {0x318DF905079926A8, 0xCC963FEE10B7D1B3}, // 1e-147 + {0xFDF17746497F7052, 0xFFBBCFE994E5C61F}, // 1e-146 + {0xFEB6EA8BEDEFA633, 0x9FD561F1FD0F9BD3}, // 1e-145 + {0xFE64A52EE96B8FC0, 0xC7CABA6E7C5382C8}, // 1e-144 + {0x3DFDCE7AA3C673B0, 0xF9BD690A1B68637B}, // 1e-143 + {0x06BEA10CA65C084E, 0x9C1661A651213E2D}, // 1e-142 + {0x486E494FCFF30A62, 0xC31BFA0FE5698DB8}, // 1e-141 + {0x5A89DBA3C3EFCCFA, 0xF3E2F893DEC3F126}, // 1e-140 + {0xF89629465A75E01C, 0x986DDB5C6B3A76B7}, // 1e-139 + {0xF6BBB397F1135823, 0xBE89523386091465}, // 1e-138 + {0x746AA07DED582E2C, 0xEE2BA6C0678B597F}, // 1e-137 + {0xA8C2A44EB4571CDC, 0x94DB483840B717EF}, // 1e-136 + {0x92F34D62616CE413, 0xBA121A4650E4DDEB}, // 1e-135 + {0x77B020BAF9C81D17, 0xE896A0D7E51E1566}, // 1e-134 + {0x0ACE1474DC1D122E, 0x915E2486EF32CD60}, // 1e-133 + {0x0D819992132456BA, 0xB5B5ADA8AAFF80B8}, // 1e-132 + {0x10E1FFF697ED6C69, 0xE3231912D5BF60E6}, // 1e-131 + {0xCA8D3FFA1EF463C1, 0x8DF5EFABC5979C8F}, // 1e-130 + {0xBD308FF8A6B17CB2, 0xB1736B96B6FD83B3}, // 1e-129 + {0xAC7CB3F6D05DDBDE, 0xDDD0467C64BCE4A0}, // 1e-128 + {0x6BCDF07A423AA96B, 0x8AA22C0DBEF60EE4}, // 1e-127 + {0x86C16C98D2C953C6, 0xAD4AB7112EB3929D}, // 1e-126 + {0xE871C7BF077BA8B7, 0xD89D64D57A607744}, // 1e-125 + {0x11471CD764AD4972, 0x87625F056C7C4A8B}, // 1e-124 + {0xD598E40D3DD89BCF, 0xA93AF6C6C79B5D2D}, // 1e-123 + {0x4AFF1D108D4EC2C3, 0xD389B47879823479}, // 1e-122 + {0xCEDF722A585139BA, 0x843610CB4BF160CB}, // 1e-121 + {0xC2974EB4EE658828, 0xA54394FE1EEDB8FE}, // 1e-120 + {0x733D226229FEEA32, 0xCE947A3DA6A9273E}, // 1e-119 + {0x0806357D5A3F525F, 0x811CCC668829B887}, // 1e-118 + {0xCA07C2DCB0CF26F7, 0xA163FF802A3426A8}, // 1e-117 + {0xFC89B393DD02F0B5, 0xC9BCFF6034C13052}, // 1e-116 + {0xBBAC2078D443ACE2, 0xFC2C3F3841F17C67}, // 1e-115 + {0xD54B944B84AA4C0D, 0x9D9BA7832936EDC0}, // 1e-114 + {0x0A9E795E65D4DF11, 0xC5029163F384A931}, // 1e-113 + {0x4D4617B5FF4A16D5, 0xF64335BCF065D37D}, // 1e-112 + {0x504BCED1BF8E4E45, 0x99EA0196163FA42E}, // 1e-111 + {0xE45EC2862F71E1D6, 0xC06481FB9BCF8D39}, // 1e-110 + {0x5D767327BB4E5A4C, 0xF07DA27A82C37088}, // 1e-109 + {0x3A6A07F8D510F86F, 0x964E858C91BA2655}, // 1e-108 + {0x890489F70A55368B, 0xBBE226EFB628AFEA}, // 1e-107 + {0x2B45AC74CCEA842E, 0xEADAB0ABA3B2DBE5}, // 1e-106 + {0x3B0B8BC90012929D, 0x92C8AE6B464FC96F}, // 1e-105 + {0x09CE6EBB40173744, 0xB77ADA0617E3BBCB}, // 1e-104 + {0xCC420A6A101D0515, 0xE55990879DDCAABD}, // 1e-103 + {0x9FA946824A12232D, 0x8F57FA54C2A9EAB6}, // 1e-102 + {0x47939822DC96ABF9, 0xB32DF8E9F3546564}, // 1e-101 + {0x59787E2B93BC56F7, 0xDFF9772470297EBD}, // 1e-100 + {0x57EB4EDB3C55B65A, 0x8BFBEA76C619EF36}, // 1e-99 + {0xEDE622920B6B23F1, 0xAEFAE51477A06B03}, // 1e-98 + {0xE95FAB368E45ECED, 0xDAB99E59958885C4}, // 1e-97 + {0x11DBCB0218EBB414, 0x88B402F7FD75539B}, // 1e-96 + {0xD652BDC29F26A119, 0xAAE103B5FCD2A881}, // 1e-95 + {0x4BE76D3346F0495F, 0xD59944A37C0752A2}, // 1e-94 + {0x6F70A4400C562DDB, 0x857FCAE62D8493A5}, // 1e-93 + {0xCB4CCD500F6BB952, 0xA6DFBD9FB8E5B88E}, // 1e-92 + {0x7E2000A41346A7A7, 0xD097AD07A71F26B2}, // 1e-91 + {0x8ED400668C0C28C8, 0x825ECC24C873782F}, // 1e-90 + {0x728900802F0F32FA, 0xA2F67F2DFA90563B}, // 1e-89 + {0x4F2B40A03AD2FFB9, 0xCBB41EF979346BCA}, // 1e-88 + {0xE2F610C84987BFA8, 0xFEA126B7D78186BC}, // 1e-87 + {0x0DD9CA7D2DF4D7C9, 0x9F24B832E6B0F436}, // 1e-86 + {0x91503D1C79720DBB, 0xC6EDE63FA05D3143}, // 1e-85 + {0x75A44C6397CE912A, 0xF8A95FCF88747D94}, // 1e-84 + {0xC986AFBE3EE11ABA, 0x9B69DBE1B548CE7C}, // 1e-83 + {0xFBE85BADCE996168, 0xC24452DA229B021B}, // 1e-82 + {0xFAE27299423FB9C3, 0xF2D56790AB41C2A2}, // 1e-81 + {0xDCCD879FC967D41A, 0x97C560BA6B0919A5}, // 1e-80 + {0x5400E987BBC1C920, 0xBDB6B8E905CB600F}, // 1e-79 + {0x290123E9AAB23B68, 0xED246723473E3813}, // 1e-78 + {0xF9A0B6720AAF6521, 0x9436C0760C86E30B}, // 1e-77 + {0xF808E40E8D5B3E69, 0xB94470938FA89BCE}, // 1e-76 + {0xB60B1D1230B20E04, 0xE7958CB87392C2C2}, // 1e-75 + {0xB1C6F22B5E6F48C2, 0x90BD77F3483BB9B9}, // 1e-74 + {0x1E38AEB6360B1AF3, 0xB4ECD5F01A4AA828}, // 1e-73 + {0x25C6DA63C38DE1B0, 0xE2280B6C20DD5232}, // 1e-72 + {0x579C487E5A38AD0E, 0x8D590723948A535F}, // 1e-71 + {0x2D835A9DF0C6D851, 0xB0AF48EC79ACE837}, // 1e-70 + {0xF8E431456CF88E65, 0xDCDB1B2798182244}, // 1e-69 + {0x1B8E9ECB641B58FF, 0x8A08F0F8BF0F156B}, // 1e-68 + {0xE272467E3D222F3F, 0xAC8B2D36EED2DAC5}, // 1e-67 + {0x5B0ED81DCC6ABB0F, 0xD7ADF884AA879177}, // 1e-66 + {0x98E947129FC2B4E9, 0x86CCBB52EA94BAEA}, // 1e-65 + {0x3F2398D747B36224, 0xA87FEA27A539E9A5}, // 1e-64 + {0x8EEC7F0D19A03AAD, 0xD29FE4B18E88640E}, // 1e-63 + {0x1953CF68300424AC, 0x83A3EEEEF9153E89}, // 1e-62 + {0x5FA8C3423C052DD7, 0xA48CEAAAB75A8E2B}, // 1e-61 + {0x3792F412CB06794D, 0xCDB02555653131B6}, // 1e-60 + {0xE2BBD88BBEE40BD0, 0x808E17555F3EBF11}, // 1e-59 + {0x5B6ACEAEAE9D0EC4, 0xA0B19D2AB70E6ED6}, // 1e-58 + {0xF245825A5A445275, 0xC8DE047564D20A8B}, // 1e-57 + {0xEED6E2F0F0D56712, 0xFB158592BE068D2E}, // 1e-56 + {0x55464DD69685606B, 0x9CED737BB6C4183D}, // 1e-55 + {0xAA97E14C3C26B886, 0xC428D05AA4751E4C}, // 1e-54 + {0xD53DD99F4B3066A8, 0xF53304714D9265DF}, // 1e-53 + {0xE546A8038EFE4029, 0x993FE2C6D07B7FAB}, // 1e-52 + {0xDE98520472BDD033, 0xBF8FDB78849A5F96}, // 1e-51 + {0x963E66858F6D4440, 0xEF73D256A5C0F77C}, // 1e-50 + {0xDDE7001379A44AA8, 0x95A8637627989AAD}, // 1e-49 + {0x5560C018580D5D52, 0xBB127C53B17EC159}, // 1e-48 + {0xAAB8F01E6E10B4A6, 0xE9D71B689DDE71AF}, // 1e-47 + {0xCAB3961304CA70E8, 0x9226712162AB070D}, // 1e-46 + {0x3D607B97C5FD0D22, 0xB6B00D69BB55C8D1}, // 1e-45 + {0x8CB89A7DB77C506A, 0xE45C10C42A2B3B05}, // 1e-44 + {0x77F3608E92ADB242, 0x8EB98A7A9A5B04E3}, // 1e-43 + {0x55F038B237591ED3, 0xB267ED1940F1C61C}, // 1e-42 + {0x6B6C46DEC52F6688, 0xDF01E85F912E37A3}, // 1e-41 + {0x2323AC4B3B3DA015, 0x8B61313BBABCE2C6}, // 1e-40 + {0xABEC975E0A0D081A, 0xAE397D8AA96C1B77}, // 1e-39 + {0x96E7BD358C904A21, 0xD9C7DCED53C72255}, // 1e-38 + {0x7E50D64177DA2E54, 0x881CEA14545C7575}, // 1e-37 + {0xDDE50BD1D5D0B9E9, 0xAA242499697392D2}, // 1e-36 + {0x955E4EC64B44E864, 0xD4AD2DBFC3D07787}, // 1e-35 + {0xBD5AF13BEF0B113E, 0x84EC3C97DA624AB4}, // 1e-34 + {0xECB1AD8AEACDD58E, 0xA6274BBDD0FADD61}, // 1e-33 + {0x67DE18EDA5814AF2, 0xCFB11EAD453994BA}, // 1e-32 + {0x80EACF948770CED7, 0x81CEB32C4B43FCF4}, // 1e-31 + {0xA1258379A94D028D, 0xA2425FF75E14FC31}, // 1e-30 + {0x096EE45813A04330, 0xCAD2F7F5359A3B3E}, // 1e-29 + {0x8BCA9D6E188853FC, 0xFD87B5F28300CA0D}, // 1e-28 + {0x775EA264CF55347D, 0x9E74D1B791E07E48}, // 1e-27 + {0x95364AFE032A819D, 0xC612062576589DDA}, // 1e-26 + {0x3A83DDBD83F52204, 0xF79687AED3EEC551}, // 1e-25 + {0xC4926A9672793542, 0x9ABE14CD44753B52}, // 1e-24 + {0x75B7053C0F178293, 0xC16D9A0095928A27}, // 1e-23 + {0x5324C68B12DD6338, 0xF1C90080BAF72CB1}, // 1e-22 + {0xD3F6FC16EBCA5E03, 0x971DA05074DA7BEE}, // 1e-21 + {0x88F4BB1CA6BCF584, 0xBCE5086492111AEA}, // 1e-20 + {0x2B31E9E3D06C32E5, 0xEC1E4A7DB69561A5}, // 1e-19 + {0x3AFF322E62439FCF, 0x9392EE8E921D5D07}, // 1e-18 + {0x09BEFEB9FAD487C2, 0xB877AA3236A4B449}, // 1e-17 + {0x4C2EBE687989A9B3, 0xE69594BEC44DE15B}, // 1e-16 + {0x0F9D37014BF60A10, 0x901D7CF73AB0ACD9}, // 1e-15 + {0x538484C19EF38C94, 0xB424DC35095CD80F}, // 1e-14 + {0x2865A5F206B06FB9, 0xE12E13424BB40E13}, // 1e-13 + {0xF93F87B7442E45D3, 0x8CBCCC096F5088CB}, // 1e-12 + {0xF78F69A51539D748, 0xAFEBFF0BCB24AAFE}, // 1e-11 + {0xB573440E5A884D1B, 0xDBE6FECEBDEDD5BE}, // 1e-10 + {0x31680A88F8953030, 0x89705F4136B4A597}, // 1e-9 + {0xFDC20D2B36BA7C3D, 0xABCC77118461CEFC}, // 1e-8 + {0x3D32907604691B4C, 0xD6BF94D5E57A42BC}, // 1e-7 + {0xA63F9A49C2C1B10F, 0x8637BD05AF6C69B5}, // 1e-6 + {0x0FCF80DC33721D53, 0xA7C5AC471B478423}, // 1e-5 + {0xD3C36113404EA4A8, 0xD1B71758E219652B}, // 1e-4 + {0x645A1CAC083126E9, 0x83126E978D4FDF3B}, // 1e-3 + {0x3D70A3D70A3D70A3, 0xA3D70A3D70A3D70A}, // 1e-2 + {0xCCCCCCCCCCCCCCCC, 0xCCCCCCCCCCCCCCCC}, // 1e-1 + {0x0000000000000000, 0x8000000000000000}, // 1e0 + {0x0000000000000000, 0xA000000000000000}, // 1e1 + {0x0000000000000000, 0xC800000000000000}, // 1e2 + {0x0000000000000000, 0xFA00000000000000}, // 1e3 + {0x0000000000000000, 0x9C40000000000000}, // 1e4 + {0x0000000000000000, 0xC350000000000000}, // 1e5 + {0x0000000000000000, 0xF424000000000000}, // 1e6 + {0x0000000000000000, 0x9896800000000000}, // 1e7 + {0x0000000000000000, 0xBEBC200000000000}, // 1e8 + {0x0000000000000000, 0xEE6B280000000000}, // 1e9 + {0x0000000000000000, 0x9502F90000000000}, // 1e10 + {0x0000000000000000, 0xBA43B74000000000}, // 1e11 + {0x0000000000000000, 0xE8D4A51000000000}, // 1e12 + {0x0000000000000000, 0x9184E72A00000000}, // 1e13 + {0x0000000000000000, 0xB5E620F480000000}, // 1e14 + {0x0000000000000000, 0xE35FA931A0000000}, // 1e15 + {0x0000000000000000, 0x8E1BC9BF04000000}, // 1e16 + {0x0000000000000000, 0xB1A2BC2EC5000000}, // 1e17 + {0x0000000000000000, 0xDE0B6B3A76400000}, // 1e18 + {0x0000000000000000, 0x8AC7230489E80000}, // 1e19 + {0x0000000000000000, 0xAD78EBC5AC620000}, // 1e20 + {0x0000000000000000, 0xD8D726B7177A8000}, // 1e21 + {0x0000000000000000, 0x878678326EAC9000}, // 1e22 + {0x0000000000000000, 0xA968163F0A57B400}, // 1e23 + {0x0000000000000000, 0xD3C21BCECCEDA100}, // 1e24 + {0x0000000000000000, 0x84595161401484A0}, // 1e25 + {0x0000000000000000, 0xA56FA5B99019A5C8}, // 1e26 + {0x0000000000000000, 0xCECB8F27F4200F3A}, // 1e27 + {0x4000000000000000, 0x813F3978F8940984}, // 1e28 + {0x5000000000000000, 0xA18F07D736B90BE5}, // 1e29 + {0xA400000000000000, 0xC9F2C9CD04674EDE}, // 1e30 + {0x4D00000000000000, 0xFC6F7C4045812296}, // 1e31 + {0xF020000000000000, 0x9DC5ADA82B70B59D}, // 1e32 + {0x6C28000000000000, 0xC5371912364CE305}, // 1e33 + {0xC732000000000000, 0xF684DF56C3E01BC6}, // 1e34 + {0x3C7F400000000000, 0x9A130B963A6C115C}, // 1e35 + {0x4B9F100000000000, 0xC097CE7BC90715B3}, // 1e36 + {0x1E86D40000000000, 0xF0BDC21ABB48DB20}, // 1e37 + {0x1314448000000000, 0x96769950B50D88F4}, // 1e38 + {0x17D955A000000000, 0xBC143FA4E250EB31}, // 1e39 + {0x5DCFAB0800000000, 0xEB194F8E1AE525FD}, // 1e40 + {0x5AA1CAE500000000, 0x92EFD1B8D0CF37BE}, // 1e41 + {0xF14A3D9E40000000, 0xB7ABC627050305AD}, // 1e42 + {0x6D9CCD05D0000000, 0xE596B7B0C643C719}, // 1e43 + {0xE4820023A2000000, 0x8F7E32CE7BEA5C6F}, // 1e44 + {0xDDA2802C8A800000, 0xB35DBF821AE4F38B}, // 1e45 + {0xD50B2037AD200000, 0xE0352F62A19E306E}, // 1e46 + {0x4526F422CC340000, 0x8C213D9DA502DE45}, // 1e47 + {0x9670B12B7F410000, 0xAF298D050E4395D6}, // 1e48 + {0x3C0CDD765F114000, 0xDAF3F04651D47B4C}, // 1e49 + {0xA5880A69FB6AC800, 0x88D8762BF324CD0F}, // 1e50 + {0x8EEA0D047A457A00, 0xAB0E93B6EFEE0053}, // 1e51 + {0x72A4904598D6D880, 0xD5D238A4ABE98068}, // 1e52 + {0x47A6DA2B7F864750, 0x85A36366EB71F041}, // 1e53 + {0x999090B65F67D924, 0xA70C3C40A64E6C51}, // 1e54 + {0xFFF4B4E3F741CF6D, 0xD0CF4B50CFE20765}, // 1e55 + {0xBFF8F10E7A8921A4, 0x82818F1281ED449F}, // 1e56 + {0xAFF72D52192B6A0D, 0xA321F2D7226895C7}, // 1e57 + {0x9BF4F8A69F764490, 0xCBEA6F8CEB02BB39}, // 1e58 + {0x02F236D04753D5B4, 0xFEE50B7025C36A08}, // 1e59 + {0x01D762422C946590, 0x9F4F2726179A2245}, // 1e60 + {0x424D3AD2B7B97EF5, 0xC722F0EF9D80AAD6}, // 1e61 + {0xD2E0898765A7DEB2, 0xF8EBAD2B84E0D58B}, // 1e62 + {0x63CC55F49F88EB2F, 0x9B934C3B330C8577}, // 1e63 + {0x3CBF6B71C76B25FB, 0xC2781F49FFCFA6D5}, // 1e64 + {0x8BEF464E3945EF7A, 0xF316271C7FC3908A}, // 1e65 + {0x97758BF0E3CBB5AC, 0x97EDD871CFDA3A56}, // 1e66 + {0x3D52EEED1CBEA317, 0xBDE94E8E43D0C8EC}, // 1e67 + {0x4CA7AAA863EE4BDD, 0xED63A231D4C4FB27}, // 1e68 + {0x8FE8CAA93E74EF6A, 0x945E455F24FB1CF8}, // 1e69 + {0xB3E2FD538E122B44, 0xB975D6B6EE39E436}, // 1e70 + {0x60DBBCA87196B616, 0xE7D34C64A9C85D44}, // 1e71 + {0xBC8955E946FE31CD, 0x90E40FBEEA1D3A4A}, // 1e72 + {0x6BABAB6398BDBE41, 0xB51D13AEA4A488DD}, // 1e73 + {0xC696963C7EED2DD1, 0xE264589A4DCDAB14}, // 1e74 + {0xFC1E1DE5CF543CA2, 0x8D7EB76070A08AEC}, // 1e75 + {0x3B25A55F43294BCB, 0xB0DE65388CC8ADA8}, // 1e76 + {0x49EF0EB713F39EBE, 0xDD15FE86AFFAD912}, // 1e77 + {0x6E3569326C784337, 0x8A2DBF142DFCC7AB}, // 1e78 + {0x49C2C37F07965404, 0xACB92ED9397BF996}, // 1e79 + {0xDC33745EC97BE906, 0xD7E77A8F87DAF7FB}, // 1e80 + {0x69A028BB3DED71A3, 0x86F0AC99B4E8DAFD}, // 1e81 + {0xC40832EA0D68CE0C, 0xA8ACD7C0222311BC}, // 1e82 + {0xF50A3FA490C30190, 0xD2D80DB02AABD62B}, // 1e83 + {0x792667C6DA79E0FA, 0x83C7088E1AAB65DB}, // 1e84 + {0x577001B891185938, 0xA4B8CAB1A1563F52}, // 1e85 + {0xED4C0226B55E6F86, 0xCDE6FD5E09ABCF26}, // 1e86 + {0x544F8158315B05B4, 0x80B05E5AC60B6178}, // 1e87 + {0x696361AE3DB1C721, 0xA0DC75F1778E39D6}, // 1e88 + {0x03BC3A19CD1E38E9, 0xC913936DD571C84C}, // 1e89 + {0x04AB48A04065C723, 0xFB5878494ACE3A5F}, // 1e90 + {0x62EB0D64283F9C76, 0x9D174B2DCEC0E47B}, // 1e91 + {0x3BA5D0BD324F8394, 0xC45D1DF942711D9A}, // 1e92 + {0xCA8F44EC7EE36479, 0xF5746577930D6500}, // 1e93 + {0x7E998B13CF4E1ECB, 0x9968BF6ABBE85F20}, // 1e94 + {0x9E3FEDD8C321A67E, 0xBFC2EF456AE276E8}, // 1e95 + {0xC5CFE94EF3EA101E, 0xEFB3AB16C59B14A2}, // 1e96 + {0xBBA1F1D158724A12, 0x95D04AEE3B80ECE5}, // 1e97 + {0x2A8A6E45AE8EDC97, 0xBB445DA9CA61281F}, // 1e98 + {0xF52D09D71A3293BD, 0xEA1575143CF97226}, // 1e99 + {0x593C2626705F9C56, 0x924D692CA61BE758}, // 1e100 + {0x6F8B2FB00C77836C, 0xB6E0C377CFA2E12E}, // 1e101 + {0x0B6DFB9C0F956447, 0xE498F455C38B997A}, // 1e102 + {0x4724BD4189BD5EAC, 0x8EDF98B59A373FEC}, // 1e103 + {0x58EDEC91EC2CB657, 0xB2977EE300C50FE7}, // 1e104 + {0x2F2967B66737E3ED, 0xDF3D5E9BC0F653E1}, // 1e105 + {0xBD79E0D20082EE74, 0x8B865B215899F46C}, // 1e106 + {0xECD8590680A3AA11, 0xAE67F1E9AEC07187}, // 1e107 + {0xE80E6F4820CC9495, 0xDA01EE641A708DE9}, // 1e108 + {0x3109058D147FDCDD, 0x884134FE908658B2}, // 1e109 + {0xBD4B46F0599FD415, 0xAA51823E34A7EEDE}, // 1e110 + {0x6C9E18AC7007C91A, 0xD4E5E2CDC1D1EA96}, // 1e111 + {0x03E2CF6BC604DDB0, 0x850FADC09923329E}, // 1e112 + {0x84DB8346B786151C, 0xA6539930BF6BFF45}, // 1e113 + {0xE612641865679A63, 0xCFE87F7CEF46FF16}, // 1e114 + {0x4FCB7E8F3F60C07E, 0x81F14FAE158C5F6E}, // 1e115 + {0xE3BE5E330F38F09D, 0xA26DA3999AEF7749}, // 1e116 + {0x5CADF5BFD3072CC5, 0xCB090C8001AB551C}, // 1e117 + {0x73D9732FC7C8F7F6, 0xFDCB4FA002162A63}, // 1e118 + {0x2867E7FDDCDD9AFA, 0x9E9F11C4014DDA7E}, // 1e119 + {0xB281E1FD541501B8, 0xC646D63501A1511D}, // 1e120 + {0x1F225A7CA91A4226, 0xF7D88BC24209A565}, // 1e121 + {0x3375788DE9B06958, 0x9AE757596946075F}, // 1e122 + {0x0052D6B1641C83AE, 0xC1A12D2FC3978937}, // 1e123 + {0xC0678C5DBD23A49A, 0xF209787BB47D6B84}, // 1e124 + {0xF840B7BA963646E0, 0x9745EB4D50CE6332}, // 1e125 + {0xB650E5A93BC3D898, 0xBD176620A501FBFF}, // 1e126 + {0xA3E51F138AB4CEBE, 0xEC5D3FA8CE427AFF}, // 1e127 + {0xC66F336C36B10137, 0x93BA47C980E98CDF}, // 1e128 + {0xB80B0047445D4184, 0xB8A8D9BBE123F017}, // 1e129 + {0xA60DC059157491E5, 0xE6D3102AD96CEC1D}, // 1e130 + {0x87C89837AD68DB2F, 0x9043EA1AC7E41392}, // 1e131 + {0x29BABE4598C311FB, 0xB454E4A179DD1877}, // 1e132 + {0xF4296DD6FEF3D67A, 0xE16A1DC9D8545E94}, // 1e133 + {0x1899E4A65F58660C, 0x8CE2529E2734BB1D}, // 1e134 + {0x5EC05DCFF72E7F8F, 0xB01AE745B101E9E4}, // 1e135 + {0x76707543F4FA1F73, 0xDC21A1171D42645D}, // 1e136 + {0x6A06494A791C53A8, 0x899504AE72497EBA}, // 1e137 + {0x0487DB9D17636892, 0xABFA45DA0EDBDE69}, // 1e138 + {0x45A9D2845D3C42B6, 0xD6F8D7509292D603}, // 1e139 + {0x0B8A2392BA45A9B2, 0x865B86925B9BC5C2}, // 1e140 + {0x8E6CAC7768D7141E, 0xA7F26836F282B732}, // 1e141 + {0x3207D795430CD926, 0xD1EF0244AF2364FF}, // 1e142 + {0x7F44E6BD49E807B8, 0x8335616AED761F1F}, // 1e143 + {0x5F16206C9C6209A6, 0xA402B9C5A8D3A6E7}, // 1e144 + {0x36DBA887C37A8C0F, 0xCD036837130890A1}, // 1e145 + {0xC2494954DA2C9789, 0x802221226BE55A64}, // 1e146 + {0xF2DB9BAA10B7BD6C, 0xA02AA96B06DEB0FD}, // 1e147 + {0x6F92829494E5ACC7, 0xC83553C5C8965D3D}, // 1e148 + {0xCB772339BA1F17F9, 0xFA42A8B73ABBF48C}, // 1e149 + {0xFF2A760414536EFB, 0x9C69A97284B578D7}, // 1e150 + {0xFEF5138519684ABA, 0xC38413CF25E2D70D}, // 1e151 + {0x7EB258665FC25D69, 0xF46518C2EF5B8CD1}, // 1e152 + {0xEF2F773FFBD97A61, 0x98BF2F79D5993802}, // 1e153 + {0xAAFB550FFACFD8FA, 0xBEEEFB584AFF8603}, // 1e154 + {0x95BA2A53F983CF38, 0xEEAABA2E5DBF6784}, // 1e155 + {0xDD945A747BF26183, 0x952AB45CFA97A0B2}, // 1e156 + {0x94F971119AEEF9E4, 0xBA756174393D88DF}, // 1e157 + {0x7A37CD5601AAB85D, 0xE912B9D1478CEB17}, // 1e158 + {0xAC62E055C10AB33A, 0x91ABB422CCB812EE}, // 1e159 + {0x577B986B314D6009, 0xB616A12B7FE617AA}, // 1e160 + {0xED5A7E85FDA0B80B, 0xE39C49765FDF9D94}, // 1e161 + {0x14588F13BE847307, 0x8E41ADE9FBEBC27D}, // 1e162 + {0x596EB2D8AE258FC8, 0xB1D219647AE6B31C}, // 1e163 + {0x6FCA5F8ED9AEF3BB, 0xDE469FBD99A05FE3}, // 1e164 + {0x25DE7BB9480D5854, 0x8AEC23D680043BEE}, // 1e165 + {0xAF561AA79A10AE6A, 0xADA72CCC20054AE9}, // 1e166 + {0x1B2BA1518094DA04, 0xD910F7FF28069DA4}, // 1e167 + {0x90FB44D2F05D0842, 0x87AA9AFF79042286}, // 1e168 + {0x353A1607AC744A53, 0xA99541BF57452B28}, // 1e169 + {0x42889B8997915CE8, 0xD3FA922F2D1675F2}, // 1e170 + {0x69956135FEBADA11, 0x847C9B5D7C2E09B7}, // 1e171 + {0x43FAB9837E699095, 0xA59BC234DB398C25}, // 1e172 + {0x94F967E45E03F4BB, 0xCF02B2C21207EF2E}, // 1e173 + {0x1D1BE0EEBAC278F5, 0x8161AFB94B44F57D}, // 1e174 + {0x6462D92A69731732, 0xA1BA1BA79E1632DC}, // 1e175 + {0x7D7B8F7503CFDCFE, 0xCA28A291859BBF93}, // 1e176 + {0x5CDA735244C3D43E, 0xFCB2CB35E702AF78}, // 1e177 + {0x3A0888136AFA64A7, 0x9DEFBF01B061ADAB}, // 1e178 + {0x088AAA1845B8FDD0, 0xC56BAEC21C7A1916}, // 1e179 + {0x8AAD549E57273D45, 0xF6C69A72A3989F5B}, // 1e180 + {0x36AC54E2F678864B, 0x9A3C2087A63F6399}, // 1e181 + {0x84576A1BB416A7DD, 0xC0CB28A98FCF3C7F}, // 1e182 + {0x656D44A2A11C51D5, 0xF0FDF2D3F3C30B9F}, // 1e183 + {0x9F644AE5A4B1B325, 0x969EB7C47859E743}, // 1e184 + {0x873D5D9F0DDE1FEE, 0xBC4665B596706114}, // 1e185 + {0xA90CB506D155A7EA, 0xEB57FF22FC0C7959}, // 1e186 + {0x09A7F12442D588F2, 0x9316FF75DD87CBD8}, // 1e187 + {0x0C11ED6D538AEB2F, 0xB7DCBF5354E9BECE}, // 1e188 + {0x8F1668C8A86DA5FA, 0xE5D3EF282A242E81}, // 1e189 + {0xF96E017D694487BC, 0x8FA475791A569D10}, // 1e190 + {0x37C981DCC395A9AC, 0xB38D92D760EC4455}, // 1e191 + {0x85BBE253F47B1417, 0xE070F78D3927556A}, // 1e192 + {0x93956D7478CCEC8E, 0x8C469AB843B89562}, // 1e193 + {0x387AC8D1970027B2, 0xAF58416654A6BABB}, // 1e194 + {0x06997B05FCC0319E, 0xDB2E51BFE9D0696A}, // 1e195 + {0x441FECE3BDF81F03, 0x88FCF317F22241E2}, // 1e196 + {0xD527E81CAD7626C3, 0xAB3C2FDDEEAAD25A}, // 1e197 + {0x8A71E223D8D3B074, 0xD60B3BD56A5586F1}, // 1e198 + {0xF6872D5667844E49, 0x85C7056562757456}, // 1e199 + {0xB428F8AC016561DB, 0xA738C6BEBB12D16C}, // 1e200 + {0xE13336D701BEBA52, 0xD106F86E69D785C7}, // 1e201 + {0xECC0024661173473, 0x82A45B450226B39C}, // 1e202 + {0x27F002D7F95D0190, 0xA34D721642B06084}, // 1e203 + {0x31EC038DF7B441F4, 0xCC20CE9BD35C78A5}, // 1e204 + {0x7E67047175A15271, 0xFF290242C83396CE}, // 1e205 + {0x0F0062C6E984D386, 0x9F79A169BD203E41}, // 1e206 + {0x52C07B78A3E60868, 0xC75809C42C684DD1}, // 1e207 + {0xA7709A56CCDF8A82, 0xF92E0C3537826145}, // 1e208 + {0x88A66076400BB691, 0x9BBCC7A142B17CCB}, // 1e209 + {0x6ACFF893D00EA435, 0xC2ABF989935DDBFE}, // 1e210 + {0x0583F6B8C4124D43, 0xF356F7EBF83552FE}, // 1e211 + {0xC3727A337A8B704A, 0x98165AF37B2153DE}, // 1e212 + {0x744F18C0592E4C5C, 0xBE1BF1B059E9A8D6}, // 1e213 + {0x1162DEF06F79DF73, 0xEDA2EE1C7064130C}, // 1e214 + {0x8ADDCB5645AC2BA8, 0x9485D4D1C63E8BE7}, // 1e215 + {0x6D953E2BD7173692, 0xB9A74A0637CE2EE1}, // 1e216 + {0xC8FA8DB6CCDD0437, 0xE8111C87C5C1BA99}, // 1e217 + {0x1D9C9892400A22A2, 0x910AB1D4DB9914A0}, // 1e218 + {0x2503BEB6D00CAB4B, 0xB54D5E4A127F59C8}, // 1e219 + {0x2E44AE64840FD61D, 0xE2A0B5DC971F303A}, // 1e220 + {0x5CEAECFED289E5D2, 0x8DA471A9DE737E24}, // 1e221 + {0x7425A83E872C5F47, 0xB10D8E1456105DAD}, // 1e222 + {0xD12F124E28F77719, 0xDD50F1996B947518}, // 1e223 + {0x82BD6B70D99AAA6F, 0x8A5296FFE33CC92F}, // 1e224 + {0x636CC64D1001550B, 0xACE73CBFDC0BFB7B}, // 1e225 + {0x3C47F7E05401AA4E, 0xD8210BEFD30EFA5A}, // 1e226 + {0x65ACFAEC34810A71, 0x8714A775E3E95C78}, // 1e227 + {0x7F1839A741A14D0D, 0xA8D9D1535CE3B396}, // 1e228 + {0x1EDE48111209A050, 0xD31045A8341CA07C}, // 1e229 + {0x934AED0AAB460432, 0x83EA2B892091E44D}, // 1e230 + {0xF81DA84D5617853F, 0xA4E4B66B68B65D60}, // 1e231 + {0x36251260AB9D668E, 0xCE1DE40642E3F4B9}, // 1e232 + {0xC1D72B7C6B426019, 0x80D2AE83E9CE78F3}, // 1e233 + {0xB24CF65B8612F81F, 0xA1075A24E4421730}, // 1e234 + {0xDEE033F26797B627, 0xC94930AE1D529CFC}, // 1e235 + {0x169840EF017DA3B1, 0xFB9B7CD9A4A7443C}, // 1e236 + {0x8E1F289560EE864E, 0x9D412E0806E88AA5}, // 1e237 + {0xF1A6F2BAB92A27E2, 0xC491798A08A2AD4E}, // 1e238 + {0xAE10AF696774B1DB, 0xF5B5D7EC8ACB58A2}, // 1e239 + {0xACCA6DA1E0A8EF29, 0x9991A6F3D6BF1765}, // 1e240 + {0x17FD090A58D32AF3, 0xBFF610B0CC6EDD3F}, // 1e241 + {0xDDFC4B4CEF07F5B0, 0xEFF394DCFF8A948E}, // 1e242 + {0x4ABDAF101564F98E, 0x95F83D0A1FB69CD9}, // 1e243 + {0x9D6D1AD41ABE37F1, 0xBB764C4CA7A4440F}, // 1e244 + {0x84C86189216DC5ED, 0xEA53DF5FD18D5513}, // 1e245 + {0x32FD3CF5B4E49BB4, 0x92746B9BE2F8552C}, // 1e246 + {0x3FBC8C33221DC2A1, 0xB7118682DBB66A77}, // 1e247 + {0x0FABAF3FEAA5334A, 0xE4D5E82392A40515}, // 1e248 + {0x29CB4D87F2A7400E, 0x8F05B1163BA6832D}, // 1e249 + {0x743E20E9EF511012, 0xB2C71D5BCA9023F8}, // 1e250 + {0x914DA9246B255416, 0xDF78E4B2BD342CF6}, // 1e251 + {0x1AD089B6C2F7548E, 0x8BAB8EEFB6409C1A}, // 1e252 + {0xA184AC2473B529B1, 0xAE9672ABA3D0C320}, // 1e253 + {0xC9E5D72D90A2741E, 0xDA3C0F568CC4F3E8}, // 1e254 + {0x7E2FA67C7A658892, 0x8865899617FB1871}, // 1e255 + {0xDDBB901B98FEEAB7, 0xAA7EEBFB9DF9DE8D}, // 1e256 + {0x552A74227F3EA565, 0xD51EA6FA85785631}, // 1e257 + {0xD53A88958F87275F, 0x8533285C936B35DE}, // 1e258 + {0x8A892ABAF368F137, 0xA67FF273B8460356}, // 1e259 + {0x2D2B7569B0432D85, 0xD01FEF10A657842C}, // 1e260 + {0x9C3B29620E29FC73, 0x8213F56A67F6B29B}, // 1e261 + {0x8349F3BA91B47B8F, 0xA298F2C501F45F42}, // 1e262 + {0x241C70A936219A73, 0xCB3F2F7642717713}, // 1e263 + {0xED238CD383AA0110, 0xFE0EFB53D30DD4D7}, // 1e264 + {0xF4363804324A40AA, 0x9EC95D1463E8A506}, // 1e265 + {0xB143C6053EDCD0D5, 0xC67BB4597CE2CE48}, // 1e266 + {0xDD94B7868E94050A, 0xF81AA16FDC1B81DA}, // 1e267 + {0xCA7CF2B4191C8326, 0x9B10A4E5E9913128}, // 1e268 + {0xFD1C2F611F63A3F0, 0xC1D4CE1F63F57D72}, // 1e269 + {0xBC633B39673C8CEC, 0xF24A01A73CF2DCCF}, // 1e270 + {0xD5BE0503E085D813, 0x976E41088617CA01}, // 1e271 + {0x4B2D8644D8A74E18, 0xBD49D14AA79DBC82}, // 1e272 + {0xDDF8E7D60ED1219E, 0xEC9C459D51852BA2}, // 1e273 + {0xCABB90E5C942B503, 0x93E1AB8252F33B45}, // 1e274 + {0x3D6A751F3B936243, 0xB8DA1662E7B00A17}, // 1e275 + {0x0CC512670A783AD4, 0xE7109BFBA19C0C9D}, // 1e276 + {0x27FB2B80668B24C5, 0x906A617D450187E2}, // 1e277 + {0xB1F9F660802DEDF6, 0xB484F9DC9641E9DA}, // 1e278 + {0x5E7873F8A0396973, 0xE1A63853BBD26451}, // 1e279 + {0xDB0B487B6423E1E8, 0x8D07E33455637EB2}, // 1e280 + {0x91CE1A9A3D2CDA62, 0xB049DC016ABC5E5F}, // 1e281 + {0x7641A140CC7810FB, 0xDC5C5301C56B75F7}, // 1e282 + {0xA9E904C87FCB0A9D, 0x89B9B3E11B6329BA}, // 1e283 + {0x546345FA9FBDCD44, 0xAC2820D9623BF429}, // 1e284 + {0xA97C177947AD4095, 0xD732290FBACAF133}, // 1e285 + {0x49ED8EABCCCC485D, 0x867F59A9D4BED6C0}, // 1e286 + {0x5C68F256BFFF5A74, 0xA81F301449EE8C70}, // 1e287 + {0x73832EEC6FFF3111, 0xD226FC195C6A2F8C}, // 1e288 + {0xC831FD53C5FF7EAB, 0x83585D8FD9C25DB7}, // 1e289 + {0xBA3E7CA8B77F5E55, 0xA42E74F3D032F525}, // 1e290 + {0x28CE1BD2E55F35EB, 0xCD3A1230C43FB26F}, // 1e291 + {0x7980D163CF5B81B3, 0x80444B5E7AA7CF85}, // 1e292 + {0xD7E105BCC332621F, 0xA0555E361951C366}, // 1e293 + {0x8DD9472BF3FEFAA7, 0xC86AB5C39FA63440}, // 1e294 + {0xB14F98F6F0FEB951, 0xFA856334878FC150}, // 1e295 + {0x6ED1BF9A569F33D3, 0x9C935E00D4B9D8D2}, // 1e296 + {0x0A862F80EC4700C8, 0xC3B8358109E84F07}, // 1e297 + {0xCD27BB612758C0FA, 0xF4A642E14C6262C8}, // 1e298 + {0x8038D51CB897789C, 0x98E7E9CCCFBD7DBD}, // 1e299 + {0xE0470A63E6BD56C3, 0xBF21E44003ACDD2C}, // 1e300 + {0x1858CCFCE06CAC74, 0xEEEA5D5004981478}, // 1e301 + {0x0F37801E0C43EBC8, 0x95527A5202DF0CCB}, // 1e302 + {0xD30560258F54E6BA, 0xBAA718E68396CFFD}, // 1e303 + {0x47C6B82EF32A2069, 0xE950DF20247C83FD}, // 1e304 + {0x4CDC331D57FA5441, 0x91D28B7416CDD27E}, // 1e305 + {0xE0133FE4ADF8E952, 0xB6472E511C81471D}, // 1e306 + {0x58180FDDD97723A6, 0xE3D8F9E563A198E5}, // 1e307 + {0x570F09EAA7EA7648, 0x8E679C2F5E44FF8F}, // 1e308 + {0x2CD2CC6551E513DA, 0xB201833B35D63F73}, // 1e309 + {0xF8077F7EA65E58D1, 0xDE81E40A034BCF4F}, // 1e310 + {0xFB04AFAF27FAF782, 0x8B112E86420F6191}, // 1e311 + {0x79C5DB9AF1F9B563, 0xADD57A27D29339F6}, // 1e312 + {0x18375281AE7822BC, 0xD94AD8B1C7380874}, // 1e313 + {0x8F2293910D0B15B5, 0x87CEC76F1C830548}, // 1e314 + {0xB2EB3875504DDB22, 0xA9C2794AE3A3C69A}, // 1e315 + {0x5FA60692A46151EB, 0xD433179D9C8CB841}, // 1e316 + {0xDBC7C41BA6BCD333, 0x849FEEC281D7F328}, // 1e317 + {0x12B9B522906C0800, 0xA5C7EA73224DEFF3}, // 1e318 + {0xD768226B34870A00, 0xCF39E50FEAE16BEF}, // 1e319 + {0xE6A1158300D46640, 0x81842F29F2CCE375}, // 1e320 + {0x60495AE3C1097FD0, 0xA1E53AF46F801C53}, // 1e321 + {0x385BB19CB14BDFC4, 0xCA5E89B18B602368}, // 1e322 + {0x46729E03DD9ED7B5, 0xFCF62C1DEE382C42}, // 1e323 + {0x6C07A2C26A8346D1, 0x9E19DB92B4E31BA9}, // 1e324 + {0xC7098B7305241885, 0xC5A05277621BE293}, // 1e325 + {0xB8CBEE4FC66D1EA7, 0xF70867153AA2DB38}, // 1e326 + {0x737F74F1DC043328, 0x9A65406D44A5C903}, // 1e327 + {0x505F522E53053FF2, 0xC0FE908895CF3B44}, // 1e328 + {0x647726B9E7C68FEF, 0xF13E34AABB430A15}, // 1e329 + {0x5ECA783430DC19F5, 0x96C6E0EAB509E64D}, // 1e330 + {0xB67D16413D132072, 0xBC789925624C5FE0}, // 1e331 + {0xE41C5BD18C57E88F, 0xEB96BF6EBADF77D8}, // 1e332 + {0x8E91B962F7B6F159, 0x933E37A534CBAAE7}, // 1e333 + {0x723627BBB5A4ADB0, 0xB80DC58E81FE95A1}, // 1e334 + {0xCEC3B1AAA30DD91C, 0xE61136F2227E3B09}, // 1e335 + {0x213A4F0AA5E8A7B1, 0x8FCAC257558EE4E6}, // 1e336 + {0xA988E2CD4F62D19D, 0xB3BD72ED2AF29E1F}, // 1e337 + {0x93EB1B80A33B8605, 0xE0ACCFA875AF45A7}, // 1e338 + {0xBC72F130660533C3, 0x8C6C01C9498D8B88}, // 1e339 + {0xEB8FAD7C7F8680B4, 0xAF87023B9BF0EE6A}, // 1e340 + {0xA67398DB9F6820E1, 0xDB68C2CA82ED2A05}, // 1e341 + {0x88083F8943A1148C, 0x892179BE91D43A43}, // 1e342 + {0x6A0A4F6B948959B0, 0xAB69D82E364948D4}, // 1e343 + {0x848CE34679ABB01C, 0xD6444E39C3DB9B09}, // 1e344 + {0xF2D80E0C0C0B4E11, 0x85EAB0E41A6940E5}, // 1e345 + {0x6F8E118F0F0E2195, 0xA7655D1D2103911F}, // 1e346 + {0x4B7195F2D2D1A9FB, 0xD13EB46469447567}, // 1e347 +} diff --git a/examples/gno.land/p/demo/json/eisel_lemire_test.gno b/examples/gno.land/p/demo/json/eisel_lemire_test.gno new file mode 100644 index 00000000000..512bf5b748e --- /dev/null +++ b/examples/gno.land/p/demo/json/eisel_lemire_test.gno @@ -0,0 +1,43 @@ +package json + +import ( + "testing" + "math" +) + +type elTestCase struct { + man uint64 + exp10 int + neg bool + want float64 + ok bool +} + +var testCases = []elTestCase { + {12345, 3, false, 12345e3, true}, + {12345, -3, false, 12345e-3, true}, +} + +func TestEiselLemire64(t *testing.T) { + for _, tc := range testCases { + got, ok := eiselLemire64(tc.man, tc.exp10, tc.neg) + if ok != tc.ok || (ok && got != tc.want) { + t.Errorf("eiselLemire64(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, tc.want, tc.ok) + } + } +} + +func TestEiselLemire32(t *testing.T) { + testCases := []elTestCase { + {12345, 3, false, 12345e3, true}, + {12345, -3, false, 12345e-3, true}, + } + + for _, tc := range testCases { + got, ok := eiselLemire32(tc.man, tc.exp10, tc.neg) + want := float32(tc.want) + if ok != tc.ok || (ok && got != want) { + t.Errorf("eiselLemire32(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, want, tc.ok) + } + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/number.gno b/examples/gno.land/p/demo/json/number.gno index cde03b2f2db..adf2ea0b4b7 100644 --- a/examples/gno.land/p/demo/json/number.gno +++ b/examples/gno.land/p/demo/json/number.gno @@ -1,77 +1,82 @@ package json +import ( + "strconv" +) + const ( absMinInt64 = 1 << 63 maxInt64 = 1<<63 - 1 maxUint64 = 1<<64 - 1 ) -func ParseNumberKind(bytes []byte) (value float64, isFloat, ok, overflow bool) { - if len(bytes) == 0 { - return 0, false, false, false - } +func ParseNumberKind(bytes []byte) (value float64, isFloat, ok bool) { + if len(bytes) == 0 { + panic("invalid number: empty string") + } - neg := bytes[0] == '-' - if neg { - bytes = bytes[1:] - } + neg := bytes[0] == '-' + if neg { + bytes = bytes[1:] + } - var intPart, fracPart uint64 - var decimalFound bool - var fracDivisor uint64 = 1 - - for _, c := range bytes { - if c == '.' { - if decimalFound { - return 0, false, false, false - } - decimalFound = true - continue - } - - if c < '0' || c > '9' { - return 0, false, false, false - } - - digit := uint64(c - '0') - if !decimalFound { - if intPart > (maxUint64-uint64(digit))/10 { - return 0, false, false, true - } - intPart = intPart*10 + digit - } else { - if fracPart > (maxUint64-uint64(digit))/10 { - return 0, false, false, true - } - fracPart = fracPart*10 + digit - fracDivisor *= 10 - } + var exponentPart []byte + for i, c := range bytes { + if c == 'e' || c == 'E' { + exponentPart = bytes[i+1:] + bytes = bytes[:i] + break + } } - if decimalFound { - isFloat = true - } + man, exp10 := extractMantissaAndExp10(bytes) - totalValue := float64(intPart) + float64(fracPart)/float64(fracDivisor) - if neg { - totalValue = -totalValue - } + if len(exponentPart) > 0 { + exp, err := strconv.Atoi(string(exponentPart)) + if err != nil { + panic("invalid exponent") + } + exp10 += exp + } - // Overflow check for integer part - if !isFloat { - if neg { - if intPart > absMinInt64 { - return 0, false, false, true - } - } else { - if intPart > maxInt64 { - return 0, false, false, true - } - } - - // return totalValue instead of float64(intPart) - return totalValue, isFloat, true, false - } + f, success := eiselLemire64(man, exp10, neg) + if !success { + return 0, false, false + } + + return f, true, true +} + +func extractMantissaAndExp10(bytes []byte) (uint64, int) { + var man uint64 + var exp10 int + decimalFound := false + + for _, c := range bytes { + if c == '.' { + if decimalFound { + panic("invalid number: multiple decimal points") + } + decimalFound = true + continue + } + + if c < '0' || c > '9' { + panic("invalid number: non-digit characters") + } + + digit := uint64(c - '0') + + if man > (maxUint64-digit)/10 { + panic("invalid number: overflow") + } + + man = man*10 + digit + + if decimalFound { + exp10-- + } + } - return totalValue, isFloat, true, false + return man, exp10 } diff --git a/examples/gno.land/p/demo/json/number_test.gno b/examples/gno.land/p/demo/json/number_test.gno index 89350ee5cd0..40ebf81de51 100644 --- a/examples/gno.land/p/demo/json/number_test.gno +++ b/examples/gno.land/p/demo/json/number_test.gno @@ -5,31 +5,73 @@ import ( ) func TestParseNumberKind(t *testing.T) { - cases := []struct { - name string - input []byte - expected float64 - isFloat bool - ok bool - overflow bool - }{ - {"Positive Integer", []byte("12345"), 12345, false, true, false}, - {"Negative Integer", []byte("-12345"), -12345, false, true, false}, - {"Positive Float", []byte("123.45"), 123.45, true, true, false}, - {"Negative Float", []byte("-123.45"), -123.45, true, true, false}, - {"Integer Overflow", []byte("18446744073709551616"), 0, false, false, true}, - {"Invalid Format", []byte("123a45"), 0, false, false, false}, - {"Multiple Decimal Points", []byte("123.45.67"), 0, false, false, false}, - {"Empty Input", []byte(""), 0, false, false, false}, - } + testCases := []struct { + input string + expected float64 + expectedPanic bool + }{ + {"123", 123, false}, + {"-123", -123, false}, + {"123.456", 123.456, false}, + {"-123.456", -123.456, false}, + {"0.123", 0.123, false}, + {"-0.123", -0.123, false}, + {"", 0, true}, + {"abc", 0, true}, + {"123.45.6", 0, true}, + {"999999999999999999999", 0, true}, + } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - value, isFloat, ok, overflow := ParseNumberKind(c.input) - if value != c.expected || isFloat != c.isFloat || ok != c.ok || overflow != c.overflow { - t.Errorf("ParseNumberKind(%s) == (%f, %t, %t, %t), expected (%f, %t, %t, %t)", - c.input, value, isFloat, ok, overflow, c.expected, c.isFloat, c.ok, c.overflow) - } - }) - } + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + defer func() { + r := recover() + if (r != nil) != tc.expectedPanic { + t.Errorf("ParseNumberKind(%s): expected panic=%v, got panic=%v", + tc.input, tc.expectedPanic, r != nil) + } + }() + + got, _, _ := ParseNumberKind([]byte(tc.input)) + if got != tc.expected && !tc.expectedPanic { + t.Errorf("ParseNumberKind(%s): expected %v, got %v", + tc.input, tc.expected, got) + } + }) + } +} + +func TestParseNumberKindWithScientificNotation(t *testing.T) { + testCases := []struct { + input string + expected float64 + }{ + {"1e6", 1000000}, + {"1E6", 1000000}, + {"1.23e10", 1.23e10}, + {"1.23E10", 1.23e10}, + {"-1.23e10", -1.23e10}, + {"-1.23E10", -1.23e10}, + {"2.45e-8", 2.45e-8}, + {"2.45E-8", 2.45e-8}, + {"-2.45e-8", -2.45e-8}, + {"-2.45E-8", -2.45e-8}, + {"5e0", 5}, + {"-5e0", -5}, + {"5E+0", 5}, + {"5e+1", 50}, + {"1e-1", 0.1}, + {"1E-1", 0.1}, + {"-1e-1", -0.1}, + {"-1E-1", -0.1}, + } + + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + got, _, _ := ParseNumberKind([]byte(tc.input)) + if got != tc.expected { + t.Errorf("ParseNumberKind(%s): got %v, want %v", tc.input, got, tc.expected) + } + }) + } } From d39ad5221f8590f7f70ae1885038b2663a3a5a7f Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 11 Dec 2023 15:16:19 +0900 Subject: [PATCH 06/72] basic lexer --- .../gno.land/p/demo/json/eisel_lemire.gno | 8 +- .../p/demo/json/eisel_lemire_test.gno | 56 ++++---- examples/gno.land/p/demo/json/lexer.gno | 80 +++++++++++ examples/gno.land/p/demo/json/lexer_test.gno | 81 +++++++++++ examples/gno.land/p/demo/json/number.gno | 100 +++++++------- examples/gno.land/p/demo/json/number_test.gno | 126 +++++++++--------- examples/gno.land/p/demo/json/table.gno | 25 ++++ examples/gno.land/p/demo/json/value.gno | 41 ++++++ examples/gno.land/p/demo/json/value_test.gno | 30 +++++ 9 files changed, 403 insertions(+), 144 deletions(-) create mode 100644 examples/gno.land/p/demo/json/lexer.gno create mode 100644 examples/gno.land/p/demo/json/lexer_test.gno create mode 100644 examples/gno.land/p/demo/json/table.gno create mode 100644 examples/gno.land/p/demo/json/value.gno create mode 100644 examples/gno.land/p/demo/json/value_test.gno diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno index 3c13c0e30e7..4e16e2d6124 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -21,14 +21,14 @@ import ( // The Eisel-Lemire algorithm accomplishes its task through the following methods: // // 1. Extracting Mantissa and Exponent -// It involves separating the mantissa, or the significant part of the number, +// It involves separating the mantissa, or the significant part of the number, // from the exponent when converting a string into a floating-point number. // // 2. Normalization and Range checking // This step normalizes the mantissa and ensures the exponent falls within an acceptable range. // // 3. Conversion to a Floating-Point Number -// Finally, the algorithm uses the normalized mantissa and exponent to convert the data +// Finally, the algorithm uses the normalized mantissa and exponent to convert the data // into an actual floating-point number. // eiselLemire64 parses a floating-point number from its mantissa and exponent representation. @@ -204,8 +204,8 @@ const ( // detailedPowersOfTen contains 128-bit mantissa approximations (rounded down) // to the powers of 10. For example: // -// - 1e43 ≈ (0xE596B7B0_C643C719 * (2 ** 79)) -// - 1e43 = (0xE596B7B0_C643C719_6D9CCD05_D0000000 * (2 ** 15)) +// - 1e43 ≈ (0xE596B7B0_C643C719 * (2 ** 79)) +// - 1e43 = (0xE596B7B0_C643C719_6D9CCD05_D0000000 * (2 ** 15)) // // The mantissas are explicitly listed. The exponents are implied by a linear // expression with slope 217706.0/65536.0 ≈ log(10)/log(2). diff --git a/examples/gno.land/p/demo/json/eisel_lemire_test.gno b/examples/gno.land/p/demo/json/eisel_lemire_test.gno index 512bf5b748e..8cfa79169ef 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire_test.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire_test.gno @@ -1,43 +1,43 @@ package json import ( - "testing" - "math" + "math" + "testing" ) type elTestCase struct { - man uint64 - exp10 int - neg bool - want float64 - ok bool + man uint64 + exp10 int + neg bool + want float64 + ok bool } -var testCases = []elTestCase { - {12345, 3, false, 12345e3, true}, - {12345, -3, false, 12345e-3, true}, +var testCases = []elTestCase{ + {12345, 3, false, 12345e3, true}, + {12345, -3, false, 12345e-3, true}, } func TestEiselLemire64(t *testing.T) { - for _, tc := range testCases { - got, ok := eiselLemire64(tc.man, tc.exp10, tc.neg) - if ok != tc.ok || (ok && got != tc.want) { - t.Errorf("eiselLemire64(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, tc.want, tc.ok) - } - } + for _, tc := range testCases { + got, ok := eiselLemire64(tc.man, tc.exp10, tc.neg) + if ok != tc.ok || (ok && got != tc.want) { + t.Errorf("eiselLemire64(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, tc.want, tc.ok) + } + } } func TestEiselLemire32(t *testing.T) { - testCases := []elTestCase { - {12345, 3, false, 12345e3, true}, - {12345, -3, false, 12345e-3, true}, - } + testCases := []elTestCase{ + {12345, 3, false, 12345e3, true}, + {12345, -3, false, 12345e-3, true}, + } - for _, tc := range testCases { - got, ok := eiselLemire32(tc.man, tc.exp10, tc.neg) - want := float32(tc.want) - if ok != tc.ok || (ok && got != want) { - t.Errorf("eiselLemire32(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, want, tc.ok) - } - } -} \ No newline at end of file + for _, tc := range testCases { + got, ok := eiselLemire32(tc.man, tc.exp10, tc.neg) + want := float32(tc.want) + if ok != tc.ok || (ok && got != want) { + t.Errorf("eiselLemire32(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, want, tc.ok) + } + } +} diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno new file mode 100644 index 00000000000..67fc70401e2 --- /dev/null +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -0,0 +1,80 @@ +package json + +type Lexer struct { + data []byte + token byte + pos int + depth int +} + +func New(data []byte) *Lexer { + return &Lexer{ + data: data, + } +} + +func (l *Lexer) findTokenStart(token byte) int { + for i := len(l.data) - 1; i >= 0; i-- { + switch l.data[i] { + case token: + return i + case '[', '{': + l.depth++ + return 0 + } + } + + return 0 +} + +func (l *Lexer) tokenEnd() int { + for i, tok := range l.data { + switch tok { + case ' ', '\n', '\r', '\t', ',', '}', ']': + return i + } + } + + return len(l.data) +} + +func (l *Lexer) tokenStart() int { + for i := len(l.data) - 1; i >= 0; i-- { + switch l.data[i] { + case '\n', '\r', '\t', ',', '{', '[': + return i + } + } + + return 0 +} + +func (l *Lexer) nextToken() int { + for i, b := range l.data { + switch b { + case '{', '[': + l.depth++ + continue + case '}', ']': + l.depth-- + continue + default: + return i + } + } + + return -1 +} + +func (l *Lexer) findLastTokenPosition() int { + for i := len(l.data) - 1; i >= 0; i-- { + switch l.data[i] { + case ' ', '\n', '\r', '\t': + continue + default: + return i + } + } + + return -1 +} diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno new file mode 100644 index 00000000000..b5078a3caed --- /dev/null +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -0,0 +1,81 @@ +package json + +import ( + "testing" +) + +type lexerTest struct { + name string + data []byte + token byte + expected int +} + +func TestFindTokenStart(t *testing.T) { + tests := []lexerTest{ + {token: '{', expected: 5}, + {token: '[', expected: 0}, + {token: 'w', expected: 6}, + {token: 'l', expected: 9}, + } + + for _, test := range tests { + l := New([]byte("hello{world}")) + if got := l.findTokenStart(test.token); got != test.expected { + t.Errorf("findTokenStart('%c') = %v, want %v", test.token, got, test.expected) + } + } +} + +func TestTokenEnd(t *testing.T) { + tests := []lexerTest{ + {name: "whitespace", data: []byte("hello world"), expected: 5}, + {name: "newline", data: []byte("hello\nworld"), expected: 5}, + {name: "tab", data: []byte("hello\tworld"), expected: 5}, + {name: "multiple words", data: []byte("hello world foo bar"), expected: 5}, + {name: "no space", data: []byte("helloworld"), expected: 10}, + } + + for _, test := range tests { + l := New(test.data) + if got := l.tokenEnd(); got != test.expected { + t.Errorf("tokenEnd() = %v, want %v", got, test.expected) + } + } +} + +func TestLexer(t *testing.T) { + tests := []struct { + name string + data string + start int + next int + last int + }{ + { + name: "Test 1", + data: "{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}", + start: 38, + next: 1, + last: 39, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := &Lexer{data: []byte(tt.data)} + + if got := l.tokenStart(); got != tt.start { + t.Errorf("tokenStart() = %v, want %v", got, tt.start) + } + + if got := l.nextToken(); got != tt.next { + t.Errorf("nextToken() = %v, want %v", got, tt.next) + } + + if got := l.findLastTokenPosition(); got != tt.last { + t.Errorf("findLastTokenPosition() = %v, want %v", got, tt.last) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/number.gno b/examples/gno.land/p/demo/json/number.gno index adf2ea0b4b7..42539fd74d4 100644 --- a/examples/gno.land/p/demo/json/number.gno +++ b/examples/gno.land/p/demo/json/number.gno @@ -1,6 +1,8 @@ package json import ( + "errors" + "math" "strconv" ) @@ -11,72 +13,72 @@ const ( ) func ParseNumberKind(bytes []byte) (value float64, isFloat, ok bool) { - if len(bytes) == 0 { + if len(bytes) == 0 { panic("invalid number: empty string") - } + } - neg := bytes[0] == '-' - if neg { - bytes = bytes[1:] - } + neg := bytes[0] == '-' + if neg { + bytes = bytes[1:] + } var exponentPart []byte for i, c := range bytes { if c == 'e' || c == 'E' { - exponentPart = bytes[i+1:] - bytes = bytes[:i] - break - } + exponentPart = bytes[i+1:] + bytes = bytes[:i] + break + } } - man, exp10 := extractMantissaAndExp10(bytes) + man, exp10 := extractMantissaAndExp10(bytes) if len(exponentPart) > 0 { - exp, err := strconv.Atoi(string(exponentPart)) - if err != nil { - panic("invalid exponent") - } - exp10 += exp - } - - f, success := eiselLemire64(man, exp10, neg) - if !success { - return 0, false, false - } - - return f, true, true + exp, err := strconv.Atoi(string(exponentPart)) + if err != nil { + panic("invalid exponent") + } + exp10 += exp + } + + f, success := eiselLemire64(man, exp10, neg) + if !success { + return 0, false, false + } + + return f, true, true } func extractMantissaAndExp10(bytes []byte) (uint64, int) { - var man uint64 - var exp10 int - decimalFound := false - - for _, c := range bytes { - if c == '.' { - if decimalFound { - panic("invalid number: multiple decimal points") - } - decimalFound = true - continue - } - - if c < '0' || c > '9' { + var man uint64 + var exp10 int + decimalFound := false + + for _, c := range bytes { + if c == '.' { + if decimalFound { + panic("invalid number: multiple decimal points") + } + decimalFound = true + continue + } + + if c < '0' || c > '9' { panic("invalid number: non-digit characters") - } + } - digit := uint64(c - '0') + digit := uint64(c - '0') - if man > (maxUint64-digit)/10 { - panic("invalid number: overflow") - } + if man > (maxUint64-digit)/10 { + panic("invalid number: overflow") + } - man = man*10 + digit + man = man*10 + digit - if decimalFound { - exp10-- - } - } + if decimalFound { + exp10-- + } + } - return man, exp10 + return man, exp10 } diff --git a/examples/gno.land/p/demo/json/number_test.gno b/examples/gno.land/p/demo/json/number_test.gno index 40ebf81de51..f8e9fc9d2d1 100644 --- a/examples/gno.land/p/demo/json/number_test.gno +++ b/examples/gno.land/p/demo/json/number_test.gno @@ -5,73 +5,73 @@ import ( ) func TestParseNumberKind(t *testing.T) { - testCases := []struct { - input string - expected float64 - expectedPanic bool - }{ - {"123", 123, false}, - {"-123", -123, false}, - {"123.456", 123.456, false}, - {"-123.456", -123.456, false}, - {"0.123", 0.123, false}, - {"-0.123", -0.123, false}, - {"", 0, true}, - {"abc", 0, true}, - {"123.45.6", 0, true}, - {"999999999999999999999", 0, true}, - } + testCases := []struct { + input string + expected float64 + expectedPanic bool + }{ + {"123", 123, false}, + {"-123", -123, false}, + {"123.456", 123.456, false}, + {"-123.456", -123.456, false}, + {"0.123", 0.123, false}, + {"-0.123", -0.123, false}, + {"", 0, true}, + {"abc", 0, true}, + {"123.45.6", 0, true}, + {"999999999999999999999", 0, true}, + } - for _, tc := range testCases { - t.Run(tc.input, func(t *testing.T) { - defer func() { - r := recover() - if (r != nil) != tc.expectedPanic { - t.Errorf("ParseNumberKind(%s): expected panic=%v, got panic=%v", - tc.input, tc.expectedPanic, r != nil) - } - }() + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + defer func() { + r := recover() + if (r != nil) != tc.expectedPanic { + t.Errorf("ParseNumberKind(%s): expected panic=%v, got panic=%v", + tc.input, tc.expectedPanic, r != nil) + } + }() - got, _, _ := ParseNumberKind([]byte(tc.input)) - if got != tc.expected && !tc.expectedPanic { - t.Errorf("ParseNumberKind(%s): expected %v, got %v", - tc.input, tc.expected, got) - } - }) - } + got, _, _ := ParseNumberKind([]byte(tc.input)) + if got != tc.expected && !tc.expectedPanic { + t.Errorf("ParseNumberKind(%s): expected %v, got %v", + tc.input, tc.expected, got) + } + }) + } } func TestParseNumberKindWithScientificNotation(t *testing.T) { - testCases := []struct { - input string - expected float64 - }{ - {"1e6", 1000000}, - {"1E6", 1000000}, - {"1.23e10", 1.23e10}, - {"1.23E10", 1.23e10}, - {"-1.23e10", -1.23e10}, - {"-1.23E10", -1.23e10}, - {"2.45e-8", 2.45e-8}, - {"2.45E-8", 2.45e-8}, - {"-2.45e-8", -2.45e-8}, - {"-2.45E-8", -2.45e-8}, - {"5e0", 5}, - {"-5e0", -5}, - {"5E+0", 5}, - {"5e+1", 50}, - {"1e-1", 0.1}, - {"1E-1", 0.1}, - {"-1e-1", -0.1}, - {"-1E-1", -0.1}, - } + testCases := []struct { + input string + expected float64 + }{ + {"1e6", 1000000}, + {"1E6", 1000000}, + {"1.23e10", 1.23e10}, + {"1.23E10", 1.23e10}, + {"-1.23e10", -1.23e10}, + {"-1.23E10", -1.23e10}, + {"2.45e-8", 2.45e-8}, + {"2.45E-8", 2.45e-8}, + {"-2.45e-8", -2.45e-8}, + {"-2.45E-8", -2.45e-8}, + {"5e0", 5}, + {"-5e0", -5}, + {"5E+0", 5}, + {"5e+1", 50}, + {"1e-1", 0.1}, + {"1E-1", 0.1}, + {"-1e-1", -0.1}, + {"-1E-1", -0.1}, + } - for _, tc := range testCases { - t.Run(tc.input, func(t *testing.T) { - got, _, _ := ParseNumberKind([]byte(tc.input)) - if got != tc.expected { - t.Errorf("ParseNumberKind(%s): got %v, want %v", tc.input, got, tc.expected) - } - }) - } + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + got, _, _ := ParseNumberKind([]byte(tc.input)) + if got != tc.expected { + t.Errorf("ParseNumberKind(%s): got %v, want %v", tc.input, got, tc.expected) + } + }) + } } diff --git a/examples/gno.land/p/demo/json/table.gno b/examples/gno.land/p/demo/json/table.gno new file mode 100644 index 00000000000..ca29bc501aa --- /dev/null +++ b/examples/gno.land/p/demo/json/table.gno @@ -0,0 +1,25 @@ +package json + +const ( + Comma = 1 + Colon = 2 + OpenParenKind = 4 + CloseParenKind = 4 + Escape = 8 + Space = 16 +) + +var lookupTable [256]byte + +func init() { + lookupTable[0x2c] = Comma + lookupTable[0x3a] = Colon + lookupTable[0x5b] = OpenParenKind + lookupTable[0x7b] = OpenParenKind + lookupTable[0x5d] = CloseParenKind + lookupTable[0x7d] = CloseParenKind + lookupTable[0x9d] = Escape + lookupTable[0x0a] = Escape + lookupTable[0x0d] = Escape + lookupTable[0x20] = Space +} diff --git a/examples/gno.land/p/demo/json/value.gno b/examples/gno.land/p/demo/json/value.gno new file mode 100644 index 00000000000..760c6d6751c --- /dev/null +++ b/examples/gno.land/p/demo/json/value.gno @@ -0,0 +1,41 @@ +package json + +type ValueType int + +const ( + NotExist ValueType = iota + String + Number + Object + Array + Boolean + Null + Unknown +) + +func (v ValueType) ToString() string { + switch v { + case NotExist: + return "not-exist" + case String: + return "string" + case Number: + return "number" + case Object: + return "object" + case Array: + return "array" + case Boolean: + return "boolean" + case Null: + return "null" + case Unknown: + return "unknown" + } +} + +var ( + TrueLiteral = []byte("true") + FalseLiteral = []byte("false") + NullLiteral = []byte("null") +) diff --git a/examples/gno.land/p/demo/json/value_test.gno b/examples/gno.land/p/demo/json/value_test.gno new file mode 100644 index 00000000000..9973a4987e3 --- /dev/null +++ b/examples/gno.land/p/demo/json/value_test.gno @@ -0,0 +1,30 @@ +package json + +import ( + "testing" +) + +type valueTypeTest struct { + valueType ValueType + expected string +} + +func TestValueTypeToString(t *testing.T) { + tests := []valueTypeTest{ + {NotExist, "not-exist"}, + {String, "string"}, + {Number, "number"}, + {Object, "object"}, + {Array, "array"}, + {Boolean, "boolean"}, + {Null, "null"}, + {Unknown, "unknown"}, + } + + for _, test := range tests { + result := test.valueType.ToString() + if result != test.expected { + t.Errorf("Expected %s, but got %s", test.expected, result) + } + } +} From 1d2a6009ce8af4cc2c786716b095abde5b374a48 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 11 Dec 2023 23:23:43 +0900 Subject: [PATCH 07/72] update lexer --- examples/gno.land/p/demo/json/escape.gno | 27 ++--- examples/gno.land/p/demo/json/escape_test.gno | 56 ++++++++- examples/gno.land/p/demo/json/lexer.gno | 113 +++++++++++++++++- examples/gno.land/p/demo/json/lexer_test.gno | 60 +++++++++- examples/gno.land/p/demo/json/utils.gno | 18 +++ 5 files changed, 245 insertions(+), 29 deletions(-) create mode 100644 examples/gno.land/p/demo/json/utils.gno diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index b7ce647b9fd..b4759bf8f70 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -2,6 +2,7 @@ package json import ( "bytes" + "errors" "unicode/utf8" ) @@ -27,20 +28,6 @@ var escapeMap = map[byte]byte{ 't': '\t', } -// hexToInt converts a hex character to its integer value. -func hexToInt(c byte) int { - switch { - case c >= '0' && c <= '9': - return int(c - '0') - case c >= 'A' && c <= 'F': - return int(c - 'A' + 10) - case c >= 'a' && c <= 'f': - return int(c - 'a' + 10) - } - - return BadHex -} - // isSurrogatePair returns true if the rune is a surrogate pair. // // A surrogate pairs are used in UTF-16 encoding to encode characters @@ -68,7 +55,7 @@ func decodeSingleUnicodeEscape(b []byte) (rune, bool) { } // convert hex to decimal - h1, h2, h3, h4 := hexToInt(b[2]), hexToInt(b[3]), hexToInt(b[4]), hexToInt(b[5]) + h1, h2, h3, h4 := h2i(b[2]), h2i(b[3]), h2i(b[4]), h2i(b[5]) if h1 == BadHex || h2 == BadHex || h3 == BadHex || h4 == BadHex { return utf8.RuneError, false } @@ -133,16 +120,16 @@ func processEscapedUTF8(in, out []byte) (inLen int, outLen int) { return -1, -1 } -func Unescape(input, output []byte) []byte { +func Unescape(input, output []byte) ([]byte, error) { firstBackslash := bytes.IndexByte(input, '\\') if firstBackslash == -1 { - return input + return input, nil } if cap(output) < len(input) { output = make([]byte, len(input)) } else { - output = output[:len(input)] + output = output[0:len(input)] } copy(output, input[:firstBackslash]) @@ -152,7 +139,7 @@ func Unescape(input, output []byte) []byte { for len(input) > 0 { inLen, bufLen := processEscapedUTF8(input, buf) if inLen == -1 { - return nil + return nil, errors.New("Encountered an invalid escape sequence in a string") } input = input[inLen:] @@ -171,5 +158,5 @@ func Unescape(input, output []byte) []byte { } } - return output[:len(output)-len(buf)] + return output[:len(output)-len(buf)], nil } diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index 1c9dea6e7be..f9faf361ac5 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -25,8 +25,8 @@ func TestHexToInt(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := hexToInt(tt.c); got != tt.want { - t.Errorf("hexToInt() = %v, want %v", got, tt.want) + if got := h2i(tt.c); got != tt.want { + t.Errorf("h2i() = %v, want %v", got, tt.want) } }) } @@ -177,10 +177,60 @@ func TestUnescape(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - output := Unescape(tc.input, make([]byte, len(tc.input)+10)) + output, _ := Unescape(tc.input, make([]byte, len(tc.input)+10)) if !bytes.Equal(output, tc.expected) { t.Errorf("unescape(%q) = %q; want %q", tc.input, output, tc.expected) } }) } } + +func TestStringEnd(t *testing.T) { + lexer := &Lexer{} + tests := []struct { + name string + input []byte + expected int + escaped bool + }{ + {"Simple String", []byte(`"Hello"`), 1, false}, + {"Escaped Quote", []byte(`"Hello\"World"`), 1, false}, + {"No Closing Quote", []byte(`"Hello`), 1, false}, + {"Backslashes", []byte(`"Hello\\World"`), 1, false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + idx, escaped := lexer.stringEnd(test.input) + if idx != test.expected || escaped != test.escaped { + t.Errorf("Failed %s: expected (%d, %v), got (%d, %v)", test.name, test.expected, test.escaped, idx, escaped) + } + }) + } +} + + +func TestBlockEnd(t *testing.T) { + lexer := &Lexer{} + tests := []struct { + name string + data []byte + open byte + close byte + expected int + }{ + {"Empty Object", []byte("{}"), '{', '}', 2}, + {"Nested Object", []byte(`{"key": {"nestedKey": "value"}}`), '{', '}', 31}, + {"Array", []byte(`["item1", "item2"]`), '[', ']', 18}, + {"Complex Object", []byte(`{"key": [1, 2, 3], "anotherKey": "value"}`), '{', '}', 41}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := lexer.blockEnd(test.data, test.open, test.close) + if result != test.expected { + t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 67fc70401e2..b020291aeeb 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -1,5 +1,17 @@ package json +import ( + "errors" +) + +const ( + KeyPathNotFoundError error = errors.New("key path not found") + ClosingBracketError error = errors.New("closing bracket not found") + ClosingQuoteError error = errors.New("closing quote not found") +) + +const unescapeStackBufSize = 64 + type Lexer struct { data []byte token byte @@ -49,8 +61,8 @@ func (l *Lexer) tokenStart() int { return 0 } -func (l *Lexer) nextToken() int { - for i, b := range l.data { +func (l *Lexer) nextToken(data []byte) int { + for i, b := range data { switch b { case '{', '[': l.depth++ @@ -78,3 +90,100 @@ func (l *Lexer) findLastTokenPosition() int { return -1 } + +func (l *Lexer) stringEnd(data []byte) (idx int, escaped bool) { + backSlashes := 0 + escaped = isEven(backSlashes) + + for idx, b := range data { + if b == '\\' { + backSlashes++ + continue + } + + if b != '"' { + backSlashes = 0 + continue + } + + return idx + 1, !isEven(backSlashes) + } + + return -1, escaped +} + +func (l *Lexer) blockEnd(data []byte, open, close byte) int { + level := 0 + i := 0 + + for i < len(data) { + switch data[i] { + case '"': + end, _ := l.stringEnd(data[i+1:]) + if end == -1 { + return -1 + } + + i += end + case open: + level++ + case close: + level-- + + if level == 0 { + return i + 1 + } + } + + i++ + } + + return -1 +} + +func (l *Lexer) findKeyPosition(data []byte, key string) (int, error) { + return searchKey(data, key, 0) +} + +func searchKey(data []byte, key string, offset int) (int, error) { + for i := offset; i < len(data); i++ { + switch data[i] { + case '"': + begin, end, nextIndex, err := findNextKey(data, i) + if err != nil { + return -1, err + } + + if string(data[begin:end]) == key { + return nextIndex, nil + } + + i = nextIndex + } + } + + return -1, KeyPathNotFoundError +} + +func findNextKey(data []byte, start int) (keyBegin, keyEnd, nextIndex int, err error) { + keyBegin = -1 + keyEnd = -1 + nextIndex = start + + for i := start; i < len(data); i++ { + switch data[i] { + case '"': + if keyBegin == -1 { + keyBegin = i + 1 + } else { + keyEnd = i + nextIndex = i + 1 + return keyBegin, keyEnd, nextIndex, nil + } + case '\\': + i++ + } + } + + return -1, -1, -1, KeyPathNotFoundError +} diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index b5078a3caed..1af1f4be1e9 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -1,6 +1,7 @@ package json import ( + "errors" "testing" ) @@ -8,6 +9,7 @@ type lexerTest struct { name string data []byte token byte + escaped bool expected int } @@ -69,13 +71,63 @@ func TestLexer(t *testing.T) { t.Errorf("tokenStart() = %v, want %v", got, tt.start) } - if got := l.nextToken(); got != tt.next { - t.Errorf("nextToken() = %v, want %v", got, tt.next) - } - if got := l.findLastTokenPosition(); got != tt.last { t.Errorf("findLastTokenPosition() = %v, want %v", got, tt.last) } }) } } + +func TestFindKeyStart(t *testing.T) { + tests := []struct { + name string + data string + key string + want int + wantErr bool + }{ + { + name: "Simple key", + data: `{"key1": "value1", "key2": "value2"}`, + key: "key1", + want: 7, + wantErr: false, + }, + { + name: "Nested key", + data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, + key: "key1", + want: 17, + wantErr: false, + }, + { + name: "Nested key 2", + data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, + key: "key2", + want: 36, + wantErr: false, + }, + { + name: "Key not found", + data: `{"key1": "value1"}`, + key: "key2", + want: -1, + wantErr: true, + }, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := &Lexer{data: []byte(tt.data)} + got, err := l.findKeyPosition([]byte(tt.data), tt.key) + if (err != nil) != tt.wantErr { + t.Errorf("findKeyStart(%d) error = %v, wantErr %v", i, err, tt.wantErr) + return + } + + if got != tt.want { + t.Errorf("findKeyStart(%d) = %v, want %v", i, got, tt.want) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno new file mode 100644 index 00000000000..d88fb87a079 --- /dev/null +++ b/examples/gno.land/p/demo/json/utils.gno @@ -0,0 +1,18 @@ +package json + +func isEven(n int) bool { + return n%2 == 0 +} + +func h2i(c byte) int { + switch { + case c >= '0' && c <= '9': + return int(c - '0') + case c >= 'A' && c <= 'F': + return int(c - 'A' + 10) + case c >= 'a' && c <= 'f': + return int(c - 'a' + 10) + } + + return BadHex +} \ No newline at end of file From 26f9c7001778e105287b0fe8f433cb8e22163a5a Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Dec 2023 11:56:15 +0900 Subject: [PATCH 08/72] key position range finder --- examples/gno.land/p/demo/json/escape_test.gno | 87 ++++++++-------- examples/gno.land/p/demo/json/lexer.gno | 84 +++++++++------- examples/gno.land/p/demo/json/lexer_test.gno | 99 ++++++++++--------- examples/gno.land/p/demo/json/utils.gno | 4 +- 4 files changed, 146 insertions(+), 128 deletions(-) diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index f9faf361ac5..87aed70cdb1 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -186,51 +186,50 @@ func TestUnescape(t *testing.T) { } func TestStringEnd(t *testing.T) { - lexer := &Lexer{} - tests := []struct { - name string - input []byte - expected int - escaped bool - }{ - {"Simple String", []byte(`"Hello"`), 1, false}, - {"Escaped Quote", []byte(`"Hello\"World"`), 1, false}, - {"No Closing Quote", []byte(`"Hello`), 1, false}, - {"Backslashes", []byte(`"Hello\\World"`), 1, false}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - idx, escaped := lexer.stringEnd(test.input) - if idx != test.expected || escaped != test.escaped { - t.Errorf("Failed %s: expected (%d, %v), got (%d, %v)", test.name, test.expected, test.escaped, idx, escaped) - } - }) - } -} + lexer := &Lexer{} + tests := []struct { + name string + input []byte + expected int + escaped bool + }{ + {"Simple String", []byte(`"Hello"`), 1, false}, + {"Escaped Quote", []byte(`"Hello\"World"`), 1, false}, + {"No Closing Quote", []byte(`"Hello`), 1, false}, + {"Backslashes", []byte(`"Hello\\World"`), 1, false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + idx, escaped := lexer.stringEnd(test.input) + if idx != test.expected || escaped != test.escaped { + t.Errorf("Failed %s: expected (%d, %v), got (%d, %v)", test.name, test.expected, test.escaped, idx, escaped) + } + }) + } +} func TestBlockEnd(t *testing.T) { - lexer := &Lexer{} - tests := []struct { - name string - data []byte - open byte - close byte - expected int - }{ - {"Empty Object", []byte("{}"), '{', '}', 2}, - {"Nested Object", []byte(`{"key": {"nestedKey": "value"}}`), '{', '}', 31}, - {"Array", []byte(`["item1", "item2"]`), '[', ']', 18}, - {"Complex Object", []byte(`{"key": [1, 2, 3], "anotherKey": "value"}`), '{', '}', 41}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result := lexer.blockEnd(test.data, test.open, test.close) - if result != test.expected { - t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) - } - }) - } + lexer := &Lexer{} + tests := []struct { + name string + data []byte + open byte + close byte + expected int + }{ + {"Empty Object", []byte("{}"), '{', '}', 2}, + {"Nested Object", []byte(`{"key": {"nestedKey": "value"}}`), '{', '}', 31}, + {"Array", []byte(`["item1", "item2"]`), '[', ']', 18}, + {"Complex Object", []byte(`{"key": [1, 2, 3], "anotherKey": "value"}`), '{', '}', 41}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := lexer.blockEnd(test.data, test.open, test.close) + if result != test.expected { + t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) + } + }) + } } diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index b020291aeeb..ac7e87a2a00 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -6,6 +6,7 @@ import ( const ( KeyPathNotFoundError error = errors.New("key path not found") + NotSpecifiedKeyError error = errors.New("key not specified") ClosingBracketError error = errors.New("closing bracket not found") ClosingQuoteError error = errors.New("closing quote not found") ) @@ -92,24 +93,24 @@ func (l *Lexer) findLastTokenPosition() int { } func (l *Lexer) stringEnd(data []byte) (idx int, escaped bool) { - backSlashes := 0 + backSlashes := 0 escaped = isEven(backSlashes) - for idx, b := range data { - if b == '\\' { - backSlashes++ - continue - } + for idx, b := range data { + if b == '\\' { + backSlashes++ + continue + } - if b != '"' { - backSlashes = 0 - continue - } + if b != '"' { + backSlashes = 0 + continue + } return idx + 1, !isEven(backSlashes) - } + } - return -1, escaped + return -1, escaped } func (l *Lexer) blockEnd(data []byte, open, close byte) int { @@ -141,8 +142,23 @@ func (l *Lexer) blockEnd(data []byte, open, close byte) int { return -1 } -func (l *Lexer) findKeyPosition(data []byte, key string) (int, error) { - return searchKey(data, key, 0) +// findKeyPositionRange identifies the start and end positions of a key within the given data, +// including positions enclosed in double quotes (`"`). +func (l *Lexer) findKeyPositionRange(data []byte, key string) (start, end int, err error) { + keyLen := len(key) + if keyLen == 0 { + return -1, -1, NotSpecifiedKeyError + } + + keyLastPos, err := searchKey(data, key, 0) + if err != nil { + return -1, -1, err + } + + start = keyLastPos - keyLen - 1 + end = keyLastPos + + return start, end, nil } func searchKey(data []byte, key string, offset int) (int, error) { @@ -166,24 +182,24 @@ func searchKey(data []byte, key string, offset int) (int, error) { } func findNextKey(data []byte, start int) (keyBegin, keyEnd, nextIndex int, err error) { - keyBegin = -1 - keyEnd = -1 - nextIndex = start - - for i := start; i < len(data); i++ { - switch data[i] { - case '"': - if keyBegin == -1 { - keyBegin = i + 1 - } else { - keyEnd = i - nextIndex = i + 1 - return keyBegin, keyEnd, nextIndex, nil - } - case '\\': - i++ - } - } - - return -1, -1, -1, KeyPathNotFoundError + keyBegin = -1 + keyEnd = -1 + nextIndex = -1 + + for i := start; i < len(data); i++ { + switch data[i] { + case '"': + if keyBegin != -1 { + keyEnd = i + nextIndex = i + 1 + return keyBegin, keyEnd, nextIndex, nil + } + + keyBegin = i + 1 + case '\\': + i++ + } + } + + return keyBegin, keyEnd, nextIndex, KeyPathNotFoundError } diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 1af1f4be1e9..f492c51de94 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -78,56 +78,59 @@ func TestLexer(t *testing.T) { } } -func TestFindKeyStart(t *testing.T) { - tests := []struct { - name string - data string - key string - want int - wantErr bool - }{ - { - name: "Simple key", - data: `{"key1": "value1", "key2": "value2"}`, - key: "key1", - want: 7, - wantErr: false, - }, - { - name: "Nested key", - data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, - key: "key1", - want: 17, - wantErr: false, - }, +func TestFindKeyPositionRange(t *testing.T) { + tests := []struct { + name string + data string + key string + want []int + wantErr bool + }{ { - name: "Nested key 2", - data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, - key: "key2", - want: 36, - wantErr: false, - }, - { - name: "Key not found", - data: `{"key1": "value1"}`, - key: "key2", - want: -1, - wantErr: true, - }, - } + name: "Simple key", + data: `{"key1": "value1", "key2": "value2"}`, + key: "key1", + want: []int{2, 7}, + }, + { + name: "Nested key", + data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, + key: "key1", + want: []int{12, 17}, + }, + { + name: "Nested key 2", + data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, + key: "key2", + want: []int{31, 36}, + }, + { + name: "Nested key 3", + data: `{\n"outer": {"key1": "value1"},\n"key2": "value2",\n"key3": "value3"\n}`, + key: "key3", + want: []int{53, 58}, + }, + { + name: "Key not found", + data: `{"key1": "value1"}`, + key: "key2", + want: []int{-1, -1}, + wantErr: true, + }, + } - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { l := &Lexer{data: []byte(tt.data)} - got, err := l.findKeyPosition([]byte(tt.data), tt.key) - if (err != nil) != tt.wantErr { - t.Errorf("findKeyStart(%d) error = %v, wantErr %v", i, err, tt.wantErr) - return - } + start, end, err := l.findKeyPositionRange([]byte(tt.data), tt.key) + if (err != nil) != tt.wantErr { + t.Errorf("findKeyPositionRange(%d) error = %v, wantErr %v", i, err, tt.wantErr) + return + } - if got != tt.want { - t.Errorf("findKeyStart(%d) = %v, want %v", i, got, tt.want) - } - }) - } + if start != tt.want[0] || end != tt.want[1] { + t.Errorf("findKeyPosition(%d) = %v, want %v", i, []int{start, end}, tt.want) + } + }) + } } diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index d88fb87a079..a6679744acd 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -1,7 +1,7 @@ package json func isEven(n int) bool { - return n%2 == 0 + return n%2 == 0 } func h2i(c byte) int { @@ -15,4 +15,4 @@ func h2i(c byte) int { } return BadHex -} \ No newline at end of file +} From 32760c4851e84c869087976a3d5dc94f2a02287d Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Dec 2023 15:01:12 +0900 Subject: [PATCH 09/72] find multiple key positions in JSON --- examples/gno.land/p/demo/json/lexer.gno | 33 ++++++--- examples/gno.land/p/demo/json/lexer_test.gno | 72 ++++++++++++++++++-- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index ac7e87a2a00..595972be6e5 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -144,24 +144,41 @@ func (l *Lexer) blockEnd(data []byte, open, close byte) int { // findKeyPositionRange identifies the start and end positions of a key within the given data, // including positions enclosed in double quotes (`"`). -func (l *Lexer) findKeyPositionRange(data []byte, key string) (start, end int, err error) { +func (l *Lexer) findKeyPositionRange(data []byte, key string) (keyRange []int, err error) { keyLen := len(key) if keyLen == 0 { - return -1, -1, NotSpecifiedKeyError + return []int{-1, -1}, NotSpecifiedKeyError } - keyLastPos, err := searchKey(data, key, 0) + keyLastPos, err := searchSingleKey(data, key, 0) if err != nil { - return -1, -1, err + return []int{-1, -1}, err } - start = keyLastPos - keyLen - 1 - end = keyLastPos + start := keyLastPos - keyLen - 1 + end := keyLastPos - return start, end, nil + keyRange = append(keyRange, start, end) + + return keyRange, nil +} + +func (l *Lexer) findMultipleKeys(data []byte, keys ...string) []int { + var positions []int + + for _, key := range keys { + keyRange, err := l.findKeyPositionRange(data, key) + if err != nil { + continue + } + + positions = append(positions, keyRange...) + } + + return positions } -func searchKey(data []byte, key string, offset int) (int, error) { +func searchSingleKey(data []byte, key string, offset int) (int, error) { for i := offset; i < len(data); i++ { switch data[i] { case '"': diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index f492c51de94..8ed7887c802 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -122,15 +122,79 @@ func TestFindKeyPositionRange(t *testing.T) { for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { l := &Lexer{data: []byte(tt.data)} - start, end, err := l.findKeyPositionRange([]byte(tt.data), tt.key) + key, err := l.findKeyPositionRange([]byte(tt.data), tt.key) + if (err != nil) != tt.wantErr { - t.Errorf("findKeyPositionRange(%d) error = %v, wantErr %v", i, err, tt.wantErr) + t.Errorf("findKeyPositionRange() error = %v, wantErr %v", err, tt.wantErr) return } - if start != tt.want[0] || end != tt.want[1] { - t.Errorf("findKeyPosition(%d) = %v, want %v", i, []int{start, end}, tt.want) + if key[0] != tt.want[0] || key[1] != tt.want[1] { + t.Errorf("findKeyPositionRange() = %v, want %v", key, tt.want) } }) } } + +func TestFindMultipleKeys(t *testing.T) { + tests := []struct { + name string + data []byte + keys []string + expected []int + }{ + { + name: "Empty data and keys", + data: []byte{}, + keys: []string{}, + expected: []int{}, + }, + { + name: "Empty data with keys", + data: []byte{}, + keys: []string{"key1", "key2"}, + expected: []int{}, + }, + { + name: "Data with no keys", + data: []byte(`{"foo": "bar"}`), + keys: []string{"key1", "key2"}, + expected: []int{}, + }, + { + name: "Data with single key", + data: []byte(`{"key1": "value1"}`), + keys: []string{"key1"}, + expected: []int{2, 7}, + }, + { + name: "Data with multiple keys", + data: []byte(`{"key1": "value1", "key2": "value2"}`), + keys: []string{"key1", "key2"}, + expected: []int{2, 7, 20, 25}, + }, + { + name: "Data with nested keys", + data: []byte(`[{"outer": {"key1": "value1"}, "key2": "value2"}]`), + keys: []string{"key1", "key2"}, + expected: []int{13, 18, 32, 37}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := &Lexer{} + got := l.findMultipleKeys(tt.data, tt.keys...) + + if len(got) != len(tt.expected) { + t.Errorf("findMultipleKeys() = %v, want %v", got, tt.expected) + } + + for i, v := range got { + if v != tt.expected[i] { + t.Errorf("findMultipleKeys() = %v, want %v", got, tt.expected) + } + } + }) + } +} \ No newline at end of file From 1de7f0c3ad38b90810d24a9a6ba344601f0f674c Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Dec 2023 15:24:11 +0900 Subject: [PATCH 10/72] parsing Integer and float values --- examples/gno.land/p/demo/json/lexer_test.gno | 4 +- examples/gno.land/p/demo/json/number.gno | 57 ++++++++- examples/gno.land/p/demo/json/number_test.gno | 120 +++++++++++++++++- examples/gno.land/p/demo/json/parser.gno | 5 + 4 files changed, 173 insertions(+), 13 deletions(-) diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 8ed7887c802..9bd72107bcd 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -123,7 +123,7 @@ func TestFindKeyPositionRange(t *testing.T) { t.Run(tt.name, func(t *testing.T) { l := &Lexer{data: []byte(tt.data)} key, err := l.findKeyPositionRange([]byte(tt.data), tt.key) - + if (err != nil) != tt.wantErr { t.Errorf("findKeyPositionRange() error = %v, wantErr %v", err, tt.wantErr) return @@ -197,4 +197,4 @@ func TestFindMultipleKeys(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/number.gno b/examples/gno.land/p/demo/json/number.gno index 42539fd74d4..d2b46d51710 100644 --- a/examples/gno.land/p/demo/json/number.gno +++ b/examples/gno.land/p/demo/json/number.gno @@ -12,15 +12,12 @@ const ( maxUint64 = 1<<64 - 1 ) -func ParseNumberKind(bytes []byte) (value float64, isFloat, ok bool) { +func ParseFloat(bytes []byte) (value float64, isFloat, ok bool) { if len(bytes) == 0 { panic("invalid number: empty string") } - neg := bytes[0] == '-' - if neg { - bytes = bytes[1:] - } + neg, bytes := trimNegativeSign(bytes) var exponentPart []byte for i, c := range bytes { @@ -49,6 +46,56 @@ func ParseNumberKind(bytes []byte) (value float64, isFloat, ok bool) { return f, true, true } +func ParseInt(bytes []byte) (v int64, ok, overflow bool) { + if len(bytes) == 0 { + return 0, false, false + } + + neg, bytes := trimNegativeSign(bytes) + + var n uint64 = 0 + for _, c := range bytes { + if c < '0' || c > '9' { + return 0, false, false + } + + if n > maxUint64/10 { + return 0, false, true + } + + n *= 10 + + n1 := n + uint64(c-'0') + if n1 < n { + return 0, false, true + } + + n = n1 + } + + if n > maxInt64 { + if neg && n == absMinInt64 { + return -absMinInt64, true, false + } + + return 0, false, true + } + + if neg { + return -int64(n), true, false + } + + return int64(n), true, false +} + +func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { + if bytes[0] == '-' { + return true, bytes[1:] + } + + return false, bytes +} + func extractMantissaAndExp10(bytes []byte) (uint64, int) { var man uint64 var exp10 int diff --git a/examples/gno.land/p/demo/json/number_test.gno b/examples/gno.land/p/demo/json/number_test.gno index f8e9fc9d2d1..af8433d6f0a 100644 --- a/examples/gno.land/p/demo/json/number_test.gno +++ b/examples/gno.land/p/demo/json/number_test.gno @@ -4,7 +4,7 @@ import ( "testing" ) -func TestParseNumberKind(t *testing.T) { +func TestParseFloat(t *testing.T) { testCases := []struct { input string expected float64 @@ -27,14 +27,14 @@ func TestParseNumberKind(t *testing.T) { defer func() { r := recover() if (r != nil) != tc.expectedPanic { - t.Errorf("ParseNumberKind(%s): expected panic=%v, got panic=%v", + t.Errorf("ParseFloat(%s): expected panic=%v, got panic=%v", tc.input, tc.expectedPanic, r != nil) } }() - got, _, _ := ParseNumberKind([]byte(tc.input)) + got, _, _ := ParseFloat([]byte(tc.input)) if got != tc.expected && !tc.expectedPanic { - t.Errorf("ParseNumberKind(%s): expected %v, got %v", + t.Errorf("ParseFloat(%s): expected %v, got %v", tc.input, tc.expected, got) } }) @@ -68,10 +68,118 @@ func TestParseNumberKindWithScientificNotation(t *testing.T) { for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { - got, _, _ := ParseNumberKind([]byte(tc.input)) + got, _, _ := ParseFloat([]byte(tc.input)) if got != tc.expected { - t.Errorf("ParseNumberKind(%s): got %v, want %v", tc.input, got, tc.expected) + t.Errorf("ParseFloat(%s): got %v, want %v", tc.input, got, tc.expected) } }) } } + +type ParseIntTest struct { + in string + out int64 + isErr bool + isOverflow bool +} + +var parseIntTests = []ParseIntTest{ + { + in: "0", + out: 0, + }, + { + in: "1", + out: 1, + }, + { + in: "-1", + out: -1, + }, + { + in: "12345", + out: 12345, + }, + { + in: "-12345", + out: -12345, + }, + { + in: "9223372036854775807", // = math.MaxInt64 + out: 9223372036854775807, + }, + { + in: "-9223372036854775808", // = math.MinInt64 + out: -9223372036854775808, + }, + { + in: "-92233720368547758081", + out: 0, + isErr: true, + isOverflow: true, + }, + { + in: "18446744073709551616", // = 2^64 + out: 0, + isErr: true, + isOverflow: true, + }, + { + in: "9223372036854775808", // = math.MaxInt64 - 1 + out: 0, + isErr: true, + isOverflow: true, + }, + { + in: "-9223372036854775809", // = math.MaxInt64 - 1 + out: 0, + isErr: true, + isOverflow: true, + }, + { + in: "", + isErr: true, + }, + { + in: "abc", + isErr: true, + }, + { + in: "12345x", + isErr: true, + }, + { + in: "123e5", + isErr: true, + }, + { + in: "9223372036854775807x", + isErr: true, + }, + { + in: "27670116110564327410", + out: 0, + isErr: true, + isOverflow: true, + }, + { + in: "-27670116110564327410", + out: 0, + isErr: true, + isOverflow: true, + }, +} + +func TestBytesParseInt(t *testing.T) { + for _, test := range parseIntTests { + out, ok, overflow := ParseInt([]byte(test.in)) + if overflow != test.isOverflow { + t.Errorf("Test '%s' error return did not overflow expectation (obtained %t, expected %t)", test.in, overflow, test.isOverflow) + } + if ok != !test.isErr { + t.Errorf("Test '%s' error return did not match expectation (obtained %t, expected %t)", test.in, !ok, test.isErr) + } else if ok && out != test.out { + t.Errorf("Test '%s' did not return the expected value (obtained %d, expected %d)", test.in, out, test.out) + } + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index a5b981cc690..545d83377b7 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -1 +1,6 @@ package json + +import ( + "bytes" + "errors" +) From e19a06d440958bddd14bb38d8094ae289210deab Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Dec 2023 15:47:25 +0900 Subject: [PATCH 11/72] number parser refactoring --- examples/gno.land/p/demo/json/number.gno | 59 +++--- examples/gno.land/p/demo/json/number_test.gno | 183 +++++------------- examples/gno.land/p/demo/json/utils.gno | 8 + 3 files changed, 91 insertions(+), 159 deletions(-) diff --git a/examples/gno.land/p/demo/json/number.gno b/examples/gno.land/p/demo/json/number.gno index d2b46d51710..6200cda4176 100644 --- a/examples/gno.land/p/demo/json/number.gno +++ b/examples/gno.land/p/demo/json/number.gno @@ -6,15 +6,23 @@ import ( "strconv" ) +const ( + EmptyBytes error = errors.New("empty bytes") + InvalidExponents = errors.New("invalid exponents") + NonDigitCharacters = errors.New("non-digit characters") + Overflow = errors.New("overflow") + MultipleDecimalPoints = errors.New("multiple decimal points") +) + const ( absMinInt64 = 1 << 63 maxInt64 = 1<<63 - 1 maxUint64 = 1<<64 - 1 ) -func ParseFloat(bytes []byte) (value float64, isFloat, ok bool) { +func ParseFloat(bytes []byte) (value float64, err error) { if len(bytes) == 0 { - panic("invalid number: empty string") + return -1, EmptyBytes } neg, bytes := trimNegativeSign(bytes) @@ -28,27 +36,30 @@ func ParseFloat(bytes []byte) (value float64, isFloat, ok bool) { } } - man, exp10 := extractMantissaAndExp10(bytes) + man, exp10, err := extractMantissaAndExp10(bytes) + if err != nil { + return -1, err + } if len(exponentPart) > 0 { exp, err := strconv.Atoi(string(exponentPart)) if err != nil { - panic("invalid exponent") + return -1, InvalidExponents } exp10 += exp } f, success := eiselLemire64(man, exp10, neg) if !success { - return 0, false, false + return 0, nil } - return f, true, true + return f, nil } -func ParseInt(bytes []byte) (v int64, ok, overflow bool) { +func ParseInt(bytes []byte) (v int64, err error) { if len(bytes) == 0 { - return 0, false, false + return 0, EmptyBytes } neg, bytes := trimNegativeSign(bytes) @@ -56,18 +67,18 @@ func ParseInt(bytes []byte) (v int64, ok, overflow bool) { var n uint64 = 0 for _, c := range bytes { if c < '0' || c > '9' { - return 0, false, false + return 0, NonDigitCharacters } if n > maxUint64/10 { - return 0, false, true + return 0, Overflow } n *= 10 n1 := n + uint64(c-'0') if n1 < n { - return 0, false, true + return 0, Overflow } n = n1 @@ -75,28 +86,20 @@ func ParseInt(bytes []byte) (v int64, ok, overflow bool) { if n > maxInt64 { if neg && n == absMinInt64 { - return -absMinInt64, true, false + return -absMinInt64, nil } - return 0, false, true + return 0, Overflow } if neg { - return -int64(n), true, false - } - - return int64(n), true, false -} - -func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { - if bytes[0] == '-' { - return true, bytes[1:] + return -int64(n), nil } - return false, bytes + return int64(n), nil } -func extractMantissaAndExp10(bytes []byte) (uint64, int) { +func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { var man uint64 var exp10 int decimalFound := false @@ -104,20 +107,20 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int) { for _, c := range bytes { if c == '.' { if decimalFound { - panic("invalid number: multiple decimal points") + return 0, 0, MultipleDecimalPoints } decimalFound = true continue } if c < '0' || c > '9' { - panic("invalid number: non-digit characters") + return 0, 0, NonDigitCharacters } digit := uint64(c - '0') if man > (maxUint64-digit)/10 { - panic("invalid number: overflow") + return 0, 0, Overflow } man = man*10 + digit @@ -127,5 +130,5 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int) { } } - return man, exp10 + return man, exp10, nil } diff --git a/examples/gno.land/p/demo/json/number_test.gno b/examples/gno.land/p/demo/json/number_test.gno index af8433d6f0a..0a6f6662945 100644 --- a/examples/gno.land/p/demo/json/number_test.gno +++ b/examples/gno.land/p/demo/json/number_test.gno @@ -6,42 +6,32 @@ import ( func TestParseFloat(t *testing.T) { testCases := []struct { - input string - expected float64 - expectedPanic bool + input string + expected float64 }{ - {"123", 123, false}, - {"-123", -123, false}, - {"123.456", 123.456, false}, - {"-123.456", -123.456, false}, - {"0.123", 0.123, false}, - {"-0.123", -0.123, false}, - {"", 0, true}, - {"abc", 0, true}, - {"123.45.6", 0, true}, - {"999999999999999999999", 0, true}, + {"123", 123}, + {"-123", -123}, + {"123.456", 123.456}, + {"-123.456", -123.456}, + {"0.123", 0.123}, + {"-0.123", -0.123}, + {"", -1}, + {"abc", -1}, + {"123.45.6", -1}, + {"999999999999999999999", -1}, } for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { - defer func() { - r := recover() - if (r != nil) != tc.expectedPanic { - t.Errorf("ParseFloat(%s): expected panic=%v, got panic=%v", - tc.input, tc.expectedPanic, r != nil) - } - }() - - got, _, _ := ParseFloat([]byte(tc.input)) - if got != tc.expected && !tc.expectedPanic { - t.Errorf("ParseFloat(%s): expected %v, got %v", - tc.input, tc.expected, got) + got, _ := ParseFloat([]byte(tc.input)) + if got != tc.expected { + t.Errorf("ParseFloat(%s): got %v, want %v", tc.input, got, tc.expected) } }) } } -func TestParseNumberKindWithScientificNotation(t *testing.T) { +func TestParseFloatWithScientificNotation(t *testing.T) { testCases := []struct { input string expected float64 @@ -68,118 +58,49 @@ func TestParseNumberKindWithScientificNotation(t *testing.T) { for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { - got, _, _ := ParseFloat([]byte(tc.input)) + got, err := ParseFloat([]byte(tc.input)) if got != tc.expected { t.Errorf("ParseFloat(%s): got %v, want %v", tc.input, got, tc.expected) } + + if err != nil { + t.Errorf("ParseFloat(%s): got error %v", tc.input, err) + } }) } } -type ParseIntTest struct { - in string - out int64 - isErr bool - isOverflow bool -} - -var parseIntTests = []ParseIntTest{ - { - in: "0", - out: 0, - }, - { - in: "1", - out: 1, - }, - { - in: "-1", - out: -1, - }, - { - in: "12345", - out: 12345, - }, - { - in: "-12345", - out: -12345, - }, - { - in: "9223372036854775807", // = math.MaxInt64 - out: 9223372036854775807, - }, - { - in: "-9223372036854775808", // = math.MinInt64 - out: -9223372036854775808, - }, - { - in: "-92233720368547758081", - out: 0, - isErr: true, - isOverflow: true, - }, - { - in: "18446744073709551616", // = 2^64 - out: 0, - isErr: true, - isOverflow: true, - }, - { - in: "9223372036854775808", // = math.MaxInt64 - 1 - out: 0, - isErr: true, - isOverflow: true, - }, - { - in: "-9223372036854775809", // = math.MaxInt64 - 1 - out: 0, - isErr: true, - isOverflow: true, - }, - { - in: "", - isErr: true, - }, - { - in: "abc", - isErr: true, - }, - { - in: "12345x", - isErr: true, - }, - { - in: "123e5", - isErr: true, - }, - { - in: "9223372036854775807x", - isErr: true, - }, - { - in: "27670116110564327410", - out: 0, - isErr: true, - isOverflow: true, - }, - { - in: "-27670116110564327410", - out: 0, - isErr: true, - isOverflow: true, - }, -} +func TestParseInt(t *testing.T) { + testCases := []struct { + input string + expected int64 + }{ + {"0", 0}, + {"1", 1}, + {"-1", -1}, + {"12345", 12345}, + {"-12345", -12345}, + {"9223372036854775807", 9223372036854775807}, + {"-9223372036854775808", -9223372036854775808}, + {"-92233720368547758081", 0}, + {"18446744073709551616", 0}, + {"9223372036854775808", 0}, + {"-9223372036854775809", 0}, + {"", 0}, + {"abc", 0}, + {"12345x", 0}, + {"123e5", 0}, + {"9223372036854775807x", 0}, + {"27670116110564327410", 0}, + {"-27670116110564327410", 0}, + } -func TestBytesParseInt(t *testing.T) { - for _, test := range parseIntTests { - out, ok, overflow := ParseInt([]byte(test.in)) - if overflow != test.isOverflow { - t.Errorf("Test '%s' error return did not overflow expectation (obtained %t, expected %t)", test.in, overflow, test.isOverflow) - } - if ok != !test.isErr { - t.Errorf("Test '%s' error return did not match expectation (obtained %t, expected %t)", test.in, !ok, test.isErr) - } else if ok && out != test.out { - t.Errorf("Test '%s' did not return the expected value (obtained %d, expected %d)", test.in, out, test.out) - } + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + got, _ := ParseInt([]byte(tc.input)) + if got != tc.expected { + t.Errorf("ParseInt(%s): got %v, want %v", tc.input, got, tc.expected) + } + }) } } \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index a6679744acd..21bc7c19c52 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -16,3 +16,11 @@ func h2i(c byte) int { return BadHex } + +func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { + if bytes[0] == '-' { + return true, bytes[1:] + } + + return false, bytes +} \ No newline at end of file From 9ba880834e02ba76f463d08b6771b15627b85272 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Dec 2023 19:18:51 +0900 Subject: [PATCH 12/72] parse primitive types --- examples/gno.land/p/demo/json/number.gno | 134 --------------- examples/gno.land/p/demo/json/parser.gno | 153 ++++++++++++++++++ .../json/{number_test.gno => parser_test.gno} | 65 +++++++- examples/gno.land/p/demo/json/utils.gno | 6 +- 4 files changed, 222 insertions(+), 136 deletions(-) delete mode 100644 examples/gno.land/p/demo/json/number.gno rename examples/gno.land/p/demo/json/{number_test.gno => parser_test.gno} (62%) diff --git a/examples/gno.land/p/demo/json/number.gno b/examples/gno.land/p/demo/json/number.gno deleted file mode 100644 index 6200cda4176..00000000000 --- a/examples/gno.land/p/demo/json/number.gno +++ /dev/null @@ -1,134 +0,0 @@ -package json - -import ( - "errors" - "math" - "strconv" -) - -const ( - EmptyBytes error = errors.New("empty bytes") - InvalidExponents = errors.New("invalid exponents") - NonDigitCharacters = errors.New("non-digit characters") - Overflow = errors.New("overflow") - MultipleDecimalPoints = errors.New("multiple decimal points") -) - -const ( - absMinInt64 = 1 << 63 - maxInt64 = 1<<63 - 1 - maxUint64 = 1<<64 - 1 -) - -func ParseFloat(bytes []byte) (value float64, err error) { - if len(bytes) == 0 { - return -1, EmptyBytes - } - - neg, bytes := trimNegativeSign(bytes) - - var exponentPart []byte - for i, c := range bytes { - if c == 'e' || c == 'E' { - exponentPart = bytes[i+1:] - bytes = bytes[:i] - break - } - } - - man, exp10, err := extractMantissaAndExp10(bytes) - if err != nil { - return -1, err - } - - if len(exponentPart) > 0 { - exp, err := strconv.Atoi(string(exponentPart)) - if err != nil { - return -1, InvalidExponents - } - exp10 += exp - } - - f, success := eiselLemire64(man, exp10, neg) - if !success { - return 0, nil - } - - return f, nil -} - -func ParseInt(bytes []byte) (v int64, err error) { - if len(bytes) == 0 { - return 0, EmptyBytes - } - - neg, bytes := trimNegativeSign(bytes) - - var n uint64 = 0 - for _, c := range bytes { - if c < '0' || c > '9' { - return 0, NonDigitCharacters - } - - if n > maxUint64/10 { - return 0, Overflow - } - - n *= 10 - - n1 := n + uint64(c-'0') - if n1 < n { - return 0, Overflow - } - - n = n1 - } - - if n > maxInt64 { - if neg && n == absMinInt64 { - return -absMinInt64, nil - } - - return 0, Overflow - } - - if neg { - return -int64(n), nil - } - - return int64(n), nil -} - -func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { - var man uint64 - var exp10 int - decimalFound := false - - for _, c := range bytes { - if c == '.' { - if decimalFound { - return 0, 0, MultipleDecimalPoints - } - decimalFound = true - continue - } - - if c < '0' || c > '9' { - return 0, 0, NonDigitCharacters - } - - digit := uint64(c - '0') - - if man > (maxUint64-digit)/10 { - return 0, 0, Overflow - } - - man = man*10 + digit - - if decimalFound { - exp10-- - } - } - - return man, exp10, nil -} diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 545d83377b7..f11a22f95f3 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -3,4 +3,157 @@ package json import ( "bytes" "errors" + "math" + "strconv" ) + +const ( + Overflow error = errors.New("overflow") + EmptyBytes = errors.New("empty bytes") + InvalidExponents = errors.New("invalid exponents") + NonDigitCharacters = errors.New("non-digit characters") + MultipleDecimalPoints = errors.New("multiple decimal points") + MalformedStringError = errors.New("malformed string") + MalformedVlaueError = errors.New("malformed value") +) + +const ( + absMinInt64 = 1 << 63 + maxInt64 = 1<<63 - 1 + maxUint64 = 1<<64 - 1 +) + +func ParseString(data []byte) (string, error) { + var buf [unescapeStackBufSize]byte + + bf, err := Unescape(data, buf[:]) + if err != nil { + return "", MalformedStringError + } + + return string(bf), nil +} + +func ParseBool(data []byte) (bool, error) { + switch { + case bytes.Equal(data, []byte("true")): + return true, nil + case bytes.Equal(data, []byte("false")): + return false, nil + default: + return false, MalformedVlaueError + } +} + +func ParseFloat(bytes []byte) (value float64, err error) { + if len(bytes) == 0 { + return -1, EmptyBytes + } + + neg, bytes := trimNegativeSign(bytes) + + var exponentPart []byte + for i, c := range bytes { + if c == 'e' || c == 'E' { + exponentPart = bytes[i+1:] + bytes = bytes[:i] + break + } + } + + man, exp10, err := extractMantissaAndExp10(bytes) + if err != nil { + return -1, err + } + + if len(exponentPart) > 0 { + exp, err := strconv.Atoi(string(exponentPart)) + if err != nil { + return -1, InvalidExponents + } + exp10 += exp + } + + f, success := eiselLemire64(man, exp10, neg) + if !success { + return 0, nil + } + + return f, nil +} + +func ParseInt(bytes []byte) (v int64, err error) { + if len(bytes) == 0 { + return 0, EmptyBytes + } + + neg, bytes := trimNegativeSign(bytes) + + var n uint64 = 0 + for _, c := range bytes { + if notDigit(c) { + return 0, NonDigitCharacters + } + + if n > maxUint64/10 { + return 0, Overflow + } + + n *= 10 + + n1 := n + uint64(c-'0') + if n1 < n { + return 0, Overflow + } + + n = n1 + } + + if n > maxInt64 { + if neg && n == absMinInt64 { + return -absMinInt64, nil + } + + return 0, Overflow + } + + if neg { + return -int64(n), nil + } + + return int64(n), nil +} + +func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { + var man uint64 + var exp10 int + decimalFound := false + + for _, c := range bytes { + if c == '.' { + if decimalFound { + return 0, 0, MultipleDecimalPoints + } + decimalFound = true + continue + } + + if notDigit(c) { + return 0, 0, NonDigitCharacters + } + + digit := uint64(c - '0') + + if man > (maxUint64-digit)/10 { + return 0, 0, Overflow + } + + man = man*10 + digit + + if decimalFound { + exp10-- + } + } + + return man, exp10, nil +} diff --git a/examples/gno.land/p/demo/json/number_test.gno b/examples/gno.land/p/demo/json/parser_test.gno similarity index 62% rename from examples/gno.land/p/demo/json/number_test.gno rename to examples/gno.land/p/demo/json/parser_test.gno index 0a6f6662945..d6da9f85a8d 100644 --- a/examples/gno.land/p/demo/json/number_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -4,6 +4,69 @@ import ( "testing" ) +func TestParseString(t *testing.T) { + tests := []struct { + input string + expected string + isError bool + }{ + {`"Hello, World!"`, "\"Hello, World!\"", false}, + {`\uFF11`, "\uFF11", false}, + {`\uFFFF`, "\uFFFF", false}, + {`true`, "true", false}, + {`false`, "false", false}, + {`\uDF00`, "", true}, + } + + for i, tt := range tests { + s, err := ParseString([]byte(tt.input)) + + if !tt.isError && err != nil { + t.Errorf("%d. unexpected error: %s", i, err) + } + + if tt.isError && err == nil { + t.Errorf("%d. expected error, but not error", i) + } + + if s != tt.expected { + t.Errorf("%d. expected=%s, but actual=%s", i, tt.expected, s) + } + } +} + +func TestParseBool(t *testing.T) { + tests := []struct { + input string + expected bool + isError bool + }{ + {`true`, true, false}, + {`false`, false, false}, + {`TRUE`, false, true}, + {`FALSE`, false, true}, + {`foo`, false, true}, + {`"true"`, false, true}, + {`"false"`, false, true}, + } + + for i, tt := range tests { + b, err := ParseBool([]byte(tt.input)) + + if !tt.isError && err != nil { + t.Errorf("%d. unexpected error: %s", i, err) + } + + if tt.isError && err == nil { + t.Errorf("%d. expected error, but not error", i) + } + + if b != tt.expected { + t.Errorf("%d. expected=%t, but actual=%t", i, tt.expected, b) + } + } +} + func TestParseFloat(t *testing.T) { testCases := []struct { input string @@ -103,4 +166,4 @@ func TestParseInt(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index 21bc7c19c52..f506468b7ab 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -1,5 +1,9 @@ package json +func notDigit(c byte) bool { + return c < '0' || c > '9' +} + func isEven(n int) bool { return n%2 == 0 } @@ -23,4 +27,4 @@ func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { } return false, bytes -} \ No newline at end of file +} From 534f0f787ed3191d812115abf9b572c1c06dde62 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Dec 2023 15:11:18 +0900 Subject: [PATCH 13/72] create searchKeys --- examples/gno.land/p/demo/json/errors.gno | 24 ++ examples/gno.land/p/demo/json/escape.gno | 82 +++---- examples/gno.land/p/demo/json/escape_test.gno | 4 +- examples/gno.land/p/demo/json/lexer.gno | 220 ++++++++++-------- examples/gno.land/p/demo/json/lexer_test.gno | 94 ++------ examples/gno.land/p/demo/json/parser.gno | 10 - examples/gno.land/p/demo/json/utils.gno | 4 - 7 files changed, 215 insertions(+), 223 deletions(-) create mode 100644 examples/gno.land/p/demo/json/errors.gno diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno new file mode 100644 index 00000000000..3a965036685 --- /dev/null +++ b/examples/gno.land/p/demo/json/errors.gno @@ -0,0 +1,24 @@ +package json + +import "errors" + +const ( + KeyPathNotFoundError error = errors.New("key path not found") + NotSpecifiedKeyError = errors.New("key not specified") + ClosingBracketError = errors.New("closing bracket not found") + ClosingQuoteError = errors.New("closing quote not found") + StringNotClosed = errors.New("string not closed") + BlockIsNotClosed = errors.New("block is not closed") + TokenNotFound = errors.New("token not found") + OutOfLevel = errors.New("out of level") + KeyLevelNotMatched = errors.New("key level not matched") + Overflow = errors.New("overflow") + EmptyBytes = errors.New("empty bytes") + InvalidExponents = errors.New("invalid exponents") + NonDigitCharacters = errors.New("non-digit characters") + MultipleDecimalPoints = errors.New("multiple decimal points") + MalformedStringError = errors.New("malformed string") + MalformedVlaueError = errors.New("malformed value") + MalformedObjectError = errors.New("malformed object") + MalformedJSON = errors.New("malformed json array") +) diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index b4759bf8f70..6577f89cf88 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -17,6 +17,47 @@ const ( BadHex = -1 ) +func Unescape(input, output []byte) ([]byte, error) { + firstBackslash := bytes.IndexByte(input, '\\') + if firstBackslash == -1 { + return input, nil + } + + if cap(output) < len(input) { + output = make([]byte, len(input)) + } else { + output = output[0:len(input)] + } + + copy(output, input[:firstBackslash]) + input = input[firstBackslash:] + buf := output[firstBackslash:] + + for len(input) > 0 { + inLen, bufLen := processEscapedUTF8(input, buf) + if inLen == -1 { + return nil, errors.New("Encountered an invalid escape sequence in a string") + } + + input = input[inLen:] + buf = buf[bufLen:] + + // copy everything until the next backslash + nextBackslash := bytes.IndexByte(input, '\\') + if nextBackslash == -1 { + copy(buf, input) + buf = buf[len(input):] + break + } else { + copy(buf, input[:nextBackslash]) + buf = buf[nextBackslash:] + input = input[nextBackslash:] + } + } + + return output[:len(output)-len(buf)], nil +} + var escapeMap = map[byte]byte{ '"': '"', '\\': '\\', @@ -119,44 +160,3 @@ func processEscapedUTF8(in, out []byte) (inLen int, outLen int) { return -1, -1 } - -func Unescape(input, output []byte) ([]byte, error) { - firstBackslash := bytes.IndexByte(input, '\\') - if firstBackslash == -1 { - return input, nil - } - - if cap(output) < len(input) { - output = make([]byte, len(input)) - } else { - output = output[0:len(input)] - } - - copy(output, input[:firstBackslash]) - input = input[firstBackslash:] - buf := output[firstBackslash:] - - for len(input) > 0 { - inLen, bufLen := processEscapedUTF8(input, buf) - if inLen == -1 { - return nil, errors.New("Encountered an invalid escape sequence in a string") - } - - input = input[inLen:] - buf = buf[bufLen:] - - // copy everything until the next backslash - nextBackslash := bytes.IndexByte(input, '\\') - if nextBackslash == -1 { - copy(buf, input) - buf = buf[len(input):] - break - } else { - copy(buf, input[:nextBackslash]) - buf = buf[nextBackslash:] - input = input[nextBackslash:] - } - } - - return output[:len(output)-len(buf)], nil -} diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index 87aed70cdb1..067b75ae526 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -201,7 +201,7 @@ func TestStringEnd(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - idx, escaped := lexer.stringEnd(test.input) + idx, escaped, _ := lexer.stringEnd(test.input) if idx != test.expected || escaped != test.escaped { t.Errorf("Failed %s: expected (%d, %v), got (%d, %v)", test.name, test.expected, test.escaped, idx, escaped) } @@ -226,7 +226,7 @@ func TestBlockEnd(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result := lexer.blockEnd(test.data, test.open, test.close) + result, _ := lexer.blockEnd(test.data, test.open, test.close) if result != test.expected { t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) } diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 595972be6e5..80d4f586dec 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -1,28 +1,21 @@ package json -import ( - "errors" -) - -const ( - KeyPathNotFoundError error = errors.New("key path not found") - NotSpecifiedKeyError error = errors.New("key not specified") - ClosingBracketError error = errors.New("closing bracket not found") - ClosingQuoteError error = errors.New("closing quote not found") -) +import "errors" const unescapeStackBufSize = 64 type Lexer struct { - data []byte - token byte - pos int - depth int + data []byte + token byte + level int + keyLevel int } func New(data []byte) *Lexer { return &Lexer{ - data: data, + data: data, + level: 0, + keyLevel: 0, } } @@ -32,7 +25,6 @@ func (l *Lexer) findTokenStart(token byte) int { case token: return i case '[', '{': - l.depth++ return 0 } } @@ -66,10 +58,8 @@ func (l *Lexer) nextToken(data []byte) int { for i, b := range data { switch b { case '{', '[': - l.depth++ continue case '}', ']': - l.depth-- continue default: return i @@ -92,37 +82,45 @@ func (l *Lexer) findLastTokenPosition() int { return -1 } -func (l *Lexer) stringEnd(data []byte) (idx int, escaped bool) { - backSlashes := 0 - escaped = isEven(backSlashes) +// TODO: refactor this function +func (l *Lexer) stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { + escaped = false + for lastCharIdx, c := range data { + if c == '"' { + if !escaped { + return lastCharIdx + 1, false, nil + } - for idx, b := range data { - if b == '\\' { - backSlashes++ - continue - } + prevCharIdx := lastCharIdx - 1 + for prevCharIdx >= 0 && data[prevCharIdx] == '\\' { + prevCharIdx-- + } - if b != '"' { - backSlashes = 0 - continue + if (lastCharIdx-prevCharIdx)%2 == 0 { + return lastCharIdx + 1, true, nil + } + + return lastCharIdx + 1, false, nil } - return idx + 1, !isEven(backSlashes) + if c == '\\' { + escaped = true + } } - return -1, escaped + return -1, escaped, StringNotClosed } -func (l *Lexer) blockEnd(data []byte, open, close byte) int { - level := 0 +func (l *Lexer) blockEnd(data []byte, open, close byte) (int, error) { + level := l.level i := 0 for i < len(data) { switch data[i] { case '"': - end, _ := l.stringEnd(data[i+1:]) - if end == -1 { - return -1 + end, _, err := l.stringEnd(data[i+1:]) + if err != nil { + return -1, StringNotClosed } i += end @@ -132,91 +130,131 @@ func (l *Lexer) blockEnd(data []byte, open, close byte) int { level-- if level == 0 { - return i + 1 + return i + 1, nil } } i++ } - return -1 + return -1, BlockIsNotClosed } -// findKeyPositionRange identifies the start and end positions of a key within the given data, -// including positions enclosed in double quotes (`"`). -func (l *Lexer) findKeyPositionRange(data []byte, key string) (keyRange []int, err error) { - keyLen := len(key) - if keyLen == 0 { - return []int{-1, -1}, NotSpecifiedKeyError - } - - keyLastPos, err := searchSingleKey(data, key, 0) - if err != nil { - return []int{-1, -1}, err - } +func findNextKey(data []byte, start int) (keyBegin, keyEnd, nextIndex int, err error) { + keyBegin = -1 + keyEnd = -1 + nextIndex = -1 - start := keyLastPos - keyLen - 1 - end := keyLastPos + for i := start; i < len(data); i++ { + switch data[i] { + case '"': + if keyBegin != -1 { + keyEnd = i + nextIndex = i + 1 + return keyBegin, keyEnd, nextIndex, nil + } - keyRange = append(keyRange, start, end) + keyBegin = i + 1 + case '\\': + i++ + } + } - return keyRange, nil + return keyBegin, keyEnd, nextIndex, KeyPathNotFoundError } -func (l *Lexer) findMultipleKeys(data []byte, keys ...string) []int { - var positions []int +// TODO: refactor this function +func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error) { + lastMatched := true + keyLevel, level := l.keyLevel, l.level + dataLenghth, numKeys := len(data), len(keys) - for _, key := range keys { - keyRange, err := l.findKeyPositionRange(data, key) - if err != nil { - continue - } - - positions = append(positions, keyRange...) + if numKeys == 0 { + return 0, nil } - return positions -} + var stackbuf [unescapeStackBufSize]byte -func searchSingleKey(data []byte, key string, offset int) (int, error) { - for i := offset; i < len(data); i++ { - switch data[i] { + for keyIndex < dataLenghth { + switch data[keyIndex] { case '"': - begin, end, nextIndex, err := findNextKey(data, i) + keyIndex++ + keyBegin := keyIndex + + strEnd, keyEscaped, err := l.stringEnd(data[keyIndex:]) + if err != nil { + return -1, StringNotClosed + } + keyIndex += strEnd + keyEnd := keyIndex - 1 + + valueOffset := l.nextToken(data[keyIndex:]) + if valueOffset == -1 { + return -1, TokenNotFound + } + + keyIndex += valueOffset + if data[keyIndex] != ':' { + keyIndex-- + } + + if level < 1 { + return -1, OutOfLevel + } + + key := data[keyBegin:keyEnd] + keyUnesc, err := Unescape(key, stackbuf[:]) if err != nil { return -1, err } - if string(data[begin:end]) == key { - return nextIndex, nil + if !keyEscaped { + keyUnesc = key } - i = nextIndex - } - } + if level > numKeys { + return -1, KeyPathNotFoundError + } - return -1, KeyPathNotFoundError -} + if string(keyUnesc) == keys[level-1] { + lastMatched = true + if keyLevel == level-1 { + keyLevel++ -func findNextKey(data []byte, start int) (keyBegin, keyEnd, nextIndex int, err error) { - keyBegin = -1 - keyEnd = -1 - nextIndex = -1 + if keyLevel == numKeys { + return keyIndex + 1, nil + } + } + } - for i := start; i < len(data); i++ { - switch data[i] { - case '"': - if keyBegin != -1 { - keyEnd = i - nextIndex = i + 1 - return keyBegin, keyEnd, nextIndex, nil + lastMatched = false + + case '{': + // in case parent key is matched then only we will increase the level otherwise can directly + // can move to the end of this block + if !lastMatched { + end, err := l.blockEnd(data[keyIndex:], '{', '}') + if err != nil { + return -1, err + } + keyIndex += end - 1 + } else { + level++ } - keyBegin = i + 1 - case '\\': - i++ + case '}': + level-- + if level == keyLevel { + keyLevel-- + } + case '[': + panic(errors.New("arrays are not supported")) + case ':': + return -1, MalformedJSON } + + keyIndex++ } - return keyBegin, keyEnd, nextIndex, KeyPathNotFoundError + return -1, KeyPathNotFoundError } diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 9bd72107bcd..952bbba667f 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -78,122 +78,66 @@ func TestLexer(t *testing.T) { } } -func TestFindKeyPositionRange(t *testing.T) { - tests := []struct { - name string - data string - key string - want []int - wantErr bool - }{ - { - name: "Simple key", - data: `{"key1": "value1", "key2": "value2"}`, - key: "key1", - want: []int{2, 7}, - }, - { - name: "Nested key", - data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, - key: "key1", - want: []int{12, 17}, - }, - { - name: "Nested key 2", - data: `{"outer": {"key1": "value1"}, "key2": "value2"}`, - key: "key2", - want: []int{31, 36}, - }, - { - name: "Nested key 3", - data: `{\n"outer": {"key1": "value1"},\n"key2": "value2",\n"key3": "value3"\n}`, - key: "key3", - want: []int{53, 58}, - }, - { - name: "Key not found", - data: `{"key1": "value1"}`, - key: "key2", - want: []int{-1, -1}, - wantErr: true, - }, - } - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - l := &Lexer{data: []byte(tt.data)} - key, err := l.findKeyPositionRange([]byte(tt.data), tt.key) - - if (err != nil) != tt.wantErr { - t.Errorf("findKeyPositionRange() error = %v, wantErr %v", err, tt.wantErr) - return - } - - if key[0] != tt.want[0] || key[1] != tt.want[1] { - t.Errorf("findKeyPositionRange() = %v, want %v", key, tt.want) - } - }) - } -} - -func TestFindMultipleKeys(t *testing.T) { +func TestSearchKeys(t *testing.T) { tests := []struct { name string data []byte keys []string - expected []int + expected int + isErr bool }{ { name: "Empty data and keys", data: []byte{}, keys: []string{}, - expected: []int{}, + expected: 0, }, { name: "Empty data with keys", data: []byte{}, keys: []string{"key1", "key2"}, - expected: []int{}, + expected: -1, + isErr: true, }, { name: "Data with no keys", data: []byte(`{"foo": "bar"}`), keys: []string{"key1", "key2"}, - expected: []int{}, + expected: -1, + isErr: true, }, { name: "Data with single key", data: []byte(`{"key1": "value1"}`), keys: []string{"key1"}, - expected: []int{2, 7}, + expected: 8, }, { name: "Data with multiple keys", data: []byte(`{"key1": "value1", "key2": "value2"}`), - keys: []string{"key1", "key2"}, - expected: []int{2, 7, 20, 25}, + keys: []string{"key2"}, + expected: 26, }, { name: "Data with nested keys", - data: []byte(`[{"outer": {"key1": "value1"}, "key2": "value2"}]`), - keys: []string{"key1", "key2"}, - expected: []int{13, 18, 32, 37}, + data: []byte(`{"outer": {"key1": "value1", "key2": "value2"}, "key3": "value3"}`), + keys: []string{"key3"}, + expected: 55, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { l := &Lexer{} - got := l.findMultipleKeys(tt.data, tt.keys...) + got, err := l.searchKeys(tt.data, tt.keys...) - if len(got) != len(tt.expected) { - t.Errorf("findMultipleKeys() = %v, want %v", got, tt.expected) + if got != tt.expected { + t.Errorf("searchKeys(%s) = %v, want %v", tt.name, got, tt.expected) } - for i, v := range got { - if v != tt.expected[i] { - t.Errorf("findMultipleKeys() = %v, want %v", got, tt.expected) - } + if (err != nil) != tt.isErr { + t.Errorf("searchKeys(%s) error = %v, wantErr %v", tt.name, err, tt.isErr) } }) } diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index f11a22f95f3..792071ff89f 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -7,16 +7,6 @@ import ( "strconv" ) -const ( - Overflow error = errors.New("overflow") - EmptyBytes = errors.New("empty bytes") - InvalidExponents = errors.New("invalid exponents") - NonDigitCharacters = errors.New("non-digit characters") - MultipleDecimalPoints = errors.New("multiple decimal points") - MalformedStringError = errors.New("malformed string") - MalformedVlaueError = errors.New("malformed value") -) - const ( absMinInt64 = 1 << 63 maxInt64 = 1<<63 - 1 diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index f506468b7ab..b739ee3b21f 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -4,10 +4,6 @@ func notDigit(c byte) bool { return c < '0' || c > '9' } -func isEven(n int) bool { - return n%2 == 0 -} - func h2i(c byte) int { switch { case c >= '0' && c <= '9': From f12375d24594a76629b913ff018fa2f78195eb74 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Dec 2023 16:41:45 +0900 Subject: [PATCH 14/72] get type value --- examples/gno.land/p/demo/json/errors.gno | 6 +- examples/gno.land/p/demo/json/lexer.gno | 257 +++++++++++++++++-- examples/gno.land/p/demo/json/lexer_test.gno | 9 +- examples/gno.land/p/demo/json/parser.gno | 21 +- 4 files changed, 265 insertions(+), 28 deletions(-) diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index 3a965036685..075637d5aeb 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -18,7 +18,9 @@ const ( NonDigitCharacters = errors.New("non-digit characters") MultipleDecimalPoints = errors.New("multiple decimal points") MalformedStringError = errors.New("malformed string") - MalformedVlaueError = errors.New("malformed value") + MalformedValueError = errors.New("malformed value") MalformedObjectError = errors.New("malformed object") - MalformedJSON = errors.New("malformed json array") + MalformedArrayError = errors.New("malformed array") + MalformedJsonError = errors.New("malformed json array") + UnknownValueTypeError = errors.New("unknown value type") ) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 80d4f586dec..c40981cb6d4 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -1,6 +1,10 @@ package json -import "errors" +import ( + "bytes" + "errors" + "strconv" +) const unescapeStackBufSize = 64 @@ -28,31 +32,30 @@ func (l *Lexer) findTokenStart(token byte) int { return 0 } } - return 0 } -func (l *Lexer) tokenEnd() int { - for i, tok := range l.data { +func tokenEnd(data []byte) int { + for i, tok := range data { switch tok { case ' ', '\n', '\r', '\t', ',', '}', ']': return i } } - return len(l.data) + return len(data) } -func (l *Lexer) tokenStart() int { - for i := len(l.data) - 1; i >= 0; i-- { - switch l.data[i] { - case '\n', '\r', '\t', ',', '{', '[': - return i - } - } +// func (l *Lexer) tokenStart() int { +// for i := len(l.data) - 1; i >= 0; i-- { +// switch l.data[i] { +// case '\n', '\r', '\t', ',', '{', '[': +// return i +// } +// } - return 0 -} +// return 0 +// } func (l *Lexer) nextToken(data []byte) int { for i, b := range data { @@ -243,14 +246,59 @@ func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error } case '}': - level-- - if level == keyLevel { - keyLevel-- - } + decreaseLevel(level, keyLevel) case '[': - panic(errors.New("arrays are not supported")) + if keyLevel == level && keys[level][0] == '[' { + keyLen := len(keys[level]) + if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { + return -1, MalformedArrayError + } + + arrIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) + if err != nil { + return -1, MalformedArrayError + } + + var valueFound []byte + var valueOffset int + + currIdx := keyIndex + l.ArrayEach(data[keyIndex:], func(value []byte, dataType ValueType, offset int, err error) { + if arrIdx == currIdx { + valueFound = value + valueOffset = offset + + if dataType == String { + valueOffset = valueOffset - 2 + start := currIdx + valueOffset + end := start + len(value) + 2 + + valueFound = valueFound[start:end] + } + } + currIdx++ + }) + + if valueFound == nil { + return -1, KeyPathNotFoundError + } + + subIndex, err := l.searchKeys(valueFound, keys[level+1:]...) + if err != nil { + return -1, err + } + + return keyIndex + valueOffset + subIndex, nil + } + + skip, err := l.blockEnd(data[keyIndex:], '[', ']') + if err != nil { + return -1, err + } + + keyIndex += skip - 1 case ':': - return -1, MalformedJSON + return -1, MalformedJsonError } keyIndex++ @@ -258,3 +306,172 @@ func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error return -1, KeyPathNotFoundError } + +func decreaseLevel(level, keyLevel int) { + level-- + if level == keyLevel { + keyLevel-- + } +} + +// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. +func (l *Lexer) ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { + if len(data) == 0 { + return -1, MalformedObjectError + } + + nT := l.nextToken(data) + if nT == -1 { + return -1, MalformedJsonError + } + + offset = nT + 1 + + if len(keys) > 0 { + offset, err = l.searchKeys(data, keys...) + if err != nil { + return offset, err + } + + // Go to closest value + nO := l.nextToken(data[offset:]) + if nO == -1 { + return offset, MalformedJsonError + } + + offset += nO + + if data[offset] != '[' { + return offset, MalformedArrayError + } + + offset++ + } + + nO := l.nextToken(data[offset:]) + if nO == -1 { + return offset, MalformedJsonError + } + + offset += nO + + if data[offset] == ']' { + return offset, nil + } + + for true { + v, t, o, e := l.Get(data[offset:]) + + if e != nil { + return offset, e + } + + if o == 0 { + break + } + + if t != NotExist { + cb(v, t, offset+o-len(v), e) + } + + if e != nil { + break + } + + offset += o + + skipToToken := l.nextToken(data[offset:]) + if skipToToken == -1 { + return offset, MalformedArrayError + } + offset += skipToToken + + if data[offset] == ']' { + break + } + + if data[offset] != ',' { + return offset, MalformedArrayError + } + + offset++ + } + + return offset, nil +} + +func (l *Lexer) Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { + a, b, _, d, e := l.internalGet(data, keys...) + return a, b, d, e +} + +func (l *Lexer) internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { + if len(keys) > 0 { + offset, err = l.searchKeys(data, keys...) + if err != nil { + return nil, NotExist, -1, -1, err + } + } + + // Go to closest value + nO := l.nextToken(data[offset:]) + if nO == -1 { + return nil, NotExist, offset, -1, MalformedJsonError + } + + offset += nO + value, dataType, endOffset, err = l.getType(data, offset) + if err != nil { + return value, dataType, offset, endOffset, err + } + + // Strip quotes from string values + if dataType == String { + value = value[1 : len(value)-1] + } + + return value[:len(value):len(value)], dataType, offset, endOffset, nil +} + +func (l *Lexer) getType(data []byte, offset int) ([]byte, ValueType, int, error) { + var dataType ValueType + endOffset := offset + + switch data[offset] { + case '"': + dataType = String + idx, _, err := l.stringEnd(data[offset+1:]) + if err != nil { + return nil, dataType, offset, MalformedStringError + } + endOffset += idx + 1 + case '[': + dataType = Array + // break label, for stopping nested loops + endOffset, err := l.blockEnd(data[offset:], '[', ']') + if err != nil { + return nil, dataType, offset, MalformedArrayError + } + endOffset += offset + case '{': + dataType = Object + + endOffset, err := l.blockEnd(data[offset:], '{', '}') + if err != nil { + return nil, dataType, offset, MalformedObjectError + } + + endOffset += offset + default: + end := tokenEnd(data[endOffset:]) + if end == -1 { + return nil, dataType, offset, MalformedValueError + } + + value := data[offset : endOffset+end] + dataType, offset, err := extractValueTypeFromToken(data[offset], value) + endOffset += end + } + + return data[offset:endOffset], dataType, endOffset, nil +} diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 952bbba667f..b92cbbe3068 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -39,8 +39,7 @@ func TestTokenEnd(t *testing.T) { } for _, test := range tests { - l := New(test.data) - if got := l.tokenEnd(); got != test.expected { + if got := tokenEnd(test.data); got != test.expected { t.Errorf("tokenEnd() = %v, want %v", got, test.expected) } } @@ -67,9 +66,9 @@ func TestLexer(t *testing.T) { t.Run(tt.name, func(t *testing.T) { l := &Lexer{data: []byte(tt.data)} - if got := l.tokenStart(); got != tt.start { - t.Errorf("tokenStart() = %v, want %v", got, tt.start) - } + // if got := l.tokenStart(); got != tt.start { + // t.Errorf("tokenStart() = %v, want %v", got, tt.start) + // } if got := l.findLastTokenPosition(); got != tt.last { t.Errorf("findLastTokenPosition() = %v, want %v", got, tt.last) diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 792071ff89f..32b950f1c5f 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -31,7 +31,7 @@ func ParseBool(data []byte) (bool, error) { case bytes.Equal(data, []byte("false")): return false, nil default: - return false, MalformedVlaueError + return false, MalformedValueError } } @@ -147,3 +147,22 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { return man, exp10, nil } + +func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { + switch b { + case 't', 'f': + if bytes.Equal(value, TrueLiteral) || bytes.Equal(value, FalseLiteral) { + dataType = Boolean + } + return Unknown, offset, UnknownValueTypeError + case 'u', 'n': + if bytes.Equal(value, NullLiteral) { + dataType = Null + } + return Unknown, offset, UnknownValueTypeError + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': + dataType = Number + default: + return Unknown, offset, UnknownValueTypeError + } +} \ No newline at end of file From 6868c109a2bca5a07b675ec8c3ff9c3026fcb2b7 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Dec 2023 18:35:16 +0900 Subject: [PATCH 15/72] parse array --- examples/gno.land/p/demo/json/lexer.gno | 25 ++++++++------ examples/gno.land/p/demo/json/lexer_test.gno | 35 +++++++++++++++++--- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index c40981cb6d4..0b3bafa7a46 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -46,16 +46,16 @@ func tokenEnd(data []byte) int { return len(data) } -// func (l *Lexer) tokenStart() int { -// for i := len(l.data) - 1; i >= 0; i-- { -// switch l.data[i] { -// case '\n', '\r', '\t', ',', '{', '[': -// return i -// } -// } +func (l *Lexer) tokenStart() int { + for i := len(l.data) - 1; i >= 0; i-- { + switch l.data[i] { + case '\n', '\r', '\t', ',', '{', '[': + return i + } + } -// return 0 -// } + return 0 +} func (l *Lexer) nextToken(data []byte) int { for i, b := range data { @@ -315,7 +315,11 @@ func decreaseLevel(level, keyLevel int) { } // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. -func (l *Lexer) ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { +func (l *Lexer) ArrayEach( + data []byte, + cb func(value []byte, dataType ValueType, offset int, err error), + keys ...string, +) (offset int, err error) { if len(data) == 0 { return -1, MalformedObjectError } @@ -447,7 +451,6 @@ func (l *Lexer) getType(data []byte, offset int) ([]byte, ValueType, int, error) endOffset += idx + 1 case '[': dataType = Array - // break label, for stopping nested loops endOffset, err := l.blockEnd(data[offset:], '[', ']') if err != nil { return nil, dataType, offset, MalformedArrayError diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index b92cbbe3068..cf0245b7e76 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -66,10 +66,6 @@ func TestLexer(t *testing.T) { t.Run(tt.name, func(t *testing.T) { l := &Lexer{data: []byte(tt.data)} - // if got := l.tokenStart(); got != tt.start { - // t.Errorf("tokenStart() = %v, want %v", got, tt.start) - // } - if got := l.findLastTokenPosition(); got != tt.last { t.Errorf("findLastTokenPosition() = %v, want %v", got, tt.last) } @@ -141,3 +137,34 @@ func TestSearchKeys(t *testing.T) { }) } } + +func TestArrayEach(t *testing.T) { + data := []byte(`{"a": { "b":[{"x": 1} ,{"x":2},{ "x":3}, {"x":4} ]}}`) + count := 0 + + l := &Lexer{} + l.ArrayEach(data, func(value []byte, typ ValueType, offset int, err error) { + count += 1 + + switch count { + case 1: + if string(value) != `{"x": 1}` { + t.Errorf("Wrong first item: %s", string(value)) + } + case 2: + if string(value) != `{"x":2}` { + t.Errorf("Wrong second item: %s", string(value)) + } + case 3: + if string(value) != `{ "x":3}` { + t.Errorf("Wrong third item: %s", string(value)) + } + case 4: + if string(value) != `{"x":4}` { + t.Errorf("Wrong fourth item: %s", string(value)) + } + default: + t.Errorf("Too many items: %d", count) + } + }, "a", "b") +} \ No newline at end of file From 1a1657f7c1ca559b68c8f0f6f271a25b72d8572a Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Dec 2023 18:55:33 +0900 Subject: [PATCH 16/72] remove Lexer struct --- examples/gno.land/p/demo/json/const.gno | 14 +++ examples/gno.land/p/demo/json/escape.gno | 11 -- examples/gno.land/p/demo/json/escape_test.gno | 6 +- examples/gno.land/p/demo/json/lexer.gno | 102 +++++++----------- examples/gno.land/p/demo/json/lexer_test.gno | 21 ++-- examples/gno.land/p/demo/json/parser.gno | 2 +- 6 files changed, 63 insertions(+), 93 deletions(-) create mode 100644 examples/gno.land/p/demo/json/const.gno diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno new file mode 100644 index 00000000000..ee14c5d6386 --- /dev/null +++ b/examples/gno.land/p/demo/json/const.gno @@ -0,0 +1,14 @@ +package json + +const ( + SupplementalPlanesOffset = 0x10000 + HighSurrogateOffset = 0xD800 + LowSurrogateOffset = 0xDC00 + + SurrogateEnd = 0xDFFF + BasicMultilingualPlaneOffset = 0xFFFF + + BadHex = -1 + + UnescapeStackBufSize = 64 +) \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 6577f89cf88..222f1ef984f 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -6,17 +6,6 @@ import ( "unicode/utf8" ) -const ( - SupplementalPlanesOffset = 0x10000 - HighSurrogateOffset = 0xD800 - LowSurrogateOffset = 0xDC00 - - SurrogateEnd = 0xDFFF - BasicMultilingualPlaneOffset = 0xFFFF - - BadHex = -1 -) - func Unescape(input, output []byte) ([]byte, error) { firstBackslash := bytes.IndexByte(input, '\\') if firstBackslash == -1 { diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index 067b75ae526..e402f73001c 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -186,7 +186,6 @@ func TestUnescape(t *testing.T) { } func TestStringEnd(t *testing.T) { - lexer := &Lexer{} tests := []struct { name string input []byte @@ -201,7 +200,7 @@ func TestStringEnd(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - idx, escaped, _ := lexer.stringEnd(test.input) + idx, escaped, _ := stringEnd(test.input) if idx != test.expected || escaped != test.escaped { t.Errorf("Failed %s: expected (%d, %v), got (%d, %v)", test.name, test.expected, test.escaped, idx, escaped) } @@ -210,7 +209,6 @@ func TestStringEnd(t *testing.T) { } func TestBlockEnd(t *testing.T) { - lexer := &Lexer{} tests := []struct { name string data []byte @@ -226,7 +224,7 @@ func TestBlockEnd(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result, _ := lexer.blockEnd(test.data, test.open, test.close) + result, _ := blockEnd(test.data, test.open, test.close) if result != test.expected { t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) } diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 0b3bafa7a46..408633c1a78 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -6,26 +6,9 @@ import ( "strconv" ) -const unescapeStackBufSize = 64 - -type Lexer struct { - data []byte - token byte - level int - keyLevel int -} - -func New(data []byte) *Lexer { - return &Lexer{ - data: data, - level: 0, - keyLevel: 0, - } -} - -func (l *Lexer) findTokenStart(token byte) int { - for i := len(l.data) - 1; i >= 0; i-- { - switch l.data[i] { +func findTokenStart(data []byte, token byte) int { + for i := len(data) - 1; i >= 0; i-- { + switch data[i] { case token: return i case '[', '{': @@ -46,18 +29,7 @@ func tokenEnd(data []byte) int { return len(data) } -func (l *Lexer) tokenStart() int { - for i := len(l.data) - 1; i >= 0; i-- { - switch l.data[i] { - case '\n', '\r', '\t', ',', '{', '[': - return i - } - } - - return 0 -} - -func (l *Lexer) nextToken(data []byte) int { +func nextToken(data []byte) int { for i, b := range data { switch b { case '{', '[': @@ -72,9 +44,9 @@ func (l *Lexer) nextToken(data []byte) int { return -1 } -func (l *Lexer) findLastTokenPosition() int { - for i := len(l.data) - 1; i >= 0; i-- { - switch l.data[i] { +func findLastTokenPosition(data []byte) int { + for i := len(data) - 1; i >= 0; i-- { + switch data[i] { case ' ', '\n', '\r', '\t': continue default: @@ -86,7 +58,7 @@ func (l *Lexer) findLastTokenPosition() int { } // TODO: refactor this function -func (l *Lexer) stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { +func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { escaped = false for lastCharIdx, c := range data { if c == '"' { @@ -114,14 +86,14 @@ func (l *Lexer) stringEnd(data []byte) (lastCharIdx int, escaped bool, err error return -1, escaped, StringNotClosed } -func (l *Lexer) blockEnd(data []byte, open, close byte) (int, error) { - level := l.level +func blockEnd(data []byte, open, close byte) (int, error) { + level := 0 i := 0 for i < len(data) { switch data[i] { case '"': - end, _, err := l.stringEnd(data[i+1:]) + end, _, err := stringEnd(data[i+1:]) if err != nil { return -1, StringNotClosed } @@ -167,16 +139,16 @@ func findNextKey(data []byte, start int) (keyBegin, keyEnd, nextIndex int, err e } // TODO: refactor this function -func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error) { +func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { lastMatched := true - keyLevel, level := l.keyLevel, l.level + keyLevel, level := 0, 0 dataLenghth, numKeys := len(data), len(keys) if numKeys == 0 { return 0, nil } - var stackbuf [unescapeStackBufSize]byte + var stackbuf [UnescapeStackBufSize]byte for keyIndex < dataLenghth { switch data[keyIndex] { @@ -184,14 +156,14 @@ func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error keyIndex++ keyBegin := keyIndex - strEnd, keyEscaped, err := l.stringEnd(data[keyIndex:]) + strEnd, keyEscaped, err := stringEnd(data[keyIndex:]) if err != nil { return -1, StringNotClosed } keyIndex += strEnd keyEnd := keyIndex - 1 - valueOffset := l.nextToken(data[keyIndex:]) + valueOffset := nextToken(data[keyIndex:]) if valueOffset == -1 { return -1, TokenNotFound } @@ -236,7 +208,7 @@ func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error // in case parent key is matched then only we will increase the level otherwise can directly // can move to the end of this block if !lastMatched { - end, err := l.blockEnd(data[keyIndex:], '{', '}') + end, err := blockEnd(data[keyIndex:], '{', '}') if err != nil { return -1, err } @@ -263,7 +235,7 @@ func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error var valueOffset int currIdx := keyIndex - l.ArrayEach(data[keyIndex:], func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(data[keyIndex:], func(value []byte, dataType ValueType, offset int, err error) { if arrIdx == currIdx { valueFound = value valueOffset = offset @@ -283,7 +255,7 @@ func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error return -1, KeyPathNotFoundError } - subIndex, err := l.searchKeys(valueFound, keys[level+1:]...) + subIndex, err := searchKeys(valueFound, keys[level+1:]...) if err != nil { return -1, err } @@ -291,7 +263,7 @@ func (l *Lexer) searchKeys(data []byte, keys ...string) (keyIndex int, err error return keyIndex + valueOffset + subIndex, nil } - skip, err := l.blockEnd(data[keyIndex:], '[', ']') + skip, err := blockEnd(data[keyIndex:], '[', ']') if err != nil { return -1, err } @@ -315,7 +287,7 @@ func decreaseLevel(level, keyLevel int) { } // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. -func (l *Lexer) ArrayEach( +func ArrayEach( data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string, @@ -324,7 +296,7 @@ func (l *Lexer) ArrayEach( return -1, MalformedObjectError } - nT := l.nextToken(data) + nT := nextToken(data) if nT == -1 { return -1, MalformedJsonError } @@ -332,13 +304,13 @@ func (l *Lexer) ArrayEach( offset = nT + 1 if len(keys) > 0 { - offset, err = l.searchKeys(data, keys...) + offset, err = searchKeys(data, keys...) if err != nil { return offset, err } // Go to closest value - nO := l.nextToken(data[offset:]) + nO := nextToken(data[offset:]) if nO == -1 { return offset, MalformedJsonError } @@ -352,7 +324,7 @@ func (l *Lexer) ArrayEach( offset++ } - nO := l.nextToken(data[offset:]) + nO := nextToken(data[offset:]) if nO == -1 { return offset, MalformedJsonError } @@ -364,7 +336,7 @@ func (l *Lexer) ArrayEach( } for true { - v, t, o, e := l.Get(data[offset:]) + v, t, o, e := Get(data[offset:]) if e != nil { return offset, e @@ -384,7 +356,7 @@ func (l *Lexer) ArrayEach( offset += o - skipToToken := l.nextToken(data[offset:]) + skipToToken := nextToken(data[offset:]) if skipToToken == -1 { return offset, MalformedArrayError } @@ -404,27 +376,27 @@ func (l *Lexer) ArrayEach( return offset, nil } -func (l *Lexer) Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { - a, b, _, d, e := l.internalGet(data, keys...) +func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { + a, b, _, d, e := internalGet(data, keys...) return a, b, d, e } -func (l *Lexer) internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { +func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { if len(keys) > 0 { - offset, err = l.searchKeys(data, keys...) + offset, err = searchKeys(data, keys...) if err != nil { return nil, NotExist, -1, -1, err } } // Go to closest value - nO := l.nextToken(data[offset:]) + nO := nextToken(data[offset:]) if nO == -1 { return nil, NotExist, offset, -1, MalformedJsonError } offset += nO - value, dataType, endOffset, err = l.getType(data, offset) + value, dataType, endOffset, err = getType(data, offset) if err != nil { return value, dataType, offset, endOffset, err } @@ -437,21 +409,21 @@ func (l *Lexer) internalGet(data []byte, keys ...string) (value []byte, dataType return value[:len(value):len(value)], dataType, offset, endOffset, nil } -func (l *Lexer) getType(data []byte, offset int) ([]byte, ValueType, int, error) { +func getType(data []byte, offset int) ([]byte, ValueType, int, error) { var dataType ValueType endOffset := offset switch data[offset] { case '"': dataType = String - idx, _, err := l.stringEnd(data[offset+1:]) + idx, _, err := stringEnd(data[offset+1:]) if err != nil { return nil, dataType, offset, MalformedStringError } endOffset += idx + 1 case '[': dataType = Array - endOffset, err := l.blockEnd(data[offset:], '[', ']') + endOffset, err := blockEnd(data[offset:], '[', ']') if err != nil { return nil, dataType, offset, MalformedArrayError } @@ -459,7 +431,7 @@ func (l *Lexer) getType(data []byte, offset int) ([]byte, ValueType, int, error) case '{': dataType = Object - endOffset, err := l.blockEnd(data[offset:], '{', '}') + endOffset, err := blockEnd(data[offset:], '{', '}') if err != nil { return nil, dataType, offset, MalformedObjectError } diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index cf0245b7e76..2893f61f880 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -21,10 +21,11 @@ func TestFindTokenStart(t *testing.T) { {token: 'l', expected: 9}, } + data := []byte(`hello{world}`) + for _, test := range tests { - l := New([]byte("hello{world}")) - if got := l.findTokenStart(test.token); got != test.expected { - t.Errorf("findTokenStart('%c') = %v, want %v", test.token, got, test.expected) + if got := findTokenStart(data, test.token); got != test.expected { + t.Errorf("findTokenStart() = %v, want %v", got, test.expected) } } } @@ -48,14 +49,14 @@ func TestTokenEnd(t *testing.T) { func TestLexer(t *testing.T) { tests := []struct { name string - data string + data []byte start int next int last int }{ { name: "Test 1", - data: "{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), start: 38, next: 1, last: 39, @@ -64,9 +65,7 @@ func TestLexer(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l := &Lexer{data: []byte(tt.data)} - - if got := l.findLastTokenPosition(); got != tt.last { + if got := findLastTokenPosition(tt.data); got != tt.last { t.Errorf("findLastTokenPosition() = %v, want %v", got, tt.last) } }) @@ -124,8 +123,7 @@ func TestSearchKeys(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l := &Lexer{} - got, err := l.searchKeys(tt.data, tt.keys...) + got, err := searchKeys(tt.data, tt.keys...) if got != tt.expected { t.Errorf("searchKeys(%s) = %v, want %v", tt.name, got, tt.expected) @@ -142,8 +140,7 @@ func TestArrayEach(t *testing.T) { data := []byte(`{"a": { "b":[{"x": 1} ,{"x":2},{ "x":3}, {"x":4} ]}}`) count := 0 - l := &Lexer{} - l.ArrayEach(data, func(value []byte, typ ValueType, offset int, err error) { + ArrayEach(data, func(value []byte, typ ValueType, offset int, err error) { count += 1 switch count { diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 32b950f1c5f..1dd77e5c562 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -14,7 +14,7 @@ const ( ) func ParseString(data []byte) (string, error) { - var buf [unescapeStackBufSize]byte + var buf [UnescapeStackBufSize]byte bf, err := Unescape(data, buf[:]) if err != nil { From 39bfa64a43a339bdcfd8c8893796c727c27a1fcd Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Dec 2023 20:49:47 +0900 Subject: [PATCH 17/72] refactor --- examples/gno.land/p/demo/json/lexer.gno | 204 ++++++++++++++---------- 1 file changed, 116 insertions(+), 88 deletions(-) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 408633c1a78..4927ed8a347 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -29,7 +29,7 @@ func tokenEnd(data []byte) int { return len(data) } -func nextToken(data []byte) int { +func nextToken(data []byte) (int, error) { for i, b := range data { switch b { case '{', '[': @@ -37,11 +37,11 @@ func nextToken(data []byte) int { case '}', ']': continue default: - return i + return i, nil } } - return -1 + return -1, TokenNotFound } func findLastTokenPosition(data []byte) int { @@ -155,58 +155,56 @@ func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { case '"': keyIndex++ keyBegin := keyIndex - + strEnd, keyEscaped, err := stringEnd(data[keyIndex:]) if err != nil { return -1, StringNotClosed } keyIndex += strEnd keyEnd := keyIndex - 1 - - valueOffset := nextToken(data[keyIndex:]) - if valueOffset == -1 { - return -1, TokenNotFound + + valueOffset, err := nextToken(data[keyIndex:]) + if err != nil { + return -1, err } - + keyIndex += valueOffset if data[keyIndex] != ':' { keyIndex-- } - + if level < 1 { return -1, OutOfLevel } - + key := data[keyBegin:keyEnd] keyUnesc, err := Unescape(key, stackbuf[:]) if err != nil { return -1, err } - + if !keyEscaped { keyUnesc = key } - + if level > numKeys { return -1, KeyPathNotFoundError } - + if string(keyUnesc) == keys[level-1] { lastMatched = true if keyLevel == level-1 { keyLevel++ - + if keyLevel == numKeys { return keyIndex + 1, nil } } } - + lastMatched = false - + case '{': - // in case parent key is matched then only we will increase the level otherwise can directly - // can move to the end of this block if !lastMatched { end, err := blockEnd(data[keyIndex:], '{', '}') if err != nil { @@ -216,7 +214,7 @@ func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { } else { level++ } - + case '}': decreaseLevel(level, keyLevel) case '[': @@ -225,21 +223,21 @@ func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { return -1, MalformedArrayError } - + arrIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) if err != nil { return -1, MalformedArrayError } - + var valueFound []byte var valueOffset int - + currIdx := keyIndex ArrayEach(data[keyIndex:], func(value []byte, dataType ValueType, offset int, err error) { if arrIdx == currIdx { valueFound = value valueOffset = offset - + if dataType == String { valueOffset = valueOffset - 2 start := currIdx + valueOffset @@ -250,30 +248,31 @@ func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { } currIdx++ }) - + if valueFound == nil { return -1, KeyPathNotFoundError } - + subIndex, err := searchKeys(valueFound, keys[level+1:]...) if err != nil { return -1, err } - + return keyIndex + valueOffset + subIndex, nil } - + skip, err := blockEnd(data[keyIndex:], '[', ']') if err != nil { return -1, err } - + keyIndex += skip - 1 case ':': return -1, MalformedJsonError } - + keyIndex++ + } return -1, KeyPathNotFoundError @@ -289,19 +288,19 @@ func decreaseLevel(level, keyLevel int) { // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. func ArrayEach( data []byte, - cb func(value []byte, dataType ValueType, offset int, err error), + f func(value []byte, dataType ValueType, offset int, err error), keys ...string, ) (offset int, err error) { if len(data) == 0 { return -1, MalformedObjectError } - nT := nextToken(data) - if nT == -1 { + next, err := nextToken(data) + if err != nil { return -1, MalformedJsonError } - offset = nT + 1 + offset = next + 1 if len(keys) > 0 { offset, err = searchKeys(data, keys...) @@ -309,13 +308,12 @@ func ArrayEach( return offset, err } - // Go to closest value - nO := nextToken(data[offset:]) - if nO == -1 { + nextOffset, err := nextToken(data[offset:]) + if err != nil { return offset, MalformedJsonError } - offset += nO + offset += nextOffset if data[offset] != '[' { return offset, MalformedArrayError @@ -324,53 +322,32 @@ func ArrayEach( offset++ } - nO := nextToken(data[offset:]) - if nO == -1 { + nextOffset, err := nextToken(data[offset:]) + if err != nil { return offset, MalformedJsonError } - offset += nO + offset += nextOffset if data[offset] == ']' { return offset, nil } for true { - v, t, o, e := Get(data[offset:]) - - if e != nil { - return offset, e - } - - if o == 0 { - break - } - - if t != NotExist { - cb(v, t, offset+o-len(v), e) - } - - if e != nil { - break - } - - offset += o - - skipToToken := nextToken(data[offset:]) - if skipToToken == -1 { - return offset, MalformedArrayError + offset, err := handleGetResult(data, offset, f) + if err != nil { + return offset, err } - offset += skipToToken - if data[offset] == ']' { - break + offset, err = handleNextTokenResult(data, offset) + if err != nil { + return offset, err } - if data[offset] != ',' { - return offset, MalformedArrayError + offset, err = handleArrayError(data, offset) + if err != nil { + return offset, err } - - offset++ } return offset, nil @@ -390,12 +367,12 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, } // Go to closest value - nO := nextToken(data[offset:]) - if nO == -1 { + nextOffset, err := nextToken(data[offset:]) + if err != nil { return nil, NotExist, offset, -1, MalformedJsonError } - offset += nO + offset += nextOffset value, dataType, endOffset, err = getType(data, offset) if err != nil { return value, dataType, offset, endOffset, err @@ -422,21 +399,9 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { } endOffset += idx + 1 case '[': - dataType = Array - endOffset, err := blockEnd(data[offset:], '[', ']') - if err != nil { - return nil, dataType, offset, MalformedArrayError - } - endOffset += offset + return processNextCollectionElement(data, offset, Array) case '{': - dataType = Object - - endOffset, err := blockEnd(data[offset:], '{', '}') - if err != nil { - return nil, dataType, offset, MalformedObjectError - } - - endOffset += offset + return processNextCollectionElement(data, offset, Object) default: end := tokenEnd(data[endOffset:]) if end == -1 { @@ -450,3 +415,66 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { return data[offset:endOffset], dataType, endOffset, nil } + +func handleGetResult( + data []byte, + offset int, + f func(value []byte, dataType ValueType, offset int, err error), +) (int, error) { + v, t, o, e := Get(data[offset:]) + if e != nil { + return offset, e + } + if o == 0 { + return offset, nil + } + if t != NotExist { + f(v, t, offset+o-len(v), e) + } + if e != nil { + return offset, e + } + return offset + o, nil +} + +func handleNextTokenResult(data []byte, offset int) (int, error) { + nextTokenOffset, err := nextToken(data[offset:]) + if err != nil { + return offset, MalformedArrayError + } + + return offset + nextTokenOffset, nil +} + +func handleArrayError(data []byte, offset int) (int, error) { + if data[offset] == ']' { + return offset, nil + } + + if data[offset] != ',' { + return offset, MalformedArrayError + } + + return offset + 1, nil +} + +func processNextCollectionElement(data []byte, offset int, dataType ValueType) ([]byte, ValueType, int, error) { + switch dataType { + case Array: + endOffset, err := blockEnd(data[offset:], '[', ']') + if err != nil { + return nil, NotExist, offset, MalformedArrayError + } + + return data[offset : offset+endOffset], Array, offset + endOffset, nil + case Object: + endOffset, err := blockEnd(data[offset:], '{', '}') + if err != nil { + return nil, NotExist, offset, MalformedObjectError + } + + return data[offset : offset+endOffset], Object, offset + endOffset, nil + default: + return nil, NotExist, offset, errors.New("not a collection") + } +} From 33ccb143526dacee5c6806a02a4d4ddf5b4cad39 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Dec 2023 21:16:53 +0900 Subject: [PATCH 18/72] re-refactor --- examples/gno.land/p/demo/json/const.gno | 2 +- examples/gno.land/p/demo/json/errors.gno | 4 +- examples/gno.land/p/demo/json/lexer.gno | 141 ++++++++++--------- examples/gno.land/p/demo/json/lexer_test.gno | 3 +- examples/gno.land/p/demo/json/parser.gno | 2 +- 5 files changed, 77 insertions(+), 75 deletions(-) diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno index ee14c5d6386..20beccfbc21 100644 --- a/examples/gno.land/p/demo/json/const.gno +++ b/examples/gno.land/p/demo/json/const.gno @@ -11,4 +11,4 @@ const ( BadHex = -1 UnescapeStackBufSize = 64 -) \ No newline at end of file +) diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index 075637d5aeb..6ce26cee629 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -20,7 +20,7 @@ const ( MalformedStringError = errors.New("malformed string") MalformedValueError = errors.New("malformed value") MalformedObjectError = errors.New("malformed object") - MalformedArrayError = errors.New("malformed array") + MalformedArrayError = errors.New("malformed array") MalformedJsonError = errors.New("malformed json array") - UnknownValueTypeError = errors.New("unknown value type") + UnknownValueTypeError = errors.New("unknown value type") ) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 4927ed8a347..a388e9838d7 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -61,7 +61,8 @@ func findLastTokenPosition(data []byte) int { func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { escaped = false for lastCharIdx, c := range data { - if c == '"' { + switch c { + case '"': if !escaped { return lastCharIdx + 1, false, nil } @@ -76,9 +77,7 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { } return lastCharIdx + 1, false, nil - } - - if c == '\\' { + case '\\': escaped = true } } @@ -155,55 +154,53 @@ func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { case '"': keyIndex++ keyBegin := keyIndex - + strEnd, keyEscaped, err := stringEnd(data[keyIndex:]) if err != nil { return -1, StringNotClosed } keyIndex += strEnd keyEnd := keyIndex - 1 - + valueOffset, err := nextToken(data[keyIndex:]) if err != nil { return -1, err } - + keyIndex += valueOffset if data[keyIndex] != ':' { keyIndex-- } - + if level < 1 { return -1, OutOfLevel } - + key := data[keyBegin:keyEnd] keyUnesc, err := Unescape(key, stackbuf[:]) if err != nil { return -1, err } - + if !keyEscaped { keyUnesc = key } - + if level > numKeys { return -1, KeyPathNotFoundError } - - if string(keyUnesc) == keys[level-1] { - lastMatched = true - if keyLevel == level-1 { - keyLevel++ - - if keyLevel == numKeys { - return keyIndex + 1, nil - } - } + + if !bytes.Equal(keyUnesc, []byte(keys[level-1])) || keyLevel+1 != numKeys { + lastMatched = false + break } - - lastMatched = false - + + if keyLevel+1 == numKeys { + return keyIndex + 1, nil + } + + lastMatched = true + case '{': if !lastMatched { end, err := blockEnd(data[keyIndex:], '{', '}') @@ -214,7 +211,7 @@ func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { } else { level++ } - + case '}': decreaseLevel(level, keyLevel) case '[': @@ -223,56 +220,56 @@ func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { return -1, MalformedArrayError } - + arrIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) if err != nil { return -1, MalformedArrayError } - - var valueFound []byte + + var valueFound []byte var valueOffset int - + currIdx := keyIndex ArrayEach(data[keyIndex:], func(value []byte, dataType ValueType, offset int, err error) { if arrIdx == currIdx { valueFound = value valueOffset = offset - + if dataType == String { valueOffset = valueOffset - 2 start := currIdx + valueOffset end := start + len(value) + 2 - + valueFound = valueFound[start:end] } } currIdx++ }) - + if valueFound == nil { return -1, KeyPathNotFoundError } - + subIndex, err := searchKeys(valueFound, keys[level+1:]...) if err != nil { return -1, err } - + return keyIndex + valueOffset + subIndex, nil } - + skip, err := blockEnd(data[keyIndex:], '[', ']') if err != nil { return -1, err } - + keyIndex += skip - 1 case ':': return -1, MalformedJsonError } - + keyIndex++ - + } return -1, KeyPathNotFoundError @@ -359,11 +356,13 @@ func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset } func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { - if len(keys) > 0 { - offset, err = searchKeys(data, keys...) - if err != nil { - return nil, NotExist, -1, -1, err - } + if len(keys) <= 0 { + return nil, NotExist, -1, -1, errors.New("no keys") + } + + offset, err = searchKeys(data, keys...) + if err != nil { + return nil, NotExist, -1, -1, err } // Go to closest value @@ -421,41 +420,45 @@ func handleGetResult( offset int, f func(value []byte, dataType ValueType, offset int, err error), ) (int, error) { - v, t, o, e := Get(data[offset:]) - if e != nil { - return offset, e - } - if o == 0 { - return offset, nil - } - if t != NotExist { - f(v, t, offset+o-len(v), e) - } - if e != nil { - return offset, e - } - return offset + o, nil + v, t, o, e := Get(data[offset:]) + if e != nil { + return offset, e + } + + if o == 0 { + return offset, nil + } + + if t != NotExist { + f(v, t, offset+o-len(v), e) + } + + if e != nil { + return offset, e + } + + return offset + o, nil } func handleNextTokenResult(data []byte, offset int) (int, error) { - nextTokenOffset, err := nextToken(data[offset:]) - if err != nil { - return offset, MalformedArrayError - } + nextTokenOffset, err := nextToken(data[offset:]) + if err != nil { + return offset, MalformedArrayError + } - return offset + nextTokenOffset, nil + return offset + nextTokenOffset, nil } func handleArrayError(data []byte, offset int) (int, error) { - if data[offset] == ']' { - return offset, nil - } + if data[offset] == ']' { + return offset, nil + } - if data[offset] != ',' { - return offset, MalformedArrayError - } + if data[offset] != ',' { + return offset, MalformedArrayError + } - return offset + 1, nil + return offset + 1, nil } func processNextCollectionElement(data []byte, offset int, dataType ValueType) ([]byte, ValueType, int, error) { diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 2893f61f880..c6e6ba881b3 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -72,7 +72,6 @@ func TestLexer(t *testing.T) { } } - func TestSearchKeys(t *testing.T) { tests := []struct { name string @@ -164,4 +163,4 @@ func TestArrayEach(t *testing.T) { t.Errorf("Too many items: %d", count) } }, "a", "b") -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 1dd77e5c562..0a1180b5e19 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -165,4 +165,4 @@ func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset default: return Unknown, offset, UnknownValueTypeError } -} \ No newline at end of file +} From 789abfd1217cc6a3a8c4ff7e5f50fa3f69d07ae4 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 14 Dec 2023 01:41:40 +0900 Subject: [PATCH 19/72] parse uint value --- examples/gno.land/p/demo/json/parser.gno | 149 ++++++++++++++++++ examples/gno.land/p/demo/json/parser_test.gno | 78 +++++++++ 2 files changed, 227 insertions(+) diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 0a1180b5e19..6db79b6f0e4 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -11,6 +11,8 @@ const ( absMinInt64 = 1 << 63 maxInt64 = 1<<63 - 1 maxUint64 = 1<<64 - 1 + intSize = 32 << (^uint(0) >> 63) + IntSize = intSize ) func ParseString(data []byte) (string, error) { @@ -114,6 +116,103 @@ func ParseInt(bytes []byte) (v int64, err error) { return int64(n), nil } +func ParseUint(b []byte, base, bitSize int) (v uint64, err error) { + if len(b) == 0 { + return 0, EmptyBytes + } + + base0 := base == 0 + s0 := string(b) + + switch { + case 2 <= base && base <= 36: + // do nothing + case base == 0: + base = 10 + if b[0] == '0' { + switch { + case len(b) >= 3 && lower(b[1]) == 'b': + base = 2 + b = b[2:] + case len(b) >= 3 && lower(b[1]) == 'o': + base = 8 + b = b[2:] + case len(b) >= 3 && lower(b[1]) == 'x': + base = 16 + b = b[2:] + default: + base = 8 + b = b[1:] + } + } + default: + return 0, errors.New("Base Error") + } + + if bitSize == 0 { + bitSize = IntSize + } else if bitSize < 0 || bitSize > 64 { + return 0, errors.New("BitSize Error") + } + + var cutoff uint64 + switch base { + case 10: + cutoff = maxUint64/10 + 1 + case 16: + cutoff = maxUint64/16 + 1 + default: + cutoff = maxUint64/uint64(base) + 1 + } + + maxVal := uint64(1)<= byte(base) { + return 0, errors.New("ParseUint Syntax Error") + } + + if n >= cutoff { + return maxVal, errors.New("ParseUint Range Error") + } + n *= uint64(base) + + n1 := n + uint64(d) + if n1 < n || n1 > maxVal { + return maxVal, errors.New("ParseUint Range Error") + } + n = n1 + } + + if underscores && !underscoreOK(s0) { + return 0, errors.New("ParseUint Syntax Error") + } + + return n, nil +} + +// lower(c) is a lower-case letter if and only if +// c is either that lower-case letter or the equivalent upper-case letter. +// Instead of writing c == 'x' || c == 'X' one can write lower(c) == 'x'. +// Note that lower of non-letters can produce other non-letters. +func lower(c byte) byte { + return c | ('x' - 'X') +} + func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { var man uint64 var exp10 int @@ -166,3 +265,53 @@ func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset return Unknown, offset, UnknownValueTypeError } } + +// underscoreOK reports whether the underscores in s are allowed. +// Checking them in this one function lets all the parsers skip over them simply. +// Underscore must appear only between digits or between a base prefix and a digit. +func underscoreOK(s string) bool { + // saw tracks the last character (class) we saw: + // ^ for beginning of number, + // 0 for a digit or base prefix, + // _ for an underscore, + // ! for none of the above. + saw := '^' + i := 0 + + // Optional sign. + if len(s) >= 1 && (s[0] == '-' || s[0] == '+') { + s = s[1:] + } + + // Optional base prefix. + hex := false + if len(s) >= 2 && s[0] == '0' && (lower(s[1]) == 'b' || lower(s[1]) == 'o' || lower(s[1]) == 'x') { + i = 2 + saw = '0' // base prefix counts as a digit for "underscore as digit separator" + hex = lower(s[1]) == 'x' + } + + // Number proper. + for ; i < len(s); i++ { + // Digits are always okay. + if '0' <= s[i] && s[i] <= '9' || hex && 'a' <= lower(s[i]) && lower(s[i]) <= 'f' { + saw = '0' + continue + } + // Underscore must follow digit. + if s[i] == '_' { + if saw != '0' { + return false + } + saw = '_' + continue + } + // Underscore must also be followed by digit. + if saw == '_' { + return false + } + // Saw non-digit, non-underscore. + saw = '!' + } + return saw != '_' +} diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index d6da9f85a8d..d0839bdbb5a 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -167,3 +167,81 @@ func TestParseInt(t *testing.T) { }) } } + +type parseUint64Test struct { + in string + out uint64 + err bool +} + +var parseUint64Tests = []parseUint64Test{ + {"", 0, true}, + {"0", 0, nil}, + {"1", 1, nil}, + {"12345", 12345, nil}, + {"012345", 12345, nil}, + {"12345x", 0, true}, + {"98765432100", 98765432100, nil}, + {"18446744073709551615", 1<<64 - 1, nil}, + {"18446744073709551616", 1<<64 - 1, true}, + {"18446744073709551620", 1<<64 - 1, true}, + {"1_2_3_4_5", 0, true}, // base=10 so no underscores allowed + {"_12345", 0, true}, + {"1__2345", 0, true}, + {"12345_", 0, true}, + {"-0", 0, true}, + {"-1", 0, true}, + {"+1", 0, true}, +} + +type parseUint32Test struct { + in string + out uint32 + err bool +} + +var parseUint32Tests = []parseUint32Test{ + {"", 0, true}, + {"0", 0, nil}, + {"1", 1, nil}, + {"12345", 12345, nil}, + {"012345", 12345, nil}, + {"12345x", 0, true}, + {"987654321", 987654321, nil}, + {"4294967295", 1<<32 - 1, nil}, + {"4294967296", 1<<32 - 1, true}, + {"1_2_3_4_5", 0, true}, // base=10 so no underscores allowed + {"_12345", 0, true}, + {"_12345", 0, true}, + {"1__2345", 0, true}, + {"12345_", 0, true}, +} + +func TestParseUint(t *testing.T) { + switch IntSize { + case 32: + for _, tc := range parseUint32Tests { + t.Run(tc.in, func(t *testing.T) { + got, err := ParseUint([]byte(tc.in), 10, 32) + if got != tc.out { + t.Errorf("ParseUint(%s): got %v, want %v", tc.in, got, tc.out) + } + if (err != nil) != tc.err { + t.Errorf("ParseUint(%s): got error %v", tc.in, err) + } + }) + } + case 64: + for _, tc := range parseUint64Tests { + t.Run(tc.in, func(t *testing.T) { + got, err := ParseUint([]byte(tc.in), 10, 64) + if got != tc.out { + t.Errorf("ParseUint(%s): got %v, want %v", tc.in, got, tc.out) + } + if (err != nil) != tc.err { + t.Errorf("ParseUint(%s): got error %v", tc.in, err) + } + }) + } + } +} From 7de39d59f47ea7343d85f1a66e1a494660d4119e Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 14 Dec 2023 15:45:49 +0900 Subject: [PATCH 20/72] revert --- examples/gno.land/p/demo/json/escape_test.gno | 47 -- examples/gno.land/p/demo/json/lexer.gno | 498 ++++-------------- examples/gno.land/p/demo/json/lexer_test.gno | 191 +++---- 3 files changed, 179 insertions(+), 557 deletions(-) diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index e402f73001c..ca4d1ca98cd 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -184,50 +184,3 @@ func TestUnescape(t *testing.T) { }) } } - -func TestStringEnd(t *testing.T) { - tests := []struct { - name string - input []byte - expected int - escaped bool - }{ - {"Simple String", []byte(`"Hello"`), 1, false}, - {"Escaped Quote", []byte(`"Hello\"World"`), 1, false}, - {"No Closing Quote", []byte(`"Hello`), 1, false}, - {"Backslashes", []byte(`"Hello\\World"`), 1, false}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - idx, escaped, _ := stringEnd(test.input) - if idx != test.expected || escaped != test.escaped { - t.Errorf("Failed %s: expected (%d, %v), got (%d, %v)", test.name, test.expected, test.escaped, idx, escaped) - } - }) - } -} - -func TestBlockEnd(t *testing.T) { - tests := []struct { - name string - data []byte - open byte - close byte - expected int - }{ - {"Empty Object", []byte("{}"), '{', '}', 2}, - {"Nested Object", []byte(`{"key": {"nestedKey": "value"}}`), '{', '}', 31}, - {"Array", []byte(`["item1", "item2"]`), '[', ']', 18}, - {"Complex Object", []byte(`{"key": [1, 2, 3], "anotherKey": "value"}`), '{', '}', 41}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result, _ := blockEnd(test.data, test.open, test.close) - if result != test.expected { - t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) - } - }) - } -} diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index a388e9838d7..71a5c120b1a 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -29,12 +29,89 @@ func tokenEnd(data []byte) int { return len(data) } +func findKeyStart(data []byte, key string) (int, error) { + i, _ := nextToken(data) + if i == -1 { + return i, KeyPathNotFoundError + } + ln := len(data) + if ln > 0 && (data[i] == '{' || data[i] == '[') { + i += 1 + } + var stackbuf [UnescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings + + if ku, err := Unescape([]byte(key), stackbuf[:]); err == nil { + key = string(ku) + } + + for i < ln { + switch data[i] { + case '"': + i++ + keyBegin := i + + strEnd, keyEscaped := stringEnd(data[i:]) + if strEnd == -1 { + break + } + i += strEnd + keyEnd := i - 1 + + valueOffset, _ := nextToken(data[i:]) + if valueOffset == -1 { + break + } + + i += valueOffset + + // if string is a key, and key level match + k := data[keyBegin:keyEnd] + // for unescape: if there are no escape sequences, this is cheap; if there are, it is a + // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize + if keyEscaped { + if ku, err := Unescape(k, stackbuf[:]); err != nil { + break + } else { + k = ku + } + } + + if data[i] == ':' && len(key) == len(k) && string(k) == key { + return keyBegin - 1, nil + } + + case '[': + end := blockEnd(data[i:], data[i], ']') + if end != -1 { + i = i + end + } + case '{': + end := blockEnd(data[i:], data[i], '}') + if end != -1 { + i = i + end + } + } + i++ + } + + return -1, KeyPathNotFoundError +} + +func tokenStart(data []byte) int { + for i := len(data) - 1; i >= 0; i-- { + switch data[i] { + case '\n', '\r', '\t', ',', '{', '[': + return i + } + } + + return 0 +} + func nextToken(data []byte) (int, error) { - for i, b := range data { - switch b { - case '{', '[': - continue - case '}', ']': + for i, c := range data { + switch c { + case ' ', '\n', '\r', '\t': continue default: return i, nil @@ -44,7 +121,7 @@ func nextToken(data []byte) (int, error) { return -1, TokenNotFound } -func findLastTokenPosition(data []byte) int { +func lastToken(data []byte) int { for i := len(data) - 1; i >= 0; i-- { switch data[i] { case ' ', '\n', '\r', '\t': @@ -57,14 +134,13 @@ func findLastTokenPosition(data []byte) int { return -1 } -// TODO: refactor this function -func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { +func stringEnd(data []byte) (lastCharIdx int, escaped bool) { escaped = false for lastCharIdx, c := range data { switch c { case '"': if !escaped { - return lastCharIdx + 1, false, nil + return lastCharIdx + 1, false } prevCharIdx := lastCharIdx - 1 @@ -73,411 +149,45 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { } if (lastCharIdx-prevCharIdx)%2 == 0 { - return lastCharIdx + 1, true, nil + return lastCharIdx + 1, true } - return lastCharIdx + 1, false, nil + return lastCharIdx + 1, false case '\\': escaped = true } } - return -1, escaped, StringNotClosed + return -1, escaped } -func blockEnd(data []byte, open, close byte) (int, error) { +// Find end of the data structure, array or object. +// For array openSym and closeSym will be '[' and ']', for object '{' and '}' +func blockEnd(data []byte, openSym byte, closeSym byte) int { level := 0 i := 0 + ln := len(data) - for i < len(data) { + for i < ln { switch data[i] { - case '"': - end, _, err := stringEnd(data[i+1:]) - if err != nil { - return -1, StringNotClosed + case '"': // If inside string, skip it + se, _ := stringEnd(data[i+1:]) + if se == -1 { + return -1 } - - i += end - case open: + i += se + case openSym: // If open symbol, increase level level++ - case close: + case closeSym: // If close symbol, increase level level-- + // If we have returned to the original level, we're done if level == 0 { - return i + 1, nil + return i + 1 } } - i++ } - return -1, BlockIsNotClosed -} - -func findNextKey(data []byte, start int) (keyBegin, keyEnd, nextIndex int, err error) { - keyBegin = -1 - keyEnd = -1 - nextIndex = -1 - - for i := start; i < len(data); i++ { - switch data[i] { - case '"': - if keyBegin != -1 { - keyEnd = i - nextIndex = i + 1 - return keyBegin, keyEnd, nextIndex, nil - } - - keyBegin = i + 1 - case '\\': - i++ - } - } - - return keyBegin, keyEnd, nextIndex, KeyPathNotFoundError -} - -// TODO: refactor this function -func searchKeys(data []byte, keys ...string) (keyIndex int, err error) { - lastMatched := true - keyLevel, level := 0, 0 - dataLenghth, numKeys := len(data), len(keys) - - if numKeys == 0 { - return 0, nil - } - - var stackbuf [UnescapeStackBufSize]byte - - for keyIndex < dataLenghth { - switch data[keyIndex] { - case '"': - keyIndex++ - keyBegin := keyIndex - - strEnd, keyEscaped, err := stringEnd(data[keyIndex:]) - if err != nil { - return -1, StringNotClosed - } - keyIndex += strEnd - keyEnd := keyIndex - 1 - - valueOffset, err := nextToken(data[keyIndex:]) - if err != nil { - return -1, err - } - - keyIndex += valueOffset - if data[keyIndex] != ':' { - keyIndex-- - } - - if level < 1 { - return -1, OutOfLevel - } - - key := data[keyBegin:keyEnd] - keyUnesc, err := Unescape(key, stackbuf[:]) - if err != nil { - return -1, err - } - - if !keyEscaped { - keyUnesc = key - } - - if level > numKeys { - return -1, KeyPathNotFoundError - } - - if !bytes.Equal(keyUnesc, []byte(keys[level-1])) || keyLevel+1 != numKeys { - lastMatched = false - break - } - - if keyLevel+1 == numKeys { - return keyIndex + 1, nil - } - - lastMatched = true - - case '{': - if !lastMatched { - end, err := blockEnd(data[keyIndex:], '{', '}') - if err != nil { - return -1, err - } - keyIndex += end - 1 - } else { - level++ - } - - case '}': - decreaseLevel(level, keyLevel) - case '[': - if keyLevel == level && keys[level][0] == '[' { - keyLen := len(keys[level]) - if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { - return -1, MalformedArrayError - } - - arrIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) - if err != nil { - return -1, MalformedArrayError - } - - var valueFound []byte - var valueOffset int - - currIdx := keyIndex - ArrayEach(data[keyIndex:], func(value []byte, dataType ValueType, offset int, err error) { - if arrIdx == currIdx { - valueFound = value - valueOffset = offset - - if dataType == String { - valueOffset = valueOffset - 2 - start := currIdx + valueOffset - end := start + len(value) + 2 - - valueFound = valueFound[start:end] - } - } - currIdx++ - }) - - if valueFound == nil { - return -1, KeyPathNotFoundError - } - - subIndex, err := searchKeys(valueFound, keys[level+1:]...) - if err != nil { - return -1, err - } - - return keyIndex + valueOffset + subIndex, nil - } - - skip, err := blockEnd(data[keyIndex:], '[', ']') - if err != nil { - return -1, err - } - - keyIndex += skip - 1 - case ':': - return -1, MalformedJsonError - } - - keyIndex++ - - } - - return -1, KeyPathNotFoundError -} - -func decreaseLevel(level, keyLevel int) { - level-- - if level == keyLevel { - keyLevel-- - } -} - -// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. -func ArrayEach( - data []byte, - f func(value []byte, dataType ValueType, offset int, err error), - keys ...string, -) (offset int, err error) { - if len(data) == 0 { - return -1, MalformedObjectError - } - - next, err := nextToken(data) - if err != nil { - return -1, MalformedJsonError - } - - offset = next + 1 - - if len(keys) > 0 { - offset, err = searchKeys(data, keys...) - if err != nil { - return offset, err - } - - nextOffset, err := nextToken(data[offset:]) - if err != nil { - return offset, MalformedJsonError - } - - offset += nextOffset - - if data[offset] != '[' { - return offset, MalformedArrayError - } - - offset++ - } - - nextOffset, err := nextToken(data[offset:]) - if err != nil { - return offset, MalformedJsonError - } - - offset += nextOffset - - if data[offset] == ']' { - return offset, nil - } - - for true { - offset, err := handleGetResult(data, offset, f) - if err != nil { - return offset, err - } - - offset, err = handleNextTokenResult(data, offset) - if err != nil { - return offset, err - } - - offset, err = handleArrayError(data, offset) - if err != nil { - return offset, err - } - } - - return offset, nil -} - -func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { - a, b, _, d, e := internalGet(data, keys...) - return a, b, d, e -} - -func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { - if len(keys) <= 0 { - return nil, NotExist, -1, -1, errors.New("no keys") - } - - offset, err = searchKeys(data, keys...) - if err != nil { - return nil, NotExist, -1, -1, err - } - - // Go to closest value - nextOffset, err := nextToken(data[offset:]) - if err != nil { - return nil, NotExist, offset, -1, MalformedJsonError - } - - offset += nextOffset - value, dataType, endOffset, err = getType(data, offset) - if err != nil { - return value, dataType, offset, endOffset, err - } - - // Strip quotes from string values - if dataType == String { - value = value[1 : len(value)-1] - } - - return value[:len(value):len(value)], dataType, offset, endOffset, nil -} - -func getType(data []byte, offset int) ([]byte, ValueType, int, error) { - var dataType ValueType - endOffset := offset - - switch data[offset] { - case '"': - dataType = String - idx, _, err := stringEnd(data[offset+1:]) - if err != nil { - return nil, dataType, offset, MalformedStringError - } - endOffset += idx + 1 - case '[': - return processNextCollectionElement(data, offset, Array) - case '{': - return processNextCollectionElement(data, offset, Object) - default: - end := tokenEnd(data[endOffset:]) - if end == -1 { - return nil, dataType, offset, MalformedValueError - } - - value := data[offset : endOffset+end] - dataType, offset, err := extractValueTypeFromToken(data[offset], value) - endOffset += end - } - - return data[offset:endOffset], dataType, endOffset, nil -} - -func handleGetResult( - data []byte, - offset int, - f func(value []byte, dataType ValueType, offset int, err error), -) (int, error) { - v, t, o, e := Get(data[offset:]) - if e != nil { - return offset, e - } - - if o == 0 { - return offset, nil - } - - if t != NotExist { - f(v, t, offset+o-len(v), e) - } - - if e != nil { - return offset, e - } - - return offset + o, nil -} - -func handleNextTokenResult(data []byte, offset int) (int, error) { - nextTokenOffset, err := nextToken(data[offset:]) - if err != nil { - return offset, MalformedArrayError - } - - return offset + nextTokenOffset, nil -} - -func handleArrayError(data []byte, offset int) (int, error) { - if data[offset] == ']' { - return offset, nil - } - - if data[offset] != ',' { - return offset, MalformedArrayError - } - - return offset + 1, nil -} - -func processNextCollectionElement(data []byte, offset int, dataType ValueType) ([]byte, ValueType, int, error) { - switch dataType { - case Array: - endOffset, err := blockEnd(data[offset:], '[', ']') - if err != nil { - return nil, NotExist, offset, MalformedArrayError - } - - return data[offset : offset+endOffset], Array, offset + endOffset, nil - case Object: - endOffset, err := blockEnd(data[offset:], '{', '}') - if err != nil { - return nil, NotExist, offset, MalformedObjectError - } - - return data[offset : offset+endOffset], Object, offset + endOffset, nil - default: - return nil, NotExist, offset, errors.New("not a collection") - } -} + return -1 +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index c6e6ba881b3..8677b44abe3 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -5,47 +5,6 @@ import ( "testing" ) -type lexerTest struct { - name string - data []byte - token byte - escaped bool - expected int -} - -func TestFindTokenStart(t *testing.T) { - tests := []lexerTest{ - {token: '{', expected: 5}, - {token: '[', expected: 0}, - {token: 'w', expected: 6}, - {token: 'l', expected: 9}, - } - - data := []byte(`hello{world}`) - - for _, test := range tests { - if got := findTokenStart(data, test.token); got != test.expected { - t.Errorf("findTokenStart() = %v, want %v", got, test.expected) - } - } -} - -func TestTokenEnd(t *testing.T) { - tests := []lexerTest{ - {name: "whitespace", data: []byte("hello world"), expected: 5}, - {name: "newline", data: []byte("hello\nworld"), expected: 5}, - {name: "tab", data: []byte("hello\tworld"), expected: 5}, - {name: "multiple words", data: []byte("hello world foo bar"), expected: 5}, - {name: "no space", data: []byte("helloworld"), expected: 10}, - } - - for _, test := range tests { - if got := tokenEnd(test.data); got != test.expected { - t.Errorf("tokenEnd() = %v, want %v", got, test.expected) - } - } -} - func TestLexer(t *testing.T) { tests := []struct { name string @@ -57,7 +16,6 @@ func TestLexer(t *testing.T) { { name: "Test 1", data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - start: 38, next: 1, last: 39, }, @@ -65,102 +23,103 @@ func TestLexer(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := findLastTokenPosition(tt.data); got != tt.last { - t.Errorf("findLastTokenPosition() = %v, want %v", got, tt.last) + if got := findTokenStart(tt.data, '{'); got != 0 { + t.Errorf("findTokenStart() = %v, want %v", got, 0) + } + + if got := findTokenStart(tt.data, '['); got != 29 { + t.Errorf("findTokenStart() = %v, want %v", got, 29) + } + + if _, err := findKeyStart(tt.data, "key"); err != nil { + t.Errorf("findKeyStart() returned an error: %v", err) + } + }) + } +} + +func TestLastToken(t *testing.T) { + tests := []struct { + name string + data []byte + expected int + }{ + {"Empty data", []byte{}, -1}, + {"Spaces only", []byte(" "), -1}, + {"Newlines only", []byte("\n\n\n\n"), -1}, + {"Mixed whitespace", []byte(" \n\r\t "), -1}, + {"One character", []byte("a"), 0}, + {"Mixed characters", []byte("abc def"), 6}, + {"Mixed characters with trailing whitespace", []byte("abc def \n\r\t "), 6}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := lastToken(test.data); got != test.expected { + t.Errorf("lastToken(%q) = %v, want %v", test.data, got, test.expected) + } + }) + } +} + +func TestBlockEnd(t *testing.T) { + tests := []struct { + name string + data []byte + open byte + close byte + expected int + }{ + {"Empty Object", []byte("{}"), '{', '}', 2}, + {"Nested Object", []byte(`{"key": {"nestedKey": "value"}}`), '{', '}', 31}, + {"Array", []byte(`["item1", "item2"]`), '[', ']', 18}, + {"Complex Object", []byte(`{"key": [1, 2, 3], "anotherKey": "value"}`), '{', '}', 41}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := blockEnd(test.data, test.open, test.close) + if result != test.expected { + t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) } }) } } -func TestSearchKeys(t *testing.T) { + +func TestFindKeyStart(t *testing.T) { tests := []struct { name string data []byte - keys []string + key string expected int - isErr bool }{ { - name: "Empty data and keys", - data: []byte{}, - keys: []string{}, - expected: 0, + name: "Test 1", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), + key: "key", + expected: 3, }, { - name: "Empty data with keys", - data: []byte{}, - keys: []string{"key1", "key2"}, - expected: -1, - isErr: true, + name: "Test 2", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), + key: "array", + expected: 20, }, { - name: "Data with no keys", - data: []byte(`{"foo": "bar"}`), - keys: []string{"key1", "key2"}, + name: "Test 3", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), + key: "value", expected: -1, - isErr: true, - }, - { - name: "Data with single key", - data: []byte(`{"key1": "value1"}`), - keys: []string{"key1"}, - expected: 8, - }, - { - name: "Data with multiple keys", - data: []byte(`{"key1": "value1", "key2": "value2"}`), - keys: []string{"key2"}, - expected: 26, - }, - { - name: "Data with nested keys", - data: []byte(`{"outer": {"key1": "value1", "key2": "value2"}, "key3": "value3"}`), - keys: []string{"key3"}, - expected: 55, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := searchKeys(tt.data, tt.keys...) - + got, _ := findKeyStart(tt.data, tt.key) if got != tt.expected { - t.Errorf("searchKeys(%s) = %v, want %v", tt.name, got, tt.expected) - } - - if (err != nil) != tt.isErr { - t.Errorf("searchKeys(%s) error = %v, wantErr %v", tt.name, err, tt.isErr) + t.Errorf("findKeyStart() = %v, want %v", got, tt.expected) } }) } -} - -func TestArrayEach(t *testing.T) { - data := []byte(`{"a": { "b":[{"x": 1} ,{"x":2},{ "x":3}, {"x":4} ]}}`) - count := 0 - - ArrayEach(data, func(value []byte, typ ValueType, offset int, err error) { - count += 1 - - switch count { - case 1: - if string(value) != `{"x": 1}` { - t.Errorf("Wrong first item: %s", string(value)) - } - case 2: - if string(value) != `{"x":2}` { - t.Errorf("Wrong second item: %s", string(value)) - } - case 3: - if string(value) != `{ "x":3}` { - t.Errorf("Wrong third item: %s", string(value)) - } - case 4: - if string(value) != `{"x":4}` { - t.Errorf("Wrong fourth item: %s", string(value)) - } - default: - t.Errorf("Too many items: %d", count) - } - }, "a", "b") -} +} \ No newline at end of file From cc136d571f6860f8018cd7e636de781edf938d13 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 15 Dec 2023 13:09:25 +0900 Subject: [PATCH 21/72] JSON PoC --- examples/gno.land/p/demo/json/errors.gno | 2 + examples/gno.land/p/demo/json/lexer.gno | 160 +++++++++++++++ examples/gno.land/p/demo/json/lexer_test.gno | 83 ++++++++ examples/gno.land/p/demo/json/parser.gno | 189 ++++++++++++++++++ examples/gno.land/p/demo/json/parser_test.gno | 34 ++++ 5 files changed, 468 insertions(+) diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index 6ce26cee629..f34e1da798a 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -4,6 +4,7 @@ import "errors" const ( KeyPathNotFoundError error = errors.New("key path not found") + ArrayIndexNotFound = errors.New("array index not found") NotSpecifiedKeyError = errors.New("key not specified") ClosingBracketError = errors.New("closing bracket not found") ClosingQuoteError = errors.New("closing quote not found") @@ -14,6 +15,7 @@ const ( KeyLevelNotMatched = errors.New("key level not matched") Overflow = errors.New("overflow") EmptyBytes = errors.New("empty bytes") + InvalidArrayIndex = errors.New("invalid array index") InvalidExponents = errors.New("invalid exponents") NonDigitCharacters = errors.New("non-digit characters") MultipleDecimalPoints = errors.New("multiple decimal points") diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 71a5c120b1a..fb93c1b8398 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -190,4 +190,164 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int { } return -1 +} + +// TODO: change function name to something more appropriate +// +// searchKeys is a function that takes a byte slice of JSON data and a series of keys. +// It traverses the JSON data to find the position where the value of the final key in the series begins. +// The position is returned as an integer representing the index in the byte slice. +// If the key series is empty, it returns 0. +// If the keys are not found or the JSON data is malformed, it returns -1. +func searchKeys(data []byte, keys ...string) (int, error) { + keyLevel := 0 + level := 0 + i := 0 + ln := len(data) + lk := len(keys) + lastMatched := true + + if lk == 0 { + return 0, nil + } + + var stackbuf [UnescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings + + for i < ln { + switch data[i] { + case '"': + i++ + keyBegin := i + + strEnd, keyEscaped := stringEnd(data[i:]) + if strEnd == -1 { + return -1, MalformedJsonError + } + i += strEnd + keyEnd := i - 1 + + valueOffset, err := nextToken(data[i:]) + if err != nil { + return -1, err + } + + i += valueOffset + + // if string is a key + + if data[i] == ':' { + if level < 1 { + return -1, MalformedJsonError + } + + key := data[keyBegin:keyEnd] + + // for unescape: if there are no escape sequences, this is cheap; if there are, it is a + // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize + var keyUnesc []byte + if !keyEscaped { + keyUnesc = key + } else if ku, err := Unescape(key, stackbuf[:]); err != nil { + return -1, err + } else { + keyUnesc = ku + } + + if level > len(keys) { + return -1, KeyLevelNotMatched + } + + if bytes.Equal(keyUnesc, []byte(keys[level-1])) { + lastMatched = true + + // if key level match + if keyLevel == level-1 { + keyLevel++ + // If we found all keys in path + if keyLevel == lk { + return i + 1, nil + } + } + } else { + lastMatched = false + } + } else { + i-- + } + case '{': + // in case parent key is matched then only we will increase the level otherwise can directly + // can move to the end of this block + if !lastMatched { + end := blockEnd(data[i:], '{', '}') + if end == -1 { + return -1, MalformedJsonError + } + i += end - 1 + } else { + level++ + } + case '}': + level-- + if level == keyLevel { + keyLevel-- + } + case '[': + // If we want to get array element by index + if keyLevel == level && keys[level][0] == '[' { + keyLen := len(keys[level]) + if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { + return -1, MalformedArrayError + } + + arrIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) + if err != nil { + return -1, InvalidArrayIndex + } + var curIdx int + var valueFound []byte + var valueOffset int + curI := i + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + if curIdx == arrIdx { + valueFound = value + valueOffset = offset + if dataType == String { + valueOffset = valueOffset - 2 + valueFound = data[curI+valueOffset : curI+valueOffset+len(value)+2] + } + } + curIdx += 1 + }) + + if valueFound == nil { + return -1, ArrayIndexNotFound + } + + subIndex, err := searchKeys(valueFound, keys[level+1:]...) + if err != nil { + return -1, err + } + + return i + valueOffset + subIndex, nil + } else { + arraySkip := blockEnd(data[i:], '[', ']') + if arraySkip < 0 { + return -1, MalformedJsonError + } + + i += arraySkip - 1 + } + case ':': // If encountered, JSON data is malformed + return -1, MalformedJsonError + } + + i++ + } + + return -1, KeyPathNotFoundError +} + +func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { + a, b, _, d, e := internalGet(data, keys...) + return a, b, d, e } \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 8677b44abe3..d3282df996c 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -122,4 +122,87 @@ func TestFindKeyStart(t *testing.T) { } }) } +} + +func TestSearchKeys(t *testing.T) { + tests := []struct { + name string + data []byte + keys []string + expected int + }{ + { + name: "Test 1", + data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), + keys: []string{"key1"}, + expected: 8, + }, + { + name: "Test 2", + data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), + keys: []string{"key2"}, + expected: 26, + }, + { + name: "Test 3", + data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), + keys: []string{"key4"}, + expected: -1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := searchKeys(tt.data, tt.keys...); got != tt.expected { + t.Errorf("searchKeys() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestGet(t *testing.T) { + data := []byte(`{ + "person": { + "name": { + "first": "Alex", + "last": "Johnson", + "fullName": "Alex Johnson" + }, + "github": { + "handle": "alexj", + "followers": 152 + }, + "avatars": [ + { "url": "https://avatars.example.com/u/12345?v=4&s=460", "type": "thumbnail" } + ] + }, + "company": { + "name": "company name", + } + } + `) + + tests := []struct { + path []string + expected string + }{ + {[]string{"person", "name", "first"}, "Alex"}, + {[]string{"person", "name", "last"}, "Johnson"}, + {[]string{"person", "name", "fullName"}, "Alex Johnson"}, + {[]string{"person", "github", "handle"}, "alexj"}, + {[]string{"person", "github", "followers"}, "152"}, + {[]string{"company", "name"}, "company name"}, + {[]string{"company"}, "{\"name\": \"company name\"}"}, + } + + for _, test := range tests { + value, _, _, err := Get(data, test.path...) + if err != nil { + t.Errorf("Got error: %v", err) + } + + if string(value) != test.expected { + t.Errorf("Expected '%s', got '%s'", test.expected, value) + } + } } \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 6db79b6f0e4..8634d42a00f 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -315,3 +315,192 @@ func underscoreOK(s string) bool { } return saw != '_' } + +var ( + trueLiteral = []byte("true") + falseLiteral = []byte("false") + nullLiteral = []byte("null") +) + +func getType(data []byte, offset int) ([]byte, ValueType, int, error) { + if len(data) == 0 { + panic("No JSON data provided") + } + + var dataType ValueType + endOffset := offset + + // if string value + if data[offset] == '"' { + dataType = String + if idx, _ := stringEnd(data[offset+1:]); idx != -1 { + endOffset += idx + 1 + } else { + return nil, dataType, offset, MalformedStringError + } + } else if data[offset] == '[' { // if array value + dataType = Array + // break label, for stopping nested loops + endOffset = blockEnd(data[offset:], '[', ']') + + if endOffset == -1 { + return nil, dataType, offset, MalformedArrayError + } + + endOffset += offset + } else if data[offset] == '{' { // if object value + dataType = Object + // break label, for stopping nested loops + endOffset = blockEnd(data[offset:], '{', '}') + + if endOffset == -1 { + return nil, dataType, offset, MalformedObjectError + } + + endOffset += offset + } else { + // Number, Boolean or None + end := tokenEnd(data[endOffset:]) + + if end == -1 { + return nil, dataType, offset, MalformedValueError + } + + value := data[offset : endOffset+end] + + switch data[offset] { + case 't', 'f': // true or false + if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { + dataType = Boolean + } else { + return nil, Unknown, offset, UnknownValueTypeError + } + case 'u', 'n': // undefined or null + if bytes.Equal(value, nullLiteral) { + dataType = Null + } else { + return nil, Unknown, offset, UnknownValueTypeError + } + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': + dataType = Number + default: + return nil, Unknown, offset, UnknownValueTypeError + } + + endOffset += end + } + return data[offset:endOffset], dataType, endOffset, nil +} + +// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. +func ArrayEach(data []byte, f func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { + if len(data) == 0 { + return -1, MalformedObjectError + } + + nT, err := nextToken(data) + if err != nil { + return -1, MalformedJsonError + } + + offset = nT + 1 + + if len(keys) > 0 { + if offset, _ = searchKeys(data, keys...); offset == -1 { + return offset, KeyPathNotFoundError + } + + // Go to closest value + nO, err := nextToken(data[offset:]) + if err != nil { + return offset, MalformedJsonError + } + + offset += nO + + if data[offset] != '[' { + return offset, MalformedArrayError + } + + offset++ + } + + nO, err := nextToken(data[offset:]) + if err != nil { + return offset, MalformedJsonError + } + + offset += nO + + if data[offset] == ']' { + return offset, nil + } + + for true { + v, t, o, e := Get(data[offset:]) + + if e != nil { + return offset, e + } + + if o == 0 { + break + } + + if t != NotExist { + f(v, t, offset+o-len(v), e) + } + + if e != nil { + break + } + + offset += o + + skipToToken, err := nextToken(data[offset:]) + if err != nil { + return offset, MalformedArrayError + } + offset += skipToToken + + if data[offset] == ']' { + break + } + + if data[offset] != ',' { + return offset, MalformedArrayError + } + + offset++ + } + + return offset, nil +} + +func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { + if len(keys) > 0 { + offset, err = searchKeys(data, keys...) + if err != nil { + return nil, NotExist, -1, -1, err + } + } + + // Go to closest value + nO, err := nextToken(data[offset:]) + if err != nil { + return nil, NotExist, offset, -1, MalformedJsonError + } + + offset += nO + value, dataType, endOffset, err = getType(data, offset) + if err != nil { + return value, dataType, offset, endOffset, err + } + + // Strip quotes from string values + if dataType == String { + value = value[1 : len(value)-1] + } + + return value[:len(value):len(value)], dataType, offset, endOffset, nil +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index d0839bdbb5a..aa73ff3df14 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -1,6 +1,7 @@ package json import ( + "bytes" "testing" ) @@ -245,3 +246,36 @@ func TestParseUint(t *testing.T) { } } } +func TestGetType(t *testing.T) { + tests := []struct { + data []byte + offset int + expected []byte + vt ValueType + err error + }{ + {[]byte(`{"name": "John", "age": 30}`), 0, []byte(`{"name": "John", "age": 30}`), Object, nil}, + {[]byte(`"Hello, World!"`), 0, []byte(`"Hello, World!"`), String, nil}, + {[]byte(`12345`), 0, []byte(`12345`), Number, nil}, + {[]byte(`true`), 0, []byte(`true`), Boolean, nil}, + {[]byte(`null`), 0, []byte(`null`), Null, nil}, + {[]byte(`[1, 2, 3]`), 0, []byte(`[1, 2, 3]`), Array, nil}, + {[]byte(`{}`), 0, []byte(`{}`), Object, nil}, + } + + for i, tt := range tests { + gotData, gotVT, _, gotErr := getType(tt.data, tt.offset) + + if !bytes.Equal(gotData, tt.expected) { + t.Errorf("%d. expected data=%s, but got data=%s", i, tt.expected, gotData) + } + + if gotVT != tt.vt { + t.Errorf("%d. expected value type=%s, but got value type=%s", i, tt.vt, gotVT) + } + + if gotErr != tt.err { + t.Errorf("%d. expected error=%v, but got error=%v", i, tt.err, gotErr) + } + } +} \ No newline at end of file From 7e9ccb85b6f977ff1b30ab1c8a5bd12e5e55b915 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 15 Dec 2023 13:16:33 +0900 Subject: [PATCH 22/72] remove unused errors --- examples/gno.land/p/demo/json/errors.gno | 18 ++++------- examples/gno.land/p/demo/json/lexer.gno | 12 +++---- examples/gno.land/p/demo/json/parser.gno | 40 ++++++++++++------------ 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index f34e1da798a..c73c91f9365 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -5,13 +5,7 @@ import "errors" const ( KeyPathNotFoundError error = errors.New("key path not found") ArrayIndexNotFound = errors.New("array index not found") - NotSpecifiedKeyError = errors.New("key not specified") - ClosingBracketError = errors.New("closing bracket not found") - ClosingQuoteError = errors.New("closing quote not found") - StringNotClosed = errors.New("string not closed") - BlockIsNotClosed = errors.New("block is not closed") TokenNotFound = errors.New("token not found") - OutOfLevel = errors.New("out of level") KeyLevelNotMatched = errors.New("key level not matched") Overflow = errors.New("overflow") EmptyBytes = errors.New("empty bytes") @@ -19,10 +13,10 @@ const ( InvalidExponents = errors.New("invalid exponents") NonDigitCharacters = errors.New("non-digit characters") MultipleDecimalPoints = errors.New("multiple decimal points") - MalformedStringError = errors.New("malformed string") - MalformedValueError = errors.New("malformed value") - MalformedObjectError = errors.New("malformed object") - MalformedArrayError = errors.New("malformed array") - MalformedJsonError = errors.New("malformed json array") - UnknownValueTypeError = errors.New("unknown value type") + MalformedString = errors.New("malformed string") + MalformedValue = errors.New("malformed value") + MalformedObject = errors.New("malformed object") + MalformedArray = errors.New("malformed array") + MalformedJson = errors.New("malformed json array") + UnknownValueType = errors.New("unknown value type") ) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index fb93c1b8398..b3a88485b89 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -221,7 +221,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { strEnd, keyEscaped := stringEnd(data[i:]) if strEnd == -1 { - return -1, MalformedJsonError + return -1, MalformedJson } i += strEnd keyEnd := i - 1 @@ -237,7 +237,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { if data[i] == ':' { if level < 1 { - return -1, MalformedJsonError + return -1, MalformedJson } key := data[keyBegin:keyEnd] @@ -280,7 +280,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { if !lastMatched { end := blockEnd(data[i:], '{', '}') if end == -1 { - return -1, MalformedJsonError + return -1, MalformedJson } i += end - 1 } else { @@ -296,7 +296,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { if keyLevel == level && keys[level][0] == '[' { keyLen := len(keys[level]) if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { - return -1, MalformedArrayError + return -1, MalformedArray } arrIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) @@ -332,13 +332,13 @@ func searchKeys(data []byte, keys ...string) (int, error) { } else { arraySkip := blockEnd(data[i:], '[', ']') if arraySkip < 0 { - return -1, MalformedJsonError + return -1, MalformedJson } i += arraySkip - 1 } case ':': // If encountered, JSON data is malformed - return -1, MalformedJsonError + return -1, MalformedJson } i++ diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 8634d42a00f..ec2b86521e0 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -20,7 +20,7 @@ func ParseString(data []byte) (string, error) { bf, err := Unescape(data, buf[:]) if err != nil { - return "", MalformedStringError + return "", MalformedString } return string(bf), nil @@ -33,7 +33,7 @@ func ParseBool(data []byte) (bool, error) { case bytes.Equal(data, []byte("false")): return false, nil default: - return false, MalformedValueError + return false, MalformedValue } } @@ -253,16 +253,16 @@ func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset if bytes.Equal(value, TrueLiteral) || bytes.Equal(value, FalseLiteral) { dataType = Boolean } - return Unknown, offset, UnknownValueTypeError + return Unknown, offset, UnknownValueType case 'u', 'n': if bytes.Equal(value, NullLiteral) { dataType = Null } - return Unknown, offset, UnknownValueTypeError + return Unknown, offset, UnknownValueType case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': dataType = Number default: - return Unknown, offset, UnknownValueTypeError + return Unknown, offset, UnknownValueType } } @@ -336,7 +336,7 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { if idx, _ := stringEnd(data[offset+1:]); idx != -1 { endOffset += idx + 1 } else { - return nil, dataType, offset, MalformedStringError + return nil, dataType, offset, MalformedString } } else if data[offset] == '[' { // if array value dataType = Array @@ -344,7 +344,7 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { endOffset = blockEnd(data[offset:], '[', ']') if endOffset == -1 { - return nil, dataType, offset, MalformedArrayError + return nil, dataType, offset, MalformedArray } endOffset += offset @@ -354,7 +354,7 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { endOffset = blockEnd(data[offset:], '{', '}') if endOffset == -1 { - return nil, dataType, offset, MalformedObjectError + return nil, dataType, offset, MalformedObject } endOffset += offset @@ -363,7 +363,7 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { end := tokenEnd(data[endOffset:]) if end == -1 { - return nil, dataType, offset, MalformedValueError + return nil, dataType, offset, MalformedValue } value := data[offset : endOffset+end] @@ -373,18 +373,18 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { dataType = Boolean } else { - return nil, Unknown, offset, UnknownValueTypeError + return nil, Unknown, offset, UnknownValueType } case 'u', 'n': // undefined or null if bytes.Equal(value, nullLiteral) { dataType = Null } else { - return nil, Unknown, offset, UnknownValueTypeError + return nil, Unknown, offset, UnknownValueType } case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': dataType = Number default: - return nil, Unknown, offset, UnknownValueTypeError + return nil, Unknown, offset, UnknownValueType } endOffset += end @@ -395,12 +395,12 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. func ArrayEach(data []byte, f func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { if len(data) == 0 { - return -1, MalformedObjectError + return -1, MalformedObject } nT, err := nextToken(data) if err != nil { - return -1, MalformedJsonError + return -1, MalformedJson } offset = nT + 1 @@ -413,13 +413,13 @@ func ArrayEach(data []byte, f func(value []byte, dataType ValueType, offset int, // Go to closest value nO, err := nextToken(data[offset:]) if err != nil { - return offset, MalformedJsonError + return offset, MalformedJson } offset += nO if data[offset] != '[' { - return offset, MalformedArrayError + return offset, MalformedArray } offset++ @@ -427,7 +427,7 @@ func ArrayEach(data []byte, f func(value []byte, dataType ValueType, offset int, nO, err := nextToken(data[offset:]) if err != nil { - return offset, MalformedJsonError + return offset, MalformedJson } offset += nO @@ -459,7 +459,7 @@ func ArrayEach(data []byte, f func(value []byte, dataType ValueType, offset int, skipToToken, err := nextToken(data[offset:]) if err != nil { - return offset, MalformedArrayError + return offset, MalformedArray } offset += skipToToken @@ -468,7 +468,7 @@ func ArrayEach(data []byte, f func(value []byte, dataType ValueType, offset int, } if data[offset] != ',' { - return offset, MalformedArrayError + return offset, MalformedArray } offset++ @@ -488,7 +488,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, // Go to closest value nO, err := nextToken(data[offset:]) if err != nil { - return nil, NotExist, offset, -1, MalformedJsonError + return nil, NotExist, offset, -1, MalformedJson } offset += nO From b69eadbfab7b1869efef166c55f3b4ac25740598 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 15 Dec 2023 15:35:58 +0900 Subject: [PATCH 23/72] flatten --- examples/gno.land/p/demo/json/encoder.gno | 48 +++++++ .../gno.land/p/demo/json/encoder_test.gno | 134 ++++++++++++++++++ examples/gno.land/p/demo/json/errors.gno | 17 +-- examples/gno.land/p/demo/json/lexer.gno | 8 +- examples/gno.land/p/demo/json/lexer_test.gno | 97 ++++++------- examples/gno.land/p/demo/json/parser.gno | 127 ++++++++++------- examples/gno.land/p/demo/json/parser_test.gno | 3 +- examples/gno.land/p/demo/json/value.gno | 1 + 8 files changed, 318 insertions(+), 117 deletions(-) create mode 100644 examples/gno.land/p/demo/json/encoder.gno create mode 100644 examples/gno.land/p/demo/json/encoder_test.gno diff --git a/examples/gno.land/p/demo/json/encoder.gno b/examples/gno.land/p/demo/json/encoder.gno new file mode 100644 index 00000000000..4c0942ef4dc --- /dev/null +++ b/examples/gno.land/p/demo/json/encoder.gno @@ -0,0 +1,48 @@ +package json + +import ( + "strings" +) + +// Flatten takes a map and returns a new one where nested maps are replaced +// by dot-delimited keys. +func Flatten(m map[string]interface{}) map[string]interface{} { + o := make(map[string]interface{}) + for k, v := range m { + switch child := v.(type) { + case map[string]interface{}: + nm := Flatten(child) + for nk, nv := range nm { + o[k+"."+nk] = nv + } + default: + o[k] = v + } + } + return o +} + +func Unflatten(flatMap map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}) + + for k, v := range flatMap { + parts := strings.Split(k, ".") + lastIdx := len(parts) - 1 + + currentMap := result + + for i, part := range parts { + if i == lastIdx { + currentMap[part] = v + } else { + if _, ok := currentMap[part]; !ok { + currentMap[part] = make(map[string]interface{}) + } + + currentMap = currentMap[part].(map[string]interface{}) + } + } + } + + return result +} diff --git a/examples/gno.land/p/demo/json/encoder_test.gno b/examples/gno.land/p/demo/json/encoder_test.gno new file mode 100644 index 00000000000..8e879dc9da7 --- /dev/null +++ b/examples/gno.land/p/demo/json/encoder_test.gno @@ -0,0 +1,134 @@ +package json + +import ( + "strings" + "testing" +) + +func equal(a, b map[string]interface{}) bool { + if len(a) != len(b) { + return false + } + + for k, v := range a { + if b[k] != v { + return false + } + } + + return true +} + +func mapsAreEqual(a, b map[string]interface{}) bool { + if len(a) != len(b) { + return false + } + + for ka, va := range a { + vb, ok := b[ka] + if !ok { + return false + } + + switch vaTyped := va.(type) { + case map[string]interface{}: + if vbTyped, ok := vb.(map[string]interface{}); ok { + if !mapsAreEqual(vaTyped, vbTyped) { + return false + } + } else { + return false + } + default: + if va != vb { + return false + } + } + } + + return true +} + +func TestFlatten(t *testing.T) { + tests := []struct { + name string + input map[string]interface{} + want map[string]interface{} + }{ + { + name: "Simple map", + input: map[string]interface{}{"a": 1, "b": 2}, + want: map[string]interface{}{"a": 1, "b": 2}, + }, + { + name: "Nested map", + input: map[string]interface{}{"a": map[string]interface{}{"b": 2, "c": 3}}, + want: map[string]interface{}{"a.b": 2, "a.c": 3}, + }, + { + name: "Empty map", + input: map[string]interface{}{}, + want: map[string]interface{}{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := Flatten(tt.input) + if !equal(got, tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestUnflatten(t *testing.T) { + tests := []struct { + name string + input map[string]interface{} + want map[string]interface{} + }{ + { + name: "Simple nested map", + input: map[string]interface{}{ + "a.b": 2, + "a.c": 3, + }, + want: map[string]interface{}{ + "a": map[string]interface{}{ + "b": 2, + "c": 3, + }, + }, + }, + { + name: "Multi-level nested map", + input: map[string]interface{}{ + "a.b.c": 1, + "a.b.d": 2, + }, + want: map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": 1, + "d": 2, + }, + }, + }, + }, + { + name: "Empty map", + input: map[string]interface{}{}, + want: map[string]interface{}{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := Unflatten(tt.input) + if !mapsAreEqual(got, tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index c73c91f9365..f484441e972 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -4,19 +4,20 @@ import "errors" const ( KeyPathNotFoundError error = errors.New("key path not found") - ArrayIndexNotFound = errors.New("array index not found") + ArrayIndexNotFound = errors.New("array index not found") TokenNotFound = errors.New("token not found") KeyLevelNotMatched = errors.New("key level not matched") Overflow = errors.New("overflow") EmptyBytes = errors.New("empty bytes") - InvalidArrayIndex = errors.New("invalid array index") + InvalidArrayIndex = errors.New("invalid array index") InvalidExponents = errors.New("invalid exponents") NonDigitCharacters = errors.New("non-digit characters") MultipleDecimalPoints = errors.New("multiple decimal points") - MalformedString = errors.New("malformed string") - MalformedValue = errors.New("malformed value") - MalformedObject = errors.New("malformed object") - MalformedArray = errors.New("malformed array") - MalformedJson = errors.New("malformed json array") - UnknownValueType = errors.New("unknown value type") + MalformedType = errors.New("malformed type") + MalformedString = errors.New("malformed string") + MalformedValue = errors.New("malformed value") + MalformedObject = errors.New("malformed object") + MalformedArray = errors.New("malformed array") + MalformedJson = errors.New("malformed json array") + UnknownValueType = errors.New("unknown value type") ) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index b3a88485b89..b12b6afecb3 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -284,12 +284,12 @@ func searchKeys(data []byte, keys ...string) (int, error) { } i += end - 1 } else { - level++ + level += 1 } case '}': - level-- + level -= 1 if level == keyLevel { - keyLevel-- + keyLevel -= 1 } case '[': // If we want to get array element by index @@ -350,4 +350,4 @@ func searchKeys(data []byte, keys ...string) (int, error) { func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { a, b, _, d, e := internalGet(data, keys...) return a, b, d, e -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index d3282df996c..52f4732b5f1 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -14,10 +14,10 @@ func TestLexer(t *testing.T) { last int }{ { - name: "Test 1", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - next: 1, - last: 39, + name: "Test 1", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), + next: 1, + last: 39, }, } @@ -39,27 +39,27 @@ func TestLexer(t *testing.T) { } func TestLastToken(t *testing.T) { - tests := []struct { - name string - data []byte - expected int - }{ - {"Empty data", []byte{}, -1}, - {"Spaces only", []byte(" "), -1}, - {"Newlines only", []byte("\n\n\n\n"), -1}, - {"Mixed whitespace", []byte(" \n\r\t "), -1}, - {"One character", []byte("a"), 0}, - {"Mixed characters", []byte("abc def"), 6}, - {"Mixed characters with trailing whitespace", []byte("abc def \n\r\t "), 6}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got := lastToken(test.data); got != test.expected { - t.Errorf("lastToken(%q) = %v, want %v", test.data, got, test.expected) - } - }) - } + tests := []struct { + name string + data []byte + expected int + }{ + {"Empty data", []byte{}, -1}, + {"Spaces only", []byte(" "), -1}, + {"Newlines only", []byte("\n\n\n\n"), -1}, + {"Mixed whitespace", []byte(" \n\r\t "), -1}, + {"One character", []byte("a"), 0}, + {"Mixed characters", []byte("abc def"), 6}, + {"Mixed characters with trailing whitespace", []byte("abc def \n\r\t "), 6}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := lastToken(test.data); got != test.expected { + t.Errorf("lastToken(%q) = %v, want %v", test.data, got, test.expected) + } + }) + } } func TestBlockEnd(t *testing.T) { @@ -86,7 +86,6 @@ func TestBlockEnd(t *testing.T) { } } - func TestFindKeyStart(t *testing.T) { tests := []struct { name string @@ -161,7 +160,7 @@ func TestSearchKeys(t *testing.T) { } func TestGet(t *testing.T) { - data := []byte(`{ + data := []byte(`{ "person": { "name": { "first": "Alex", @@ -171,38 +170,34 @@ func TestGet(t *testing.T) { "github": { "handle": "alexj", "followers": 152 - }, - "avatars": [ - { "url": "https://avatars.example.com/u/12345?v=4&s=460", "type": "thumbnail" } - ] + } }, "company": { "name": "company name", - } + }, } `) - tests := []struct { - path []string - expected string - }{ + tests := []struct { + path []string + expected string + }{ {[]string{"person", "name", "first"}, "Alex"}, {[]string{"person", "name", "last"}, "Johnson"}, - {[]string{"person", "name", "fullName"}, "Alex Johnson"}, + {[]string{"person", "name", "fullName"}, "Alex Johnson"}, {[]string{"person", "github", "handle"}, "alexj"}, {[]string{"person", "github", "followers"}, "152"}, {[]string{"company", "name"}, "company name"}, - {[]string{"company"}, "{\"name\": \"company name\"}"}, - } - - for _, test := range tests { - value, _, _, err := Get(data, test.path...) - if err != nil { - t.Errorf("Got error: %v", err) - } - - if string(value) != test.expected { - t.Errorf("Expected '%s', got '%s'", test.expected, value) - } - } -} \ No newline at end of file + } + + for _, test := range tests { + value, _, _, err := Get(data, test.path...) + if err != nil { + t.Errorf("Got error: %v", err) + } + + if string(value) != test.expected { + t.Errorf("Expected '%s', got '%s'", test.expected, value) + } + } +} diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index ec2b86521e0..9786fb8921e 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -205,7 +205,7 @@ func ParseUint(b []byte, base, bitSize int) (v uint64, err error) { return n, nil } -// lower(c) is a lower-case letter if and only if +// lower is a lower-case letter if and only if // c is either that lower-case letter or the equivalent upper-case letter. // Instead of writing c == 'x' || c == 'X' one can write lower(c) == 'x'. // Note that lower of non-letters can produce other non-letters. @@ -313,6 +313,7 @@ func underscoreOK(s string) bool { // Saw non-digit, non-underscore. saw = '!' } + return saw != '_' } @@ -324,72 +325,92 @@ var ( func getType(data []byte, offset int) ([]byte, ValueType, int, error) { if len(data) == 0 { - panic("No JSON data provided") + return nil, Unknown, offset, errors.New("no JSON data provided") } - var dataType ValueType - endOffset := offset + dataType, endOffset, err := parseValue(data, offset) + if err != nil { + return nil, dataType, offset, err + } - // if string value - if data[offset] == '"' { - dataType = String - if idx, _ := stringEnd(data[offset+1:]); idx != -1 { - endOffset += idx + 1 - } else { - return nil, dataType, offset, MalformedString - } - } else if data[offset] == '[' { // if array value - dataType = Array - // break label, for stopping nested loops - endOffset = blockEnd(data[offset:], '[', ']') + return data[offset:endOffset], dataType, endOffset, nil +} - if endOffset == -1 { - return nil, dataType, offset, MalformedArray - } +func parseValue(data []byte, offset int) (ValueType, int, error) { + switch data[offset] { + case '"': + return parseString(data, offset) + case '[', '{': + return parseContainer(data, offset) + } - endOffset += offset - } else if data[offset] == '{' { // if object value - dataType = Object - // break label, for stopping nested loops - endOffset = blockEnd(data[offset:], '{', '}') + return parsePrimitive(data, offset) +} - if endOffset == -1 { - return nil, dataType, offset, MalformedObject - } +// parseString parses a JSON string and returns its type and the end position. +func parseString(data []byte, offset int) (ValueType, int, error) { + if idx, _ := stringEnd(data[offset+1:]); idx != -1 { + return String, offset + idx + 1, nil + } + return String, offset, errors.New("malformed string") +} - endOffset += offset +// parseContainer parses a JSON array or object and returns its type and the end position. +func parseContainer(data []byte, offset int) (ValueType, int, error) { + var containerType ValueType + var closing byte + + if data[offset] == '[' { + containerType = Array + closing = ']' } else { - // Number, Boolean or None - end := tokenEnd(data[endOffset:]) + containerType = Object + closing = '}' + } - if end == -1 { - return nil, dataType, offset, MalformedValue - } + endOffset := blockEnd(data[offset:], data[offset], closing) + if endOffset == -1 { + return containerType, offset, MalformedType + } - value := data[offset : endOffset+end] + return containerType, offset + endOffset, nil +} - switch data[offset] { - case 't', 'f': // true or false - if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { - dataType = Boolean - } else { - return nil, Unknown, offset, UnknownValueType - } - case 'u', 'n': // undefined or null - if bytes.Equal(value, nullLiteral) { - dataType = Null - } else { - return nil, Unknown, offset, UnknownValueType +func parsePrimitive(data []byte, offset int) (ValueType, int, error) { + end := tokenEnd(data[offset:]) + if end == -1 { + return Unknown, offset, errors.New("malformed value") + } + value := data[offset : offset+end] + + switch data[offset] { + case 't', 'f': + _boolValue, err := ParseBool(value) + if err != nil { + return Unknown, offset, err + } + return Boolean, offset + end, nil + case 'n': + if bytes.Equal(value, nullLiteral) { + return Null, offset + end, nil + } + return Unknown, offset, errors.New("unknown null type") + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': + if !bytes.ContainsAny(value, ".eE") { + _intValue, err := ParseInt(value) + if err != nil { + return Number, offset + end, err } - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': - dataType = Number - default: - return nil, Unknown, offset, UnknownValueType } - endOffset += end + if _floatValue, err := ParseFloat(value); err != nil { + return Number, offset + end, err + } + + return Number, offset + end, nil } - return data[offset:endOffset], dataType, endOffset, nil + + return Unknown, offset, errors.New("unknown value type") } // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. @@ -503,4 +524,4 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, } return value[:len(value):len(value)], dataType, offset, endOffset, nil -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index aa73ff3df14..b02dfa6a3f6 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -246,6 +246,7 @@ func TestParseUint(t *testing.T) { } } } + func TestGetType(t *testing.T) { tests := []struct { data []byte @@ -278,4 +279,4 @@ func TestGetType(t *testing.T) { t.Errorf("%d. expected error=%v, but got error=%v", i, tt.err, gotErr) } } -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/value.gno b/examples/gno.land/p/demo/json/value.gno index 760c6d6751c..fab0e4a70db 100644 --- a/examples/gno.land/p/demo/json/value.gno +++ b/examples/gno.land/p/demo/json/value.gno @@ -6,6 +6,7 @@ const ( NotExist ValueType = iota String Number + Float Object Array Boolean From 6e72466792c1bcf59eb41c9add5108d2ceef522c Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 15 Dec 2023 17:42:16 +0900 Subject: [PATCH 24/72] investigate flattening --- examples/gno.land/p/demo/json/encoder.gno | 18 +++++++++ .../gno.land/p/demo/json/encoder_test.gno | 38 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/examples/gno.land/p/demo/json/encoder.gno b/examples/gno.land/p/demo/json/encoder.gno index 4c0942ef4dc..557c176377f 100644 --- a/examples/gno.land/p/demo/json/encoder.gno +++ b/examples/gno.land/p/demo/json/encoder.gno @@ -1,6 +1,7 @@ package json import ( + "fmt" "strings" ) @@ -46,3 +47,20 @@ func Unflatten(flatMap map[string]interface{}) map[string]interface{} { return result } + +func MapToJSONString(m map[string]interface{}) string { + var sb strings.Builder + sb.WriteString("{") + + isFirst := true + for k, v := range m { + if !isFirst { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("\"%s\": %v", k, v)) + isFirst = false + } + + sb.WriteString("}") + return sb.String() +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/encoder_test.gno b/examples/gno.land/p/demo/json/encoder_test.gno index 8e879dc9da7..732e904ed11 100644 --- a/examples/gno.land/p/demo/json/encoder_test.gno +++ b/examples/gno.land/p/demo/json/encoder_test.gno @@ -132,3 +132,41 @@ func TestUnflatten(t *testing.T) { }) } } + +func TestMapToJSONString(t *testing.T) { + cases := []struct { + name string + input map[string]interface{} + expected string + }{ + { + name: "Simple Map", + input: map[string]interface{}{ + "a": 1, + "b": 2, + }, + expected: "{\"a\": 1, \"b\": 2}", + }, + { + name: "Nested Map", + input: map[string]interface{}{ + "a": map[string]interface{}{ + "b": 2, + "c": 3, + }, + "d": 4, + }, + expected: "{\"a.b\": 2, \"a.c\": 3, \"d\": 4}", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + flattened := Flatten(tc.input) + result := MapToJSONString(flattened) + if result != tc.expected { + t.Errorf("Test %s failed. Expected %s, got %s", tc.name, tc.expected, result) + } + }) + } +} From 83a58eb2411cda541269e6a07c3356174dfaeb35 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 15 Dec 2023 19:00:26 +0900 Subject: [PATCH 25/72] refactor --- examples/gno.land/p/demo/json/const.gno | 8 + examples/gno.land/p/demo/json/lexer.gno | 75 ++++--- examples/gno.land/p/demo/json/lexer_test.gno | 4 +- examples/gno.land/p/demo/json/parser.gno | 196 ++++++++----------- examples/gno.land/p/demo/json/table.gno | 25 --- examples/gno.land/p/demo/json/token.gno | 30 +++ gnovm/stdlibs/unicode/utf16/utf16_test.gno | 1 - 7 files changed, 159 insertions(+), 180 deletions(-) delete mode 100644 examples/gno.land/p/demo/json/table.gno create mode 100644 examples/gno.land/p/demo/json/token.gno diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno index 20beccfbc21..9834ae8b51d 100644 --- a/examples/gno.land/p/demo/json/const.gno +++ b/examples/gno.land/p/demo/json/const.gno @@ -12,3 +12,11 @@ const ( UnescapeStackBufSize = 64 ) + +const ( + absMinInt64 = 1 << 63 + maxInt64 = 1<<63 - 1 + maxUint64 = 1<<64 - 1 + intSize = 32 << (^uint(0) >> 63) + IntSize = intSize +) \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index b12b6afecb3..8b171d450e6 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -11,7 +11,7 @@ func findTokenStart(data []byte, token byte) int { switch data[i] { case token: return i - case '[', '{': + case ArrayStartToken, ObjectStartToken: return 0 } } @@ -21,7 +21,7 @@ func findTokenStart(data []byte, token byte) int { func tokenEnd(data []byte) int { for i, tok := range data { switch tok { - case ' ', '\n', '\r', '\t', ',', '}', ']': + case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken, ValueSeparatorToken, ObjectEndToken, ArrayEndToken: return i } } @@ -34,19 +34,20 @@ func findKeyStart(data []byte, key string) (int, error) { if i == -1 { return i, KeyPathNotFoundError } + ln := len(data) - if ln > 0 && (data[i] == '{' || data[i] == '[') { + if ln > 0 && (data[i] == ObjectStartToken || data[i] == ArrayStartToken) { i += 1 } - var stackbuf [UnescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings + var stackbuf [UnescapeStackBufSize]byte if ku, err := Unescape([]byte(key), stackbuf[:]); err == nil { key = string(ku) } for i < ln { switch data[i] { - case '"': + case DoublyQuoteToken: i++ keyBegin := i @@ -76,18 +77,17 @@ func findKeyStart(data []byte, key string) (int, error) { } } - if data[i] == ':' && len(key) == len(k) && string(k) == key { + if data[i] == ColonToken && len(key) == len(k) && string(k) == key { return keyBegin - 1, nil } - case '[': - end := blockEnd(data[i:], data[i], ']') - if end != -1 { + case ArrayStartToken: + if end := blockEnd(data[i:], data[i], ']'); end != -1 { i = i + end } - case '{': - end := blockEnd(data[i:], data[i], '}') - if end != -1 { + + case ObjectStartToken: + if end := blockEnd(data[i:], data[i], '}'); end != -1 { i = i + end } } @@ -100,7 +100,7 @@ func findKeyStart(data []byte, key string) (int, error) { func tokenStart(data []byte) int { for i := len(data) - 1; i >= 0; i-- { switch data[i] { - case '\n', '\r', '\t', ',', '{', '[': + case NewLineToken, CarriageReturnToken, TabToken, ValueSeparatorToken, ObjectStartToken, ArrayStartToken: return i } } @@ -111,7 +111,7 @@ func tokenStart(data []byte) int { func nextToken(data []byte) (int, error) { for i, c := range data { switch c { - case ' ', '\n', '\r', '\t': + case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken: continue default: return i, nil @@ -124,7 +124,7 @@ func nextToken(data []byte) (int, error) { func lastToken(data []byte) int { for i := len(data) - 1; i >= 0; i-- { switch data[i] { - case ' ', '\n', '\r', '\t': + case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken: continue default: return i @@ -138,7 +138,7 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool) { escaped = false for lastCharIdx, c := range data { switch c { - case '"': + case DoublyQuoteToken: if !escaped { return lastCharIdx + 1, false } @@ -153,7 +153,7 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool) { } return lastCharIdx + 1, false - case '\\': + case BackSlashToken: escaped = true } } @@ -170,18 +170,16 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int { for i < ln { switch data[i] { - case '"': // If inside string, skip it + case DoublyQuoteToken: se, _ := stringEnd(data[i+1:]) if se == -1 { return -1 } i += se - case openSym: // If open symbol, increase level + case openSym: level++ - case closeSym: // If close symbol, increase level + case closeSym: level-- - - // If we have returned to the original level, we're done if level == 0 { return i + 1 } @@ -192,14 +190,12 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int { return -1 } -// TODO: change function name to something more appropriate -// -// searchKeys is a function that takes a byte slice of JSON data and a series of keys. +// findValueIndex is a function that takes a byte slice of JSON data and a series of keys. // It traverses the JSON data to find the position where the value of the final key in the series begins. // The position is returned as an integer representing the index in the byte slice. // If the key series is empty, it returns 0. // If the keys are not found or the JSON data is malformed, it returns -1. -func searchKeys(data []byte, keys ...string) (int, error) { +func findValueIndex(data []byte, keys ...string) (int, error) { keyLevel := 0 level := 0 i := 0 @@ -211,11 +207,10 @@ func searchKeys(data []byte, keys ...string) (int, error) { return 0, nil } - var stackbuf [UnescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings - + var stackbuf [UnescapeStackBufSize]byte for i < ln { switch data[i] { - case '"': + case DoublyQuoteToken: i++ keyBegin := i @@ -232,10 +227,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { } i += valueOffset - - // if string is a key - - if data[i] == ':' { + if data[i] == ColonToken { if level < 1 { return -1, MalformedJson } @@ -274,7 +266,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { } else { i-- } - case '{': + case ObjectStartToken: // in case parent key is matched then only we will increase the level otherwise can directly // can move to the end of this block if !lastMatched { @@ -286,16 +278,15 @@ func searchKeys(data []byte, keys ...string) (int, error) { } else { level += 1 } - case '}': - level -= 1 - if level == keyLevel { + case ObjectEndToken: + if level -= 1; level == keyLevel { keyLevel -= 1 } - case '[': + case ArrayStartToken: // If we want to get array element by index - if keyLevel == level && keys[level][0] == '[' { + if keyLevel == level && keys[level][0] == ArrayEndToken { keyLen := len(keys[level]) - if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { + if keyLen < 3 || keys[level][0] != ArrayStartToken || keys[level][keyLen-1] != ArrayEndToken { return -1, MalformedArray } @@ -323,7 +314,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { return -1, ArrayIndexNotFound } - subIndex, err := searchKeys(valueFound, keys[level+1:]...) + subIndex, err := findValueIndex(valueFound, keys[level+1:]...) if err != nil { return -1, err } @@ -337,7 +328,7 @@ func searchKeys(data []byte, keys ...string) (int, error) { i += arraySkip - 1 } - case ':': // If encountered, JSON data is malformed + case ColonToken: // If encountered, JSON data is malformed return -1, MalformedJson } diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 52f4732b5f1..be66779157a 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -152,8 +152,8 @@ func TestSearchKeys(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, _ := searchKeys(tt.data, tt.keys...); got != tt.expected { - t.Errorf("searchKeys() = %v, want %v", got, tt.expected) + if got, _ := findValueIndex(tt.data, tt.keys...); got != tt.expected { + t.Errorf("findValueIndex() = %v, want %v", got, tt.expected) } }) } diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 9786fb8921e..347a99684fb 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -7,14 +7,6 @@ import ( "strconv" ) -const ( - absMinInt64 = 1 << 63 - maxInt64 = 1<<63 - 1 - maxUint64 = 1<<64 - 1 - intSize = 32 << (^uint(0) >> 63) - IntSize = intSize -) - func ParseString(data []byte) (string, error) { var buf [UnescapeStackBufSize]byte @@ -171,7 +163,7 @@ func ParseUint(b []byte, base, bitSize int) (v uint64, err error) { for _, c := range b { var d byte switch { - case c == '_' && base0: + case c == UnderscoreToken && base0: underscores = true continue case '0' <= c && c <= '9': @@ -279,7 +271,7 @@ func underscoreOK(s string) bool { i := 0 // Optional sign. - if len(s) >= 1 && (s[0] == '-' || s[0] == '+') { + if len(s) >= 1 && (s[0] == MinusToken || s[0] == PlusToken) { s = s[1:] } @@ -291,38 +283,29 @@ func underscoreOK(s string) bool { hex = lower(s[1]) == 'x' } - // Number proper. for ; i < len(s); i++ { - // Digits are always okay. - if '0' <= s[i] && s[i] <= '9' || hex && 'a' <= lower(s[i]) && lower(s[i]) <= 'f' { + currentChar := s[i] + isDigit := '0' <= currentChar && currentChar <= '9' + isHexChar := hex && 'a' <= lower(currentChar) && lower(currentChar) <= 'f' + + if isDigit || isHexChar { saw = '0' - continue - } - // Underscore must follow digit. - if s[i] == '_' { + } else if currentChar == UnderscoreToken { if saw != '0' { return false } - saw = '_' - continue - } - // Underscore must also be followed by digit. - if saw == '_' { - return false + saw = UnderscoreToken + } else { + if saw == UnderscoreToken { + return false + } + saw = BangToken } - // Saw non-digit, non-underscore. - saw = '!' } - return saw != '_' + return saw != UnderscoreToken } -var ( - trueLiteral = []byte("true") - falseLiteral = []byte("false") - nullLiteral = []byte("null") -) - func getType(data []byte, offset int) ([]byte, ValueType, int, error) { if len(data) == 0 { return nil, Unknown, offset, errors.New("no JSON data provided") @@ -338,9 +321,9 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { func parseValue(data []byte, offset int) (ValueType, int, error) { switch data[offset] { - case '"': + case DoublyQuoteToken: return parseString(data, offset) - case '[', '{': + case ArrayStartToken, ObjectStartToken: return parseContainer(data, offset) } @@ -360,12 +343,12 @@ func parseContainer(data []byte, offset int) (ValueType, int, error) { var containerType ValueType var closing byte - if data[offset] == '[' { + if data[offset] == ArrayStartToken { containerType = Array - closing = ']' + closing = ArrayEndToken } else { containerType = Object - closing = '}' + closing = ObjectEndToken } endOffset := blockEnd(data[offset:], data[offset], closing) @@ -385,8 +368,7 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { switch data[offset] { case 't', 'f': - _boolValue, err := ParseBool(value) - if err != nil { + if _boolValue, err := ParseBool(value); err != nil { return Unknown, offset, err } return Boolean, offset + end, nil @@ -397,8 +379,7 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { return Unknown, offset, errors.New("unknown null type") case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': if !bytes.ContainsAny(value, ".eE") { - _intValue, err := ParseInt(value) - if err != nil { + if _intValue, err := ParseInt(value); err != nil { return Number, offset + end, err } } @@ -413,94 +394,90 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { return Unknown, offset, errors.New("unknown value type") } -// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. -func ArrayEach(data []byte, f func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { - if len(data) == 0 { - return -1, MalformedObject - } - - nT, err := nextToken(data) - if err != nil { - return -1, MalformedJson - } - - offset = nT + 1 - - if len(keys) > 0 { - if offset, _ = searchKeys(data, keys...); offset == -1 { - return offset, KeyPathNotFoundError - } - - // Go to closest value - nO, err := nextToken(data[offset:]) - if err != nil { - return offset, MalformedJson - } +func ArrayEach(data []byte, callback func(value []byte, dataType ValueType, offset int, err error), keys ...string) (int, error) { + if len(data) == 0 { + return -1, MalformedObject + } - offset += nO + offset, err := parseInitialOffset(data, keys) + if err != nil { + return -1, err + } - if data[offset] != '[' { - return offset, MalformedArray - } + return parseArrayElements(data, offset, callback) +} - offset++ - } +func parseInitialOffset(data []byte, keys []string) (int, error) { + offset, err := nextToken(data) + if err != nil { + return -1, MalformedJson + } - nO, err := nextToken(data[offset:]) - if err != nil { - return offset, MalformedJson - } + if len(keys) > 0 { + offset, err = findValueIndex(data, keys...) + if err != nil { + return -1, KeyPathNotFoundError + } - offset += nO + if offset, err = parseUntilArrayStart(data, offset); err != nil { + return -1, err + } + } - if data[offset] == ']' { - return offset, nil - } + return offset, nil +} - for true { - v, t, o, e := Get(data[offset:]) +func parseUntilArrayStart(data []byte, offset int) (int, error) { + offset, err := nextToken(data[offset:]) + if err != nil { + return -1, MalformedJson + } - if e != nil { - return offset, e - } + if data[offset] != ArrayStartToken { + return -1, MalformedArray + } - if o == 0 { - break - } + return offset + 1, nil +} - if t != NotExist { - f(v, t, offset+o-len(v), e) - } +func parseArrayElements(data []byte, offset int, callback func(value []byte, dataType ValueType, offset int, err error)) (int, error) { + for data[offset] != ArrayEndToken { + value, valueType, valueOffset, err := Get(data[offset:]) + if err != nil { + return offset, err + } - if e != nil { - break - } + if valueOffset == 0 || valueType == NotExist { + break + } - offset += o + callback(value, valueType, offset+valueOffset-len(value), err) - skipToToken, err := nextToken(data[offset:]) - if err != nil { - return offset, MalformedArray - } - offset += skipToToken + if offset, err = parseNextElementOrEndArray(data, offset+valueOffset); err != nil { + return offset, err + } + } - if data[offset] == ']' { - break - } + return offset, nil +} - if data[offset] != ',' { - return offset, MalformedArray - } +func parseNextElementOrEndArray(data []byte, offset int) (int, error) { + offset, err := nextToken(data[offset:]) + if err != nil { + return -1, MalformedArray + } - offset++ - } + if data[offset] != ValueSeparatorToken && data[offset] != ArrayEndToken { + return -1, MalformedArray + } - return offset, nil + return offset + 1, nil } + func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { if len(keys) > 0 { - offset, err = searchKeys(data, keys...) + offset, err = findValueIndex(data, keys...) if err != nil { return nil, NotExist, -1, -1, err } @@ -513,8 +490,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, } offset += nO - value, dataType, endOffset, err = getType(data, offset) - if err != nil { + if value, dataType, endOffset, err = getType(data, offset); err != nil { return value, dataType, offset, endOffset, err } diff --git a/examples/gno.land/p/demo/json/table.gno b/examples/gno.land/p/demo/json/table.gno deleted file mode 100644 index ca29bc501aa..00000000000 --- a/examples/gno.land/p/demo/json/table.gno +++ /dev/null @@ -1,25 +0,0 @@ -package json - -const ( - Comma = 1 - Colon = 2 - OpenParenKind = 4 - CloseParenKind = 4 - Escape = 8 - Space = 16 -) - -var lookupTable [256]byte - -func init() { - lookupTable[0x2c] = Comma - lookupTable[0x3a] = Colon - lookupTable[0x5b] = OpenParenKind - lookupTable[0x7b] = OpenParenKind - lookupTable[0x5d] = CloseParenKind - lookupTable[0x7d] = CloseParenKind - lookupTable[0x9d] = Escape - lookupTable[0x0a] = Escape - lookupTable[0x0d] = Escape - lookupTable[0x20] = Space -} diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno new file mode 100644 index 00000000000..3df3a1ad986 --- /dev/null +++ b/examples/gno.land/p/demo/json/token.gno @@ -0,0 +1,30 @@ +package json + +const ( + ArrayStartToken = '[' + ArrayEndToken = ']' + ObjectStartToken = '{' + ObjectEndToken = '}' + ValueSeparatorToken = ',' + DotToken = '.' + ColonToken = ':' + QuoteToken = '\'' + DoublyQuoteToken = '"' + EmptyStringToken = "" + WhiteSpaceToken = ' ' + PlusToken = '+' + MinusToken = '-' + BangToken = '!' + NewLineToken = '\n' + TabToken = '\t' + CarriageReturnToken = '\r' + SlashToken = '/' + BackSlashToken = '\\' + UnderscoreToken = '_' +) + +const ( + trueLiteral = []byte("true") + falseLiteral = []byte("false") + nullLiteral = []byte("null") +) \ No newline at end of file diff --git a/gnovm/stdlibs/unicode/utf16/utf16_test.gno b/gnovm/stdlibs/unicode/utf16/utf16_test.gno index 5dc577af805..bc2f39622ae 100644 --- a/gnovm/stdlibs/unicode/utf16/utf16_test.gno +++ b/gnovm/stdlibs/unicode/utf16/utf16_test.gno @@ -5,7 +5,6 @@ package utf16 import ( - // "reflect" "testing" "unicode" ) From 57916bc1c6808fc79ba8b32e44c5df964b07d7d7 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Sun, 17 Dec 2023 21:30:52 +0900 Subject: [PATCH 26/72] type assertion in flatten --- examples/gno.land/p/demo/json/const.gno | 2 +- examples/gno.land/p/demo/json/encoder.gno | 90 +++++------- .../gno.land/p/demo/json/encoder_test.gno | 134 ++++-------------- examples/gno.land/p/demo/json/lexer.gno | 2 +- examples/gno.land/p/demo/json/parser.gno | 113 ++++++++------- examples/gno.land/p/demo/json/token.gno | 40 +++--- 6 files changed, 143 insertions(+), 238 deletions(-) diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno index 9834ae8b51d..f45cee90b48 100644 --- a/examples/gno.land/p/demo/json/const.gno +++ b/examples/gno.land/p/demo/json/const.gno @@ -19,4 +19,4 @@ const ( maxUint64 = 1<<64 - 1 intSize = 32 << (^uint(0) >> 63) IntSize = intSize -) \ No newline at end of file +) diff --git a/examples/gno.land/p/demo/json/encoder.gno b/examples/gno.land/p/demo/json/encoder.gno index 557c176377f..6d3b9edfc5e 100644 --- a/examples/gno.land/p/demo/json/encoder.gno +++ b/examples/gno.land/p/demo/json/encoder.gno @@ -1,66 +1,52 @@ package json import ( + "errors" "fmt" + "strconv" "strings" ) -// Flatten takes a map and returns a new one where nested maps are replaced -// by dot-delimited keys. -func Flatten(m map[string]interface{}) map[string]interface{} { - o := make(map[string]interface{}) +// TODO: should use byte slice as input +func Flatten(m map[string]interface{}) map[string]string { + flattened := make(map[string]string) + var builder strings.Builder + for k, v := range m { - switch child := v.(type) { - case map[string]interface{}: - nm := Flatten(child) - for nk, nv := range nm { - o[k+"."+nk] = nv - } - default: - o[k] = v - } + flattenValue(k, v, flattened, &builder) } - return o -} - -func Unflatten(flatMap map[string]interface{}) map[string]interface{} { - result := make(map[string]interface{}) - - for k, v := range flatMap { - parts := strings.Split(k, ".") - lastIdx := len(parts) - 1 - - currentMap := result - for i, part := range parts { - if i == lastIdx { - currentMap[part] = v - } else { - if _, ok := currentMap[part]; !ok { - currentMap[part] = make(map[string]interface{}) - } + return flattened +} - currentMap = currentMap[part].(map[string]interface{}) - } +func flattenValue(prefix string, val interface{}, flattened map[string]string, builder *strings.Builder) { + switch child := val.(type) { + case map[string]interface{}: + for k, v := range child { + builder.Reset() + builder.WriteString(prefix) + builder.WriteString(".") + builder.WriteString(k) + flattenValue(builder.String(), v, flattened, builder) + } + case []interface{}: + for i, item := range child { + builder.Reset() + builder.WriteString(prefix) + builder.WriteString("[") + builder.WriteString(strconv.Itoa(i)) + builder.WriteString("]") + flattenValue(builder.String(), item, flattened, builder) } + case string: + flattened[prefix] = "string:\"" + child + "\"" + case int, int32, int64: + flattened[prefix] = "number:" + fmt.Sprintf("%d", child) + case float32, float64: + flattened[prefix] = "number:" + fmt.Sprintf("%f", child) + case bool: + flattened[prefix] = "boolean:" + fmt.Sprintf("%t", child) + default: + flattened[prefix] = "unknown" } - - return result } - -func MapToJSONString(m map[string]interface{}) string { - var sb strings.Builder - sb.WriteString("{") - - isFirst := true - for k, v := range m { - if !isFirst { - sb.WriteString(", ") - } - sb.WriteString(fmt.Sprintf("\"%s\": %v", k, v)) - isFirst = false - } - - sb.WriteString("}") - return sb.String() -} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/encoder_test.gno b/examples/gno.land/p/demo/json/encoder_test.gno index 732e904ed11..f685355af76 100644 --- a/examples/gno.land/p/demo/json/encoder_test.gno +++ b/examples/gno.land/p/demo/json/encoder_test.gno @@ -1,6 +1,7 @@ package json import ( + "errors" "strings" "testing" ) @@ -50,123 +51,42 @@ func mapsAreEqual(a, b map[string]interface{}) bool { } func TestFlatten(t *testing.T) { - tests := []struct { - name string - input map[string]interface{} - want map[string]interface{} + testCases := []struct { + name string + input map[string]interface{} + expected map[string]string }{ { - name: "Simple map", - input: map[string]interface{}{"a": 1, "b": 2}, - want: map[string]interface{}{"a": 1, "b": 2}, - }, - { - name: "Nested map", - input: map[string]interface{}{"a": map[string]interface{}{"b": 2, "c": 3}}, - want: map[string]interface{}{"a.b": 2, "a.c": 3}, - }, - { - name: "Empty map", - input: map[string]interface{}{}, - want: map[string]interface{}{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := Flatten(tt.input) - if !equal(got, tt.want) { - t.Errorf("got %v, want %v", got, tt.want) - } - }) - } -} - -func TestUnflatten(t *testing.T) { - tests := []struct { - name string - input map[string]interface{} - want map[string]interface{} - }{ - { - name: "Simple nested map", + name: "Basic Test", input: map[string]interface{}{ - "a.b": 2, - "a.c": 3, - }, - want: map[string]interface{}{ - "a": map[string]interface{}{ - "b": 2, - "c": 3, - }, - }, - }, - { - name: "Multi-level nested map", - input: map[string]interface{}{ - "a.b.c": 1, - "a.b.d": 2, - }, - want: map[string]interface{}{ - "a": map[string]interface{}{ - "b": map[string]interface{}{ - "c": 1, - "d": 2, + "user": map[string]interface{}{ + "name": "John Doe", + "age": 30, + "address": map[string]interface{}{ + "street": "123 Main St", + "city": "Anytown", }, + "favoriteNumbers": []interface{}{7, 13, 42}, }, }, - }, - { - name: "Empty map", - input: map[string]interface{}{}, - want: map[string]interface{}{}, + expected: map[string]string{ + "user.name": "string:\"John Doe\"", + "user.age": "number:30", + "user.address.street": "string:\"123 Main St\"", + "user.address.city": "string:\"Anytown\"", + "user.favoriteNumbers[0]": "number:7", + "user.favoriteNumbers[1]": "number:13", + "user.favoriteNumbers[2]": "number:42", + }, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := Unflatten(tt.input) - if !mapsAreEqual(got, tt.want) { - t.Errorf("got %v, want %v", got, tt.want) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := Flatten(tc.input) + if !mapsAreEqual(result, tc.expected) { + t.Errorf("Expected %v, got %v", tc.expected, result) } }) } } - -func TestMapToJSONString(t *testing.T) { - cases := []struct { - name string - input map[string]interface{} - expected string - }{ - { - name: "Simple Map", - input: map[string]interface{}{ - "a": 1, - "b": 2, - }, - expected: "{\"a\": 1, \"b\": 2}", - }, - { - name: "Nested Map", - input: map[string]interface{}{ - "a": map[string]interface{}{ - "b": 2, - "c": 3, - }, - "d": 4, - }, - expected: "{\"a.b\": 2, \"a.c\": 3, \"d\": 4}", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - flattened := Flatten(tc.input) - result := MapToJSONString(flattened) - if result != tc.expected { - t.Errorf("Test %s failed. Expected %s, got %s", tc.name, tc.expected, result) - } - }) - } -} diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 8b171d450e6..c7ea0f5f6d4 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -11,7 +11,7 @@ func findTokenStart(data []byte, token byte) int { switch data[i] { case token: return i - case ArrayStartToken, ObjectStartToken: + case ArrayStartToken, ObjectStartToken: return 0 } } diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 347a99684fb..87567224aa5 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -287,7 +287,7 @@ func underscoreOK(s string) bool { currentChar := s[i] isDigit := '0' <= currentChar && currentChar <= '9' isHexChar := hex && 'a' <= lower(currentChar) && lower(currentChar) <= 'f' - + if isDigit || isHexChar { saw = '0' } else if currentChar == UnderscoreToken { @@ -395,86 +395,85 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { } func ArrayEach(data []byte, callback func(value []byte, dataType ValueType, offset int, err error), keys ...string) (int, error) { - if len(data) == 0 { - return -1, MalformedObject - } + if len(data) == 0 { + return -1, MalformedObject + } - offset, err := parseInitialOffset(data, keys) - if err != nil { - return -1, err - } + offset, err := parseInitialOffset(data, keys) + if err != nil { + return -1, err + } - return parseArrayElements(data, offset, callback) + return parseArrayElements(data, offset, callback) } func parseInitialOffset(data []byte, keys []string) (int, error) { - offset, err := nextToken(data) - if err != nil { - return -1, MalformedJson - } - - if len(keys) > 0 { - offset, err = findValueIndex(data, keys...) - if err != nil { - return -1, KeyPathNotFoundError - } - - if offset, err = parseUntilArrayStart(data, offset); err != nil { - return -1, err - } - } - - return offset, nil + offset, err := nextToken(data) + if err != nil { + return -1, MalformedJson + } + + if len(keys) > 0 { + offset, err = findValueIndex(data, keys...) + if err != nil { + return -1, KeyPathNotFoundError + } + + if offset, err = parseUntilArrayStart(data, offset); err != nil { + return -1, err + } + } + + return offset, nil } func parseUntilArrayStart(data []byte, offset int) (int, error) { - offset, err := nextToken(data[offset:]) - if err != nil { - return -1, MalformedJson - } + offset, err := nextToken(data[offset:]) + if err != nil { + return -1, MalformedJson + } - if data[offset] != ArrayStartToken { - return -1, MalformedArray - } + if data[offset] != ArrayStartToken { + return -1, MalformedArray + } - return offset + 1, nil + return offset + 1, nil } func parseArrayElements(data []byte, offset int, callback func(value []byte, dataType ValueType, offset int, err error)) (int, error) { - for data[offset] != ArrayEndToken { - value, valueType, valueOffset, err := Get(data[offset:]) - if err != nil { - return offset, err - } + for data[offset] != ArrayEndToken { + value, valueType, valueOffset, err := Get(data[offset:]) + if err != nil { + return offset, err + } - if valueOffset == 0 || valueType == NotExist { - break - } + if valueOffset == 0 || valueType == NotExist { + break + } - callback(value, valueType, offset+valueOffset-len(value), err) + callback(value, valueType, offset+valueOffset-len(value), err) - if offset, err = parseNextElementOrEndArray(data, offset+valueOffset); err != nil { - return offset, err - } - } + if offset, err = parseNextElementOrEndArray(data, offset+valueOffset); err != nil { + return offset, err + } + } - return offset, nil + return offset, nil } func parseNextElementOrEndArray(data []byte, offset int) (int, error) { - offset, err := nextToken(data[offset:]) - if err != nil { - return -1, MalformedArray - } + offset, err := nextToken(data[offset:]) + if err != nil { + return -1, MalformedArray + } - if data[offset] != ValueSeparatorToken && data[offset] != ArrayEndToken { - return -1, MalformedArray - } + if data[offset] != ValueSeparatorToken && data[offset] != ArrayEndToken { + return -1, MalformedArray + } - return offset + 1, nil + return offset + 1, nil } - func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { if len(keys) > 0 { offset, err = findValueIndex(data, keys...) diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 3df3a1ad986..c283ce3d9fb 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -1,30 +1,30 @@ package json const ( - ArrayStartToken = '[' - ArrayEndToken = ']' - ObjectStartToken = '{' - ObjectEndToken = '}' - ValueSeparatorToken = ',' - DotToken = '.' - ColonToken = ':' - QuoteToken = '\'' - DoublyQuoteToken = '"' - EmptyStringToken = "" - WhiteSpaceToken = ' ' - PlusToken = '+' - MinusToken = '-' - BangToken = '!' - NewLineToken = '\n' - TabToken = '\t' + ArrayStartToken = '[' + ArrayEndToken = ']' + ObjectStartToken = '{' + ObjectEndToken = '}' + ValueSeparatorToken = ',' + DotToken = '.' + ColonToken = ':' + QuoteToken = '\'' + DoublyQuoteToken = '"' + EmptyStringToken = "" + WhiteSpaceToken = ' ' + PlusToken = '+' + MinusToken = '-' + BangToken = '!' + NewLineToken = '\n' + TabToken = '\t' CarriageReturnToken = '\r' - SlashToken = '/' - BackSlashToken = '\\' - UnderscoreToken = '_' + SlashToken = '/' + BackSlashToken = '\\' + UnderscoreToken = '_' ) const ( trueLiteral = []byte("true") falseLiteral = []byte("false") nullLiteral = []byte("null") -) \ No newline at end of file +) From 5a54e9a97c54946d7f9e53e4a8c096010f5d7916 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 19 Dec 2023 15:26:15 +0900 Subject: [PATCH 27/72] save --- examples/gno.land/p/demo/json/escape.gno | 22 +- examples/gno.land/p/demo/json/lexer.gno | 271 +++++++++--------- examples/gno.land/p/demo/json/lexer_test.gno | 56 ---- examples/gno.land/p/demo/json/parser.gno | 18 +- examples/gno.land/p/demo/json/parser_test.gno | 26 +- examples/gno.land/p/demo/json/token.gno | 2 + 6 files changed, 166 insertions(+), 229 deletions(-) diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 222f1ef984f..e0073cbc50c 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -7,7 +7,7 @@ import ( ) func Unescape(input, output []byte) ([]byte, error) { - firstBackslash := bytes.IndexByte(input, '\\') + firstBackslash := bytes.IndexByte(input, BackSlashToken) if firstBackslash == -1 { return input, nil } @@ -32,7 +32,7 @@ func Unescape(input, output []byte) ([]byte, error) { buf = buf[bufLen:] // copy everything until the next backslash - nextBackslash := bytes.IndexByte(input, '\\') + nextBackslash := bytes.IndexByte(input, BackSlashToken) if nextBackslash == -1 { copy(buf, input) buf = buf[len(input):] @@ -48,14 +48,14 @@ func Unescape(input, output []byte) ([]byte, error) { } var escapeMap = map[byte]byte{ - '"': '"', - '\\': '\\', - '/': '/', - 'b': '\b', - 'f': '\f', - 'n': '\n', - 'r': '\r', - 't': '\t', + '"': DoublyQuoteToken, + '\\': BackSlashToken, + '/': SlashToken, + 'b': BackspaceToken, + 'f': FormFeedToken, + 'n': NewLineToken, + 'r': CarriageReturnToken, + 't': TabToken, } // isSurrogatePair returns true if the rune is a surrogate pair. @@ -131,7 +131,7 @@ func decodeUnicodeEscape(b []byte) (rune, int) { // If the escape sequence is invalid, or if 'in' does not completely enclose the escape sequence, // function returns (-1, -1) to indicate an error. func processEscapedUTF8(in, out []byte) (inLen int, outLen int) { - if len(in) < 2 || in[0] != '\\' { + if len(in) < 2 || in[0] != BackSlashToken { return -1, -1 } diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index c7ea0f5f6d4..eeabb8e00fe 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -6,18 +6,6 @@ import ( "strconv" ) -func findTokenStart(data []byte, token byte) int { - for i := len(data) - 1; i >= 0; i-- { - switch data[i] { - case token: - return i - case ArrayStartToken, ObjectStartToken: - return 0 - } - } - return 0 -} - func tokenEnd(data []byte) int { for i, tok := range data { switch tok { @@ -45,7 +33,7 @@ func findKeyStart(data []byte, key string) (int, error) { key = string(ku) } - for i < ln { + for _; i < ln; i++ { switch data[i] { case DoublyQuoteToken: i++ @@ -82,32 +70,20 @@ func findKeyStart(data []byte, key string) (int, error) { } case ArrayStartToken: - if end := blockEnd(data[i:], data[i], ']'); end != -1 { + if end := blockEnd(data[i:], data[i], ArrayEndToken); end != -1 { i = i + end } case ObjectStartToken: - if end := blockEnd(data[i:], data[i], '}'); end != -1 { + if end := blockEnd(data[i:], data[i], ObjectEndToken); end != -1 { i = i + end } } - i++ } return -1, KeyPathNotFoundError } -func tokenStart(data []byte) int { - for i := len(data) - 1; i >= 0; i-- { - switch data[i] { - case NewLineToken, CarriageReturnToken, TabToken, ValueSeparatorToken, ObjectStartToken, ArrayStartToken: - return i - } - } - - return 0 -} - func nextToken(data []byte) (int, error) { for i, c := range data { switch c { @@ -121,19 +97,6 @@ func nextToken(data []byte) (int, error) { return -1, TokenNotFound } -func lastToken(data []byte) int { - for i := len(data) - 1; i >= 0; i-- { - switch data[i] { - case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken: - continue - default: - return i - } - } - - return -1 -} - func stringEnd(data []byte) (lastCharIdx int, escaped bool) { escaped = false for lastCharIdx, c := range data { @@ -144,7 +107,7 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool) { } prevCharIdx := lastCharIdx - 1 - for prevCharIdx >= 0 && data[prevCharIdx] == '\\' { + for prevCharIdx >= 0 && data[prevCharIdx] == BackSlashToken { prevCharIdx-- } @@ -165,10 +128,8 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool) { // For array openSym and closeSym will be '[' and ']', for object '{' and '}' func blockEnd(data []byte, openSym byte, closeSym byte) int { level := 0 - i := 0 - ln := len(data) - for i < ln { + for i := 0; i < len(data); i++ { switch data[i] { case DoublyQuoteToken: se, _ := stringEnd(data[i+1:]) @@ -177,14 +138,13 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int { } i += se case openSym: - level++ + level += 1 case closeSym: - level-- + level -= 1 if level == 0 { return i + 1 } } - i++ } return -1 @@ -196,81 +156,65 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int { // If the key series is empty, it returns 0. // If the keys are not found or the JSON data is malformed, it returns -1. func findValueIndex(data []byte, keys ...string) (int, error) { - keyLevel := 0 - level := 0 - i := 0 - ln := len(data) - lk := len(keys) - lastMatched := true - - if lk == 0 { + if len(keys) == 0 { return 0, nil } - var stackbuf [UnescapeStackBufSize]byte - for i < ln { + var ( + i int + keyLevel int + level int + lastMatched bool = true + stackbuf [UnescapeStackBufSize]byte + ) + + for i < len(data) { switch data[i] { case DoublyQuoteToken: - i++ + i += 1 keyBegin := i - + strEnd, keyEscaped := stringEnd(data[i:]) if strEnd == -1 { return -1, MalformedJson } i += strEnd keyEnd := i - 1 - + valueOffset, err := nextToken(data[i:]) if err != nil { return -1, err } - + i += valueOffset - if data[i] == ColonToken { - if level < 1 { - return -1, MalformedJson - } - - key := data[keyBegin:keyEnd] - - // for unescape: if there are no escape sequences, this is cheap; if there are, it is a - // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize - var keyUnesc []byte - if !keyEscaped { - keyUnesc = key - } else if ku, err := Unescape(key, stackbuf[:]); err != nil { - return -1, err - } else { - keyUnesc = ku - } - - if level > len(keys) { - return -1, KeyLevelNotMatched - } - - if bytes.Equal(keyUnesc, []byte(keys[level-1])) { - lastMatched = true - - // if key level match - if keyLevel == level-1 { - keyLevel++ - // If we found all keys in path - if keyLevel == lk { - return i + 1, nil - } - } - } else { - lastMatched = false - } - } else { - i-- + if data[i] != ColonToken { + i -= 1 + continue + } + + if level < 1 { + return -1, MalformedJson + } + + key := data[keyBegin:keyEnd] + keyUnesc, err := keyMatched(key, keyEscaped, stackbuf, keys, level) + if err != nil { + return -1, err + } + + lastMatched = bytes.Equal(keyUnesc, []byte(keys[level-1])) + if (lastMatched && keyLevel == level-1 && keyLevel != len(keys)) { + keyLevel += 1 + } + + if keyLevel == len(keys) { + return i + 1, nil } case ObjectStartToken: // in case parent key is matched then only we will increase the level otherwise can directly // can move to the end of this block if !lastMatched { - end := blockEnd(data[i:], '{', '}') + end := blockEnd(data[i:], ObjectStartToken, ObjectEndToken) if end == -1 { return -1, MalformedJson } @@ -279,55 +223,56 @@ func findValueIndex(data []byte, keys ...string) (int, error) { level += 1 } case ObjectEndToken: - if level -= 1; level == keyLevel { - keyLevel -= 1 - } + level, keyLevel = decreaseLevel(level, keyLevel) case ArrayStartToken: - // If we want to get array element by index - if keyLevel == level && keys[level][0] == ArrayEndToken { - keyLen := len(keys[level]) - if keyLen < 3 || keys[level][0] != ArrayStartToken || keys[level][keyLen-1] != ArrayEndToken { - return -1, MalformedArray - } - - arrIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) - if err != nil { - return -1, InvalidArrayIndex - } - var curIdx int - var valueFound []byte - var valueOffset int - curI := i - ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { - if curIdx == arrIdx { - valueFound = value - valueOffset = offset - if dataType == String { - valueOffset = valueOffset - 2 - valueFound = data[curI+valueOffset : curI+valueOffset+len(value)+2] - } - } - curIdx += 1 - }) - - if valueFound == nil { - return -1, ArrayIndexNotFound - } + var ( + curIdx int + valueFound []byte + valueOffset int + ) - subIndex, err := findValueIndex(valueFound, keys[level+1:]...) - if err != nil { - return -1, err - } + isArrayKey, err := isArrayKey(level, keyLevel, keys) + if err != nil { + return -1, err + } - return i + valueOffset + subIndex, nil - } else { - arraySkip := blockEnd(data[i:], '[', ']') + if !isArrayKey { + arraySkip := blockEnd(data[i:], ArrayStartToken, ArrayEndToken) if arraySkip < 0 { return -1, MalformedJson } i += arraySkip - 1 } + + arrIdx, err := strconv.Atoi(keys[level][1:len(keys[level])-1]) + if err != nil { + return -1, InvalidArrayIndex + } + + currIdx := i + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + if curIdx == arrIdx { + valueFound = value + valueOffset = offset + if dataType == String { + valueOffset = valueOffset - 2 + valueFound = data[currIdx+valueOffset : currIdx+valueOffset+len(value)+2] + } + } + curIdx += 1 + }) + + if valueFound == nil { + return -1, ArrayIndexNotFound + } + + subIdx, err := findValueIndex(valueFound, keys[level+1:]...) + if err != nil { + return -1, err + } + + return i + valueOffset + subIdx, nil case ColonToken: // If encountered, JSON data is malformed return -1, MalformedJson } @@ -338,6 +283,52 @@ func findValueIndex(data []byte, keys ...string) (int, error) { return -1, KeyPathNotFoundError } +func decreaseLevel(level, keyLevel int) (int, int) { + if level -= 1; level == keyLevel { + keyLevel -= 1 + } + + return level, keyLevel +} + +func keyMatched(key []byte, keyEscaped bool, stackbuf [UnescapeStackBufSize]byte, keys []string, level int) ([]byte, error) { + var ( + keyUnesc []byte + err error + ) + + if !keyEscaped { + keyUnesc = key + } + + if ku, err := Unescape(key, stackbuf[:]); err != nil { + return nil, err + } else { + keyUnesc = ku + } + + if level > len(keys) { + return nil, KeyLevelNotMatched + } + + return keyUnesc, err +} + +func isArrayKey(level int, keyLevel int, keys []string) (bool, error) { + if keyLevel != level { + return false, nil + } + + key := keys[level] + keyLen := len(key) + + if keyLen < 3 || key[0] != ArrayStartToken || key[keyLen-1] != ArrayEndToken { + return false, MalformedArray + } + + return true, nil +} + func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { a, b, _, d, e := internalGet(data, keys...) return a, b, d, e diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index be66779157a..7485a1da399 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -5,62 +5,6 @@ import ( "testing" ) -func TestLexer(t *testing.T) { - tests := []struct { - name string - data []byte - start int - next int - last int - }{ - { - name: "Test 1", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - next: 1, - last: 39, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := findTokenStart(tt.data, '{'); got != 0 { - t.Errorf("findTokenStart() = %v, want %v", got, 0) - } - - if got := findTokenStart(tt.data, '['); got != 29 { - t.Errorf("findTokenStart() = %v, want %v", got, 29) - } - - if _, err := findKeyStart(tt.data, "key"); err != nil { - t.Errorf("findKeyStart() returned an error: %v", err) - } - }) - } -} - -func TestLastToken(t *testing.T) { - tests := []struct { - name string - data []byte - expected int - }{ - {"Empty data", []byte{}, -1}, - {"Spaces only", []byte(" "), -1}, - {"Newlines only", []byte("\n\n\n\n"), -1}, - {"Mixed whitespace", []byte(" \n\r\t "), -1}, - {"One character", []byte("a"), 0}, - {"Mixed characters", []byte("abc def"), 6}, - {"Mixed characters with trailing whitespace", []byte("abc def \n\r\t "), 6}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got := lastToken(test.data); got != test.expected { - t.Errorf("lastToken(%q) = %v, want %v", test.data, got, test.expected) - } - }) - } -} func TestBlockEnd(t *testing.T) { tests := []struct { diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 87567224aa5..3ee921cf8f7 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -7,7 +7,7 @@ import ( "strconv" ) -func ParseString(data []byte) (string, error) { +func ParseStringLiteral(data []byte) (string, error) { var buf [UnescapeStackBufSize]byte bf, err := Unescape(data, buf[:]) @@ -18,7 +18,7 @@ func ParseString(data []byte) (string, error) { return string(bf), nil } -func ParseBool(data []byte) (bool, error) { +func ParseBoolLiteral(data []byte) (bool, error) { switch { case bytes.Equal(data, []byte("true")): return true, nil @@ -29,7 +29,7 @@ func ParseBool(data []byte) (bool, error) { } } -func ParseFloat(bytes []byte) (value float64, err error) { +func ParseFloatLiteral(bytes []byte) (value float64, err error) { if len(bytes) == 0 { return -1, EmptyBytes } @@ -66,7 +66,7 @@ func ParseFloat(bytes []byte) (value float64, err error) { return f, nil } -func ParseInt(bytes []byte) (v int64, err error) { +func ParseIntLiteral(bytes []byte) (v int64, err error) { if len(bytes) == 0 { return 0, EmptyBytes } @@ -368,7 +368,7 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { switch data[offset] { case 't', 'f': - if _boolValue, err := ParseBool(value); err != nil { + if _boolValue, err := ParseBoolLiteral(value); err != nil { return Unknown, offset, err } return Boolean, offset + end, nil @@ -379,12 +379,12 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { return Unknown, offset, errors.New("unknown null type") case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': if !bytes.ContainsAny(value, ".eE") { - if _intValue, err := ParseInt(value); err != nil { + if _intValue, err := ParseIntLiteral(value); err != nil { return Number, offset + end, err } } - if _floatValue, err := ParseFloat(value); err != nil { + if _floatValue, err := ParseFloatLiteral(value); err != nil { return Number, offset + end, err } @@ -440,7 +440,7 @@ func parseUntilArrayStart(data []byte, offset int) (int, error) { return offset + 1, nil } -func parseArrayElements(data []byte, offset int, callback func(value []byte, dataType ValueType, offset int, err error)) (int, error) { +func parseArrayElements(data []byte, offset int, f func(value []byte, dataType ValueType, offset int, err error)) (int, error) { for data[offset] != ArrayEndToken { value, valueType, valueOffset, err := Get(data[offset:]) if err != nil { @@ -451,7 +451,7 @@ func parseArrayElements(data []byte, offset int, callback func(value []byte, dat break } - callback(value, valueType, offset+valueOffset-len(value), err) + f(value, valueType, offset+valueOffset-len(value), err) if offset, err = parseNextElementOrEndArray(data, offset+valueOffset); err != nil { return offset, err diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index b02dfa6a3f6..09c434f829d 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -5,7 +5,7 @@ import ( "testing" ) -func TestParseString(t *testing.T) { +func TestParseStringLiteral(t *testing.T) { tests := []struct { input string expected string @@ -20,7 +20,7 @@ func TestParseString(t *testing.T) { } for i, tt := range tests { - s, err := ParseString([]byte(tt.input)) + s, err := ParseStringLiteral([]byte(tt.input)) if !tt.isError && err != nil { t.Errorf("%d. unexpected error: %s", i, err) @@ -36,7 +36,7 @@ func TestParseString(t *testing.T) { } } -func TestParseBool(t *testing.T) { +func TestParseBoolLiteral(t *testing.T) { tests := []struct { input string expected bool @@ -52,7 +52,7 @@ func TestParseBool(t *testing.T) { } for i, tt := range tests { - b, err := ParseBool([]byte(tt.input)) + b, err := ParseBoolLiteral([]byte(tt.input)) if !tt.isError && err != nil { t.Errorf("%d. unexpected error: %s", i, err) @@ -68,7 +68,7 @@ func TestParseBool(t *testing.T) { } } -func TestParseFloat(t *testing.T) { +func TestParseFloatLiteral(t *testing.T) { testCases := []struct { input string expected float64 @@ -87,9 +87,9 @@ func TestParseFloat(t *testing.T) { for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { - got, _ := ParseFloat([]byte(tc.input)) + got, _ := ParseFloatLiteral([]byte(tc.input)) if got != tc.expected { - t.Errorf("ParseFloat(%s): got %v, want %v", tc.input, got, tc.expected) + t.Errorf("ParseFloatLiteral(%s): got %v, want %v", tc.input, got, tc.expected) } }) } @@ -122,19 +122,19 @@ func TestParseFloatWithScientificNotation(t *testing.T) { for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { - got, err := ParseFloat([]byte(tc.input)) + got, err := ParseFloatLiteral([]byte(tc.input)) if got != tc.expected { - t.Errorf("ParseFloat(%s): got %v, want %v", tc.input, got, tc.expected) + t.Errorf("ParseFloatLiteral(%s): got %v, want %v", tc.input, got, tc.expected) } if err != nil { - t.Errorf("ParseFloat(%s): got error %v", tc.input, err) + t.Errorf("ParseFloatLiteral(%s): got error %v", tc.input, err) } }) } } -func TestParseInt(t *testing.T) { +func TestParseIntLiteral(t *testing.T) { testCases := []struct { input string expected int64 @@ -161,9 +161,9 @@ func TestParseInt(t *testing.T) { for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { - got, _ := ParseInt([]byte(tc.input)) + got, _ := ParseIntLiteral([]byte(tc.input)) if got != tc.expected { - t.Errorf("ParseInt(%s): got %v, want %v", tc.input, got, tc.expected) + t.Errorf("ParseIntLiteral(%s): got %v, want %v", tc.input, got, tc.expected) } }) } diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index c283ce3d9fb..8a72264eab1 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -18,6 +18,8 @@ const ( NewLineToken = '\n' TabToken = '\t' CarriageReturnToken = '\r' + FormFeedToken = '\f' + BackspaceToken = '\b' SlashToken = '/' BackSlashToken = '\\' UnderscoreToken = '_' From 5f138b63673cc5db66cace6a4f6c7004a8e95731 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 20 Dec 2023 18:58:05 +0900 Subject: [PATCH 28/72] key extract --- examples/gno.land/p/demo/json/lexer.gno | 54 +++++++++++++++++--- examples/gno.land/p/demo/json/lexer_test.gno | 45 ++++++++++++++++ 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index eeabb8e00fe..ed622bc19f7 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -283,6 +283,53 @@ func findValueIndex(data []byte, keys ...string) (int, error) { return -1, KeyPathNotFoundError } +func ExtractKeys(data []byte) (keys []string, err error) { + var ( + stack []byte + inString bool + potentialKey string + afterString bool + ) + + for i := 0; i < len(data); i++ { + switch data[i] { + case ObjectStartToken, ArrayStartToken: + stack = append(stack, data[i]) + case ObjectEndToken, ArrayEndToken: + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + case DoublyQuoteToken: + if inString { + // End of string + inString = false + afterString = true + } else { + // Start of string + inString = true + potentialKey = "" + if len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { + afterString = false + } + } + case ColonToken: + if afterString && len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { + keys = append(keys, potentialKey) + afterString = false + } + default: + if inString { + potentialKey += string(data[i]) + } + if !inString && data[i] != WhiteSpaceToken && data[i] != NewLineToken && data[i] != CarriageReturnToken && data[i] != TabToken { + afterString = false + } + } + } + + return keys, nil +} + func decreaseLevel(level, keyLevel int) (int, int) { if level -= 1; level == keyLevel { keyLevel -= 1 @@ -291,12 +338,7 @@ func decreaseLevel(level, keyLevel int) (int, int) { return level, keyLevel } -func keyMatched(key []byte, keyEscaped bool, stackbuf [UnescapeStackBufSize]byte, keys []string, level int) ([]byte, error) { - var ( - keyUnesc []byte - err error - ) - +func keyMatched(key []byte, keyEscaped bool, stackbuf [UnescapeStackBufSize]byte, keys []string, level int) (keyUnesc []byte, err error) { if !keyEscaped { keyUnesc = key } diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 7485a1da399..41293181d61 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -145,3 +145,48 @@ func TestGet(t *testing.T) { } } } + +func TestExtractKeys(t *testing.T) { + tests := []struct { + name string + jsonData []byte + expected []string + }{ + { + name: "Simple Object", + jsonData: []byte(`{"name": "John", "age": 30}`), + expected: []string{"name", "age"}, + }, + { + name: "Nested Object", + jsonData: []byte(`{"person": {"name": "John", "age": 30}, "city": "New York"}`), + expected: []string{"person", "name", "age", "city"}, + }, + { + name: "Array of Objects", + jsonData: []byte(`[{"name": "John"}, {"age": 30}]`), + expected: []string{"name", "age"}, + }, + { + name: "Empty Object", + jsonData: []byte(`{}`), + expected: []string{}, + }, + // Additional test cases can be added here. + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := ExtractKeys(test.jsonData) + if err != nil { + t.Errorf("name: %s error = %v", test.name, err) + return + } + + if len(result) != len(test.expected) { + t.Errorf("name: %s, expected %d keys, got %d. contents: %s", test.name, len(test.expected), len(result), result) + return + } + }) + } +} \ No newline at end of file From ebef5d5a1fae3393891ad2bd48a13e6f72137298 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 20 Dec 2023 22:13:28 +0900 Subject: [PATCH 29/72] re-organize the files --- .../gno.land/p/demo/json/eisel_lemire.gno | 3 +- examples/gno.land/p/demo/json/encoder.gno | 59 ++++ .../gno.land/p/demo/json/encoder_test.gno | 103 +++++++ examples/gno.land/p/demo/json/lexer.gno | 280 +----------------- examples/gno.land/p/demo/json/lexer_test.gno | 162 ---------- examples/gno.land/p/demo/json/parser.gno | 46 --- .../gno.land/p/demo/json/state_machine.gno | 266 +++++++++++++++++ .../p/demo/json/state_machine_test.gno | 121 ++++++++ examples/gno.land/p/demo/json/token.gno | 2 +- 9 files changed, 561 insertions(+), 481 deletions(-) create mode 100644 examples/gno.land/p/demo/json/state_machine.gno create mode 100644 examples/gno.land/p/demo/json/state_machine_test.gno diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno index 4e16e2d6124..343e57739f7 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -1,5 +1,3 @@ -package json - // This file implements the Eisel-Lemire ParseFloat algorithm, published in // 2020 and discussed extensively at // https://nigeltao.github.io/blog/2020/eisel-lemire.html @@ -12,6 +10,7 @@ package json // // Additional testing (on over several million test strings) is done by // https://github.com/nigeltao/parse-number-fxx-test-data/blob/5280dcfccf6d0b02a65ae282dad0b6d9de50e039/script/test-go-strconv.go +package json import ( "math" diff --git a/examples/gno.land/p/demo/json/encoder.gno b/examples/gno.land/p/demo/json/encoder.gno index 6d3b9edfc5e..3ed1871a786 100644 --- a/examples/gno.land/p/demo/json/encoder.gno +++ b/examples/gno.land/p/demo/json/encoder.gno @@ -50,3 +50,62 @@ func flattenValue(prefix string, val interface{}, flattened map[string]string, b flattened[prefix] = "unknown" } } + +// extractKeys extracts all keys from a JSON object. +func extractKeys(data []byte) (keys []string, err error) { + var ( + // stack tracks nested structures in JSON, such as objects or arrays. + // If the top of the stack is an ObjectStartToken ({), + // it confirms we're inside an object. + stack []byte + // inString flag indicates whether the parser is currently within a string. + inString bool + potentialKey string + // afterString flag indicates whether the last parsed token was a string, + // suggesting we may be just after a key in a key-value pair. + afterString bool + ) + + for i := 0; i < len(data); i++ { + switch data[i] { + case ObjectStartToken, ArrayStartToken: + stack = append(stack, data[i]) + case ObjectEndToken, ArrayEndToken: + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + case DoublyQuoteToken: + if inString { + // End of string + inString = false + afterString = true + } else { + // Start of string + inString = true + potentialKey = "" + if len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { + afterString = false + } + } + case ColonToken: + // checks if the parser is currently within a key-value pair in a JSON object. + if afterString && len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { + keys = append(keys, potentialKey) + afterString = false + } + default: + if inString { + potentialKey += string(data[i]) + } + if isNotWhiteSpaceOrControlChar(inString, data[i]) { + afterString = false + } + } + } + + return keys, nil +} + +func isNotWhiteSpaceOrControlChar(inString bool, char byte) bool { + return !inString && char != WhiteSpaceToken && char != NewLineToken && char != CarriageReturnToken && char != TabToken +} diff --git a/examples/gno.land/p/demo/json/encoder_test.gno b/examples/gno.land/p/demo/json/encoder_test.gno index f685355af76..b2fd6b2d9d1 100644 --- a/examples/gno.land/p/demo/json/encoder_test.gno +++ b/examples/gno.land/p/demo/json/encoder_test.gno @@ -90,3 +90,106 @@ func TestFlatten(t *testing.T) { }) } } + +func TestExtractKeys(t *testing.T) { + tests := []struct { + name string + jsonData []byte + expected []string + }{ + { + name: "Simple Object", + jsonData: []byte(`{"name": "John", "age": 30}`), + expected: []string{"name", "age"}, + }, + { + name: "Nested Object", + jsonData: []byte(`{"person": {"name": "John", "age": 30}, "city": "New York"}`), + expected: []string{"person", "name", "age", "city"}, + }, + { + name: "Array of Objects", + jsonData: []byte(`[{"name": "John"}, {"age": 30}]`), + expected: []string{"name", "age"}, + }, + { + name: "Nested Object 2", + jsonData: []byte(`{ + "a": { + "b": { + "c": 1, + "d": 2 + } + }, + "e": { + "f": 3 + } + }`), + expected: []string{"a", "b", "c", "d", "e", "f"}, + }, + { + name: "Empty Object", + jsonData: []byte(`{}`), + expected: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := extractKeys(test.jsonData) + if err != nil { + t.Errorf("name: %s error = %v", test.name, err) + return + } + + if len(result) != len(test.expected) { + t.Errorf("name: %s, expected %d keys, got %d. contents: %s", test.name, len(test.expected), len(result), result) + return + } + }) + } +} + +// func TestGetTypeFromKey(t *testing.T) { +// testCases := []struct { +// name string +// data []byte +// key string +// expected ValueType +// err error +// }{ +// { +// name: "Key Exists", +// data: []byte(`{"name": "John", "age": 30}`), +// key: "name", +// expected: String, +// err: nil, +// }, +// { +// name: "Key Does Not Exist", +// data: []byte(`{"name": "John", "age": 30}`), +// key: "address", +// expected: NotExist, +// err: errors.New("key not found"), +// }, +// { +// name: "Empty Data", +// data: []byte(`{}`), +// key: "name", +// expected: NotExist, +// err: errors.New("key not found"), +// }, +// } + +// for _, tc := range testCases { +// t.Run(tc.name, func(t *testing.T) { +// result, err := getTypeFromKey(tc.data, tc.key) +// if result != tc.expected { +// t.Errorf("Expected %v, got %v", tc.expected, result) +// } +// if (err == nil && tc.err != nil) || (err != nil && tc.err == nil) || (err != nil && tc.err != nil && err.Error() != tc.err.Error()) { +// t.Errorf("Expected error: %v, got error: %v", tc.err, err) +// } +// }) +// } +// } diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index ed622bc19f7..8fb450cccb1 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -17,73 +17,6 @@ func tokenEnd(data []byte) int { return len(data) } -func findKeyStart(data []byte, key string) (int, error) { - i, _ := nextToken(data) - if i == -1 { - return i, KeyPathNotFoundError - } - - ln := len(data) - if ln > 0 && (data[i] == ObjectStartToken || data[i] == ArrayStartToken) { - i += 1 - } - - var stackbuf [UnescapeStackBufSize]byte - if ku, err := Unescape([]byte(key), stackbuf[:]); err == nil { - key = string(ku) - } - - for _; i < ln; i++ { - switch data[i] { - case DoublyQuoteToken: - i++ - keyBegin := i - - strEnd, keyEscaped := stringEnd(data[i:]) - if strEnd == -1 { - break - } - i += strEnd - keyEnd := i - 1 - - valueOffset, _ := nextToken(data[i:]) - if valueOffset == -1 { - break - } - - i += valueOffset - - // if string is a key, and key level match - k := data[keyBegin:keyEnd] - // for unescape: if there are no escape sequences, this is cheap; if there are, it is a - // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize - if keyEscaped { - if ku, err := Unescape(k, stackbuf[:]); err != nil { - break - } else { - k = ku - } - } - - if data[i] == ColonToken && len(key) == len(k) && string(k) == key { - return keyBegin - 1, nil - } - - case ArrayStartToken: - if end := blockEnd(data[i:], data[i], ArrayEndToken); end != -1 { - i = i + end - } - - case ObjectStartToken: - if end := blockEnd(data[i:], data[i], ObjectEndToken); end != -1 { - i = i + end - } - } - } - - return -1, KeyPathNotFoundError -} - func nextToken(data []byte) (int, error) { for i, c := range data { switch c { @@ -150,199 +83,26 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int { return -1 } -// findValueIndex is a function that takes a byte slice of JSON data and a series of keys. -// It traverses the JSON data to find the position where the value of the final key in the series begins. -// The position is returned as an integer representing the index in the byte slice. -// If the key series is empty, it returns 0. -// If the keys are not found or the JSON data is malformed, it returns -1. -func findValueIndex(data []byte, keys ...string) (int, error) { - if len(keys) == 0 { - return 0, nil - } - - var ( - i int - keyLevel int - level int - lastMatched bool = true - stackbuf [UnescapeStackBufSize]byte - ) - - for i < len(data) { - switch data[i] { - case DoublyQuoteToken: - i += 1 - keyBegin := i - - strEnd, keyEscaped := stringEnd(data[i:]) - if strEnd == -1 { - return -1, MalformedJson - } - i += strEnd - keyEnd := i - 1 - - valueOffset, err := nextToken(data[i:]) - if err != nil { - return -1, err - } - - i += valueOffset - if data[i] != ColonToken { - i -= 1 - continue - } - - if level < 1 { - return -1, MalformedJson - } - - key := data[keyBegin:keyEnd] - keyUnesc, err := keyMatched(key, keyEscaped, stackbuf, keys, level) - if err != nil { - return -1, err - } - - lastMatched = bytes.Equal(keyUnesc, []byte(keys[level-1])) - if (lastMatched && keyLevel == level-1 && keyLevel != len(keys)) { - keyLevel += 1 - } - - if keyLevel == len(keys) { - return i + 1, nil - } - case ObjectStartToken: - // in case parent key is matched then only we will increase the level otherwise can directly - // can move to the end of this block - if !lastMatched { - end := blockEnd(data[i:], ObjectStartToken, ObjectEndToken) - if end == -1 { - return -1, MalformedJson - } - i += end - 1 - } else { - level += 1 - } - case ObjectEndToken: - level, keyLevel = decreaseLevel(level, keyLevel) - case ArrayStartToken: - var ( - curIdx int - valueFound []byte - valueOffset int - ) - - isArrayKey, err := isArrayKey(level, keyLevel, keys) - if err != nil { - return -1, err - } - - if !isArrayKey { - arraySkip := blockEnd(data[i:], ArrayStartToken, ArrayEndToken) - if arraySkip < 0 { - return -1, MalformedJson - } - - i += arraySkip - 1 - } - - arrIdx, err := strconv.Atoi(keys[level][1:len(keys[level])-1]) - if err != nil { - return -1, InvalidArrayIndex - } - - currIdx := i - ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { - if curIdx == arrIdx { - valueFound = value - valueOffset = offset - if dataType == String { - valueOffset = valueOffset - 2 - valueFound = data[currIdx+valueOffset : currIdx+valueOffset+len(value)+2] - } - } - curIdx += 1 - }) - - if valueFound == nil { - return -1, ArrayIndexNotFound - } - - subIdx, err := findValueIndex(valueFound, keys[level+1:]...) - if err != nil { - return -1, err - } - - return i + valueOffset + subIdx, nil - case ColonToken: // If encountered, JSON data is malformed - return -1, MalformedJson - } - - i++ +func isArrayKey(level int, keyLevel int, keys []string) (bool, error) { + if keyLevel != level { + return false, nil } - return -1, KeyPathNotFoundError -} - -func ExtractKeys(data []byte) (keys []string, err error) { - var ( - stack []byte - inString bool - potentialKey string - afterString bool - ) - - for i := 0; i < len(data); i++ { - switch data[i] { - case ObjectStartToken, ArrayStartToken: - stack = append(stack, data[i]) - case ObjectEndToken, ArrayEndToken: - if len(stack) > 0 { - stack = stack[:len(stack)-1] - } - case DoublyQuoteToken: - if inString { - // End of string - inString = false - afterString = true - } else { - // Start of string - inString = true - potentialKey = "" - if len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { - afterString = false - } - } - case ColonToken: - if afterString && len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { - keys = append(keys, potentialKey) - afterString = false - } - default: - if inString { - potentialKey += string(data[i]) - } - if !inString && data[i] != WhiteSpaceToken && data[i] != NewLineToken && data[i] != CarriageReturnToken && data[i] != TabToken { - afterString = false - } - } - } + key := keys[level] + keyLen := len(key) - return keys, nil -} - -func decreaseLevel(level, keyLevel int) (int, int) { - if level -= 1; level == keyLevel { - keyLevel -= 1 + if keyLen < 3 || key[0] != ArrayStartToken || key[keyLen-1] != ArrayEndToken { + return false, MalformedArray } - return level, keyLevel + return true, nil } func keyMatched(key []byte, keyEscaped bool, stackbuf [UnescapeStackBufSize]byte, keys []string, level int) (keyUnesc []byte, err error) { if !keyEscaped { keyUnesc = key - } - + } + if ku, err := Unescape(key, stackbuf[:]); err != nil { return nil, err } else { @@ -355,23 +115,3 @@ func keyMatched(key []byte, keyEscaped bool, stackbuf [UnescapeStackBufSize]byte return keyUnesc, err } - -func isArrayKey(level int, keyLevel int, keys []string) (bool, error) { - if keyLevel != level { - return false, nil - } - - key := keys[level] - keyLen := len(key) - - if keyLen < 3 || key[0] != ArrayStartToken || key[keyLen-1] != ArrayEndToken { - return false, MalformedArray - } - - return true, nil -} - -func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { - a, b, _, d, e := internalGet(data, keys...) - return a, b, d, e -} diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno index 41293181d61..2e838cb8774 100644 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ b/examples/gno.land/p/demo/json/lexer_test.gno @@ -5,7 +5,6 @@ import ( "testing" ) - func TestBlockEnd(t *testing.T) { tests := []struct { name string @@ -29,164 +28,3 @@ func TestBlockEnd(t *testing.T) { }) } } - -func TestFindKeyStart(t *testing.T) { - tests := []struct { - name string - data []byte - key string - expected int - }{ - { - name: "Test 1", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - key: "key", - expected: 3, - }, - { - name: "Test 2", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - key: "array", - expected: 20, - }, - { - name: "Test 3", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - key: "value", - expected: -1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, _ := findKeyStart(tt.data, tt.key) - if got != tt.expected { - t.Errorf("findKeyStart() = %v, want %v", got, tt.expected) - } - }) - } -} - -func TestSearchKeys(t *testing.T) { - tests := []struct { - name string - data []byte - keys []string - expected int - }{ - { - name: "Test 1", - data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), - keys: []string{"key1"}, - expected: 8, - }, - { - name: "Test 2", - data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), - keys: []string{"key2"}, - expected: 26, - }, - { - name: "Test 3", - data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), - keys: []string{"key4"}, - expected: -1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got, _ := findValueIndex(tt.data, tt.keys...); got != tt.expected { - t.Errorf("findValueIndex() = %v, want %v", got, tt.expected) - } - }) - } -} - -func TestGet(t *testing.T) { - data := []byte(`{ - "person": { - "name": { - "first": "Alex", - "last": "Johnson", - "fullName": "Alex Johnson" - }, - "github": { - "handle": "alexj", - "followers": 152 - } - }, - "company": { - "name": "company name", - }, - } - `) - - tests := []struct { - path []string - expected string - }{ - {[]string{"person", "name", "first"}, "Alex"}, - {[]string{"person", "name", "last"}, "Johnson"}, - {[]string{"person", "name", "fullName"}, "Alex Johnson"}, - {[]string{"person", "github", "handle"}, "alexj"}, - {[]string{"person", "github", "followers"}, "152"}, - {[]string{"company", "name"}, "company name"}, - } - - for _, test := range tests { - value, _, _, err := Get(data, test.path...) - if err != nil { - t.Errorf("Got error: %v", err) - } - - if string(value) != test.expected { - t.Errorf("Expected '%s', got '%s'", test.expected, value) - } - } -} - -func TestExtractKeys(t *testing.T) { - tests := []struct { - name string - jsonData []byte - expected []string - }{ - { - name: "Simple Object", - jsonData: []byte(`{"name": "John", "age": 30}`), - expected: []string{"name", "age"}, - }, - { - name: "Nested Object", - jsonData: []byte(`{"person": {"name": "John", "age": 30}, "city": "New York"}`), - expected: []string{"person", "name", "age", "city"}, - }, - { - name: "Array of Objects", - jsonData: []byte(`[{"name": "John"}, {"age": 30}]`), - expected: []string{"name", "age"}, - }, - { - name: "Empty Object", - jsonData: []byte(`{}`), - expected: []string{}, - }, - // Additional test cases can be added here. - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result, err := ExtractKeys(test.jsonData) - if err != nil { - t.Errorf("name: %s error = %v", test.name, err) - return - } - - if len(result) != len(test.expected) { - t.Errorf("name: %s, expected %d keys, got %d. contents: %s", test.name, len(test.expected), len(result), result) - return - } - }) - } -} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 3ee921cf8f7..95ac9e92396 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -239,25 +239,6 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { return man, exp10, nil } -func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { - switch b { - case 't', 'f': - if bytes.Equal(value, TrueLiteral) || bytes.Equal(value, FalseLiteral) { - dataType = Boolean - } - return Unknown, offset, UnknownValueType - case 'u', 'n': - if bytes.Equal(value, NullLiteral) { - dataType = Null - } - return Unknown, offset, UnknownValueType - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': - dataType = Number - default: - return Unknown, offset, UnknownValueType - } -} - // underscoreOK reports whether the underscores in s are allowed. // Checking them in this one function lets all the parsers skip over them simply. // Underscore must appear only between digits or between a base prefix and a digit. @@ -473,30 +454,3 @@ func parseNextElementOrEndArray(data []byte, offset int) (int, error) { return offset + 1, nil } - -func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { - if len(keys) > 0 { - offset, err = findValueIndex(data, keys...) - if err != nil { - return nil, NotExist, -1, -1, err - } - } - - // Go to closest value - nO, err := nextToken(data[offset:]) - if err != nil { - return nil, NotExist, offset, -1, MalformedJson - } - - offset += nO - if value, dataType, endOffset, err = getType(data, offset); err != nil { - return value, dataType, offset, endOffset, err - } - - // Strip quotes from string values - if dataType == String { - value = value[1 : len(value)-1] - } - - return value[:len(value):len(value)], dataType, offset, endOffset, nil -} diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno new file mode 100644 index 00000000000..41b8e15dee2 --- /dev/null +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -0,0 +1,266 @@ +package json + +import ( + "bytes" + "errors" + "strconv" +) + +func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { + switch b { + case 't', 'f': + if bytes.Equal(value, TrueLiteral) || bytes.Equal(value, FalseLiteral) { + dataType = Boolean + } + return Unknown, offset, UnknownValueType + case 'u', 'n': + if bytes.Equal(value, NullLiteral) { + dataType = Null + } + return Unknown, offset, UnknownValueType + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': + dataType = Number + default: + return Unknown, offset, UnknownValueType + } +} + +// findValueIndex is a function that takes a byte slice of JSON data and a series of keys. +// It traverses the JSON data to find the position where the value of the final key in the series begins. +// The position is returned as an integer representing the index in the byte slice. +func findValueIndex(data []byte, keys ...string) (int, error) { + if len(keys) == 0 { + return 0, nil + } + + var ( + i int + keyLevel int + level int + lastMatched bool = true + stackbuf [UnescapeStackBufSize]byte + ) + + for i < len(data) { + switch data[i] { + case DoublyQuoteToken: + i += 1 + keyBegin := i + + strEnd, keyEscaped := stringEnd(data[i:]) + if strEnd == -1 { + return -1, MalformedJson + } + i += strEnd + keyEnd := i - 1 + + valueOffset, err := nextToken(data[i:]) + if err != nil { + return -1, err + } + + i += valueOffset + if data[i] != ColonToken { + i -= 1 + continue + } + + if level < 1 { + return -1, MalformedJson + } + + key := data[keyBegin:keyEnd] + keyUnesc, err := keyMatched(key, keyEscaped, stackbuf, keys, level) + if err != nil { + return -1, err + } + + lastMatched = bytes.Equal(keyUnesc, []byte(keys[level-1])) + if lastMatched && keyLevel == level-1 && keyLevel != len(keys) { + keyLevel += 1 + } + + if keyLevel == len(keys) { + return i + 1, nil + } + case ObjectStartToken: + // in case parent key is matched then only we will increase the level otherwise can directly + // can move to the end of this block + if !lastMatched { + end := blockEnd(data[i:], ObjectStartToken, ObjectEndToken) + if end == -1 { + return -1, MalformedJson + } + i += end - 1 + } else { + level += 1 + } + case ObjectEndToken: + level, keyLevel = decreaseLevel(level, keyLevel) + case ArrayStartToken: + var ( + curIdx int + valueFound []byte + valueOffset int + ) + + isArrayKey, err := isArrayKey(level, keyLevel, keys) + if err != nil { + return -1, err + } + + if !isArrayKey { + arraySkip := blockEnd(data[i:], ArrayStartToken, ArrayEndToken) + if arraySkip < 0 { + return -1, MalformedJson + } + + i += arraySkip - 1 + } + + arrIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1]) + if err != nil { + return -1, InvalidArrayIndex + } + + currIdx := i + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + if curIdx == arrIdx { + valueFound = value + valueOffset = offset + if dataType == String { + valueOffset = valueOffset - 2 + valueFound = data[currIdx+valueOffset : currIdx+valueOffset+len(value)+2] + } + } + curIdx += 1 + }) + + if valueFound == nil { + return -1, ArrayIndexNotFound + } + + subIdx, err := findValueIndex(valueFound, keys[level+1:]...) + if err != nil { + return -1, err + } + + return i + valueOffset + subIdx, nil + case ColonToken: // If encountered, JSON data is malformed + return -1, MalformedJson + } + + i++ + } + + return -1, KeyPathNotFoundError +} + +func decreaseLevel(level, keyLevel int) (int, int) { + if level -= 1; level == keyLevel { + keyLevel -= 1 + } + + return level, keyLevel +} + +func findKeyStart(data []byte, key string) (int, error) { + i, _ := nextToken(data) + if i == -1 { + return i, KeyPathNotFoundError + } + + ln := len(data) + if ln > 0 && (data[i] == ObjectStartToken || data[i] == ArrayStartToken) { + i += 1 + } + + var stackbuf [UnescapeStackBufSize]byte + if ku, err := Unescape([]byte(key), stackbuf[:]); err == nil { + key = string(ku) + } + + for _; i < ln; i++ { + switch data[i] { + case DoublyQuoteToken: + i++ + keyBegin := i + + strEnd, keyEscaped := stringEnd(data[i:]) + if strEnd == -1 { + break + } + i += strEnd + keyEnd := i - 1 + + valueOffset, _ := nextToken(data[i:]) + if valueOffset == -1 { + break + } + + i += valueOffset + + // if string is a key, and key level match + k := data[keyBegin:keyEnd] + // for unescape: if there are no escape sequences, this is cheap; if there are, it is a + // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize + if keyEscaped { + if ku, err := Unescape(k, stackbuf[:]); err != nil { + break + } else { + k = ku + } + } + + if data[i] == ColonToken && len(key) == len(k) && string(k) == key { + return keyBegin - 1, nil + } + + case ArrayStartToken: + if end := blockEnd(data[i:], data[i], ArrayEndToken); end != -1 { + i = i + end + } + + case ObjectStartToken: + if end := blockEnd(data[i:], data[i], ObjectEndToken); end != -1 { + i = i + end + } + } + } + + return -1, KeyPathNotFoundError +} + +func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { + if len(keys) > 0 { + offset, err = findValueIndex(data, keys...) + if err != nil { + return nil, NotExist, -1, -1, err + } + } + + // Go to closest value + nO, err := nextToken(data[offset:]) + if err != nil { + return nil, NotExist, offset, -1, MalformedJson + } + + offset += nO + if value, dataType, endOffset, err = getType(data, offset); err != nil { + return value, dataType, offset, endOffset, err + } + + // Strip quotes from string values + if dataType == String { + value = value[1 : len(value)-1] + } + + return value[:len(value):len(value)], dataType, offset, endOffset, nil +} + +// Get is a function that retrieves a value from the given data based on the provided keys. +// It returns the value, data type, offset, and any error encountered during the retrieval process. +func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { + a, b, _, d, e := internalGet(data, keys...) + return a, b, d, e +} diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno new file mode 100644 index 00000000000..44f5acd265a --- /dev/null +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -0,0 +1,121 @@ +package json + +import ( + "testing" +) + +func TestSearchKeys(t *testing.T) { + tests := []struct { + name string + data []byte + keys []string + expected int + }{ + { + name: "Test 1", + data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), + keys: []string{"key1"}, + expected: 8, + }, + { + name: "Test 2", + data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), + keys: []string{"key2"}, + expected: 26, + }, + { + name: "Test 3", + data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), + keys: []string{"key4"}, + expected: -1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := findValueIndex(tt.data, tt.keys...); got != tt.expected { + t.Errorf("findValueIndex() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestFindKeyStart(t *testing.T) { + tests := []struct { + name string + data []byte + key string + expected int + }{ + { + name: "Test 1", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), + key: "key", + expected: 3, + }, + { + name: "Test 2", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), + key: "array", + expected: 20, + }, + { + name: "Test 3", + data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), + key: "value", + expected: -1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _ := findKeyStart(tt.data, tt.key) + if got != tt.expected { + t.Errorf("findKeyStart() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestGet(t *testing.T) { + data := []byte(`{ + "person": { + "name": { + "first": "Alex", + "last": "Johnson", + "fullName": "Alex Johnson" + }, + "github": { + "handle": "alexj", + "followers": 152 + } + }, + "company": { + "name": "company name", + }, + } + `) + + tests := []struct { + path []string + expected string + }{ + {[]string{"person", "name", "first"}, "Alex"}, + {[]string{"person", "name", "last"}, "Johnson"}, + {[]string{"person", "name", "fullName"}, "Alex Johnson"}, + {[]string{"person", "github", "handle"}, "alexj"}, + {[]string{"person", "github", "followers"}, "152"}, + {[]string{"company", "name"}, "company name"}, + } + + for _, test := range tests { + value, _, _, err := Get(data, test.path...) + if err != nil { + t.Errorf("Got error: %v", err) + } + + if string(value) != test.expected { + t.Errorf("Expected '%s', got '%s'", test.expected, value) + } + } +} diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 8a72264eab1..07ef5b9e538 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -18,7 +18,7 @@ const ( NewLineToken = '\n' TabToken = '\t' CarriageReturnToken = '\r' - FormFeedToken = '\f' + FormFeedToken = '\f' BackspaceToken = '\b' SlashToken = '/' BackSlashToken = '\\' From 2b35743cef55adbb7f715707f8c9fea199b30ce0 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 21 Dec 2023 00:16:20 +0900 Subject: [PATCH 30/72] save --- examples/gno.land/p/demo/json/parser.gno | 8 ++ .../gno.land/p/demo/json/state_machine.gno | 20 +++++ .../p/demo/json/state_machine_test.gno | 79 +++++++++++++++++++ 3 files changed, 107 insertions(+) diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 95ac9e92396..4d009fe18f8 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -287,6 +287,12 @@ func underscoreOK(s string) bool { return saw != UnderscoreToken } +// getType is a function that takes a byte slice and an offset as input parameters. +// It returns a byte slice representing the data type, a ValueType indicating the type of data, +// an integer representing the end offset, and an error if any. +// If the input byte slice is empty, it returns an error indicating that no JSON data was provided. +// Otherwise, it calls the parseValue function to parse the value and returns the parsed data type, +// the end offset, and any error encountered during parsing. func getType(data []byte, offset int) ([]byte, ValueType, int, error) { if len(data) == 0 { return nil, Unknown, offset, errors.New("no JSON data provided") @@ -300,6 +306,8 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { return data[offset:endOffset], dataType, endOffset, nil } +// parseValue parses a JSON value from the given data starting at the specified offset. +// It returns the parsed value, the new offset after parsing, and an error if any. func parseValue(data []byte, offset int) (ValueType, int, error) { switch data[offset] { case DoublyQuoteToken: diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 41b8e15dee2..7f3d26408e6 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -264,3 +264,23 @@ func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset a, b, _, d, e := internalGet(data, keys...) return a, b, d, e } + +func MapKeyTypes(data []byte) (map[string]ValueType, error) { + keyTypeMap := make(map[string]ValueType) + + keys, err := extractKeys(data) + if err != nil { + return nil, err + } + + for _, key := range keys { + _, typ, _, err := Get(data, key) + if err != nil { + return nil, err + } + + keyTypeMap[key] = typ + } + + return keyTypeMap, nil +} diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index 44f5acd265a..d1c941de4a8 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -119,3 +119,82 @@ func TestGet(t *testing.T) { } } } +func TestMapKeyTypes(t *testing.T) { + tests := []struct { + name string + data []byte + expectedMap map[string]ValueType + expectedErr error + }{ + { + name: "Test 1", + data: []byte(`{"key1": "value1", "key2": 123, "key3": true}`), + expectedMap: map[string]ValueType{ + "key1": String, + "key2": Number, + "key3": Boolean, + }, + expectedErr: nil, + }, + // { + // name: "Test 2", + // data: []byte(`{"key1": "value1", "key2": [1, 2, 3], "key3": {"nested": "value"}}`), + // expectedMap: map[string]ValueType{ + // "key1": String, + // "key2": Array, + // "key3": Object, + // }, + // expectedErr: nil, + // }, + { + name: "Test 3", + data: []byte(`{"key1": "value1", "key2": null, "key3": 3.14}`), + expectedMap: map[string]ValueType{ + "key1": String, + "key2": Null, + "key3": Number, + }, + expectedErr: nil, + }, + // { + // name: "Test 4", + // data: []byte(`{"key1": "value1", "key2": {"nested": [1, 2, 3]}, "key3": true}`), + // expectedMap: map[string]ValueType{ + // "key1": String, + // "key2": Object, + // "key3": Boolean, + // }, + // expectedErr: nil, + // }, + { + name: "Test 5", + data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), + expectedMap: map[string]ValueType{ + "key1": String, + "key2": String, + "key3": String, + }, + expectedErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotMap, gotErr := MapKeyTypes(tt.data) + + if gotMap == nil { + t.Errorf("MapKeyTypes() gotMap = %v, expectedMap = %v", gotMap, tt.expectedMap) + } + + if gotErr != tt.expectedErr { + t.Errorf("MapKeyTypes() gotErr = %v, expectedErr = %v", gotErr, tt.expectedErr) + } + + for key, value := range gotMap { + if tt.expectedMap[key] != value { + t.Errorf("MapKeyTypes() gotMap = %v, expectedMap = %v", gotMap, tt.expectedMap) + } + } + }) + } +} \ No newline at end of file From 40faddb75279180b66c1dedc9af20b1d009566f5 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Sat, 23 Dec 2023 20:35:51 +0900 Subject: [PATCH 31/72] fix array --- examples/gno.land/p/demo/json/lexer.gno | 15 - .../gno.land/p/demo/json/state_machine.gno | 140 +++-- .../p/demo/json/state_machine_test.gno | 562 ++++++++++++++++-- 3 files changed, 616 insertions(+), 101 deletions(-) diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 8fb450cccb1..09ad9c3b747 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -83,21 +83,6 @@ func blockEnd(data []byte, openSym byte, closeSym byte) int { return -1 } -func isArrayKey(level int, keyLevel int, keys []string) (bool, error) { - if keyLevel != level { - return false, nil - } - - key := keys[level] - keyLen := len(key) - - if keyLen < 3 || key[0] != ArrayStartToken || key[keyLen-1] != ArrayEndToken { - return false, MalformedArray - } - - return true, nil -} - func keyMatched(key []byte, keyEscaped bool, stackbuf [UnescapeStackBufSize]byte, keys []string, level int) (keyUnesc []byte, err error) { if !keyEscaped { keyUnesc = key diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 7f3d26408e6..c3cdf041ba2 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -34,14 +34,13 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } var ( - i int keyLevel int level int lastMatched bool = true stackbuf [UnescapeStackBufSize]byte ) - for i < len(data) { + for i := 0; i < len(data); i++ { switch data[i] { case DoublyQuoteToken: i += 1 @@ -98,59 +97,106 @@ func findValueIndex(data []byte, keys ...string) (int, error) { case ObjectEndToken: level, keyLevel = decreaseLevel(level, keyLevel) case ArrayStartToken: - var ( - curIdx int - valueFound []byte - valueOffset int - ) - - isArrayKey, err := isArrayKey(level, keyLevel, keys) - if err != nil { - return -1, err - } - - if !isArrayKey { - arraySkip := blockEnd(data[i:], ArrayStartToken, ArrayEndToken) - if arraySkip < 0 { - return -1, MalformedJson + // if keyLevel == level && keys[level][0] == '[' { + // keyLen := len(keys[level]) + // if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { + // return -1, InvalidArrayIndex + // } + // } + // var ( + // curIdx int + // valueFound []byte + // valueOffset int + // ) + + // isArrayKey, err := isArrayKey(level, keyLevel, keys) + // if err != nil { + // return -1, err + // } + + // if !isArrayKey { + // arraySkip := blockEnd(data[i:], ArrayStartToken, ArrayEndToken) + // if arraySkip < 0 { + // return -1, MalformedJson + // } + + // i += arraySkip - 1 + // } + + // arrIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1]) + // if err != nil { + // return -1, InvalidArrayIndex + // } + + // currIdx := i + // ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + // if curIdx == arrIdx { + // valueFound = value + // valueOffset = offset + // if dataType == String { + // valueOffset = valueOffset - 2 + // valueFound = data[currIdx+valueOffset : currIdx+valueOffset+len(value)+2] + // } + // } + // curIdx += 1 + // }) + + // if valueFound == nil { + // return -1, ArrayIndexNotFound + // } + + // subIdx, err := findValueIndex(valueFound, keys[level+1:]...) + // if err != nil { + // return -1, err + // } + + // return i + valueOffset + subIdx, nil + // If we want to get array element by index + if keyLevel == level && keys[level][0] == '[' { + keyLen := len(keys[level]) + if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { + return -1, InvalidArrayIndex } + aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) + if err != nil { + return -1, InvalidArrayIndex + } + var curIdx int + var valueFound []byte + var valueOffset int + curI := i + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + if curIdx == aIdx { + valueFound = value + valueOffset = offset + if dataType == String { + valueOffset = valueOffset - 2 + valueFound = data[curI+valueOffset : curI+valueOffset+len(value)+2] + } + } + curIdx += 1 + }) - i += arraySkip - 1 - } - - arrIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1]) - if err != nil { - return -1, InvalidArrayIndex - } - - currIdx := i - ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { - if curIdx == arrIdx { - valueFound = value - valueOffset = offset - if dataType == String { - valueOffset = valueOffset - 2 - valueFound = data[currIdx+valueOffset : currIdx+valueOffset+len(value)+2] + if valueFound == nil { + return -1, ArrayIndexNotFound + } else { + subIndex, err := findValueIndex(valueFound, keys[level+1:]...) + if err != nil { + return -1, KeyPathNotFoundError } + return i + valueOffset + subIndex, nil + } + } else { + // Do not search for keys inside arrays + if arraySkip := blockEnd(data[i:], '[', ']'); arraySkip == -1 { + return -1, MalformedJson + } else { + i += arraySkip - 1 } - curIdx += 1 - }) - - if valueFound == nil { - return -1, ArrayIndexNotFound - } - - subIdx, err := findValueIndex(valueFound, keys[level+1:]...) - if err != nil { - return -1, err } - - return i + valueOffset + subIdx, nil case ColonToken: // If encountered, JSON data is malformed return -1, MalformedJson } - - i++ } return -1, KeyPathNotFoundError diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index d1c941de4a8..6324c708739 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -77,48 +77,49 @@ func TestFindKeyStart(t *testing.T) { } } -func TestGet(t *testing.T) { - data := []byte(`{ - "person": { - "name": { - "first": "Alex", - "last": "Johnson", - "fullName": "Alex Johnson" - }, - "github": { - "handle": "alexj", - "followers": 152 - } - }, - "company": { - "name": "company name", - }, - } - `) +// func TestGet(t *testing.T) { +// data := []byte(`{ +// "person": { +// "name": { +// "first": "Alex", +// "last": "Johnson", +// "fullName": "Alex Johnson" +// }, +// "github": { +// "handle": "alexj", +// "followers": 152 +// } +// }, +// "company": { +// "name": "company name", +// }, +// } +// `) - tests := []struct { - path []string - expected string - }{ - {[]string{"person", "name", "first"}, "Alex"}, - {[]string{"person", "name", "last"}, "Johnson"}, - {[]string{"person", "name", "fullName"}, "Alex Johnson"}, - {[]string{"person", "github", "handle"}, "alexj"}, - {[]string{"person", "github", "followers"}, "152"}, - {[]string{"company", "name"}, "company name"}, - } +// tests := []struct { +// path []string +// expected string +// }{ +// {[]string{"person", "name", "first"}, "Alex"}, +// {[]string{"person", "name", "last"}, "Johnson"}, +// {[]string{"person", "name", "fullName"}, "Alex Johnson"}, +// {[]string{"person", "github", "handle"}, "alexj"}, +// {[]string{"person", "github", "followers"}, "152"}, +// {[]string{"company", "name"}, "company name"}, +// } - for _, test := range tests { - value, _, _, err := Get(data, test.path...) - if err != nil { - t.Errorf("Got error: %v", err) - } +// for _, test := range tests { +// value, _, _, err := Get(data, test.path...) +// if err != nil { +// t.Errorf("Got error: %v", err) +// } + +// if string(value) != test.expected { +// t.Errorf("Expected '%s', got '%s'", test.expected, value) +// } +// } +// } - if string(value) != test.expected { - t.Errorf("Expected '%s', got '%s'", test.expected, value) - } - } -} func TestMapKeyTypes(t *testing.T) { tests := []struct { name string @@ -197,4 +198,487 @@ func TestMapKeyTypes(t *testing.T) { } }) } +} + +type GetTest struct { + desc string + json string + path []string + + isErr bool + isFound bool + + data interface{} +} + +var getTests = []GetTest{ + // Trivial tests + { + desc: "read string", + json: `""`, + isFound: true, + data: ``, + }, + { + desc: "read number", + json: `0`, + isFound: true, + data: `0`, + }, + { + desc: "read object", + json: `{}`, + isFound: true, + data: `{}`, + }, + { + desc: "read array", + json: `[]`, + isFound: true, + data: `[]`, + }, + { + desc: "read boolean", + json: `true`, + isFound: true, + data: `true`, + }, + { + desc: "read boolean 2", + json: `false`, + isFound: true, + data: `false`, + }, + { + desc: "read null", + json: `null`, + isFound: true, + data: `null`, + }, + + // Found key tests + { + desc: "handling multiple nested keys with same name", + json: `{"a":[{"b":1},{"b":2},3],"c":{"d":[1,2]}} }`, + path: []string{"c", "d"}, + isFound: true, + data: `[1,2]`, + }, + { + desc: "read basic key", + json: `{"a":"b"}`, + path: []string{"a"}, + isFound: true, + data: `b`, + }, + { + desc: "read basic key with space", + json: `{"a": "b"}`, + path: []string{"a"}, + isFound: true, + data: `b`, + }, + { + desc: "read composite key", + json: `{"a": { "b":{"c":"d" }}}`, + path: []string{"a", "b", "c"}, + isFound: true, + data: `d`, + }, + { + desc: `read numberic value as string`, + json: `{"a": "b", "c": 1}`, + path: []string{"c"}, + isFound: true, + data: `1`, + }, + { + desc: `handle multiple nested keys with same name`, + json: `{"a":[{"b":1},{"b":2},3],"c":{"c":[1,2]}} }`, + path: []string{"c", "c"}, + isFound: true, + data: `[1,2]`, + }, + // { + // desc: `read string values with quotes`, + // json: `{"a": "string\"with\"quotes"}`, + // path: []string{"a"}, + // isFound: true, + // data: `string\"with\"quotes`, + // }, + { + desc: `read object`, + json: `{"a": { "b":{"c":"d" }}}`, + path: []string{"a", "b"}, + isFound: true, + data: `{"c":"d" }`, + }, + { + desc: `empty path`, + json: `{"c":"d" }`, + path: []string{}, + isFound: true, + data: `{"c":"d" }`, + }, + { + desc: `formatted JSON value`, + json: "{\n \"a\": \"b\"\n}", + path: []string{"a"}, + isFound: true, + data: `b`, + }, + { + desc: `formatted JSON value 2`, + json: "{\n \"a\":\n {\n\"b\":\n {\"c\":\"d\",\n\"e\": \"f\"}\n}\n}", + path: []string{"a", "b"}, + isFound: true, + data: "{\"c\":\"d\",\n\"e\": \"f\"}", + }, + { + desc: `whitespace`, + json: " \n\r\t{ \n\r\t\"whitespace\" \n\r\t: \n\r\t333 \n\r\t} \n\r\t", + path: []string{"whitespace"}, + isFound: true, + data: "333", + }, + // { + // desc: `escaped backslash quote`, + // json: `{"a": "\\\""}`, + // path: []string{"a"}, + // isFound: true, + // data: `\\\"`, + // }, + { + desc: `unescaped backslash quote`, + json: `{"a": "\\"}`, + path: []string{"a"}, + isFound: true, + data: `\\`, + }, + { + desc: `unicode in JSON`, + json: `{"a": "15°C"}`, + path: []string{"a"}, + isFound: true, + data: `15°C`, + }, + { + desc: `no padding + nested`, + json: `{"a":{"a":"1"},"b":2}`, + path: []string{"b"}, + isFound: true, + data: `2`, + }, + { + desc: `no padding + nested + array`, + json: `{"a":{"b":[1,2]},"c":3}`, + path: []string{"c"}, + isFound: true, + data: `3`, + }, + { + desc: `empty key`, + json: `{"":{"":{"":true}}}`, + path: []string{"", "", ""}, + isFound: true, + data: `true`, + }, + + // Escaped key tests + { + desc: `key with simple escape`, + json: `{"a\\b":1}`, + path: []string{"a\\b"}, + isFound: true, + data: `1`, + }, + { + desc: `key and value with whitespace escapes`, + json: `{"key\b\f\n\r\tkey":"value\b\f\n\r\tvalue"}`, + path: []string{"key\b\f\n\r\tkey"}, + isFound: true, + data: `value\b\f\n\r\tvalue`, // value is not unescaped since this is Get(), but the key should work correctly + }, + { + desc: `key with Unicode escape`, + json: `{"a\u00B0b":1}`, + path: []string{"a\u00B0b"}, + isFound: true, + data: `1`, + }, + { + desc: `key with complex escape`, + json: `{"a\uD83D\uDE03b":1}`, + path: []string{"a\U0001F603b"}, + isFound: true, + data: `1`, + }, + + { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance + desc: `malformed with trailing whitespace`, + json: `{"a":1 `, + path: []string{"a"}, + isFound: true, + data: `1`, + }, + { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance + desc: `malformed with wrong closing bracket`, + json: `{"a":1]`, + path: []string{"a"}, + isFound: true, + data: `1`, + }, + + // Not found key tests ** current: throw reflect error. should be fixed + // { + // desc: `empty input`, + // json: ``, + // path: []string{"a"}, + // isFound: false, + // }, + // { + // desc: "non-existent key 1", + // json: `{"a":"b"}`, + // path: []string{"c"}, + // isFound: false, + // }, + // { + // desc: "non-existent key 2", + // json: `{"a":"b"}`, + // path: []string{"b"}, + // isFound: false, + // }, + // { + // desc: "non-existent key 3", + // json: `{"aa":"b"}`, + // path: []string{"a"}, + // isFound: false, + // }, + // { + // desc: "apply scope of parent when search for nested key", + // json: `{"a": { "b": 1}, "c": 2 }`, + // path: []string{"a", "b", "c"}, + // isFound: false, + // }, + // { + // desc: `apply scope to key level`, + // json: `{"a": { "b": 1}, "c": 2 }`, + // path: []string{"b"}, + // isFound: false, + // }, + // { + // desc: `handle escaped quote in key name in JSON`, + // json: `{"key\"key": 1}`, + // path: []string{"key"}, + // isFound: false, + // }, + // { + // desc: "handling multiple keys with different name", + // json: `{"a":{"a":1},"b":{"a":3,"c":[1,2]}}`, + // path: []string{"a", "c"}, + // isFound: false, + // }, + // { + // desc: "handling nested json", + // json: `{"a":{"b":{"c":1},"d":4}}`, + // path: []string{"a", "d"}, + // isFound: true, + // data: `4`, + // }, + // { // Issue #148 + // desc: `missing key in different key same level`, + // json: `{"s":"s","ic":2,"r":{"o":"invalid"}}`, + // path: []string{"ic", "o"}, + // isFound: false, + // }, + + // Error/invalid tests + // { + // desc: `handle escaped quote in key name in JSON`, + // json: `{"key\"key": 1}`, + // path: []string{"key"}, + // isFound: false, + // }, + { + desc: `missing closing brace, but can still find key`, + json: `{"a":"b"`, + path: []string{"a"}, + isFound: true, + data: `b`, + }, + { + desc: `missing value closing quote`, + json: `{"a":"b`, + path: []string{"a"}, + isErr: true, + }, + { + desc: `missing value closing curly brace`, + json: `{"a": { "b": "c"`, + path: []string{"a"}, + isErr: true, + }, + { + desc: `missing value closing square bracket`, + json: `{"a": [1, 2, 3 }`, + path: []string{"a"}, + isErr: true, + }, + { + desc: `missing value 1`, + json: `{"a":`, + path: []string{"a"}, + isErr: true, + }, + { + desc: `missing value 2`, + json: `{"a": `, + path: []string{"a"}, + isErr: true, + }, + { + desc: `missing value 3`, + json: `{"a":}`, + path: []string{"a"}, + isErr: true, + }, + { + desc: `malformed array (no closing brace)`, + json: `{"a":[, "b":123}`, + path: []string{"b"}, + isFound: false, + isErr: true, + }, + // { // Issue #81 + // desc: `missing key in object in array`, + // json: `{"p":{"a":[{"u":"abc","t":"th"}]}}`, + // path: []string{"p", "a", "[0]", "x"}, + // }, + // { // Issue #81 counter test + // desc: `existing key in object in array`, + // json: `{"p":{"a":[{"u":"abc","t":"th"}]}}`, + // path: []string{"p", "a", "[0]", "u"}, + // isFound: true, + // data: "abc", + // }, + // { // This test returns not found instead of a parse error, as checking for the malformed JSON would reduce performance + // desc: "malformed key (followed by comma followed by colon)", + // json: `{"a",:1}`, + // path: []string{"a"}, + // isFound: false, + // }, + { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance (this is not ideal) + desc: "malformed 'colon chain', lookup first string", + json: `{"a":"b":"c"}`, + path: []string{"a"}, + isFound: true, + data: "b", + }, + { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance (this is not ideal) + desc: "malformed 'colon chain', lookup second string", + json: `{"a":"b":"c"}`, + path: []string{"b"}, + isFound: true, + data: "c", + }, + // Array index paths + // { + // desc: "last key in path is index", + // json: `{"a":[{"b":1},{"b":"2"}, 3],"c":{"c":[1,2]}}`, + // path: []string{"a", "[1]"}, + // isFound: true, + // data: `{"b":"2"}`, + // }, + // { + // desc: "get string from array", + // json: `{"a":[{"b":1},"foo", 3],"c":{"c":[1,2]}}`, + // path: []string{"a", "[1]"}, + // isFound: true, + // data: "foo", + // }, + // { + // desc: "key in path is index", + // json: `{"a":[{"b":"1"},{"b":"2"},3],"c":{"c":[1,2]}}`, + // path: []string{"a", "[0]", "b"}, + // isFound: true, + // data: `1`, + // }, + // { + // desc: "last key in path is an index to value in array (formatted json)", + // json: `{ + // "a": [ + // { + // "b": 1 + // }, + // {"b":"2"}, + // 3 + // ], + // "c": { + // "c": [ + // 1, + // 2 + // ] + // } + // }`, + // path: []string{"a", "[1]"}, + // isFound: true, + // data: `{"b":"2"}`, + // }, + // { + // desc: "key in path is index (formatted json)", + // json: `{ + // "a": [ + // {"b": 1}, + // {"b": "2"}, + // 3 + // ], + // "c": { + // "c": [1, 2] + // } + // }`, + // path: []string{"a", "[0]", "b"}, + // isFound: true, + // data: `1`, + // }, + // { + // // Issue #178: Crash in searchKeys + // desc: `invalid json`, + // json: `{{{"":`, + // path: []string{"a", "b"}, + // isFound: false, + // }, + // { + // desc: `opening brace instead of closing and without key`, + // json: `{"a":1{`, + // path: []string{"b"}, + // isFound: false, + // }, +} + +func TestGet(t *testing.T) { + for _, test := range getTests { + t.Run(test.desc, func(t *testing.T) { + data := []byte(test.json) + value, _, _, err := Get(data, test.path...) + if test.isErr { + if err == nil { + t.Errorf("Expected error, got nil") + } + } else { + if err != nil { + t.Errorf("Got error: %v", err) + } + } + + if test.isFound { + if string(value) != test.data { + t.Errorf("Expected '%s', got '%s'", test.data, value) + } + } else { + if value != nil { + t.Errorf("Expected nil, got '%s'", value) + } + } + }) + } } \ No newline at end of file From fbc13fc06f2e070c06c649df43f640098e42b291 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 4 Jan 2024 11:58:00 +0900 Subject: [PATCH 32/72] struct parser --- examples/gno.land/p/demo/json/encoder.gno | 10 +- examples/gno.land/p/demo/json/escape.gno | 2 +- examples/gno.land/p/demo/json/lexer.gno | 2 +- examples/gno.land/p/demo/json/parser.gno | 24 ++-- .../gno.land/p/demo/json/state_machine.gno | 20 ++-- examples/gno.land/p/demo/json/struct.gno | 103 ++++++++++++++++++ examples/gno.land/p/demo/json/token.gno | 56 ++++++++-- examples/gno.land/p/demo/json/value.gno | 42 ------- examples/gno.land/p/demo/json/value_test.gno | 30 ----- 9 files changed, 181 insertions(+), 108 deletions(-) create mode 100644 examples/gno.land/p/demo/json/struct.gno delete mode 100644 examples/gno.land/p/demo/json/value.gno delete mode 100644 examples/gno.land/p/demo/json/value_test.gno diff --git a/examples/gno.land/p/demo/json/encoder.gno b/examples/gno.land/p/demo/json/encoder.gno index 3ed1871a786..6e5b4656041 100644 --- a/examples/gno.land/p/demo/json/encoder.gno +++ b/examples/gno.land/p/demo/json/encoder.gno @@ -55,7 +55,7 @@ func flattenValue(prefix string, val interface{}, flattened map[string]string, b func extractKeys(data []byte) (keys []string, err error) { var ( // stack tracks nested structures in JSON, such as objects or arrays. - // If the top of the stack is an ObjectStartToken ({), + // If the top of the stack is an CurlyOpenToken ({), // it confirms we're inside an object. stack []byte // inString flag indicates whether the parser is currently within a string. @@ -68,9 +68,9 @@ func extractKeys(data []byte) (keys []string, err error) { for i := 0; i < len(data); i++ { switch data[i] { - case ObjectStartToken, ArrayStartToken: + case CurlyOpenToken, SquareOpenToken: stack = append(stack, data[i]) - case ObjectEndToken, ArrayEndToken: + case CurlyCloseToken, SquareCloseToken: if len(stack) > 0 { stack = stack[:len(stack)-1] } @@ -83,13 +83,13 @@ func extractKeys(data []byte) (keys []string, err error) { // Start of string inString = true potentialKey = "" - if len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { + if len(stack) > 0 && stack[len(stack)-1] == CurlyOpenToken { afterString = false } } case ColonToken: // checks if the parser is currently within a key-value pair in a JSON object. - if afterString && len(stack) > 0 && stack[len(stack)-1] == ObjectStartToken { + if afterString && len(stack) > 0 && stack[len(stack)-1] == CurlyOpenToken { keys = append(keys, potentialKey) afterString = false } diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index e0073cbc50c..36caeee4f67 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -51,7 +51,7 @@ var escapeMap = map[byte]byte{ '"': DoublyQuoteToken, '\\': BackSlashToken, '/': SlashToken, - 'b': BackspaceToken, + 'b': BackSpaceToken, 'f': FormFeedToken, 'n': NewLineToken, 'r': CarriageReturnToken, diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno index 09ad9c3b747..7c5d3481e09 100644 --- a/examples/gno.land/p/demo/json/lexer.gno +++ b/examples/gno.land/p/demo/json/lexer.gno @@ -9,7 +9,7 @@ import ( func tokenEnd(data []byte) int { for i, tok := range data { switch tok { - case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken, ValueSeparatorToken, ObjectEndToken, ArrayEndToken: + case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken, CommaToken, CurlyCloseToken, SquareCloseToken: return i } } diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 4d009fe18f8..3d6c88d1f72 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -163,7 +163,7 @@ func ParseUint(b []byte, base, bitSize int) (v uint64, err error) { for _, c := range b { var d byte switch { - case c == UnderscoreToken && base0: + case c == UnderScoreToken && base0: underscores = true continue case '0' <= c && c <= '9': @@ -271,20 +271,20 @@ func underscoreOK(s string) bool { if isDigit || isHexChar { saw = '0' - } else if currentChar == UnderscoreToken { + } else if currentChar == UnderScoreToken { if saw != '0' { return false } - saw = UnderscoreToken + saw = UnderScoreToken } else { - if saw == UnderscoreToken { + if saw == UnderScoreToken { return false } saw = BangToken } } - return saw != UnderscoreToken + return saw != UnderScoreToken } // getType is a function that takes a byte slice and an offset as input parameters. @@ -312,7 +312,7 @@ func parseValue(data []byte, offset int) (ValueType, int, error) { switch data[offset] { case DoublyQuoteToken: return parseString(data, offset) - case ArrayStartToken, ObjectStartToken: + case SquareOpenToken, CurlyOpenToken: return parseContainer(data, offset) } @@ -332,12 +332,12 @@ func parseContainer(data []byte, offset int) (ValueType, int, error) { var containerType ValueType var closing byte - if data[offset] == ArrayStartToken { + if data[offset] == SquareOpenToken { containerType = Array - closing = ArrayEndToken + closing = SquareCloseToken } else { containerType = Object - closing = ObjectEndToken + closing = CurlyCloseToken } endOffset := blockEnd(data[offset:], data[offset], closing) @@ -422,7 +422,7 @@ func parseUntilArrayStart(data []byte, offset int) (int, error) { return -1, MalformedJson } - if data[offset] != ArrayStartToken { + if data[offset] != SquareOpenToken { return -1, MalformedArray } @@ -430,7 +430,7 @@ func parseUntilArrayStart(data []byte, offset int) (int, error) { } func parseArrayElements(data []byte, offset int, f func(value []byte, dataType ValueType, offset int, err error)) (int, error) { - for data[offset] != ArrayEndToken { + for data[offset] != SquareCloseToken { value, valueType, valueOffset, err := Get(data[offset:]) if err != nil { return offset, err @@ -456,7 +456,7 @@ func parseNextElementOrEndArray(data []byte, offset int) (int, error) { return -1, MalformedArray } - if data[offset] != ValueSeparatorToken && data[offset] != ArrayEndToken { + if data[offset] != CommaToken && data[offset] != SquareCloseToken { return -1, MalformedArray } diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index c3cdf041ba2..c71e7612e96 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -82,11 +82,11 @@ func findValueIndex(data []byte, keys ...string) (int, error) { if keyLevel == len(keys) { return i + 1, nil } - case ObjectStartToken: + case CurlyOpenToken: // in case parent key is matched then only we will increase the level otherwise can directly // can move to the end of this block if !lastMatched { - end := blockEnd(data[i:], ObjectStartToken, ObjectEndToken) + end := blockEnd(data[i:], CurlyOpenToken, CurlyCloseToken) if end == -1 { return -1, MalformedJson } @@ -94,9 +94,9 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } else { level += 1 } - case ObjectEndToken: + case CurlyCloseToken: level, keyLevel = decreaseLevel(level, keyLevel) - case ArrayStartToken: + case SquareOpenToken: // if keyLevel == level && keys[level][0] == '[' { // keyLen := len(keys[level]) // if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { @@ -115,7 +115,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { // } // if !isArrayKey { - // arraySkip := blockEnd(data[i:], ArrayStartToken, ArrayEndToken) + // arraySkip := blockEnd(data[i:], SquareOpenToken, SquareCloseToken) // if arraySkip < 0 { // return -1, MalformedJson // } @@ -217,7 +217,7 @@ func findKeyStart(data []byte, key string) (int, error) { } ln := len(data) - if ln > 0 && (data[i] == ObjectStartToken || data[i] == ArrayStartToken) { + if ln > 0 && (data[i] == CurlyOpenToken || data[i] == SquareOpenToken) { i += 1 } @@ -262,13 +262,13 @@ func findKeyStart(data []byte, key string) (int, error) { return keyBegin - 1, nil } - case ArrayStartToken: - if end := blockEnd(data[i:], data[i], ArrayEndToken); end != -1 { + case SquareOpenToken: + if end := blockEnd(data[i:], data[i], SquareCloseToken); end != -1 { i = i + end } - case ObjectStartToken: - if end := blockEnd(data[i:], data[i], ObjectEndToken); end != -1 { + case CurlyOpenToken: + if end := blockEnd(data[i:], data[i], CurlyCloseToken); end != -1 { i = i + end } } diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno new file mode 100644 index 00000000000..d0cf6dac86e --- /dev/null +++ b/examples/gno.land/p/demo/json/struct.gno @@ -0,0 +1,103 @@ +package json + +import ( + "strings" + "unicode" +) + +type FieldMetaData struct { + Name string + Tag string + Type string + IsNested bool + IsOptional bool + Depth int +} + +func ParseStruct(b []byte) []FieldMetaData { + return scanFields(b, 0) +} + +func scanFields(b []byte, depth int) []FieldMetaData { + l := len(b) + fields := []FieldMetaData + + if l < 2 { + return nil + } + + for i := 0; i < l; i++ { + // end of current struct + if b[i] == CurlyCloseToken { + break + } + + if unicode.IsLetter(rune(b[i])) { + fd, newIndex := parseField(b, i, depth) + fields = append(fields, fd) + i = newIndex + } + + if isStructIdent(b, i) { + // Skip to the opening brace + for i < l && b[i] != CurlyOpenToken { + i++ + } + + if i < l { + nested := parseFields(b[i+1:], depth+1) + fields = append(fields, nested...) + } + } + } + + return fields +} + +func parseField(b []byte, start, depth int) (FieldMetaData, int) { + field := FieldMetaData{Depth: depth} + i := start + l := len(b) + + // parse field name + for i < l && b[i] != WhiteSpaceToken { + i++ + } + field.Name = string(b[start:i]) + + // Skip to JSON tag + for i < l && b[i] != BackTickToken { + i++ + } + + if i < l { + field.Tag, i = parseTag(b, i+1) + field.IsOptional = isOptional(field.Tag) + } + + return field, i +} + +func parseTag(b []byte, start int) (string, int) { + i := start + if i+6 >= len(b) { + return "", i + } + + i += 6 // skip `json:"` + start = i + + for i < len(b) && b[i] != DoublyQuoteToken { + i++ + } + + return string(b[start:i]), i+1 +} + +func isOptional(tag string) bool { + return strings.Contains(tag, ",omitempty") +} + +func isStructIdent(b []byte, start int) bool { + return strings.Contains(string(b[start:]), "struct") +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 07ef5b9e538..816f6c3361c 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -1,13 +1,14 @@ package json const ( - ArrayStartToken = '[' - ArrayEndToken = ']' - ObjectStartToken = '{' - ObjectEndToken = '}' - ValueSeparatorToken = ',' + SquareOpenToken = '[' + SquareCloseToken = ']' + CurlyOpenToken = '{' + CurlyCloseToken = '}' + CommaToken = ',' DotToken = '.' ColonToken = ':' + BackTickToken = '`' QuoteToken = '\'' DoublyQuoteToken = '"' EmptyStringToken = "" @@ -19,10 +20,10 @@ const ( TabToken = '\t' CarriageReturnToken = '\r' FormFeedToken = '\f' - BackspaceToken = '\b' + BackSpaceToken = '\b' SlashToken = '/' BackSlashToken = '\\' - UnderscoreToken = '_' + UnderScoreToken = '_' ) const ( @@ -30,3 +31,44 @@ const ( falseLiteral = []byte("false") nullLiteral = []byte("null") ) + +type ValueType int + +const ( + NotExist ValueType = iota + String + Number + Float + Object + Array + Boolean + Null + Unknown +) + +func (v ValueType) ToString() string { + switch v { + case NotExist: + return "not-exist" + case String: + return "string" + case Number: + return "number" + case Object: + return "object" + case Array: + return "array" + case Boolean: + return "boolean" + case Null: + return "null" + case Unknown: + return "unknown" + } +} + +var ( + TrueLiteral = []byte("true") + FalseLiteral = []byte("false") + NullLiteral = []byte("null") +) diff --git a/examples/gno.land/p/demo/json/value.gno b/examples/gno.land/p/demo/json/value.gno deleted file mode 100644 index fab0e4a70db..00000000000 --- a/examples/gno.land/p/demo/json/value.gno +++ /dev/null @@ -1,42 +0,0 @@ -package json - -type ValueType int - -const ( - NotExist ValueType = iota - String - Number - Float - Object - Array - Boolean - Null - Unknown -) - -func (v ValueType) ToString() string { - switch v { - case NotExist: - return "not-exist" - case String: - return "string" - case Number: - return "number" - case Object: - return "object" - case Array: - return "array" - case Boolean: - return "boolean" - case Null: - return "null" - case Unknown: - return "unknown" - } -} - -var ( - TrueLiteral = []byte("true") - FalseLiteral = []byte("false") - NullLiteral = []byte("null") -) diff --git a/examples/gno.land/p/demo/json/value_test.gno b/examples/gno.land/p/demo/json/value_test.gno deleted file mode 100644 index 9973a4987e3..00000000000 --- a/examples/gno.land/p/demo/json/value_test.gno +++ /dev/null @@ -1,30 +0,0 @@ -package json - -import ( - "testing" -) - -type valueTypeTest struct { - valueType ValueType - expected string -} - -func TestValueTypeToString(t *testing.T) { - tests := []valueTypeTest{ - {NotExist, "not-exist"}, - {String, "string"}, - {Number, "number"}, - {Object, "object"}, - {Array, "array"}, - {Boolean, "boolean"}, - {Null, "null"}, - {Unknown, "unknown"}, - } - - for _, test := range tests { - result := test.valueType.ToString() - if result != test.expected { - t.Errorf("Expected %s, but got %s", test.expected, result) - } - } -} From 7041cfc0882b7661290f0f6a87d3d1ac3e19a1d0 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 4 Jan 2024 16:36:01 +0900 Subject: [PATCH 33/72] struct parser finish --- examples/gno.land/p/demo/json/struct.gno | 116 +++++------------- examples/gno.land/p/demo/json/struct_test.gno | 101 +++++++++++++++ 2 files changed, 135 insertions(+), 82 deletions(-) create mode 100644 examples/gno.land/p/demo/json/struct_test.gno diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index d0cf6dac86e..e9e57578f14 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -1,103 +1,55 @@ package json import ( + "regexp" "strings" - "unicode" ) type FieldMetaData struct { - Name string - Tag string - Type string - IsNested bool - IsOptional bool - Depth int + Name string + Tag string + Type string + IsNested bool + IsOptional bool + Depth int } -func ParseStruct(b []byte) []FieldMetaData { - return scanFields(b, 0) -} - -func scanFields(b []byte, depth int) []FieldMetaData { - l := len(b) - fields := []FieldMetaData +func ParseStruct(data []byte, depth int) []FieldMetaData { + var fields []FieldMetaData + dataStr := string(data) - if l < 2 { - return nil - } + // separate fields using regular expressions + re := regexp.MustCompile(`(\w+)\s+(\*?\w+|\[\]\w+|struct\s*{[^}]*})\s+` + "`" + `json:"([^"]+)"` + "`") + matches := re.FindAllStringSubmatch(dataStr, -1) - for i := 0; i < l; i++ { - // end of current struct - if b[i] == CurlyCloseToken { - break + for _, match := range matches { + field := FieldMetaData{ + Name: match[1], + Tag: match[3], + Depth: depth, } - if unicode.IsLetter(rune(b[i])) { - fd, newIndex := parseField(b, i, depth) - fields = append(fields, fd) - i = newIndex + // check if the field is a nested struct + // if it is, set the type to "struct" literal and set IsNested to true + if strings.HasPrefix(match[2], "struct") { + field.Type = "struct" + field.IsNested = true + } else { + field.Type = match[2] + field.IsNested = false } - if isStructIdent(b, i) { - // Skip to the opening brace - for i < l && b[i] != CurlyOpenToken { - i++ - } + field.IsOptional = strings.Contains(field.Tag, "omitempty") - if i < l { - nested := parseFields(b[i+1:], depth+1) - fields = append(fields, nested...) - } + fields = append(fields, field) + + // if the field is a nested struct, parse it recursively + if field.IsNested { + nestedData := []byte(match[2]) + nestedFields := ParseStruct(nestedData, depth+1) + fields = append(fields, nestedFields...) } } return fields } - -func parseField(b []byte, start, depth int) (FieldMetaData, int) { - field := FieldMetaData{Depth: depth} - i := start - l := len(b) - - // parse field name - for i < l && b[i] != WhiteSpaceToken { - i++ - } - field.Name = string(b[start:i]) - - // Skip to JSON tag - for i < l && b[i] != BackTickToken { - i++ - } - - if i < l { - field.Tag, i = parseTag(b, i+1) - field.IsOptional = isOptional(field.Tag) - } - - return field, i -} - -func parseTag(b []byte, start int) (string, int) { - i := start - if i+6 >= len(b) { - return "", i - } - - i += 6 // skip `json:"` - start = i - - for i < len(b) && b[i] != DoublyQuoteToken { - i++ - } - - return string(b[start:i]), i+1 -} - -func isOptional(tag string) bool { - return strings.Contains(tag, ",omitempty") -} - -func isStructIdent(b []byte, start int) bool { - return strings.Contains(string(b[start:]), "struct") -} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno new file mode 100644 index 00000000000..5f95b566611 --- /dev/null +++ b/examples/gno.land/p/demo/json/struct_test.gno @@ -0,0 +1,101 @@ +package json + +import ( + "fmt" + "testing" +) + +func TestParseStruct(t *testing.T) { + data := []byte(` + type Person struct { + Name string ` + "`" + `json:"name"` + "`" + ` + Age int ` + "`" + `json:"age"` + "`" + ` + } + `) + + expectedFields := []FieldMetaData{ + { + Name: "Name", + Type: "string", + Tag: "name", + Depth: 0, + IsNested: false, + IsOptional: false, + }, + { + Name: "Age", + Type: "int", + Tag: "age", + Depth: 0, + IsNested: false, + IsOptional: false, + }, + } + + fields := ParseStruct(data, 0) + + if len(fields) != len(expectedFields) { + t.Errorf("Expected %d fields, but got %d", len(expectedFields), len(fields)) + } + + for i, field := range fields { + expectedField := expectedFields[i] + + if field.Name != expectedField.Name { + t.Errorf("Expected field name %s, but got %s", expectedField.Name, field.Name) + } + + if field.Type != expectedField.Type { + t.Errorf("Expected field type %s, but got %s", expectedField.Type, field.Type) + } + + if field.Tag != expectedField.Tag { + t.Errorf("Expected field tag %s, but got %s", expectedField.Tag, field.Tag) + } + + if field.Depth != expectedField.Depth { + t.Errorf("Expected field depth %d, but got %d", expectedField.Depth, field.Depth) + } + + if field.IsNested != expectedField.IsNested { + t.Errorf("Expected field IsNested %t, but got %t", expectedField.IsNested, field.IsNested) + } + + if field.IsOptional != expectedField.IsOptional { + t.Errorf("Expected field IsOptional %t, but got %t", expectedField.IsOptional, field.IsOptional) + } + } +} + +func TestParseNestedStruct(t *testing.T) { + data := []byte(`type Server struct { + Name string ` + "`json:\"name\"`" + ` + Config struct { + Enabled bool ` + "`json:\"enabled,omitempty\"`" + ` + Enabled2 bool ` + "`json:\"enabled2\"`" + ` + } ` + "`json:\"config\"`" + ` + Age int ` + "`json:\"age\"`" + ` + }`) + + expected := []FieldMetaData{ + {Name: "Server", Type: "struct", Tag: "config", IsNested: true, IsOptional: false, Depth: 0}, + {Name: "Name", Type: "string", Tag: "name", IsNested: false, IsOptional: false, Depth: 1}, + {Name: "Enabled", Type: "bool", Tag: "enabled,omitempty", IsNested: false, IsOptional: true, Depth: 1}, + {Name: "Enabled2", Type: "bool", Tag: "enabled2", IsNested: false, IsOptional: false, Depth: 1}, + {Name: "Age", Type: "int", Tag: "age", IsNested: false, IsOptional: false, Depth: 0}, + } + + actual := ParseStruct(data, 0) + + if len(actual) != len(expected) { + t.Errorf("Expected and actual slice lengths differ. Expected %d, got %d", len(expected), len(actual)) + return + } + + for i, actualField := range actual { + expectedField := expected[i] + if actualField != expectedField { + t.Errorf("Field %d - Expected %+v, got %+v", i, expectedField, actualField) + } + } +} From 9fee2016c6cce6b5754562791034adf4c6ea24c5 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 8 Jan 2024 18:21:45 +0900 Subject: [PATCH 34/72] marshaling --- .../p/demo/json/eisel_lemire_test.gno | 43 --- examples/gno.land/p/demo/json/encoder.gno | 111 -------- .../gno.land/p/demo/json/encoder_test.gno | 195 ------------- examples/gno.land/p/demo/json/lexer_test.gno | 30 -- examples/gno.land/p/demo/json/parser.gno | 80 ------ .../gno.land/p/demo/json/state_machine.gno | 259 +++++++++++++----- .../p/demo/json/state_machine_test.gno | 185 +++---------- examples/gno.land/p/demo/json/struct.gno | 141 ++++++++-- examples/gno.land/p/demo/json/struct_test.gno | 162 ++++++++--- examples/gno.land/p/demo/json/token.gno | 20 +- examples/gno.land/p/demo/json/utils.gno | 44 +++ gnovm/tests/files/marshal.gno | 47 ++++ 12 files changed, 571 insertions(+), 746 deletions(-) delete mode 100644 examples/gno.land/p/demo/json/eisel_lemire_test.gno delete mode 100644 examples/gno.land/p/demo/json/encoder.gno delete mode 100644 examples/gno.land/p/demo/json/encoder_test.gno delete mode 100644 examples/gno.land/p/demo/json/lexer_test.gno create mode 100644 gnovm/tests/files/marshal.gno diff --git a/examples/gno.land/p/demo/json/eisel_lemire_test.gno b/examples/gno.land/p/demo/json/eisel_lemire_test.gno deleted file mode 100644 index 8cfa79169ef..00000000000 --- a/examples/gno.land/p/demo/json/eisel_lemire_test.gno +++ /dev/null @@ -1,43 +0,0 @@ -package json - -import ( - "math" - "testing" -) - -type elTestCase struct { - man uint64 - exp10 int - neg bool - want float64 - ok bool -} - -var testCases = []elTestCase{ - {12345, 3, false, 12345e3, true}, - {12345, -3, false, 12345e-3, true}, -} - -func TestEiselLemire64(t *testing.T) { - for _, tc := range testCases { - got, ok := eiselLemire64(tc.man, tc.exp10, tc.neg) - if ok != tc.ok || (ok && got != tc.want) { - t.Errorf("eiselLemire64(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, tc.want, tc.ok) - } - } -} - -func TestEiselLemire32(t *testing.T) { - testCases := []elTestCase{ - {12345, 3, false, 12345e3, true}, - {12345, -3, false, 12345e-3, true}, - } - - for _, tc := range testCases { - got, ok := eiselLemire32(tc.man, tc.exp10, tc.neg) - want := float32(tc.want) - if ok != tc.ok || (ok && got != want) { - t.Errorf("eiselLemire32(%v, %v, %v) = %v, %v; want %v, %v", tc.man, tc.exp10, tc.neg, got, ok, want, tc.ok) - } - } -} diff --git a/examples/gno.land/p/demo/json/encoder.gno b/examples/gno.land/p/demo/json/encoder.gno deleted file mode 100644 index 6e5b4656041..00000000000 --- a/examples/gno.land/p/demo/json/encoder.gno +++ /dev/null @@ -1,111 +0,0 @@ -package json - -import ( - "errors" - "fmt" - "strconv" - "strings" -) - -// TODO: should use byte slice as input -func Flatten(m map[string]interface{}) map[string]string { - flattened := make(map[string]string) - var builder strings.Builder - - for k, v := range m { - flattenValue(k, v, flattened, &builder) - } - - return flattened -} - -func flattenValue(prefix string, val interface{}, flattened map[string]string, builder *strings.Builder) { - switch child := val.(type) { - case map[string]interface{}: - for k, v := range child { - builder.Reset() - builder.WriteString(prefix) - builder.WriteString(".") - builder.WriteString(k) - flattenValue(builder.String(), v, flattened, builder) - } - case []interface{}: - for i, item := range child { - builder.Reset() - builder.WriteString(prefix) - builder.WriteString("[") - builder.WriteString(strconv.Itoa(i)) - builder.WriteString("]") - flattenValue(builder.String(), item, flattened, builder) - } - case string: - flattened[prefix] = "string:\"" + child + "\"" - case int, int32, int64: - flattened[prefix] = "number:" + fmt.Sprintf("%d", child) - case float32, float64: - flattened[prefix] = "number:" + fmt.Sprintf("%f", child) - case bool: - flattened[prefix] = "boolean:" + fmt.Sprintf("%t", child) - default: - flattened[prefix] = "unknown" - } -} - -// extractKeys extracts all keys from a JSON object. -func extractKeys(data []byte) (keys []string, err error) { - var ( - // stack tracks nested structures in JSON, such as objects or arrays. - // If the top of the stack is an CurlyOpenToken ({), - // it confirms we're inside an object. - stack []byte - // inString flag indicates whether the parser is currently within a string. - inString bool - potentialKey string - // afterString flag indicates whether the last parsed token was a string, - // suggesting we may be just after a key in a key-value pair. - afterString bool - ) - - for i := 0; i < len(data); i++ { - switch data[i] { - case CurlyOpenToken, SquareOpenToken: - stack = append(stack, data[i]) - case CurlyCloseToken, SquareCloseToken: - if len(stack) > 0 { - stack = stack[:len(stack)-1] - } - case DoublyQuoteToken: - if inString { - // End of string - inString = false - afterString = true - } else { - // Start of string - inString = true - potentialKey = "" - if len(stack) > 0 && stack[len(stack)-1] == CurlyOpenToken { - afterString = false - } - } - case ColonToken: - // checks if the parser is currently within a key-value pair in a JSON object. - if afterString && len(stack) > 0 && stack[len(stack)-1] == CurlyOpenToken { - keys = append(keys, potentialKey) - afterString = false - } - default: - if inString { - potentialKey += string(data[i]) - } - if isNotWhiteSpaceOrControlChar(inString, data[i]) { - afterString = false - } - } - } - - return keys, nil -} - -func isNotWhiteSpaceOrControlChar(inString bool, char byte) bool { - return !inString && char != WhiteSpaceToken && char != NewLineToken && char != CarriageReturnToken && char != TabToken -} diff --git a/examples/gno.land/p/demo/json/encoder_test.gno b/examples/gno.land/p/demo/json/encoder_test.gno deleted file mode 100644 index b2fd6b2d9d1..00000000000 --- a/examples/gno.land/p/demo/json/encoder_test.gno +++ /dev/null @@ -1,195 +0,0 @@ -package json - -import ( - "errors" - "strings" - "testing" -) - -func equal(a, b map[string]interface{}) bool { - if len(a) != len(b) { - return false - } - - for k, v := range a { - if b[k] != v { - return false - } - } - - return true -} - -func mapsAreEqual(a, b map[string]interface{}) bool { - if len(a) != len(b) { - return false - } - - for ka, va := range a { - vb, ok := b[ka] - if !ok { - return false - } - - switch vaTyped := va.(type) { - case map[string]interface{}: - if vbTyped, ok := vb.(map[string]interface{}); ok { - if !mapsAreEqual(vaTyped, vbTyped) { - return false - } - } else { - return false - } - default: - if va != vb { - return false - } - } - } - - return true -} - -func TestFlatten(t *testing.T) { - testCases := []struct { - name string - input map[string]interface{} - expected map[string]string - }{ - { - name: "Basic Test", - input: map[string]interface{}{ - "user": map[string]interface{}{ - "name": "John Doe", - "age": 30, - "address": map[string]interface{}{ - "street": "123 Main St", - "city": "Anytown", - }, - "favoriteNumbers": []interface{}{7, 13, 42}, - }, - }, - expected: map[string]string{ - "user.name": "string:\"John Doe\"", - "user.age": "number:30", - "user.address.street": "string:\"123 Main St\"", - "user.address.city": "string:\"Anytown\"", - "user.favoriteNumbers[0]": "number:7", - "user.favoriteNumbers[1]": "number:13", - "user.favoriteNumbers[2]": "number:42", - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := Flatten(tc.input) - if !mapsAreEqual(result, tc.expected) { - t.Errorf("Expected %v, got %v", tc.expected, result) - } - }) - } -} - -func TestExtractKeys(t *testing.T) { - tests := []struct { - name string - jsonData []byte - expected []string - }{ - { - name: "Simple Object", - jsonData: []byte(`{"name": "John", "age": 30}`), - expected: []string{"name", "age"}, - }, - { - name: "Nested Object", - jsonData: []byte(`{"person": {"name": "John", "age": 30}, "city": "New York"}`), - expected: []string{"person", "name", "age", "city"}, - }, - { - name: "Array of Objects", - jsonData: []byte(`[{"name": "John"}, {"age": 30}]`), - expected: []string{"name", "age"}, - }, - { - name: "Nested Object 2", - jsonData: []byte(`{ - "a": { - "b": { - "c": 1, - "d": 2 - } - }, - "e": { - "f": 3 - } - }`), - expected: []string{"a", "b", "c", "d", "e", "f"}, - }, - { - name: "Empty Object", - jsonData: []byte(`{}`), - expected: []string{}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result, err := extractKeys(test.jsonData) - if err != nil { - t.Errorf("name: %s error = %v", test.name, err) - return - } - - if len(result) != len(test.expected) { - t.Errorf("name: %s, expected %d keys, got %d. contents: %s", test.name, len(test.expected), len(result), result) - return - } - }) - } -} - -// func TestGetTypeFromKey(t *testing.T) { -// testCases := []struct { -// name string -// data []byte -// key string -// expected ValueType -// err error -// }{ -// { -// name: "Key Exists", -// data: []byte(`{"name": "John", "age": 30}`), -// key: "name", -// expected: String, -// err: nil, -// }, -// { -// name: "Key Does Not Exist", -// data: []byte(`{"name": "John", "age": 30}`), -// key: "address", -// expected: NotExist, -// err: errors.New("key not found"), -// }, -// { -// name: "Empty Data", -// data: []byte(`{}`), -// key: "name", -// expected: NotExist, -// err: errors.New("key not found"), -// }, -// } - -// for _, tc := range testCases { -// t.Run(tc.name, func(t *testing.T) { -// result, err := getTypeFromKey(tc.data, tc.key) -// if result != tc.expected { -// t.Errorf("Expected %v, got %v", tc.expected, result) -// } -// if (err == nil && tc.err != nil) || (err != nil && tc.err == nil) || (err != nil && tc.err != nil && err.Error() != tc.err.Error()) { -// t.Errorf("Expected error: %v, got error: %v", tc.err, err) -// } -// }) -// } -// } diff --git a/examples/gno.land/p/demo/json/lexer_test.gno b/examples/gno.land/p/demo/json/lexer_test.gno deleted file mode 100644 index 2e838cb8774..00000000000 --- a/examples/gno.land/p/demo/json/lexer_test.gno +++ /dev/null @@ -1,30 +0,0 @@ -package json - -import ( - "errors" - "testing" -) - -func TestBlockEnd(t *testing.T) { - tests := []struct { - name string - data []byte - open byte - close byte - expected int - }{ - {"Empty Object", []byte("{}"), '{', '}', 2}, - {"Nested Object", []byte(`{"key": {"nestedKey": "value"}}`), '{', '}', 31}, - {"Array", []byte(`["item1", "item2"]`), '[', ']', 18}, - {"Complex Object", []byte(`{"key": [1, 2, 3], "anotherKey": "value"}`), '{', '}', 41}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result := blockEnd(test.data, test.open, test.close) - if result != test.expected { - t.Errorf("Failed %s: expected %d, got %d", test.name, test.expected, result) - } - }) - } -} diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 3d6c88d1f72..881d1eb277e 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -382,83 +382,3 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { return Unknown, offset, errors.New("unknown value type") } - -func ArrayEach(data []byte, callback func(value []byte, dataType ValueType, offset int, err error), keys ...string) (int, error) { - if len(data) == 0 { - return -1, MalformedObject - } - - offset, err := parseInitialOffset(data, keys) - if err != nil { - return -1, err - } - - return parseArrayElements(data, offset, callback) -} - -func parseInitialOffset(data []byte, keys []string) (int, error) { - offset, err := nextToken(data) - if err != nil { - return -1, MalformedJson - } - - if len(keys) > 0 { - offset, err = findValueIndex(data, keys...) - if err != nil { - return -1, KeyPathNotFoundError - } - - if offset, err = parseUntilArrayStart(data, offset); err != nil { - return -1, err - } - } - - return offset, nil -} - -func parseUntilArrayStart(data []byte, offset int) (int, error) { - offset, err := nextToken(data[offset:]) - if err != nil { - return -1, MalformedJson - } - - if data[offset] != SquareOpenToken { - return -1, MalformedArray - } - - return offset + 1, nil -} - -func parseArrayElements(data []byte, offset int, f func(value []byte, dataType ValueType, offset int, err error)) (int, error) { - for data[offset] != SquareCloseToken { - value, valueType, valueOffset, err := Get(data[offset:]) - if err != nil { - return offset, err - } - - if valueOffset == 0 || valueType == NotExist { - break - } - - f(value, valueType, offset+valueOffset-len(value), err) - - if offset, err = parseNextElementOrEndArray(data, offset+valueOffset); err != nil { - return offset, err - } - } - - return offset, nil -} - -func parseNextElementOrEndArray(data []byte, offset int) (int, error) { - offset, err := nextToken(data[offset:]) - if err != nil { - return -1, MalformedArray - } - - if data[offset] != CommaToken && data[offset] != SquareCloseToken { - return -1, MalformedArray - } - - return offset + 1, nil -} diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index c71e7612e96..64c0e5cc314 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -9,12 +9,12 @@ import ( func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { switch b { case 't', 'f': - if bytes.Equal(value, TrueLiteral) || bytes.Equal(value, FalseLiteral) { + if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { dataType = Boolean } return Unknown, offset, UnknownValueType case 'u', 'n': - if bytes.Equal(value, NullLiteral) { + if bytes.Equal(value, nullLiteral) { dataType = Null } return Unknown, offset, UnknownValueType @@ -97,73 +97,22 @@ func findValueIndex(data []byte, keys ...string) (int, error) { case CurlyCloseToken: level, keyLevel = decreaseLevel(level, keyLevel) case SquareOpenToken: - // if keyLevel == level && keys[level][0] == '[' { - // keyLen := len(keys[level]) - // if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { - // return -1, InvalidArrayIndex - // } - // } - // var ( - // curIdx int - // valueFound []byte - // valueOffset int - // ) - - // isArrayKey, err := isArrayKey(level, keyLevel, keys) - // if err != nil { - // return -1, err - // } - - // if !isArrayKey { - // arraySkip := blockEnd(data[i:], SquareOpenToken, SquareCloseToken) - // if arraySkip < 0 { - // return -1, MalformedJson - // } - - // i += arraySkip - 1 - // } - - // arrIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1]) - // if err != nil { - // return -1, InvalidArrayIndex - // } - - // currIdx := i - // ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { - // if curIdx == arrIdx { - // valueFound = value - // valueOffset = offset - // if dataType == String { - // valueOffset = valueOffset - 2 - // valueFound = data[currIdx+valueOffset : currIdx+valueOffset+len(value)+2] - // } - // } - // curIdx += 1 - // }) - - // if valueFound == nil { - // return -1, ArrayIndexNotFound - // } - - // subIdx, err := findValueIndex(valueFound, keys[level+1:]...) - // if err != nil { - // return -1, err - // } - - // return i + valueOffset + subIdx, nil // If we want to get array element by index - if keyLevel == level && keys[level][0] == '[' { + if keyLevel == level && keys[level][0] == SquareOpenToken { keyLen := len(keys[level]) - if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' { + if keyLen < 3 || keys[level][0] != SquareOpenToken || keys[level][keyLen-1] != SquareCloseToken { return -1, InvalidArrayIndex } aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) if err != nil { return -1, InvalidArrayIndex } - var curIdx int - var valueFound []byte - var valueOffset int + + var ( + curIdx int + valueFound []byte + valueOffset int + ) curI := i ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { if curIdx == aIdx { @@ -311,22 +260,188 @@ func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset return a, b, d, e } -func MapKeyTypes(data []byte) (map[string]ValueType, error) { - keyTypeMap := make(map[string]ValueType) +func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { + if len(data) == 0 { + return -1, MalformedObject + } - keys, err := extractKeys(data) - if err != nil { - return nil, err + if next, err := nextToken(data); err != nil { + return -1, MalformedObject + } else { + offset += next } - for _, key := range keys { - _, typ, _, err := Get(data, key) - if err != nil { - return nil, err + if len(keys) > 0 { + if offset, err = findValueIndex(data, keys...); err != nil { + return -1, KeyPathNotFoundError + } + + // go to closest value + if n0, err := nextToken(data[offset:]); err != nil { + return -1, MalformedObject + } else { + offset += n0 + } + + if data[offset] != SquareOpenToken { + return -1, MalformedObject + } else { + offset += 1 + } + } + + if n0, err := nextToken(data[offset:]); err != nil { + return -1, MalformedObject + } else { + offset += n0 + } + + if data[offset] == SquareCloseToken { + return -1, nil + } + + for true { + v, t, o, e := Get(data[offset:]) + + if e != nil { + return -1, e + } + + if o == 0 { + break + } + + if t != NotExist { + cb(v, t, offset+o-len(v), e) + } + + if e != nil { + break } - keyTypeMap[key] = typ + offset += o + if skipToToken, err := nextToken(data[offset:]); err != nil { + return offset, MalformedArray + } else { + offset += skipToToken + } + + if data[offset] == SquareCloseToken { + break + } + + if data[offset] != CommaToken { + return offset, MalformedArray + } else { + offset += 1 + } } - return keyTypeMap, nil + return offset, nil } + +func ObjectEach(data []byte, cb func(key []byte, value []byte, dataType ValueType, offset int) error, keys ...string) (err error) { + offset := 0 + + // descend to the target key. + if len(keys) > 0 { + if off, err := findValueIndex(data, keys...); err != nil { + return KeyPathNotFoundError + } else { + offset = off + } + } + + if off, err := nextToken(data[offset:]); err != nil { + return MalformedObject + } else if offset += off; data[offset] != CurlyOpenToken { + return MalformedObject + } else { + offset += 1 + } + + // skip to the first token inside the object, + // or stop if we find the end of the object. + if off, err := nextToken(data[offset:]); err != nil { + return MalformedJson + } else if offset += off; data[offset] == CurlyCloseToken { + return nil + } + + for offset < len(data) { + // find the next key + var key []byte + + // check the type of next token + switch data[offset] { + case DoublyQuoteToken: + offset += 1 // accept as string and skip opening quote + case CurlyCloseToken: + return nil // found end of the object. stop + default: + return MalformedObject + } + + // find end of the string + var keyEscaped bool + if off, esc := stringEnd(data[offset:]); off == -1 { + return MalformedJson + } else { + key, keyEscaped = data[offset:offset+off-1], esc + offset += off + } + + // unescape the string if needed + if keyEscaped { + // stack allocated array for allocation-free unescaping of small string + var stackbuf [UnescapeStackBufSize]byte + if ku, err := Unescape(key, stackbuf[:]); err != nil { + return MalformedJson + } else { + key = ku + } + } + + // skip the colon + if off, err := nextToken(data[offset:]); err != nil { + return MalformedJson + } else if offset += off; data[offset] != ColonToken { + return MalformedJson + } else { + offset += 1 + } + + if value, valueType, off, err := Get(data[offset:]); err != nil { + return err + } else if err := cb(key, value, valueType, offset+off); err != nil { + return err + } else { + offset += off + } + + // skip over the next comma to the following token, + // or stop if we reach the ending brace. + if off, err := nextToken(data[offset:]); err != nil { + return MalformedJson + } else { + offset += off + switch data[offset] { + case CurlyCloseToken: + return nil // found end of the object. stop + case CommaToken: + offset += 1 // skip comma and move to the next token + default: + return MalformedObject + } + } + + // skip to the next token after the comma + if off, err := nextToken(data[offset:]); err != nil { + return MalformedJson + } else { + offset += off + } + } + + return MalformedObject +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index 6324c708739..47d9147a930 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -1,6 +1,8 @@ package json import ( + "errors" + "fmt" "testing" ) @@ -77,129 +79,6 @@ func TestFindKeyStart(t *testing.T) { } } -// func TestGet(t *testing.T) { -// data := []byte(`{ -// "person": { -// "name": { -// "first": "Alex", -// "last": "Johnson", -// "fullName": "Alex Johnson" -// }, -// "github": { -// "handle": "alexj", -// "followers": 152 -// } -// }, -// "company": { -// "name": "company name", -// }, -// } -// `) - -// tests := []struct { -// path []string -// expected string -// }{ -// {[]string{"person", "name", "first"}, "Alex"}, -// {[]string{"person", "name", "last"}, "Johnson"}, -// {[]string{"person", "name", "fullName"}, "Alex Johnson"}, -// {[]string{"person", "github", "handle"}, "alexj"}, -// {[]string{"person", "github", "followers"}, "152"}, -// {[]string{"company", "name"}, "company name"}, -// } - -// for _, test := range tests { -// value, _, _, err := Get(data, test.path...) -// if err != nil { -// t.Errorf("Got error: %v", err) -// } - -// if string(value) != test.expected { -// t.Errorf("Expected '%s', got '%s'", test.expected, value) -// } -// } -// } - -func TestMapKeyTypes(t *testing.T) { - tests := []struct { - name string - data []byte - expectedMap map[string]ValueType - expectedErr error - }{ - { - name: "Test 1", - data: []byte(`{"key1": "value1", "key2": 123, "key3": true}`), - expectedMap: map[string]ValueType{ - "key1": String, - "key2": Number, - "key3": Boolean, - }, - expectedErr: nil, - }, - // { - // name: "Test 2", - // data: []byte(`{"key1": "value1", "key2": [1, 2, 3], "key3": {"nested": "value"}}`), - // expectedMap: map[string]ValueType{ - // "key1": String, - // "key2": Array, - // "key3": Object, - // }, - // expectedErr: nil, - // }, - { - name: "Test 3", - data: []byte(`{"key1": "value1", "key2": null, "key3": 3.14}`), - expectedMap: map[string]ValueType{ - "key1": String, - "key2": Null, - "key3": Number, - }, - expectedErr: nil, - }, - // { - // name: "Test 4", - // data: []byte(`{"key1": "value1", "key2": {"nested": [1, 2, 3]}, "key3": true}`), - // expectedMap: map[string]ValueType{ - // "key1": String, - // "key2": Object, - // "key3": Boolean, - // }, - // expectedErr: nil, - // }, - { - name: "Test 5", - data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), - expectedMap: map[string]ValueType{ - "key1": String, - "key2": String, - "key3": String, - }, - expectedErr: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotMap, gotErr := MapKeyTypes(tt.data) - - if gotMap == nil { - t.Errorf("MapKeyTypes() gotMap = %v, expectedMap = %v", gotMap, tt.expectedMap) - } - - if gotErr != tt.expectedErr { - t.Errorf("MapKeyTypes() gotErr = %v, expectedErr = %v", gotErr, tt.expectedErr) - } - - for key, value := range gotMap { - if tt.expectedMap[key] != value { - t.Errorf("MapKeyTypes() gotMap = %v, expectedMap = %v", gotMap, tt.expectedMap) - } - } - }) - } -} - type GetTest struct { desc string json string @@ -244,16 +123,16 @@ var getTests = []GetTest{ data: `true`, }, { - desc: "read boolean 2", - json: `false`, + desc: "read boolean 2", + json: `false`, isFound: true, - data: `false`, + data: `false`, }, { - desc: "read null", - json: `null`, + desc: "read null", + json: `null`, isFound: true, - data: `null`, + data: `null`, }, // Found key tests @@ -299,13 +178,6 @@ var getTests = []GetTest{ isFound: true, data: `[1,2]`, }, - // { - // desc: `read string values with quotes`, - // json: `{"a": "string\"with\"quotes"}`, - // path: []string{"a"}, - // isFound: true, - // data: `string\"with\"quotes`, - // }, { desc: `read object`, json: `{"a": { "b":{"c":"d" }}}`, @@ -341,13 +213,6 @@ var getTests = []GetTest{ isFound: true, data: "333", }, - // { - // desc: `escaped backslash quote`, - // json: `{"a": "\\\""}`, - // path: []string{"a"}, - // isFound: true, - // data: `\\\"`, - // }, { desc: `unescaped backslash quote`, json: `{"a": "\\"}`, @@ -485,7 +350,7 @@ var getTests = []GetTest{ // isFound: true, // data: `4`, // }, - // { // Issue #148 + // { // desc: `missing key in different key same level`, // json: `{"s":"s","ic":2,"r":{"o":"invalid"}}`, // path: []string{"ic", "o"}, @@ -681,4 +546,34 @@ func TestGet(t *testing.T) { } }) } -} \ No newline at end of file +} + +func TestArrayEach(t *testing.T) { + mock := []byte(`{"a": { "b":[{"x": 1} ,{"x": 2},{ "x":3}, {"x":4} ]}}`) + count := 0 + + ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err error) { + count++ + + switch count { + case 1: + if string(value) != `{"x": 1}` { + t.Errorf("Wrong first item: %s", string(value)) + } + case 2: + if string(value) != `{"x": 2}` { + t.Errorf("Wrong second item: %s", string(value)) + } + case 3: + if string(value) != `{ "x":3}` { + t.Errorf("Wrong third item: %s", string(value)) + } + case 4: + if string(value) != `{"x":4}` { + t.Errorf("Wrong forth item: %s", string(value)) + } + default: + t.Errorf("Should process only 4 items") + } + }, "a", "b") +} diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index e9e57578f14..4900a90ad3b 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -1,26 +1,37 @@ package json import ( + "errors" + "fmt" "regexp" "strings" + + "gno.land/p/demo/ufmt" ) +// Tatget Struct +// -> Extract Fields and Save them in a FieldMetaData Struct +// -> Search Values from JSON based on the Extracted Fields via Get method +// -> Zip the Fields with the Values +// -> Return the Zipped Data + +// TODO: current regex approach is not robust and flexible. So, it needs to be improved while maintaining the recursive descent approach. +const fieldRegex = `(\w+)\s+(\*?\w+|\[\]\w+|struct\s*{[^}]*}|map\[\w+\]\w+|interface{})\s+` + "`" + `json:"([^"]+)"` + "`" + +type Fields []FieldMetaData + type FieldMetaData struct { Name string Tag string Type string IsNested bool - IsOptional bool Depth int } -func ParseStruct(data []byte, depth int) []FieldMetaData { - var fields []FieldMetaData - dataStr := string(data) - +func ParseStruct(data string, depth int) (fields Fields) { // separate fields using regular expressions - re := regexp.MustCompile(`(\w+)\s+(\*?\w+|\[\]\w+|struct\s*{[^}]*})\s+` + "`" + `json:"([^"]+)"` + "`") - matches := re.FindAllStringSubmatch(dataStr, -1) + re := regexp.MustCompile(fieldRegex) + matches := re.FindAllStringSubmatch(data, -1) for _, match := range matches { field := FieldMetaData{ @@ -30,22 +41,13 @@ func ParseStruct(data []byte, depth int) []FieldMetaData { } // check if the field is a nested struct - // if it is, set the type to "struct" literal and set IsNested to true - if strings.HasPrefix(match[2], "struct") { - field.Type = "struct" - field.IsNested = true - } else { - field.Type = match[2] - field.IsNested = false - } - - field.IsOptional = strings.Contains(field.Tag, "omitempty") - + fieldType := match[2] + field = field.checkFieldType(fieldType) fields = append(fields, field) - // if the field is a nested struct, parse it recursively + // apply recursion when the field is a nested structure if field.IsNested { - nestedData := []byte(match[2]) + nestedData := match[2] nestedFields := ParseStruct(nestedData, depth+1) fields = append(fields, nestedFields...) } @@ -53,3 +55,102 @@ func ParseStruct(data []byte, depth int) []FieldMetaData { return fields } + +func (f FieldMetaData) checkFieldType(hayStack string) FieldMetaData { + switch { + case strings.HasPrefix(hayStack, StructLiteral): + f.Type = "struct" + f.IsNested = true + case strings.HasPrefix(hayStack, MapLiteral): + f.Type = "map" + f.IsNested = false + panic("ParseStruct: map type not supported") + case strings.HasPrefix(hayStack, InterfaceLiteral): + f.Type = "interface" + panic("ParseStruct: interface type not supported") + default: + f.Type = hayStack + f.IsNested = false + } + + return f +} + +// Names returns the names of the fields +// which will used to search the values from JSON. +func (fs *Fields) Names() (names []string) { + for _, f := range *fs { + names = append(names, f.Name) + } + + return names +} + +// TODO: support for complex types +// Zip zips the fields with the values from JSON. +func Zip(json []byte, extractedKeys ...string) (zippedData map[string]interface{}, err error) { + zippedData = make(map[string]interface{}) + + for _, key := range extractedKeys { + value, dataType, offset, err := Get(json, key) + if err != nil { + return nil, err + } + + switch dataType { + case String: + zippedData[key] = string(value) + case Number: + if n, err := ParseFloatLiteral(value); err != nil { + return nil, err + } else { + zippedData[key] = n + } + case Boolean: + if b, err := ParseBoolLiteral(value); err != nil { + return nil, err + } else { + zippedData[key] = b + } + case Array: + panic("Zip: TODO array type not supported") + case Object: + panic("Zip: TODO object type not supported") + case Null: + zippedData[key] = nil + default: + return nil, errors.New("Zip: unknown data type") + } + } + + return zippedData, nil +} + +func Marshal(data map[string]interface{}) (string, error) { + var sb strings.Builder + sb.WriteString("{") + + isFirst := true + for key, value := range data { + if !isFirst { + sb.WriteString(", ") + } + sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) + + switch v := value.(type) { + case string: + sb.WriteString(ufmt.Sprintf("\"%s\"", v)) + case float32: + sb.WriteString(ufmt.Sprintf("%f", v)) + case bool: + sb.WriteString(ufmt.Sprintf("%t", v)) + default: + return "", errors.New("Marshal: unknown data type") + } + + isFirst = false + } + + sb.WriteString("}") + return sb.String(), nil +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno index 5f95b566611..650292c8d26 100644 --- a/examples/gno.land/p/demo/json/struct_test.gno +++ b/examples/gno.land/p/demo/json/struct_test.gno @@ -1,35 +1,22 @@ package json import ( + "errors" "fmt" "testing" ) func TestParseStruct(t *testing.T) { - data := []byte(` - type Person struct { - Name string ` + "`" + `json:"name"` + "`" + ` - Age int ` + "`" + `json:"age"` + "`" + ` - } - `) - - expectedFields := []FieldMetaData{ - { - Name: "Name", - Type: "string", - Tag: "name", - Depth: 0, - IsNested: false, - IsOptional: false, - }, - { - Name: "Age", - Type: "int", - Tag: "age", - Depth: 0, - IsNested: false, - IsOptional: false, - }, + data := ` + type Person struct { + Name string ` + "`json:\"name\"`" + ` + Age int ` + "`json:\"age\"`" + ` + } + ` + + expectedFields := Fields{ + {Name: "Name", Type: "string", Tag: "name", Depth: 0}, + {Name: "Age", Type: "int", Tag: "age", Depth: 0}, } fields := ParseStruct(data, 0) @@ -60,29 +47,25 @@ func TestParseStruct(t *testing.T) { if field.IsNested != expectedField.IsNested { t.Errorf("Expected field IsNested %t, but got %t", expectedField.IsNested, field.IsNested) } - - if field.IsOptional != expectedField.IsOptional { - t.Errorf("Expected field IsOptional %t, but got %t", expectedField.IsOptional, field.IsOptional) - } } } func TestParseNestedStruct(t *testing.T) { - data := []byte(`type Server struct { - Name string ` + "`json:\"name\"`" + ` - Config struct { - Enabled bool ` + "`json:\"enabled,omitempty\"`" + ` - Enabled2 bool ` + "`json:\"enabled2\"`" + ` - } ` + "`json:\"config\"`" + ` - Age int ` + "`json:\"age\"`" + ` - }`) + data := `type Server struct { + Name string ` + "`json:\"name\"`" + ` + Config struct { + Enabled bool ` + "`json:\"enabled,omitempty\"`" + ` + Enabled2 bool ` + "`json:\"enabled2\"`" + ` + } ` + "`json:\"config\"`" + ` + Age int ` + "`json:\"age\"`" + ` + }` expected := []FieldMetaData{ - {Name: "Server", Type: "struct", Tag: "config", IsNested: true, IsOptional: false, Depth: 0}, - {Name: "Name", Type: "string", Tag: "name", IsNested: false, IsOptional: false, Depth: 1}, - {Name: "Enabled", Type: "bool", Tag: "enabled,omitempty", IsNested: false, IsOptional: true, Depth: 1}, - {Name: "Enabled2", Type: "bool", Tag: "enabled2", IsNested: false, IsOptional: false, Depth: 1}, - {Name: "Age", Type: "int", Tag: "age", IsNested: false, IsOptional: false, Depth: 0}, + {Name: "Server", Type: "struct", Tag: "config", IsNested: true, Depth: 0}, + {Name: "Name", Type: "string", Tag: "name", IsNested: false, Depth: 1}, + {Name: "Enabled", Type: "bool", Tag: "enabled,omitempty", IsNested: false, Depth: 1}, + {Name: "Enabled2", Type: "bool", Tag: "enabled2", IsNested: false, Depth: 1}, + {Name: "Age", Type: "int", Tag: "age", IsNested: false, Depth: 0}, } actual := ParseStruct(data, 0) @@ -99,3 +82,100 @@ func TestParseNestedStruct(t *testing.T) { } } } + +func TestExtractNames(t *testing.T) { + tests := []struct { + fields Fields + expected []string + }{ + { + fields: Fields { + {Name: "Name", Type: "string", Tag: "name", Depth: 0}, + {Name: "Age", Type: "int", Tag: "age", Depth: 0}, + }, + expected: []string{"Name", "Age"}, + }, + { + fields: Fields{ + {Name: "Server", Type: "struct", Tag: "config", IsNested: true, Depth: 0}, + {Name: "Name", Type: "string", Tag: "name", IsNested: false, Depth: 1}, + {Name: "Enabled", Type: "bool", Tag: "enabled,omitempty", IsNested: false, Depth: 1}, + {Name: "Enabled2", Type: "bool", Tag: "enabled2", IsNested: false, Depth: 1}, + {Name: "Age", Type: "int", Tag: "age", IsNested: false, Depth: 0}, + }, + expected: []string{"Server", "Name", "Enabled", "Enabled2", "Age"}, + }, + } + + for i, test := range tests { + actual := test.fields.Names() + if len(actual) != len(test.expected) { + t.Errorf("Test %d - Expected and actual slice lengths differ. Expected %d, got %d", i, len(test.expected), len(actual)) + return + } + + for j, actualName := range actual { + expectedName := test.expected[j] + if actualName != expectedName { + t.Errorf("Test %d - Expected %s, got %s", i, expectedName, actualName) + } + } + } +} + +func TestZip(t *testing.T) { + cases := []struct { + name string + json []byte + keys []string + expected map[string]interface{} + expectErr bool + }{ + { + "Valid JSON with String, Number and Boolean", + []byte(`{"name":"test","age":30,"active":true, "memo": null}`), + []string{"name", "age", "active", "memo"}, + map[string]interface{}{"name": "test", "age": 30.0, "active": true, "memo": nil}, + false, + }, + { + "whitespace containing JSON", + []byte(`{"name":"value with whitespcae","age":30,"active":true, "memo": null}`), + []string{"name", "age", "active", "memo"}, + map[string]interface{}{"name": "value with whitespcae", "age": 30.0, "active": true, "memo": nil}, + false, + }, + { + "unknown type throw error", + []byte(`{"name":"test","age":30,"active":true, "memo": None}`), + []string{"name", "age", "active", "memo", "unknown"}, + map[string]interface{}{"name": "test", "age": 30.0, "active": true, "memo": nil}, + true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + result, err := Zip(tc.json, tc.keys...) + if (err != nil) != tc.expectErr { + t.Errorf("Zip(%s) error = %v, expectErr %v", tc.name, err, tc.expectErr) + return + } + + if tc.expectErr { + return + } + + for key, value := range result { + expectedValue, ok := tc.expected[key] + if !ok { + t.Errorf("Unexpected zipped field: %s", key) + } + + if value != expectedValue { + t.Errorf("Expected value %v for field %s, but got %v", expectedValue, key, value) + } + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 816f6c3361c..70aef23da0e 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -3,9 +3,9 @@ package json const ( SquareOpenToken = '[' SquareCloseToken = ']' - CurlyOpenToken = '{' + CurlyOpenToken = '{' CurlyCloseToken = '}' - CommaToken = ',' + CommaToken = ',' DotToken = '.' ColonToken = ':' BackTickToken = '`' @@ -32,6 +32,12 @@ const ( nullLiteral = []byte("null") ) +const ( + StructLiteral = "struct" + InterfaceLiteral = "interface{}" + MapLiteral = "map" +) + type ValueType int const ( @@ -46,7 +52,7 @@ const ( Unknown ) -func (v ValueType) ToString() string { +func (v ValueType) String() string { switch v { case NotExist: return "not-exist" @@ -64,11 +70,7 @@ func (v ValueType) ToString() string { return "null" case Unknown: return "unknown" + default: + return string(v) } } - -var ( - TrueLiteral = []byte("true") - FalseLiteral = []byte("false") - NullLiteral = []byte("null") -) diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index b739ee3b21f..c678ad5b6b1 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -24,3 +24,47 @@ func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { return false, bytes } + +func isEqualMap(a, b map[string]interface{}) bool { + if len(a) != len(b) { + return false + } + + for key, valueA := range a { + valueB, ok := b[key] + if !ok { + return false + } + + switch valueA := valueA.(type) { + case []interface{}: + if valueB, ok := valueB.([]interface{}); ok { + if !isEqualSlice(valueA, valueB) { + return false + } + } else { + return false + } + default: + if valueA != valueB { + return false + } + } + } + + return true +} + +func isEqualSlice(a, b []interface{}) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} diff --git a/gnovm/tests/files/marshal.gno b/gnovm/tests/files/marshal.gno new file mode 100644 index 00000000000..ced36c89a72 --- /dev/null +++ b/gnovm/tests/files/marshal.gno @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "strings" +) + +func Marshal(data map[string]interface{}) (string, error) { + var sb strings.Builder + sb.WriteString("{") + + isFirst := true + for key, value := range data { + if !isFirst { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("\"%s\": ", key)) + + switch v := value.(type) { + case string: + sb.WriteString(fmt.Sprintf("\"%s\"", v)) + case int64: + sb.WriteString(fmt.Sprintf("%d", v)) + case float64: + sb.WriteString(fmt.Sprintf("%f", v)) + case bool: + sb.WriteString(fmt.Sprintf("%t", v)) + default: + return "", fmt.Errorf("unsupported type") + } + + isFirst = false + } + + sb.WriteString("}") + return sb.String(), nil +} + +func main() { + zippedData := map[string]interface{}{ + "name": "John Doe", + "age": 30.0, + "isMember": true, + } + json, _ := Marshal(zippedData) + fmt.Println(json) +} \ No newline at end of file From 16e9cf30b5e04e20324f72bda793f27a801831b2 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 8 Jan 2024 19:25:12 +0900 Subject: [PATCH 35/72] handle type formatting --- examples/gno.land/p/demo/json/struct.gno | 7 ++++--- gnovm/tests/files/marshal.gno | 17 ++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 4900a90ad3b..dd52c565b70 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -2,7 +2,6 @@ package json import ( "errors" - "fmt" "regexp" "strings" @@ -140,8 +139,10 @@ func Marshal(data map[string]interface{}) (string, error) { switch v := value.(type) { case string: sb.WriteString(ufmt.Sprintf("\"%s\"", v)) - case float32: - sb.WriteString(ufmt.Sprintf("%f", v)) + case int, int32, int64: + sb.WriteString(ufmt.Sprintf("%d", v)) + case float32, float64: + sb.WriteString(fmt.Sprintf("%f", v)) case bool: sb.WriteString(ufmt.Sprintf("%t", v)) default: diff --git a/gnovm/tests/files/marshal.gno b/gnovm/tests/files/marshal.gno index ced36c89a72..c4d5929b4a6 100644 --- a/gnovm/tests/files/marshal.gno +++ b/gnovm/tests/files/marshal.gno @@ -1,8 +1,11 @@ package main import ( + "errors" "fmt" "strings" + + "gno.land/p/demo/ufmt" ) func Marshal(data map[string]interface{}) (string, error) { @@ -14,19 +17,19 @@ func Marshal(data map[string]interface{}) (string, error) { if !isFirst { sb.WriteString(", ") } - sb.WriteString(fmt.Sprintf("\"%s\": ", key)) + sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) switch v := value.(type) { case string: - sb.WriteString(fmt.Sprintf("\"%s\"", v)) - case int64: - sb.WriteString(fmt.Sprintf("%d", v)) - case float64: + sb.WriteString(ufmt.Sprintf("\"%s\"", v)) + case int, int32, int64: + sb.WriteString(ufmt.Sprintf("%d", v)) + case float32, float64: sb.WriteString(fmt.Sprintf("%f", v)) case bool: - sb.WriteString(fmt.Sprintf("%t", v)) + sb.WriteString(ufmt.Sprintf("%t", v)) default: - return "", fmt.Errorf("unsupported type") + return "", errors.New("Marshal: unknown data type") } isFirst = false From e5964bd7531f149841d0d31ff69f3fabadf23906 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 8 Jan 2024 19:30:52 +0900 Subject: [PATCH 36/72] handle type formatting --- examples/gno.land/p/demo/json/struct.gno | 7 +++---- gnovm/tests/files/marshal.gno | 15 +++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index dd52c565b70..4900a90ad3b 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -2,6 +2,7 @@ package json import ( "errors" + "fmt" "regexp" "strings" @@ -139,10 +140,8 @@ func Marshal(data map[string]interface{}) (string, error) { switch v := value.(type) { case string: sb.WriteString(ufmt.Sprintf("\"%s\"", v)) - case int, int32, int64: - sb.WriteString(ufmt.Sprintf("%d", v)) - case float32, float64: - sb.WriteString(fmt.Sprintf("%f", v)) + case float32: + sb.WriteString(ufmt.Sprintf("%f", v)) case bool: sb.WriteString(ufmt.Sprintf("%t", v)) default: diff --git a/gnovm/tests/files/marshal.gno b/gnovm/tests/files/marshal.gno index c4d5929b4a6..892806b96a8 100644 --- a/gnovm/tests/files/marshal.gno +++ b/gnovm/tests/files/marshal.gno @@ -1,11 +1,8 @@ package main import ( - "errors" "fmt" "strings" - - "gno.land/p/demo/ufmt" ) func Marshal(data map[string]interface{}) (string, error) { @@ -17,19 +14,21 @@ func Marshal(data map[string]interface{}) (string, error) { if !isFirst { sb.WriteString(", ") } - sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) + sb.WriteString(fmt.Sprintf("\"%s\": ", key)) switch v := value.(type) { case string: - sb.WriteString(ufmt.Sprintf("\"%s\"", v)) + sb.WriteString(fmt.Sprintf("\"%s\"", v)) case int, int32, int64: - sb.WriteString(ufmt.Sprintf("%d", v)) + sb.WriteString(fmt.Sprintf("%d", v)) case float32, float64: sb.WriteString(fmt.Sprintf("%f", v)) case bool: - sb.WriteString(ufmt.Sprintf("%t", v)) + sb.WriteString(fmt.Sprintf("%t", v)) + case nil: + sb.WriteString("null") default: - return "", errors.New("Marshal: unknown data type") + return "", fmt.Errorf("unsupported type") } isFirst = false From d3d0ff7c3ea477a3f5e68e33aacba112884e2d4d Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 8 Jan 2024 19:35:09 +0900 Subject: [PATCH 37/72] fmt --- .../gno.land/p/demo/json/state_machine.gno | 8 +-- examples/gno.land/p/demo/json/struct.gno | 22 ++++--- examples/gno.land/p/demo/json/struct_test.gno | 54 +++++++-------- examples/gno.land/p/demo/json/utils.gno | 66 +++++++++---------- gnovm/tests/files/marshal.gno | 17 +++-- 5 files changed, 88 insertions(+), 79 deletions(-) diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 64c0e5cc314..45a82644d4f 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -109,8 +109,8 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } var ( - curIdx int - valueFound []byte + curIdx int + valueFound []byte valueOffset int ) curI := i @@ -419,7 +419,7 @@ func ObjectEach(data []byte, cb func(key []byte, value []byte, dataType ValueTyp offset += off } - // skip over the next comma to the following token, + // skip over the next comma to the following token, // or stop if we reach the ending brace. if off, err := nextToken(data[offset:]); err != nil { return MalformedJson @@ -444,4 +444,4 @@ func ObjectEach(data []byte, cb func(key []byte, value []byte, dataType ValueTyp } return MalformedObject -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 4900a90ad3b..b9fe8785877 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -9,7 +9,7 @@ import ( "gno.land/p/demo/ufmt" ) -// Tatget Struct +// Tatget Struct // -> Extract Fields and Save them in a FieldMetaData Struct // -> Search Values from JSON based on the Extracted Fields via Get method // -> Zip the Fields with the Values @@ -21,11 +21,11 @@ const fieldRegex = `(\w+)\s+(\*?\w+|\[\]\w+|struct\s*{[^}]*}|map\[\w+\]\w+|inter type Fields []FieldMetaData type FieldMetaData struct { - Name string - Tag string - Type string - IsNested bool - Depth int + Name string + Tag string + Type string + IsNested bool + Depth int } func ParseStruct(data string, depth int) (fields Fields) { @@ -140,10 +140,14 @@ func Marshal(data map[string]interface{}) (string, error) { switch v := value.(type) { case string: sb.WriteString(ufmt.Sprintf("\"%s\"", v)) - case float32: - sb.WriteString(ufmt.Sprintf("%f", v)) + case int, int32, int64: + sb.WriteString(ufmt.Sprintf("%d", v)) + case float32.float64: + sb.WriteString(fmt.Sprintf("%f", v)) case bool: sb.WriteString(ufmt.Sprintf("%t", v)) + case nil: + sb.WriteString("null") default: return "", errors.New("Marshal: unknown data type") } @@ -153,4 +157,4 @@ func Marshal(data map[string]interface{}) (string, error) { sb.WriteString("}") return sb.String(), nil -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno index 650292c8d26..546673b3805 100644 --- a/examples/gno.land/p/demo/json/struct_test.gno +++ b/examples/gno.land/p/demo/json/struct_test.gno @@ -85,11 +85,11 @@ func TestParseNestedStruct(t *testing.T) { func TestExtractNames(t *testing.T) { tests := []struct { - fields Fields - expected []string + fields Fields + expected []string }{ { - fields: Fields { + fields: Fields{ {Name: "Name", Type: "string", Tag: "name", Depth: 0}, {Name: "Age", Type: "int", Tag: "age", Depth: 0}, }, @@ -124,20 +124,20 @@ func TestExtractNames(t *testing.T) { } func TestZip(t *testing.T) { - cases := []struct { - name string - json []byte - keys []string - expected map[string]interface{} - expectErr bool - }{ - { - "Valid JSON with String, Number and Boolean", - []byte(`{"name":"test","age":30,"active":true, "memo": null}`), - []string{"name", "age", "active", "memo"}, - map[string]interface{}{"name": "test", "age": 30.0, "active": true, "memo": nil}, - false, - }, + cases := []struct { + name string + json []byte + keys []string + expected map[string]interface{} + expectErr bool + }{ + { + "Valid JSON with String, Number and Boolean", + []byte(`{"name":"test","age":30,"active":true, "memo": null}`), + []string{"name", "age", "active", "memo"}, + map[string]interface{}{"name": "test", "age": 30.0, "active": true, "memo": nil}, + false, + }, { "whitespace containing JSON", []byte(`{"name":"value with whitespcae","age":30,"active":true, "memo": null}`), @@ -152,15 +152,15 @@ func TestZip(t *testing.T) { map[string]interface{}{"name": "test", "age": 30.0, "active": true, "memo": nil}, true, }, - } + } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - result, err := Zip(tc.json, tc.keys...) - if (err != nil) != tc.expectErr { - t.Errorf("Zip(%s) error = %v, expectErr %v", tc.name, err, tc.expectErr) - return - } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + result, err := Zip(tc.json, tc.keys...) + if (err != nil) != tc.expectErr { + t.Errorf("Zip(%s) error = %v, expectErr %v", tc.name, err, tc.expectErr) + return + } if tc.expectErr { return @@ -176,6 +176,6 @@ func TestZip(t *testing.T) { t.Errorf("Expected value %v for field %s, but got %v", expectedValue, key, value) } } - }) - } + }) + } } diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index c678ad5b6b1..4a61280a40b 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -26,45 +26,45 @@ func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { } func isEqualMap(a, b map[string]interface{}) bool { - if len(a) != len(b) { - return false - } + if len(a) != len(b) { + return false + } - for key, valueA := range a { - valueB, ok := b[key] - if !ok { - return false - } + for key, valueA := range a { + valueB, ok := b[key] + if !ok { + return false + } - switch valueA := valueA.(type) { - case []interface{}: - if valueB, ok := valueB.([]interface{}); ok { - if !isEqualSlice(valueA, valueB) { - return false - } - } else { - return false - } - default: - if valueA != valueB { - return false - } - } - } + switch valueA := valueA.(type) { + case []interface{}: + if valueB, ok := valueB.([]interface{}); ok { + if !isEqualSlice(valueA, valueB) { + return false + } + } else { + return false + } + default: + if valueA != valueB { + return false + } + } + } - return true + return true } func isEqualSlice(a, b []interface{}) bool { - if len(a) != len(b) { - return false - } + if len(a) != len(b) { + return false + } - for i := range a { - if a[i] != b[i] { - return false - } - } + for i := range a { + if a[i] != b[i] { + return false + } + } - return true + return true } diff --git a/gnovm/tests/files/marshal.gno b/gnovm/tests/files/marshal.gno index 892806b96a8..defefe54199 100644 --- a/gnovm/tests/files/marshal.gno +++ b/gnovm/tests/files/marshal.gno @@ -1,8 +1,11 @@ package main import ( + "errors" "fmt" "strings" + + "gno.land/p/demo/ufmt" ) func Marshal(data map[string]interface{}) (string, error) { @@ -14,21 +17,21 @@ func Marshal(data map[string]interface{}) (string, error) { if !isFirst { sb.WriteString(", ") } - sb.WriteString(fmt.Sprintf("\"%s\": ", key)) + sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) switch v := value.(type) { case string: - sb.WriteString(fmt.Sprintf("\"%s\"", v)) + sb.WriteString(ufmt.Sprintf("\"%s\"", v)) case int, int32, int64: - sb.WriteString(fmt.Sprintf("%d", v)) + sb.WriteString(ufmt.Sprintf("%d", v)) case float32, float64: sb.WriteString(fmt.Sprintf("%f", v)) case bool: - sb.WriteString(fmt.Sprintf("%t", v)) + sb.WriteString(ufmt.Sprintf("%t", v)) case nil: sb.WriteString("null") default: - return "", fmt.Errorf("unsupported type") + return "", errors.New("Marshal: unknown data type") } isFirst = false @@ -41,8 +44,10 @@ func Marshal(data map[string]interface{}) (string, error) { func main() { zippedData := map[string]interface{}{ "name": "John Doe", - "age": 30.0, + "age": 30, + "height": 1.8, "isMember": true, + "address": nil, } json, _ := Marshal(zippedData) fmt.Println(json) From fffe9d12a1018053a55787573076a3492c0ee509 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 10 Jan 2024 20:49:44 +0900 Subject: [PATCH 38/72] add more types for marshaling --- examples/gno.land/p/demo/json/struct.gno | 82 ++++++++++++------ examples/gno.land/p/demo/json/struct_test.gno | 42 +++++++++ gnovm/tests/files/marshal.gno | 85 +++++++++---------- 3 files changed, 141 insertions(+), 68 deletions(-) diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index b9fe8785877..e536cfafb09 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -1,6 +1,7 @@ package json import ( + "bytes" "errors" "fmt" "regexp" @@ -126,35 +127,68 @@ func Zip(json []byte, extractedKeys ...string) (zippedData map[string]interface{ return zippedData, nil } -func Marshal(data map[string]interface{}) (string, error) { +// Marshal converts the map to JSON string. +func Marshal(data interface{}) (string, error) { var sb strings.Builder - sb.WriteString("{") - isFirst := true - for key, value := range data { - if !isFirst { - sb.WriteString(", ") + switch v := data.(type) { + case map[string]interface{}: + sb.WriteString("{") + isFirst := true + for key, value := range v { + if !isFirst { + sb.WriteString(", ") + } + sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) + + if marshaledValue, err := Marshal(value); err != nil { + return "", err + } else { + sb.WriteString(marshaledValue) + } + + isFirst = false } - sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) - - switch v := value.(type) { - case string: - sb.WriteString(ufmt.Sprintf("\"%s\"", v)) - case int, int32, int64: - sb.WriteString(ufmt.Sprintf("%d", v)) - case float32.float64: - sb.WriteString(fmt.Sprintf("%f", v)) - case bool: - sb.WriteString(ufmt.Sprintf("%t", v)) - case nil: - sb.WriteString("null") - default: - return "", errors.New("Marshal: unknown data type") + sb.WriteString("}") + case []interface{}: + sb.WriteString("[") + for i, value := range v { + if i > 0 { + sb.WriteString(", ") + } + if marshaledValue, err := Marshal(value); err != nil { + return "", err + } else { + sb.WriteString(marshaledValue) + } } - - isFirst = false + sb.WriteString("]") + case string: + sb.WriteString(fmt.Sprintf("\"%s\"", v)) + case int, int8, int16, int32, int64: + sb.WriteString(fmt.Sprintf("%d", v)) + case uint, uint8, uint16, uint32, uint64: + sb.WriteString(fmt.Sprintf("%d", v)) + case float32, float64: + sb.WriteString(fmt.Sprintf("%f", v)) + case bool: + sb.WriteString(fmt.Sprintf("%t", v)) + case nil: + sb.WriteString("null") + default: + return "", errors.New("Marshal: unknown data type") } - sb.WriteString("}") return sb.String(), nil } + +func Unmarshal(json []byte, data interface{}) error { + jsonString := string(json) + // 1. get field names from the struct + + // 2. get values from json + + // 3. zip the fields with the values +} + +// TODO: pretty print diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno index 546673b3805..754bb396bc0 100644 --- a/examples/gno.land/p/demo/json/struct_test.gno +++ b/examples/gno.land/p/demo/json/struct_test.gno @@ -179,3 +179,45 @@ func TestZip(t *testing.T) { }) } } + +func TestMarshal(t *testing.T) { + tests := []struct { + name string + data map[string]interface{} + want string + wantErr bool + }{ + { + name: "Test with various data types", + data: map[string]interface{}{ + "name": "John Doe", + "age": 30, + "admin": true, + "score": 99.5, + "attributes": map[string]interface{}{ + "intelligence": 100, + "strength": 80, + }, + "friends": []interface{}{ + "Jane Doe", + "Richard Roe", + }, + }, + want: `{"name": "John Doe", "age": 30, "admin": true, "score": 99.500000, "attributes": {"intelligence": 100, "strength": 80}, "friends": ["Jane Doe", "Richard Roe"]}`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Marshal(tt.data) + if (err != nil) != tt.wantErr { + t.Errorf("Marshal() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Marshal() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/gnovm/tests/files/marshal.gno b/gnovm/tests/files/marshal.gno index defefe54199..dbbe15f550c 100644 --- a/gnovm/tests/files/marshal.gno +++ b/gnovm/tests/files/marshal.gno @@ -1,54 +1,51 @@ package main import ( - "errors" "fmt" - "strings" - "gno.land/p/demo/ufmt" + "gno.land/p/demo/json" ) -func Marshal(data map[string]interface{}) (string, error) { - var sb strings.Builder - sb.WriteString("{") - - isFirst := true - for key, value := range data { - if !isFirst { - sb.WriteString(", ") - } - sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) - - switch v := value.(type) { - case string: - sb.WriteString(ufmt.Sprintf("\"%s\"", v)) - case int, int32, int64: - sb.WriteString(ufmt.Sprintf("%d", v)) - case float32, float64: - sb.WriteString(fmt.Sprintf("%f", v)) - case bool: - sb.WriteString(ufmt.Sprintf("%t", v)) - case nil: - sb.WriteString("null") - default: - return "", errors.New("Marshal: unknown data type") - } - - isFirst = false - } - - sb.WriteString("}") - return sb.String(), nil -} - func main() { - zippedData := map[string]interface{}{ - "name": "John Doe", - "age": 30, - "height": 1.8, - "isMember": true, - "address": nil, + character := map[string]interface{}{ + "name": "Eldon", + "class": "Wizard", + "race": "Elf", + "level": 5, + "stats": map[string]interface{}{ + "strength": 8.5, + "dexterity": 14, + "constitution": 10, + "intelligence": 18, + "wisdom": 12, + "charisma": 11, + }, + "hitPoints": 27, + "alignment": "Neutral Good", + "skills": map[string]interface{}{ + "arcana": 7, + "history": 7, + "investigation": 6, + "nature": 5, + "religion": 5, + }, + "spells": []interface{}{"Magic Missile", "Fireball", "Invisibility", "Shield"}, + "inventory": map[string]interface{}{ + "weapons": []interface{}{"Quarterstaff", "Dagger"}, + "armor": "Mage Armor", + "items": []interface{}{"Potion of Healing", "Spellbook"}, + "gold": 120, + }, + "background": map[string]interface{}{ + "story": "Raised by a secret society of wizards...", + "traits": []interface{}{"Curious", "Cautious"}, + "ideals": "Knowledge is the path to power and domination.", + "bonds": "Must protect ancient wizardry tome.", + "flaws": "Can't resist a new mystery.", + }, } - json, _ := Marshal(zippedData) + + json, err := json.Marshal(character) + if err != nil { panic(err) } fmt.Println(json) -} \ No newline at end of file +} From 188e39c3b4acd4416bcef343128852e98b7fb8f7 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 11 Jan 2024 14:41:58 +0900 Subject: [PATCH 39/72] define dynamic structs and generate JSON --- examples/gno.land/p/demo/json/struct.gno | 231 +++++++++++++---------- gnovm/tests/files/marshal.gno | 78 ++++---- 2 files changed, 172 insertions(+), 137 deletions(-) diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index e536cfafb09..9e4dc8a5c3f 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -4,127 +4,144 @@ import ( "bytes" "errors" "fmt" - "regexp" "strings" + "strconv" "gno.land/p/demo/ufmt" ) -// Tatget Struct -// -> Extract Fields and Save them in a FieldMetaData Struct -// -> Search Values from JSON based on the Extracted Fields via Get method -// -> Zip the Fields with the Values -// -> Return the Zipped Data +const ( + StringType FieldType = "string" + IntType = "int" + FloatType = "float" + BooleanType = "boolean" + NilType = "nil" +) -// TODO: current regex approach is not robust and flexible. So, it needs to be improved while maintaining the recursive descent approach. -const fieldRegex = `(\w+)\s+(\*?\w+|\[\]\w+|struct\s*{[^}]*}|map\[\w+\]\w+|interface{})\s+` + "`" + `json:"([^"]+)"` + "`" +type FieldType string -type Fields []FieldMetaData +type Field struct { + Name string + Type FieldType + Value interface{} +} -type FieldMetaData struct { - Name string - Tag string - Type string - IsNested bool - Depth int +type Struct struct { + Fields []Field } -func ParseStruct(data string, depth int) (fields Fields) { - // separate fields using regular expressions - re := regexp.MustCompile(fieldRegex) - matches := re.FindAllStringSubmatch(data, -1) +func NewStruct() *Struct { + return &Struct{} +} - for _, match := range matches { - field := FieldMetaData{ - Name: match[1], - Tag: match[3], - Depth: depth, - } +func (s *Struct) AddField(name string, fieldType FieldType, value interface{}) *Struct { + field := Field{Name: name, Type: fieldType, Value: value} + s.Fields = append(s.Fields, field) + return s +} - // check if the field is a nested struct - fieldType := match[2] - field = field.checkFieldType(fieldType) - fields = append(fields, field) +func (s *Struct) AddStringField(name string, value string) *Struct { + return s.AddField(name, StringType, value) +} - // apply recursion when the field is a nested structure - if field.IsNested { - nestedData := match[2] - nestedFields := ParseStruct(nestedData, depth+1) - fields = append(fields, nestedFields...) - } - } +func (s *Struct) AddIntField(name string, value int) *Struct { + return s.AddField(name, IntType, value) +} - return fields +func (s *Struct) AddFloatField(name string, value float64) *Struct { + return s.AddField(name, FloatType, value) } -func (f FieldMetaData) checkFieldType(hayStack string) FieldMetaData { - switch { - case strings.HasPrefix(hayStack, StructLiteral): - f.Type = "struct" - f.IsNested = true - case strings.HasPrefix(hayStack, MapLiteral): - f.Type = "map" - f.IsNested = false - panic("ParseStruct: map type not supported") - case strings.HasPrefix(hayStack, InterfaceLiteral): - f.Type = "interface" - panic("ParseStruct: interface type not supported") - default: - f.Type = hayStack - f.IsNested = false - } +func (s *Struct) AddBoolField(name string, value bool) *Struct { + return s.AddField(name, BooleanType, value) +} - return f +func (s *Struct) AddNilField(name string) *Struct { + return s.AddField(name, NilType, nil) } -// Names returns the names of the fields -// which will used to search the values from JSON. -func (fs *Fields) Names() (names []string) { - for _, f := range *fs { - names = append(names, f.Name) +func (s *Struct) Marshal() (string, error) { + var sb strings.Builder + sb.WriteString("{") + + for i, field := range s.Fields { + if i > 0 { + sb.WriteString(", ") + } + + sb.WriteString(fmt.Sprintf("\"%s\": ", field.Name)) + + if valueStr, err := marshalValue(field.Value); err != nil { + return "", err + } else { + sb.WriteString(valueStr) + } } - return names + sb.WriteString("}") + return sb.String(), nil +} + +type CustomMarshaller interface { + MarshalCustom() (string, error) } -// TODO: support for complex types -// Zip zips the fields with the values from JSON. -func Zip(json []byte, extractedKeys ...string) (zippedData map[string]interface{}, err error) { - zippedData = make(map[string]interface{}) +func marshalValue(value interface{}) (string, error) { + if customVal, ok := value.(CustomMarshaller); ok { + return customVal.MarshalCustom() + } - for _, key := range extractedKeys { - value, dataType, offset, err := Get(json, key) - if err != nil { - return nil, err - } + var sb strings.Builder - switch dataType { - case String: - zippedData[key] = string(value) - case Number: - if n, err := ParseFloatLiteral(value); err != nil { - return nil, err + switch v := value.(type) { + case map[string]interface{}: + sb.WriteString("{") + isFirst := true + for key, value := range v { + if !isFirst { + sb.WriteString(", ") + } + sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) + + if marshaledValue, err := marshalValue(value); err != nil { + return "", err } else { - zippedData[key] = n + sb.WriteString(marshaledValue) } - case Boolean: - if b, err := ParseBoolLiteral(value); err != nil { - return nil, err + + isFirst = false + } + sb.WriteString("}") + case []interface{}: + sb.WriteString("[") + for i, value := range v { + if i > 0 { + sb.WriteString(", ") + } + if marshaledValue, err := marshalValue(value); err != nil { + return "", err } else { - zippedData[key] = b + sb.WriteString(marshaledValue) } - case Array: - panic("Zip: TODO array type not supported") - case Object: - panic("Zip: TODO object type not supported") - case Null: - zippedData[key] = nil - default: - return nil, errors.New("Zip: unknown data type") } + sb.WriteString("]") + case string: + sb.WriteString(strconv.Quote(v)) + case int, int8, int16, int32, int64: + sb.WriteString(fmt.Sprintf("%d", v)) + case uint, uint8, uint16, uint32, uint64: + sb.WriteString(fmt.Sprintf("%d", v)) + case float32, float64: + sb.WriteString(fmt.Sprintf("%g", v)) + case bool: + sb.WriteString(fmt.Sprintf("%t", v)) + case nil: + sb.WriteString("null") + default: + return "", errors.New("Marshal: unknown data type") } - return zippedData, nil + return sb.String(), nil } // Marshal converts the map to JSON string. @@ -182,13 +199,31 @@ func Marshal(data interface{}) (string, error) { return sb.String(), nil } -func Unmarshal(json []byte, data interface{}) error { - jsonString := string(json) - // 1. get field names from the struct - - // 2. get values from json - - // 3. zip the fields with the values +func Unmarshal(data []byte, s *Struct) error { + for _, field := range s.Fields { + value, dataType, _, err := Get(data, field.Name) + if err != nil { + return err + } + + switch dataType { + case String: + field.Value = string(value) + case Number, Float: + if n, err := ParseFloatLiteral(value); err != nil { + return err + } else { + field.Value = n + } + case Boolean: + if b, err := ParseBoolLiteral(value); err != nil { + return err + } else { + field.Value = b + } + default: + return fmt.Errorf("unsupported type: %v", dataType) + } + } + return nil } - -// TODO: pretty print diff --git a/gnovm/tests/files/marshal.gno b/gnovm/tests/files/marshal.gno index dbbe15f550c..6a351e0e4fa 100644 --- a/gnovm/tests/files/marshal.gno +++ b/gnovm/tests/files/marshal.gno @@ -1,51 +1,51 @@ package main import ( + "errors" "fmt" + "strconv" + "strings" + "gno.land/p/demo/ufmt" "gno.land/p/demo/json" ) + func main() { - character := map[string]interface{}{ - "name": "Eldon", - "class": "Wizard", - "race": "Elf", - "level": 5, - "stats": map[string]interface{}{ - "strength": 8.5, - "dexterity": 14, - "constitution": 10, - "intelligence": 18, - "wisdom": 12, - "charisma": 11, - }, - "hitPoints": 27, - "alignment": "Neutral Good", - "skills": map[string]interface{}{ - "arcana": 7, - "history": 7, - "investigation": 6, - "nature": 5, - "religion": 5, - }, - "spells": []interface{}{"Magic Missile", "Fireball", "Invisibility", "Shield"}, - "inventory": map[string]interface{}{ - "weapons": []interface{}{"Quarterstaff", "Dagger"}, - "armor": "Mage Armor", - "items": []interface{}{"Potion of Healing", "Spellbook"}, - "gold": 120, - }, - "background": map[string]interface{}{ - "story": "Raised by a secret society of wizards...", - "traits": []interface{}{"Curious", "Cautious"}, - "ideals": "Knowledge is the path to power and domination.", - "bonds": "Must protect ancient wizardry tome.", - "flaws": "Can't resist a new mystery.", - }, + data := []byte(`{"Name": "John Doe", "Age": 30, "IsActive": true}`) + s := json.NewStruct(). + AddStringField("Name", "John Doe"). + AddIntField("Age", 30). + AddBoolField("IsActive", true). + AddFloatField("Score", 4.5) + + jsonStr, err := s.Marshal() + if err != nil { + fmt.Println("Error marshaling:", err) + return + } + println("Marshalled:") + println(jsonStr) + println("--------------------") + + if json.Unmarshal(data, s); err != nil { + fmt.Println("Error unmarshaling:", err) + return } - json, err := json.Marshal(character) - if err != nil { panic(err) } - fmt.Println(json) + println("Unmarshalled:") + for _, field := range s.Fields { + fmt.Printf("%s: %v\n", field.Name, field.Value) + } } + + +// output: +// Marshalled: +// {"Name": "John Doe", "Age": 30, "IsActive": true, "Score": 4.5} +// -------------------- +// Unmarshalled: +// Name: John Doe +// Age: 30 +// IsActive: true +// Score: 4.5 \ No newline at end of file From cc0c1ed5823afcf3b9eb5b6245941a1e9b5de521 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 12 Jan 2024 19:20:46 +0900 Subject: [PATCH 40/72] reduce memory alloc when marshal --- .../gno.land/p/demo/json/state_machine.gno | 106 -------- examples/gno.land/p/demo/json/struct.gno | 239 ++++++++++-------- examples/gno.land/p/demo/json/struct_test.gno | 223 ---------------- gnovm/tests/files/{marshal.gno => json0.gno} | 20 +- gnovm/tests/files/json1.gno | 39 +++ 5 files changed, 178 insertions(+), 449 deletions(-) delete mode 100644 examples/gno.land/p/demo/json/struct_test.gno rename gnovm/tests/files/{marshal.gno => json0.gno} (77%) create mode 100644 gnovm/tests/files/json1.gno diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 45a82644d4f..7b41e4f0dbb 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -339,109 +339,3 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int return offset, nil } - -func ObjectEach(data []byte, cb func(key []byte, value []byte, dataType ValueType, offset int) error, keys ...string) (err error) { - offset := 0 - - // descend to the target key. - if len(keys) > 0 { - if off, err := findValueIndex(data, keys...); err != nil { - return KeyPathNotFoundError - } else { - offset = off - } - } - - if off, err := nextToken(data[offset:]); err != nil { - return MalformedObject - } else if offset += off; data[offset] != CurlyOpenToken { - return MalformedObject - } else { - offset += 1 - } - - // skip to the first token inside the object, - // or stop if we find the end of the object. - if off, err := nextToken(data[offset:]); err != nil { - return MalformedJson - } else if offset += off; data[offset] == CurlyCloseToken { - return nil - } - - for offset < len(data) { - // find the next key - var key []byte - - // check the type of next token - switch data[offset] { - case DoublyQuoteToken: - offset += 1 // accept as string and skip opening quote - case CurlyCloseToken: - return nil // found end of the object. stop - default: - return MalformedObject - } - - // find end of the string - var keyEscaped bool - if off, esc := stringEnd(data[offset:]); off == -1 { - return MalformedJson - } else { - key, keyEscaped = data[offset:offset+off-1], esc - offset += off - } - - // unescape the string if needed - if keyEscaped { - // stack allocated array for allocation-free unescaping of small string - var stackbuf [UnescapeStackBufSize]byte - if ku, err := Unescape(key, stackbuf[:]); err != nil { - return MalformedJson - } else { - key = ku - } - } - - // skip the colon - if off, err := nextToken(data[offset:]); err != nil { - return MalformedJson - } else if offset += off; data[offset] != ColonToken { - return MalformedJson - } else { - offset += 1 - } - - if value, valueType, off, err := Get(data[offset:]); err != nil { - return err - } else if err := cb(key, value, valueType, offset+off); err != nil { - return err - } else { - offset += off - } - - // skip over the next comma to the following token, - // or stop if we reach the ending brace. - if off, err := nextToken(data[offset:]); err != nil { - return MalformedJson - } else { - offset += off - switch data[offset] { - case CurlyCloseToken: - return nil // found end of the object. stop - case CommaToken: - offset += 1 // skip comma and move to the next token - default: - return MalformedObject - } - } - - // skip to the next token after the comma - if off, err := nextToken(data[offset:]); err != nil { - return MalformedJson - } else { - offset += off - } - } - - return MalformedObject -} diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 9e4dc8a5c3f..0b220ee9291 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -10,196 +10,207 @@ import ( "gno.land/p/demo/ufmt" ) -const ( - StringType FieldType = "string" - IntType = "int" - FloatType = "float" - BooleanType = "boolean" - NilType = "nil" -) - type FieldType string +// Field stores each field's information. type Field struct { - Name string - Type FieldType - Value interface{} + Name string + Type ValueType + Value interface{} + ArrayValues []interface{} + NestedStruct *Struct } +// Struct holds the whole structure of target struct. type Struct struct { Fields []Field } +// NewStruct create new empty struct placeholder. func NewStruct() *Struct { return &Struct{} } -func (s *Struct) AddField(name string, fieldType FieldType, value interface{}) *Struct { +// addField adds a field to the struct. The order of the fields is preserved. +func (s *Struct) addField(name string, fieldType ValueType, value interface{}) *Struct { field := Field{Name: name, Type: fieldType, Value: value} s.Fields = append(s.Fields, field) return s } func (s *Struct) AddStringField(name string, value string) *Struct { - return s.AddField(name, StringType, value) + return s.addField(name, String, value) } func (s *Struct) AddIntField(name string, value int) *Struct { - return s.AddField(name, IntType, value) + return s.addField(name, Number, value) } func (s *Struct) AddFloatField(name string, value float64) *Struct { - return s.AddField(name, FloatType, value) + return s.addField(name, Float, value) } func (s *Struct) AddBoolField(name string, value bool) *Struct { - return s.AddField(name, BooleanType, value) + return s.addField(name, Boolean, value) +} + +func (s *Struct) AddNullField(name string) *Struct { + return s.addField(name, Null, nil) +} + +func (s *Struct) AddObjectField(name string, inner *Struct) *Struct { + f := Field { + Name: name, + Type: Object, + NestedStruct: inner, + } + s.Fields = append(s.Fields, f) + + return s } -func (s *Struct) AddNilField(name string) *Struct { - return s.AddField(name, NilType, nil) +func (s *Struct) AddArrayField(name string, array []interface{}) *Struct { + f := Field { + Name: name, + Type: Array, + ArrayValues: array, + } + s.Fields = append(s.Fields, f) + + return s } -func (s *Struct) Marshal() (string, error) { - var sb strings.Builder - sb.WriteString("{") +// Marshaler marshals the struct into a JSON byte slice +func (s *Struct) Marshaler() ([]byte, error) { + var buf bytes.Buffer + buf.WriteByte('{') for i, field := range s.Fields { + // separate fields with comma if i > 0 { - sb.WriteString(", ") + buf.WriteByte(',') + buf.WriteByte(' ') } - sb.WriteString(fmt.Sprintf("\"%s\": ", field.Name)) + // `"name": ` + buf.WriteString(strconv.Quote(field.Name)) + buf.WriteByte(':') + buf.WriteByte(' ') - if valueStr, err := marshalValue(field.Value); err != nil { - return "", err + if field.NestedStruct != nil { + if nestedJSON, err := field.NestedStruct.Marshaler(); err != nil { + return nil, err + } else { + buf.Write(nestedJSON) + } + } else if len(field.ArrayValues) > 0 { + buf.WriteByte('[') + for i, value := range field.ArrayValues { + if i > 0 { + buf.WriteByte(',') + buf.WriteByte(' ') + } + if valueStr, err := marshalValue(value); err != nil { + return nil, err + } else { + buf.Write(valueStr) + } + } + buf.WriteByte(']') } else { - sb.WriteString(valueStr) - } + if valueStr, err := marshalValue(field.Value); err != nil { + return nil, err + } else { + buf.Write(valueStr) + } + } } - sb.WriteString("}") - return sb.String(), nil + buf.WriteByte('}') + return buf.Bytes(), nil } +// CustomMarshaller interface that the custom type should implement. type CustomMarshaller interface { - MarshalCustom() (string, error) + MarshalCustom() ([]byte, error) } -func marshalValue(value interface{}) (string, error) { +func marshalValue(value interface{}) ([]byte, error) { if customVal, ok := value.(CustomMarshaller); ok { - return customVal.MarshalCustom() + if marshaled, err := customVal.MarshalCustom(); err != nil { + return nil, err + } else { + return marshaled, nil + } } - var sb strings.Builder + var buf bytes.Buffer switch v := value.(type) { + // Object case map[string]interface{}: - sb.WriteString("{") + buf.WriteByte('{') isFirst := true for key, value := range v { if !isFirst { - sb.WriteString(", ") + buf.WriteByte(',') + buf.WriteByte(' ') } - sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) - if marshaledValue, err := marshalValue(value); err != nil { - return "", err - } else { - sb.WriteString(marshaledValue) - } + buf.WriteString(strconv.Quote(key)) + buf.WriteByte(':') + buf.WriteByte(' ') - isFirst = false - } - sb.WriteString("}") - case []interface{}: - sb.WriteString("[") - for i, value := range v { - if i > 0 { - sb.WriteString(", ") - } if marshaledValue, err := marshalValue(value); err != nil { - return "", err - } else { - sb.WriteString(marshaledValue) - } - } - sb.WriteString("]") - case string: - sb.WriteString(strconv.Quote(v)) - case int, int8, int16, int32, int64: - sb.WriteString(fmt.Sprintf("%d", v)) - case uint, uint8, uint16, uint32, uint64: - sb.WriteString(fmt.Sprintf("%d", v)) - case float32, float64: - sb.WriteString(fmt.Sprintf("%g", v)) - case bool: - sb.WriteString(fmt.Sprintf("%t", v)) - case nil: - sb.WriteString("null") - default: - return "", errors.New("Marshal: unknown data type") - } - - return sb.String(), nil -} - -// Marshal converts the map to JSON string. -func Marshal(data interface{}) (string, error) { - var sb strings.Builder - - switch v := data.(type) { - case map[string]interface{}: - sb.WriteString("{") - isFirst := true - for key, value := range v { - if !isFirst { - sb.WriteString(", ") - } - sb.WriteString(ufmt.Sprintf("\"%s\": ", key)) - - if marshaledValue, err := Marshal(value); err != nil { - return "", err + return nil, err } else { - sb.WriteString(marshaledValue) + buf.Write(marshaledValue) } isFirst = false } - sb.WriteString("}") + buf.WriteByte('}') + // Array case []interface{}: - sb.WriteString("[") + buf.WriteByte('[') for i, value := range v { if i > 0 { - sb.WriteString(", ") + buf.WriteByte(',') + buf.WriteByte(' ') } - if marshaledValue, err := Marshal(value); err != nil { - return "", err + if marshaledValue, err := marshalValue(value); err != nil { + return nil, err } else { - sb.WriteString(marshaledValue) + buf.Write(marshaledValue) } } - sb.WriteString("]") + buf.WriteByte(']') case string: - sb.WriteString(fmt.Sprintf("\"%s\"", v)) + // must handle escape characters + quoted := strconv.Quote(v) + buf.WriteString(quoted) case int, int8, int16, int32, int64: - sb.WriteString(fmt.Sprintf("%d", v)) + buf.WriteString(fmt.Sprintf("%d", v)) case uint, uint8, uint16, uint32, uint64: - sb.WriteString(fmt.Sprintf("%d", v)) + buf.WriteString(fmt.Sprintf("%d", v)) case float32, float64: - sb.WriteString(fmt.Sprintf("%f", v)) + buf.WriteString(fmt.Sprintf("%g", v)) case bool: - sb.WriteString(fmt.Sprintf("%t", v)) + buf.WriteString(fmt.Sprintf("%t", v)) case nil: - sb.WriteString("null") + buf.WriteByte('n') + buf.WriteByte('u') + buf.WriteByte('l') + buf.WriteByte('l') default: - return "", errors.New("Marshal: unknown data type") + return nil, errors.New("Marshal: unknown data type") } - return sb.String(), nil + return buf.Bytes(), nil } -func Unmarshal(data []byte, s *Struct) error { +// Unmarshaler unmarshals the JSON string into the struct. +func Unmarshaler(data []byte, s *Struct) error { for _, field := range s.Fields { value, dataType, _, err := Get(data, field.Name) if err != nil { @@ -221,8 +232,18 @@ func Unmarshal(data []byte, s *Struct) error { } else { field.Value = b } + case Object: + if field.NestedStruct != nil { + if val, _, _, err := Get(data, field.Name); err != nil { + return err + } else { + if err = Unmarshaler(val, field.NestedStruct); err != nil { + return err + } + } + } default: - return fmt.Errorf("unsupported type: %v", dataType) + return fmt.Errorf("json.Unmarshaler: unsupported type: %v", dataType) } } return nil diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno deleted file mode 100644 index 754bb396bc0..00000000000 --- a/examples/gno.land/p/demo/json/struct_test.gno +++ /dev/null @@ -1,223 +0,0 @@ -package json - -import ( - "errors" - "fmt" - "testing" -) - -func TestParseStruct(t *testing.T) { - data := ` - type Person struct { - Name string ` + "`json:\"name\"`" + ` - Age int ` + "`json:\"age\"`" + ` - } - ` - - expectedFields := Fields{ - {Name: "Name", Type: "string", Tag: "name", Depth: 0}, - {Name: "Age", Type: "int", Tag: "age", Depth: 0}, - } - - fields := ParseStruct(data, 0) - - if len(fields) != len(expectedFields) { - t.Errorf("Expected %d fields, but got %d", len(expectedFields), len(fields)) - } - - for i, field := range fields { - expectedField := expectedFields[i] - - if field.Name != expectedField.Name { - t.Errorf("Expected field name %s, but got %s", expectedField.Name, field.Name) - } - - if field.Type != expectedField.Type { - t.Errorf("Expected field type %s, but got %s", expectedField.Type, field.Type) - } - - if field.Tag != expectedField.Tag { - t.Errorf("Expected field tag %s, but got %s", expectedField.Tag, field.Tag) - } - - if field.Depth != expectedField.Depth { - t.Errorf("Expected field depth %d, but got %d", expectedField.Depth, field.Depth) - } - - if field.IsNested != expectedField.IsNested { - t.Errorf("Expected field IsNested %t, but got %t", expectedField.IsNested, field.IsNested) - } - } -} - -func TestParseNestedStruct(t *testing.T) { - data := `type Server struct { - Name string ` + "`json:\"name\"`" + ` - Config struct { - Enabled bool ` + "`json:\"enabled,omitempty\"`" + ` - Enabled2 bool ` + "`json:\"enabled2\"`" + ` - } ` + "`json:\"config\"`" + ` - Age int ` + "`json:\"age\"`" + ` - }` - - expected := []FieldMetaData{ - {Name: "Server", Type: "struct", Tag: "config", IsNested: true, Depth: 0}, - {Name: "Name", Type: "string", Tag: "name", IsNested: false, Depth: 1}, - {Name: "Enabled", Type: "bool", Tag: "enabled,omitempty", IsNested: false, Depth: 1}, - {Name: "Enabled2", Type: "bool", Tag: "enabled2", IsNested: false, Depth: 1}, - {Name: "Age", Type: "int", Tag: "age", IsNested: false, Depth: 0}, - } - - actual := ParseStruct(data, 0) - - if len(actual) != len(expected) { - t.Errorf("Expected and actual slice lengths differ. Expected %d, got %d", len(expected), len(actual)) - return - } - - for i, actualField := range actual { - expectedField := expected[i] - if actualField != expectedField { - t.Errorf("Field %d - Expected %+v, got %+v", i, expectedField, actualField) - } - } -} - -func TestExtractNames(t *testing.T) { - tests := []struct { - fields Fields - expected []string - }{ - { - fields: Fields{ - {Name: "Name", Type: "string", Tag: "name", Depth: 0}, - {Name: "Age", Type: "int", Tag: "age", Depth: 0}, - }, - expected: []string{"Name", "Age"}, - }, - { - fields: Fields{ - {Name: "Server", Type: "struct", Tag: "config", IsNested: true, Depth: 0}, - {Name: "Name", Type: "string", Tag: "name", IsNested: false, Depth: 1}, - {Name: "Enabled", Type: "bool", Tag: "enabled,omitempty", IsNested: false, Depth: 1}, - {Name: "Enabled2", Type: "bool", Tag: "enabled2", IsNested: false, Depth: 1}, - {Name: "Age", Type: "int", Tag: "age", IsNested: false, Depth: 0}, - }, - expected: []string{"Server", "Name", "Enabled", "Enabled2", "Age"}, - }, - } - - for i, test := range tests { - actual := test.fields.Names() - if len(actual) != len(test.expected) { - t.Errorf("Test %d - Expected and actual slice lengths differ. Expected %d, got %d", i, len(test.expected), len(actual)) - return - } - - for j, actualName := range actual { - expectedName := test.expected[j] - if actualName != expectedName { - t.Errorf("Test %d - Expected %s, got %s", i, expectedName, actualName) - } - } - } -} - -func TestZip(t *testing.T) { - cases := []struct { - name string - json []byte - keys []string - expected map[string]interface{} - expectErr bool - }{ - { - "Valid JSON with String, Number and Boolean", - []byte(`{"name":"test","age":30,"active":true, "memo": null}`), - []string{"name", "age", "active", "memo"}, - map[string]interface{}{"name": "test", "age": 30.0, "active": true, "memo": nil}, - false, - }, - { - "whitespace containing JSON", - []byte(`{"name":"value with whitespcae","age":30,"active":true, "memo": null}`), - []string{"name", "age", "active", "memo"}, - map[string]interface{}{"name": "value with whitespcae", "age": 30.0, "active": true, "memo": nil}, - false, - }, - { - "unknown type throw error", - []byte(`{"name":"test","age":30,"active":true, "memo": None}`), - []string{"name", "age", "active", "memo", "unknown"}, - map[string]interface{}{"name": "test", "age": 30.0, "active": true, "memo": nil}, - true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - result, err := Zip(tc.json, tc.keys...) - if (err != nil) != tc.expectErr { - t.Errorf("Zip(%s) error = %v, expectErr %v", tc.name, err, tc.expectErr) - return - } - - if tc.expectErr { - return - } - - for key, value := range result { - expectedValue, ok := tc.expected[key] - if !ok { - t.Errorf("Unexpected zipped field: %s", key) - } - - if value != expectedValue { - t.Errorf("Expected value %v for field %s, but got %v", expectedValue, key, value) - } - } - }) - } -} - -func TestMarshal(t *testing.T) { - tests := []struct { - name string - data map[string]interface{} - want string - wantErr bool - }{ - { - name: "Test with various data types", - data: map[string]interface{}{ - "name": "John Doe", - "age": 30, - "admin": true, - "score": 99.5, - "attributes": map[string]interface{}{ - "intelligence": 100, - "strength": 80, - }, - "friends": []interface{}{ - "Jane Doe", - "Richard Roe", - }, - }, - want: `{"name": "John Doe", "age": 30, "admin": true, "score": 99.500000, "attributes": {"intelligence": 100, "strength": 80}, "friends": ["Jane Doe", "Richard Roe"]}`, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := Marshal(tt.data) - if (err != nil) != tt.wantErr { - t.Errorf("Marshal() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("Marshal() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/gnovm/tests/files/marshal.gno b/gnovm/tests/files/json0.gno similarity index 77% rename from gnovm/tests/files/marshal.gno rename to gnovm/tests/files/json0.gno index 6a351e0e4fa..cafae5b8d8f 100644 --- a/gnovm/tests/files/marshal.gno +++ b/gnovm/tests/files/json0.gno @@ -1,12 +1,8 @@ package main import ( - "errors" "fmt" - "strconv" - "strings" - "gno.land/p/demo/ufmt" "gno.land/p/demo/json" ) @@ -17,18 +13,19 @@ func main() { AddStringField("Name", "John Doe"). AddIntField("Age", 30). AddBoolField("IsActive", true). - AddFloatField("Score", 4.5) + AddFloatField("Score", 4.5). + AddNullField("Option") - jsonStr, err := s.Marshal() + jsonStr, err := s.Marshaler() if err != nil { fmt.Println("Error marshaling:", err) return } println("Marshalled:") - println(jsonStr) + println(string(jsonStr)) println("--------------------") - if json.Unmarshal(data, s); err != nil { + if json.Unmarshaler(data, s); err != nil { fmt.Println("Error unmarshaling:", err) return } @@ -40,12 +37,13 @@ func main() { } -// output: +// Output: // Marshalled: -// {"Name": "John Doe", "Age": 30, "IsActive": true, "Score": 4.5} +// {"Name": "John Doe", "Age": 30, "IsActive": true, "Score": 4.5, "Option": null} // -------------------- // Unmarshalled: // Name: John Doe // Age: 30 // IsActive: true -// Score: 4.5 \ No newline at end of file +// Score: 4.5 +// Option: \ No newline at end of file diff --git a/gnovm/tests/files/json1.gno b/gnovm/tests/files/json1.gno new file mode 100644 index 00000000000..6a5c9957ca5 --- /dev/null +++ b/gnovm/tests/files/json1.gno @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + + "gno.land/p/demo/json" +) + +func main() { + contactStruct := json.NewStruct(). + AddStringField("email", "john@example.com"). + AddStringField("phone", "123-456-7890") + + addressStruct := json.NewStruct(). + AddStringField("street", "123 Main St"). + AddStringField("city", "Anytown"). + AddStringField("state", "CA"). + AddIntField("zip", 12345) + + userStruct := json.NewStruct(). + AddStringField("name", "John Doe"). + AddObjectField("contact", contactStruct). + AddObjectField("address", addressStruct). + AddArrayField("roles", []interface{}{"admin", "user"}) + + s := json.NewStruct(). + AddObjectField("user", userStruct) + + jsonStr, err := s.Marshaler() + if err != nil { + fmt.Println("Error marshaling:", err) + return + } + + fmt.Println(string(jsonStr)) +} + +// Ouput: +// {"user": {"name": "John Doe", "contact": {"email": "john@example.com", "phone": "123-456-7890"}, "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": 12345}, "roles": ["admin", "user"]}} \ No newline at end of file From 31358548bde85deb49e94130a1da357db0ecc9b2 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 16 Jan 2024 16:24:47 +0900 Subject: [PATCH 41/72] CRUD for struct instance --- examples/gno.land/p/demo/json/const.gno | 2 +- .../gno.land/p/demo/json/eisel_lemire.gno | 3 +- examples/gno.land/p/demo/json/escape.gno | 2 +- examples/gno.land/p/demo/json/escape_test.gno | 2 +- examples/gno.land/p/demo/json/lexer.gno | 102 ------- examples/gno.land/p/demo/json/parser.gno | 227 ++++++++------- examples/gno.land/p/demo/json/parser_test.gno | 92 ++----- .../gno.land/p/demo/json/state_machine.gno | 10 +- .../p/demo/json/state_machine_test.gno | 9 + examples/gno.land/p/demo/json/struct.gno | 258 ++++++++++++------ examples/gno.land/p/demo/json/struct_test.gno | 211 ++++++++++++++ gnovm/tests/files/json0.gno | 27 +- gnovm/tests/files/json1.gno | 2 +- gnovm/tests/files/json2.gno | 19 ++ 14 files changed, 587 insertions(+), 379 deletions(-) delete mode 100644 examples/gno.land/p/demo/json/lexer.gno create mode 100644 examples/gno.land/p/demo/json/struct_test.gno create mode 100644 gnovm/tests/files/json2.gno diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno index f45cee90b48..7aa714e8aa2 100644 --- a/examples/gno.land/p/demo/json/const.gno +++ b/examples/gno.land/p/demo/json/const.gno @@ -10,7 +10,7 @@ const ( BadHex = -1 - UnescapeStackBufSize = 64 + unescapeStackBufSize = 64 ) const ( diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno index 343e57739f7..4e16e2d6124 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -1,3 +1,5 @@ +package json + // This file implements the Eisel-Lemire ParseFloat algorithm, published in // 2020 and discussed extensively at // https://nigeltao.github.io/blog/2020/eisel-lemire.html @@ -10,7 +12,6 @@ // // Additional testing (on over several million test strings) is done by // https://github.com/nigeltao/parse-number-fxx-test-data/blob/5280dcfccf6d0b02a65ae282dad0b6d9de50e039/script/test-go-strconv.go -package json import ( "math" diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 36caeee4f67..36fa0cd5260 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -6,7 +6,7 @@ import ( "unicode/utf8" ) -func Unescape(input, output []byte) ([]byte, error) { +func unescape(input, output []byte) ([]byte, error) { firstBackslash := bytes.IndexByte(input, BackSlashToken) if firstBackslash == -1 { return input, nil diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index ca4d1ca98cd..4d740e04bff 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -177,7 +177,7 @@ func TestUnescape(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - output, _ := Unescape(tc.input, make([]byte, len(tc.input)+10)) + output, _ := unescape(tc.input, make([]byte, len(tc.input)+10)) if !bytes.Equal(output, tc.expected) { t.Errorf("unescape(%q) = %q; want %q", tc.input, output, tc.expected) } diff --git a/examples/gno.land/p/demo/json/lexer.gno b/examples/gno.land/p/demo/json/lexer.gno deleted file mode 100644 index 7c5d3481e09..00000000000 --- a/examples/gno.land/p/demo/json/lexer.gno +++ /dev/null @@ -1,102 +0,0 @@ -package json - -import ( - "bytes" - "errors" - "strconv" -) - -func tokenEnd(data []byte) int { - for i, tok := range data { - switch tok { - case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken, CommaToken, CurlyCloseToken, SquareCloseToken: - return i - } - } - - return len(data) -} - -func nextToken(data []byte) (int, error) { - for i, c := range data { - switch c { - case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken: - continue - default: - return i, nil - } - } - - return -1, TokenNotFound -} - -func stringEnd(data []byte) (lastCharIdx int, escaped bool) { - escaped = false - for lastCharIdx, c := range data { - switch c { - case DoublyQuoteToken: - if !escaped { - return lastCharIdx + 1, false - } - - prevCharIdx := lastCharIdx - 1 - for prevCharIdx >= 0 && data[prevCharIdx] == BackSlashToken { - prevCharIdx-- - } - - if (lastCharIdx-prevCharIdx)%2 == 0 { - return lastCharIdx + 1, true - } - - return lastCharIdx + 1, false - case BackSlashToken: - escaped = true - } - } - - return -1, escaped -} - -// Find end of the data structure, array or object. -// For array openSym and closeSym will be '[' and ']', for object '{' and '}' -func blockEnd(data []byte, openSym byte, closeSym byte) int { - level := 0 - - for i := 0; i < len(data); i++ { - switch data[i] { - case DoublyQuoteToken: - se, _ := stringEnd(data[i+1:]) - if se == -1 { - return -1 - } - i += se - case openSym: - level += 1 - case closeSym: - level -= 1 - if level == 0 { - return i + 1 - } - } - } - - return -1 -} - -func keyMatched(key []byte, keyEscaped bool, stackbuf [UnescapeStackBufSize]byte, keys []string, level int) (keyUnesc []byte, err error) { - if !keyEscaped { - keyUnesc = key - } - - if ku, err := Unescape(key, stackbuf[:]); err != nil { - return nil, err - } else { - keyUnesc = ku - } - - if level > len(keys) { - return nil, KeyLevelNotMatched - } - - return keyUnesc, err -} diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 881d1eb277e..cd6001945eb 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -7,10 +7,111 @@ import ( "strconv" ) +func tokenEnd(data []byte) int { + for i, tok := range data { + switch tok { + case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken, CommaToken, CurlyCloseToken, SquareCloseToken: + return i + } + } + + return len(data) +} + +func nextToken(data []byte) (int, error) { + for i, c := range data { + switch c { + case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken: + continue + default: + return i, nil + } + } + + return -1, TokenNotFound +} + +func stringEnd(data []byte) (lastCharIdx int, escaped bool) { + escaped = false + for lastCharIdx, c := range data { + switch c { + case DoublyQuoteToken: + if !escaped { + return lastCharIdx + 1, false + } + + prevCharIdx := lastCharIdx - 1 + for prevCharIdx >= 0 && data[prevCharIdx] == BackSlashToken { + prevCharIdx-- + } + + if (lastCharIdx-prevCharIdx)%2 == 0 { + return lastCharIdx + 1, true + } + + return lastCharIdx + 1, false + case BackSlashToken: + escaped = true + } + } + + return -1, escaped +} + +// Find end of the data structure, array or object. +// For array openSym and closeSym will be '[' and ']', for object '{' and '}' +func blockEnd(data []byte, openSym byte, closeSym byte) int { + level := 0 + + for i := 0; i < len(data); i++ { + switch data[i] { + case DoublyQuoteToken: + se, _ := stringEnd(data[i+1:]) + if se == -1 { + return -1 + } + i += se + case openSym: + level += 1 + case closeSym: + level -= 1 + if level == 0 { + return i + 1 + } + } + } + + return -1 +} + +func keyMatched( + key []byte, + keyEscaped bool, + stackbuf [unescapeStackBufSize]byte, + keys []string, level int, +) (keyUnesc []byte, err error) { + if !keyEscaped { + keyUnesc = key + } + + if ku, err := unescape(key, stackbuf[:]); err != nil { + return nil, err + } else { + keyUnesc = ku + } + + if level > len(keys) { + return nil, KeyLevelNotMatched + } + + return keyUnesc, err +} + +// PaseStringLiteral parses a string from the given byte slice. func ParseStringLiteral(data []byte) (string, error) { - var buf [UnescapeStackBufSize]byte + var buf [unescapeStackBufSize]byte - bf, err := Unescape(data, buf[:]) + bf, err := unescape(data, buf[:]) if err != nil { return "", MalformedString } @@ -18,6 +119,7 @@ func ParseStringLiteral(data []byte) (string, error) { return string(bf), nil } +// ParseBoolLiteral parses a boolean value from the given byte slice. func ParseBoolLiteral(data []byte) (bool, error) { switch { case bytes.Equal(data, []byte("true")): @@ -29,6 +131,11 @@ func ParseBoolLiteral(data []byte) (bool, error) { } } +// PaseFloatLiteral parses a float64 from the given byte slice. +// +// It utilizes double-precision (64-bit) floating-point format as defined +// by the IEEE 754 standard, providing a decimal precision of approximately 15 digits. +// TODO: support for 32-bit floating-point format func ParseFloatLiteral(bytes []byte) (value float64, err error) { if len(bytes) == 0 { return -1, EmptyBytes @@ -58,6 +165,7 @@ func ParseFloatLiteral(bytes []byte) (value float64, err error) { exp10 += exp } + // for fast float64 conversion f, success := eiselLemire64(man, exp10, neg) if !success { return 0, nil @@ -108,95 +216,6 @@ func ParseIntLiteral(bytes []byte) (v int64, err error) { return int64(n), nil } -func ParseUint(b []byte, base, bitSize int) (v uint64, err error) { - if len(b) == 0 { - return 0, EmptyBytes - } - - base0 := base == 0 - s0 := string(b) - - switch { - case 2 <= base && base <= 36: - // do nothing - case base == 0: - base = 10 - if b[0] == '0' { - switch { - case len(b) >= 3 && lower(b[1]) == 'b': - base = 2 - b = b[2:] - case len(b) >= 3 && lower(b[1]) == 'o': - base = 8 - b = b[2:] - case len(b) >= 3 && lower(b[1]) == 'x': - base = 16 - b = b[2:] - default: - base = 8 - b = b[1:] - } - } - default: - return 0, errors.New("Base Error") - } - - if bitSize == 0 { - bitSize = IntSize - } else if bitSize < 0 || bitSize > 64 { - return 0, errors.New("BitSize Error") - } - - var cutoff uint64 - switch base { - case 10: - cutoff = maxUint64/10 + 1 - case 16: - cutoff = maxUint64/16 + 1 - default: - cutoff = maxUint64/uint64(base) + 1 - } - - maxVal := uint64(1)<= byte(base) { - return 0, errors.New("ParseUint Syntax Error") - } - - if n >= cutoff { - return maxVal, errors.New("ParseUint Range Error") - } - n *= uint64(base) - - n1 := n + uint64(d) - if n1 < n || n1 > maxVal { - return maxVal, errors.New("ParseUint Range Error") - } - n = n1 - } - - if underscores && !underscoreOK(s0) { - return 0, errors.New("ParseUint Syntax Error") - } - - return n, nil -} - // lower is a lower-case letter if and only if // c is either that lower-case letter or the equivalent upper-case letter. // Instead of writing c == 'x' || c == 'X' one can write lower(c) == 'x'. @@ -287,13 +306,13 @@ func underscoreOK(s string) bool { return saw != UnderScoreToken } -// getType is a function that takes a byte slice and an offset as input parameters. +// getTypeFromByteSlice is a function that takes a byte slice and an offset as input parameters. // It returns a byte slice representing the data type, a ValueType indicating the type of data, // an integer representing the end offset, and an error if any. // If the input byte slice is empty, it returns an error indicating that no JSON data was provided. // Otherwise, it calls the parseValue function to parse the value and returns the parsed data type, // the end offset, and any error encountered during parsing. -func getType(data []byte, offset int) ([]byte, ValueType, int, error) { +func getTypeFromByteSlice(data []byte, offset int) ([]byte, ValueType, int, error) { if len(data) == 0 { return nil, Unknown, offset, errors.New("no JSON data provided") } @@ -306,6 +325,28 @@ func getType(data []byte, offset int) ([]byte, ValueType, int, error) { return data[offset:endOffset], dataType, endOffset, nil } +// getTypeFromValue check the type of the given value and returns the corresponding ValueType. +func getTypeFromValue(v interface{}) ValueType { + switch v.(type) { + case string: + return String + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return Number + case float32, float64: + return Float + case bool: + return Boolean + case nil: + return Null + case map[string]interface{}: + return Object + case []interface{}: + return Array + default: + return Unknown + } +} + // parseValue parses a JSON value from the given data starting at the specified offset. // It returns the parsed value, the new offset after parsing, and an error if any. func parseValue(data []byte, offset int) (ValueType, int, error) { diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index 09c434f829d..7c7cdf22734 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -69,7 +69,7 @@ func TestParseBoolLiteral(t *testing.T) { } func TestParseFloatLiteral(t *testing.T) { - testCases := []struct { + tests := []struct { input string expected float64 }{ @@ -85,18 +85,18 @@ func TestParseFloatLiteral(t *testing.T) { {"999999999999999999999", -1}, } - for _, tc := range testCases { - t.Run(tc.input, func(t *testing.T) { - got, _ := ParseFloatLiteral([]byte(tc.input)) - if got != tc.expected { - t.Errorf("ParseFloatLiteral(%s): got %v, want %v", tc.input, got, tc.expected) + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got, _ := ParseFloatLiteral([]byte(tt.input)) + if got != tt.expected { + t.Errorf("ParseFloatLiteral(%s): got %v, want %v", tt.input, got, tt.expected) } }) } } func TestParseFloatWithScientificNotation(t *testing.T) { - testCases := []struct { + tests := []struct { input string expected float64 }{ @@ -120,22 +120,22 @@ func TestParseFloatWithScientificNotation(t *testing.T) { {"-1E-1", -0.1}, } - for _, tc := range testCases { - t.Run(tc.input, func(t *testing.T) { - got, err := ParseFloatLiteral([]byte(tc.input)) - if got != tc.expected { - t.Errorf("ParseFloatLiteral(%s): got %v, want %v", tc.input, got, tc.expected) + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got, err := ParseFloatLiteral([]byte(tt.input)) + if got != tt.expected { + t.Errorf("ParseFloatLiteral(%s): got %v, want %v", tt.input, got, tt.expected) } if err != nil { - t.Errorf("ParseFloatLiteral(%s): got error %v", tc.input, err) + t.Errorf("ParseFloatLiteral(%s): got error %v", tt.input, err) } }) } } func TestParseIntLiteral(t *testing.T) { - testCases := []struct { + tests := []struct { input string expected int64 }{ @@ -159,11 +159,11 @@ func TestParseIntLiteral(t *testing.T) { {"-27670116110564327410", 0}, } - for _, tc := range testCases { - t.Run(tc.input, func(t *testing.T) { - got, _ := ParseIntLiteral([]byte(tc.input)) - if got != tc.expected { - t.Errorf("ParseIntLiteral(%s): got %v, want %v", tc.input, got, tc.expected) + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got, _ := ParseIntLiteral([]byte(tt.input)) + if got != tt.expected { + t.Errorf("ParseIntLiteral(%s): got %v, want %v", tt.input, got, tt.expected) } }) } @@ -195,58 +195,6 @@ var parseUint64Tests = []parseUint64Test{ {"+1", 0, true}, } -type parseUint32Test struct { - in string - out uint32 - err bool -} - -var parseUint32Tests = []parseUint32Test{ - {"", 0, true}, - {"0", 0, nil}, - {"1", 1, nil}, - {"12345", 12345, nil}, - {"012345", 12345, nil}, - {"12345x", 0, true}, - {"987654321", 987654321, nil}, - {"4294967295", 1<<32 - 1, nil}, - {"4294967296", 1<<32 - 1, true}, - {"1_2_3_4_5", 0, true}, // base=10 so no underscores allowed - {"_12345", 0, true}, - {"_12345", 0, true}, - {"1__2345", 0, true}, - {"12345_", 0, true}, -} - -func TestParseUint(t *testing.T) { - switch IntSize { - case 32: - for _, tc := range parseUint32Tests { - t.Run(tc.in, func(t *testing.T) { - got, err := ParseUint([]byte(tc.in), 10, 32) - if got != tc.out { - t.Errorf("ParseUint(%s): got %v, want %v", tc.in, got, tc.out) - } - if (err != nil) != tc.err { - t.Errorf("ParseUint(%s): got error %v", tc.in, err) - } - }) - } - case 64: - for _, tc := range parseUint64Tests { - t.Run(tc.in, func(t *testing.T) { - got, err := ParseUint([]byte(tc.in), 10, 64) - if got != tc.out { - t.Errorf("ParseUint(%s): got %v, want %v", tc.in, got, tc.out) - } - if (err != nil) != tc.err { - t.Errorf("ParseUint(%s): got error %v", tc.in, err) - } - }) - } - } -} - func TestGetType(t *testing.T) { tests := []struct { data []byte @@ -265,7 +213,7 @@ func TestGetType(t *testing.T) { } for i, tt := range tests { - gotData, gotVT, _, gotErr := getType(tt.data, tt.offset) + gotData, gotVT, _, gotErr := getTypeFromByteSlice(tt.data, tt.offset) if !bytes.Equal(gotData, tt.expected) { t.Errorf("%d. expected data=%s, but got data=%s", i, tt.expected, gotData) diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 7b41e4f0dbb..35386de7e1c 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -37,7 +37,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { keyLevel int level int lastMatched bool = true - stackbuf [UnescapeStackBufSize]byte + stackbuf [unescapeStackBufSize]byte ) for i := 0; i < len(data); i++ { @@ -170,8 +170,8 @@ func findKeyStart(data []byte, key string) (int, error) { i += 1 } - var stackbuf [UnescapeStackBufSize]byte - if ku, err := Unescape([]byte(key), stackbuf[:]); err == nil { + var stackbuf [unescapeStackBufSize]byte + if ku, err := unescape([]byte(key), stackbuf[:]); err == nil { key = string(ku) } @@ -200,7 +200,7 @@ func findKeyStart(data []byte, key string) (int, error) { // for unescape: if there are no escape sequences, this is cheap; if there are, it is a // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize if keyEscaped { - if ku, err := Unescape(k, stackbuf[:]); err != nil { + if ku, err := unescape(k, stackbuf[:]); err != nil { break } else { k = ku @@ -241,7 +241,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, } offset += nO - if value, dataType, endOffset, err = getType(data, offset); err != nil { + if value, dataType, endOffset, err = getTypeFromByteSlice(data, offset); err != nil { return value, dataType, offset, endOffset, err } diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index 47d9147a930..6f5fb542dd9 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -577,3 +577,12 @@ func TestArrayEach(t *testing.T) { } }, "a", "b") } + +func TestArrayEach2(t *testing.T) { + test := []byte(`{"a": ["string", 42, true, false, null, {"b": 1}, [1, 2, 3]]}`) + ArrayEach(test, func(value []byte, dataType ValueType, offset int, err error) { + if string(value) != "string" && string(value) != "42" && string(value) != "true" && string(value) != "false" && string(value) != "null" && string(value) != `{"b": 1}` && string(value) != "[1, 2, 3]" { + t.Errorf("Wrong item: %s", string(value)) + } + }, "a") +} diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 0b220ee9291..a64759d077d 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -4,26 +4,33 @@ import ( "bytes" "errors" "fmt" - "strings" + "regexp" "strconv" + "strings" "gno.land/p/demo/ufmt" ) -type FieldType string - // Field stores each field's information. type Field struct { - Name string - Type ValueType - Value interface{} - ArrayValues []interface{} - NestedStruct *Struct + // name holds the field name. + name string + // typ holds the field type. + typ ValueType + // value holds the field value. + value interface{} + // arrayValues store the each value of the array. + arrayValues []interface{} + // nestedStruct represents the JSON object. + nestedStruct *Struct } // Struct holds the whole structure of target struct. type Struct struct { - Fields []Field + // fields holds the fields of the struct. + fields []Field + // TODO: may use map to speed up the search. + // fmap map[string]*Field } // NewStruct create new empty struct placeholder. @@ -33,8 +40,8 @@ func NewStruct() *Struct { // addField adds a field to the struct. The order of the fields is preserved. func (s *Struct) addField(name string, fieldType ValueType, value interface{}) *Struct { - field := Field{Name: name, Type: fieldType, Value: value} - s.Fields = append(s.Fields, field) + field := Field{name: name, typ: fieldType, value: value} + s.fields = append(s.fields, field) return s } @@ -59,57 +66,58 @@ func (s *Struct) AddNullField(name string) *Struct { } func (s *Struct) AddObjectField(name string, inner *Struct) *Struct { - f := Field { - Name: name, - Type: Object, - NestedStruct: inner, + f := Field{ + name: name, + typ: Object, + nestedStruct: inner, } - s.Fields = append(s.Fields, f) + s.fields = append(s.fields, f) return s } func (s *Struct) AddArrayField(name string, array []interface{}) *Struct { - f := Field { - Name: name, - Type: Array, - ArrayValues: array, + f := Field{ + name: name, + typ: Array, + arrayValues: array, } - s.Fields = append(s.Fields, f) + s.fields = append(s.fields, f) return s } -// Marshaler marshals the struct into a JSON byte slice -func (s *Struct) Marshaler() ([]byte, error) { +/* Encoder */ + +// Marshaler marshals the struct into a JSON byte slice. +func (s *Struct) Marshal() ([]byte, error) { var buf bytes.Buffer buf.WriteByte('{') - for i, field := range s.Fields { - // separate fields with comma + for i, field := range s.fields { if i > 0 { - buf.WriteByte(',') - buf.WriteByte(' ') + buf.WriteString(", ") } // `"name": ` - buf.WriteString(strconv.Quote(field.Name)) - buf.WriteByte(':') - buf.WriteByte(' ') + buf.WriteString(strconv.Quote(field.name)) + buf.WriteString(": ") - if field.NestedStruct != nil { - if nestedJSON, err := field.NestedStruct.Marshaler(); err != nil { + if field.nestedStruct != nil { + // nested struct are considered as object. + // need to apply marshaler to each field. + if nestedJSON, err := field.nestedStruct.Marshal(); err != nil { return nil, err } else { buf.Write(nestedJSON) } - } else if len(field.ArrayValues) > 0 { + } else if len(field.arrayValues) > 0 { buf.WriteByte('[') - for i, value := range field.ArrayValues { + for i, value := range field.arrayValues { if i > 0 { - buf.WriteByte(',') - buf.WriteByte(' ') + buf.WriteString(", ") } + // apply marshaler to each value if valueStr, err := marshalValue(value); err != nil { return nil, err } else { @@ -118,7 +126,7 @@ func (s *Struct) Marshaler() ([]byte, error) { } buf.WriteByte(']') } else { - if valueStr, err := marshalValue(field.Value); err != nil { + if valueStr, err := marshalValue(field.value); err != nil { return nil, err } else { buf.Write(valueStr) @@ -135,6 +143,7 @@ type CustomMarshaller interface { MarshalCustom() ([]byte, error) } +// marshalValue marshals the value by its type. func marshalValue(value interface{}) ([]byte, error) { if customVal, ok := value.(CustomMarshaller); ok { if marshaled, err := customVal.MarshalCustom(); err != nil { @@ -153,13 +162,12 @@ func marshalValue(value interface{}) ([]byte, error) { isFirst := true for key, value := range v { if !isFirst { - buf.WriteByte(',') - buf.WriteByte(' ') + buf.WriteString(", ") } + // `"": ` buf.WriteString(strconv.Quote(key)) - buf.WriteByte(':') - buf.WriteByte(' ') + buf.WriteString(": ") if marshaledValue, err := marshalValue(value); err != nil { return nil, err @@ -175,8 +183,7 @@ func marshalValue(value interface{}) ([]byte, error) { buf.WriteByte('[') for i, value := range v { if i > 0 { - buf.WriteByte(',') - buf.WriteByte(' ') + buf.WriteString(", ") } if marshaledValue, err := marshalValue(value); err != nil { return nil, err @@ -198,53 +205,146 @@ func marshalValue(value interface{}) ([]byte, error) { case bool: buf.WriteString(fmt.Sprintf("%t", v)) case nil: - buf.WriteByte('n') - buf.WriteByte('u') - buf.WriteByte('l') - buf.WriteByte('l') + buf.WriteString("null") default: - return nil, errors.New("Marshal: unknown data type") + return nil, errors.New(fmt.Sprintf("json.Marshal: unsupported type %s", value)) } return buf.Bytes(), nil } -// Unmarshaler unmarshals the JSON string into the struct. -func Unmarshaler(data []byte, s *Struct) error { - for _, field := range s.Fields { - value, dataType, _, err := Get(data, field.Name) - if err != nil { - return err - } +/* Decoder */ + +// Unmarshal unmarshals the JSON data into the struct. +// The struct should be declared as a Struct instance. +func Unmarshal(data []byte, s *Struct) error { + keys, err := extractKeysFromJSON(data) + if err != nil { + return err + } - switch dataType { - case String: - field.Value = string(value) - case Number, Float: - if n, err := ParseFloatLiteral(value); err != nil { + for _, key := range keys { + // regex-parsed key contains double quotes, so need to remove them. + // this is unnecessary if we don't use regex when retrieving keys. + key = strings.ReplaceAll(key, `"`, "") + val, typ, _, err := Get(data, key) + if err != nil { + return err + } + + switch typ { + case String: + s.AddStringField(key, string(val)) + case Number, Float: + if got, err := ParseFloatLiteral(val); err != nil { return err } else { - field.Value = n + s.AddFloatField(key, got) } - case Boolean: - if b, err := ParseBoolLiteral(value); err != nil { + case Boolean: + if got, err := ParseBoolLiteral(val); err != nil { return err } else { - field.Value = b - } - case Object: - if field.NestedStruct != nil { - if val, _, _, err := Get(data, field.Name); err != nil { - return err - } else { - if err = Unmarshaler(val, field.NestedStruct); err != nil { - return err - } - } + s.AddBoolField(key, got) } - default: - return fmt.Errorf("json.Unmarshaler: unsupported type: %v", dataType) - } - } - return nil + case Null: + s.AddNullField(key) + // TODO: Support Array and Object types + default: + return errors.New("json.Unmarshal: unknown data type") + } + } + + return nil +} + +func extractKeysFromJSON(data []byte) ([]string, error) { + // parse the JSON key names. + // TODO: reduce overhead. regex is easy to use but slow. + re := regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) + + // check if the JSON is valid. + if !re.Match(data) { + return nil, errors.New("Invalid JSON format") + } + + var keys []string + matches := re.FindAllSubmatch(data, -1) + for _, match := range matches { + key := string(match[0]) + + // reject empty keys + if key == "" { + return nil, errors.New("json: key cannot be an empty string") + } + + key = strings.TrimSuffix(key, ":") + key = strings.TrimSpace(key) + keys = append(keys, key) + } + + return keys, nil +} + +// String apply the marshaler to the struct and return the JSON string. +func (s *Struct) String() string { + if json, err := s.Marshal(); err != nil { + return err.Error() + } else { + return string(json) + } +} + +/* CRUD */ +// CRUD operations for the struct. +// by using this methods, we can modify the struct also the JSON data easily. + +// Search finds a field by name and returns it. +func (s *Struct) Search(name string) (*Field, error) { + for i, field := range s.fields { + if field.name == name { + return &s.fields[i], nil + } + } + + return nil, errors.New(fmt.Sprintf("json.Search: Field name %s not found", name)) +} + +// Remove removes a field from the struct instance. +func (s *Struct) Remove(name string) error { + for i, field := range s.fields { + if field.name == name { + // arranges the slice to remove the field. + s.fields = append(s.fields[:i], s.fields[i+1:]...) + break + } + } + + return errors.New(fmt.Sprintf("json.Search: Field name %s not found", name)) +} + +// Update updates the value of the field. +// +// The updated value must be the same type as the original one. +// Otherwise, it will return an error. +func (s *Struct) Update(name string, newVal interface{}) error { + // search field by name + f, err := s.Search(name) + if err != nil { + return errors.New(fmt.Sprintf("json.Update: Field name %s not found", name)) + } + + // type assertion + newValType := getTypeFromValue(newVal) + if f.typ != newValType { + return errors.New( + fmt.Sprintf("json.Update: Field type mismatch. Expected %s, got %s", + f.typ.String(), + newValType.String(), + )) + } + + // update the value + f.value = newVal + return nil } diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno new file mode 100644 index 00000000000..31a98791846 --- /dev/null +++ b/examples/gno.land/p/demo/json/struct_test.gno @@ -0,0 +1,211 @@ +package json + +import ( + "errors" + "testing" +) + +var dummyStruct = &Struct{ + fields: []Field{ + {name: "field1", value: "value1"}, + {name: "field2", value: "value2"}, + {name: "field3", value: "value3"}, + }, +} + +func TestStructSearch(t *testing.T) { + s := dummyStruct + tests := []struct { + name string + searchName string + expectedName string + expectedErr error + }{ + { + name: "Test 1", + searchName: "field1", + expectedName: "field1", + expectedErr: nil, + }, + { + name: "Test 2", + searchName: "field2", + expectedName: "field2", + expectedErr: nil, + }, + { + name: "Test 3", + searchName: "field4", + expectedName: "", + expectedErr: errors.New("Field name not found"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + field, err := s.Search(tt.searchName) + + if err != nil && tt.expectedErr == nil { + t.Errorf("Search() error = %v, expectedErr %v", err, tt.expectedErr) + return + } + + if field != nil && field.name != tt.expectedName { + t.Errorf("Search() got = %v, want %v", field.name, tt.expectedName) + } + }) + } +} + +func TestRemoveField(t *testing.T) { + tests := []struct { + name string + fields []Field + remove string + expectedFields []string + }{ + { + name: "remove existing field", + fields: []Field{ + {name: "a", typ: String, value: "apple"}, + {name: "b", typ: String, value: "banana"}, + }, + remove: "a", + expectedFields: []string{"b"}, + }, + { + name: "remove non-existing field", + fields: []Field{ + {name: "a", typ: String, value: "apple"}, + }, + remove: "b", + expectedFields: []string{"a"}, + }, + { + name: "remove from empty struct", + fields: []Field{}, + remove: "a", + expectedFields: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewStruct() + for _, f := range tt.fields { + s.addField(f.name, f.typ, f.value) + } + + s.Remove(tt.remove) + + if !fieldsMatch(s.fields, tt.expectedFields) { + t.Errorf("Remove() got = %v, want %v", fieldNames(s.fields), tt.expectedFields) + } + }) + } +} + +func TestUpdateField(t *testing.T) { + tests := []struct { + name string + init []Field + fName string + newValue interface{} + expectedFields []Field + expectError bool + }{ + { + name: "update existing field", + init: []Field{ + {name: "a", typ: String, value: "apple"}, + {name: "b", typ: Number, value: 10}, + }, + fName: "a", + newValue: "avocado", + expectedFields: []Field{ + {name: "a", typ: String, value: "avocado"}, + {name: "b", typ: Number, value: 10}, + }, + expectError: false, + }, + { + name: "update non-existing field", + init: []Field{ + {name: "a", typ: String, value: "apple"}, + }, + fName: "b", + newValue: "banana", + expectedFields: []Field{ + {name: "a", typ: String, value: "apple"}, + }, + expectError: true, + }, + { + name: "type mismatch update", + init: []Field{ + {name: "a", typ: String, value: "apple"}, + }, + fName: "a", + newValue: 100, // int instead of string + expectedFields: []Field{ + {name: "a", typ: String, value: "apple"}, + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewStruct() + for _, f := range tt.init { + s.addField(f.name, f.typ, f.value) + } + + err := s.Update(tt.fName, tt.newValue) + if (err != nil) != tt.expectError { + t.Errorf("Update() error = %v, expectErr %v", err, tt.expectError) + return + } + + // check if fields are equal + for _, ef := range tt.expectedFields { + f, err := s.Search(ef.name) + if err != nil { + t.Errorf("Update() error = %v, expectErr %v", err, tt.expectError) + return + } + + if f.value != ef.value { + t.Errorf("Update() got = %v, want %v", f.value, ef.value) + } + + if f.typ != ef.typ { + t.Errorf("Update() got = %v, want %v", f.typ, ef.typ) + } + } + }) + } +} + +func fieldsMatch(fs []Field, names []string) bool { + if len(fs) != len(names) { + return false + } + + for i, f := range fs { + if f.name != names[i] { + return false + } + } + + return true +} + +func fieldNames(fs []Field) []string { + names := make([]string, len(fs)) + for i, f := range fs { + names[i] = f.name + } + + return names +} diff --git a/gnovm/tests/files/json0.gno b/gnovm/tests/files/json0.gno index cafae5b8d8f..0219b8d2aef 100644 --- a/gnovm/tests/files/json0.gno +++ b/gnovm/tests/files/json0.gno @@ -8,42 +8,23 @@ import ( func main() { - data := []byte(`{"Name": "John Doe", "Age": 30, "IsActive": true}`) s := json.NewStruct(). AddStringField("Name", "John Doe"). AddIntField("Age", 30). AddBoolField("IsActive", true). AddFloatField("Score", 4.5). - AddNullField("Option") + AddNullField("Option"). + AddArrayField("arr", []interface{}{"one", 2, true, nil}) - jsonStr, err := s.Marshaler() + jsonStr, err := s.Marshal() if err != nil { fmt.Println("Error marshaling:", err) return } println("Marshalled:") println(string(jsonStr)) - println("--------------------") - - if json.Unmarshaler(data, s); err != nil { - fmt.Println("Error unmarshaling:", err) - return - } - - println("Unmarshalled:") - for _, field := range s.Fields { - fmt.Printf("%s: %v\n", field.Name, field.Value) - } } - // Output: // Marshalled: -// {"Name": "John Doe", "Age": 30, "IsActive": true, "Score": 4.5, "Option": null} -// -------------------- -// Unmarshalled: -// Name: John Doe -// Age: 30 -// IsActive: true -// Score: 4.5 -// Option: \ No newline at end of file +// {"Name": "John Doe", "Age": 30, "IsActive": true, "Score": 4.5, "Option": null, "arr": ["one", 2, true, null]} \ No newline at end of file diff --git a/gnovm/tests/files/json1.gno b/gnovm/tests/files/json1.gno index 6a5c9957ca5..0a013cfef45 100644 --- a/gnovm/tests/files/json1.gno +++ b/gnovm/tests/files/json1.gno @@ -26,7 +26,7 @@ func main() { s := json.NewStruct(). AddObjectField("user", userStruct) - jsonStr, err := s.Marshaler() + jsonStr, err := s.Marshal() if err != nil { fmt.Println("Error marshaling:", err) return diff --git a/gnovm/tests/files/json2.gno b/gnovm/tests/files/json2.gno new file mode 100644 index 00000000000..d1cc9147e3c --- /dev/null +++ b/gnovm/tests/files/json2.gno @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + + "gno.land/p/demo/json" +) + +func main() { + data := []byte(`{"name": "Foo", "age": 20, "score": 4.2, "subscribed": true, "option": null}`) + + s := json.NewStruct() + err := json.Unmarshal(data, s) + if err != nil { + panic(err) + } + + fmt.Println(s.String()) +} \ No newline at end of file From 4f93bc29c28ac6a9f3ab42d94d3bd19a5e774da5 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 16 Jan 2024 18:18:59 +0900 Subject: [PATCH 42/72] add output --- gnovm/tests/files/json2.gno | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gnovm/tests/files/json2.gno b/gnovm/tests/files/json2.gno index d1cc9147e3c..bba3a12bf1a 100644 --- a/gnovm/tests/files/json2.gno +++ b/gnovm/tests/files/json2.gno @@ -15,5 +15,9 @@ func main() { panic(err) } + // `String()` method use Marshal internally to generate human readable JSON. fmt.Println(s.String()) -} \ No newline at end of file +} + +// Output: +// {"name": "Foo", "age": 20, "score": 4.2, "subscribed": true, "option": null} \ No newline at end of file From 7a62f9391885c609b046b0a0cba42b45cfec1770 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 17 Jan 2024 15:07:48 +0900 Subject: [PATCH 43/72] add basic linter and example --- examples/gno.land/p/demo/json/parser_test.gno | 26 ------ .../gno.land/p/demo/json/state_machine.gno | 34 ++++++- .../p/demo/json/state_machine_test.gno | 6 +- gnovm/tests/files/json3.gno | 93 +++++++++++++++++++ 4 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 gnovm/tests/files/json3.gno diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index 7c7cdf22734..ff1406aac96 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -169,32 +169,6 @@ func TestParseIntLiteral(t *testing.T) { } } -type parseUint64Test struct { - in string - out uint64 - err bool -} - -var parseUint64Tests = []parseUint64Test{ - {"", 0, true}, - {"0", 0, nil}, - {"1", 1, nil}, - {"12345", 12345, nil}, - {"012345", 12345, nil}, - {"12345x", 0, true}, - {"98765432100", 98765432100, nil}, - {"18446744073709551615", 1<<64 - 1, nil}, - {"18446744073709551616", 1<<64 - 1, true}, - {"18446744073709551620", 1<<64 - 1, true}, - {"1_2_3_4_5", 0, true}, // base=10 so no underscores allowed - {"_12345", 0, true}, - {"1__2345", 0, true}, - {"12345_", 0, true}, - {"-0", 0, true}, - {"-1", 0, true}, - {"+1", 0, true}, -} - func TestGetType(t *testing.T) { tests := []struct { data []byte diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 35386de7e1c..42c3358281a 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -3,6 +3,7 @@ package json import ( "bytes" "errors" + "regexp" "strconv" ) @@ -245,16 +246,45 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, return value, dataType, offset, endOffset, err } - // Strip quotes from string values + // strip quotes from string values if dataType == String { value = value[1 : len(value)-1] } + // remove unnecessary whitespace characters to make more readable + if dataType == Object { + value = adjustIndentation(value) + } + return value[:len(value):len(value)], dataType, offset, endOffset, nil } +// adjustIndentation is a kind of hack to remove the extra tabs and spaces from the Object values. +// TODO: storing depth information while parsing step, may allows us to avoid regex. +func adjustIndentation(value []byte) []byte { + re := regexp.MustCompile(`(?m)^\t+`) + + // remove the rest, leaving only one of the tabs at the beginning of each line + value = re.ReplaceAll(value, []byte("\t")) + + // remove the last line's spaces + reLastLineSpaces := regexp.MustCompile(`(?m)^\s+\}`) + value = reLastLineSpaces.ReplaceAll(value, []byte("}")) + + return value +} + // Get is a function that retrieves a value from the given data based on the provided keys. -// It returns the value, data type, offset, and any error encountered during the retrieval process. +// It returns the value, data type, offset (position), and any error encountered during the retrieval process. +// +// For example: +// +// ``` gno +// data := []byte(`{"name":{"first":"Janet","last":"Prichard"},"age":47}`) +// value, dataType, offset, err := json.Get(data, "name", "last") +// ``` +// +// It returns the value `Prichard` with data type `String`, the offset `37`, and `nil` for the error. func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { a, b, _, d, e := internalGet(data, keys...) return a, b, d, e diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index 6f5fb542dd9..38fa4878218 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -549,7 +549,7 @@ func TestGet(t *testing.T) { } func TestArrayEach(t *testing.T) { - mock := []byte(`{"a": { "b":[{"x": 1} ,{"x": 2},{ "x":3}, {"x":4} ]}}`) + mock := []byte(`{"a": { "b":[{"x": "1"} ,{"x": 2},{ "x":3}, {"x":4} ]}}`) count := 0 ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err error) { @@ -557,7 +557,7 @@ func TestArrayEach(t *testing.T) { switch count { case 1: - if string(value) != `{"x": 1}` { + if string(value) != `{"x": "1"}` { t.Errorf("Wrong first item: %s", string(value)) } case 2: @@ -579,7 +579,7 @@ func TestArrayEach(t *testing.T) { } func TestArrayEach2(t *testing.T) { - test := []byte(`{"a": ["string", 42, true, false, null, {"b": 1}, [1, 2, 3]]}`) + test := []byte(`{"a": ["string", 42, true, false, null, [1, 2, 3]]}`) ArrayEach(test, func(value []byte, dataType ValueType, offset int, err error) { if string(value) != "string" && string(value) != "42" && string(value) != "true" && string(value) != "false" && string(value) != "null" && string(value) != `{"b": 1}` && string(value) != "[1, 2, 3]" { t.Errorf("Wrong item: %s", string(value)) diff --git a/gnovm/tests/files/json3.gno b/gnovm/tests/files/json3.gno new file mode 100644 index 00000000000..e88794c798d --- /dev/null +++ b/gnovm/tests/files/json3.gno @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + + "gno.land/p/demo/json" +) + +// demo for value retrieval from JSON data +func main() { + data := []byte(` + { + "character": { + "race": "Tenebrine", + "species": "Raccoon", + "class": "Thief", + "name": "Whisker" + "stats": { + "strength": 8, + "dexterity": 18, + "constitution": 12, + "intelligence": 11, + "wisdom": 12, + "charisma": 14 + }, + }, + "skills": { + "acrobatics": 4, + "animal_handling": 1, + "arcana": 1, + "athletics": 2, + "deception": 5, + "history": 1, + "insight": 3, + "intimidation": 2, + "investigation": 4, + "medicine": 1, + "nature": 1, + "perception": 4, + "performance": 3, + "persuasion": 3, + "religion": 1, + "sleight_of_hand": 6, + "stealth": 5, + "survival": 3 + }, + "equipment": { + "weapons": ["Daggers", "Short sword"], + "armor": "Leather armor", + "items": ["Thieves' tools", "Backpack", "Rations (5 days)", "Lock picks", "Disguise kit"], + "gold": 200 + }, + "backstory": "Raised on the streets of the shadowy Tenebrine city, ${player} learned to survive by his wits and agility.\nSkilled in the art of stealth and deception, he mastered the craft of thievery to claim what the streets denied him.\nKnown only by his alias 'Shadowpaw', he now seeks greater challenges and wealth beyond the alleyways of his upbringing." + } + `) + + // `json.Get` function takes a JSON and a path of keys (must enter each key name separately) to retrieve the value. + if val, typ, _, err := json.Get(data, "character", "name"); err != nil { + panic(err) + } else { + fmt.Printf("name: %s type: %v\n", val, typ.String()) + } + + if ab, typ, _, err := json.Get(data, "character", "stats"); err != nil { + panic(err) + } else { + fmt.Printf("stats: %s, type: %v\n\n", ab, typ.String()) + } + + if stealth, typ, _, err := json.Get(data, "skills", "stealth"); err != nil { + panic(err) + } else { + fmt.Printf("stealth: %s, type: %v\n\n", stealth, typ.String()) + } + + if weapons, typ, _, err := json.Get(data, "equipment", "weapons"); err != nil { + panic(err) + } else { + fmt.Printf("weapons: %s, type: %v\n\n", weapons, typ.String()) + } + + if armor, typ, _, err := json.Get(data, "equipment", "armor"); err != nil { + panic(err) + } else { + fmt.Printf("armor: %s, type: %v\n\n", armor, typ.String()) + } + + if backstory, typ, _, err := json.Get(data, "backstory"); err != nil { + panic(err) + } else { + fmt.Printf("backstory: %s, type: %v\n\n", backstory, typ.String()) + } +} \ No newline at end of file From 807d77db55c3a222e776b4d7a065574db6181379 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 22 Jan 2024 17:21:04 +0900 Subject: [PATCH 44/72] add lookup table --- examples/gno.land/p/demo/json/errors.gno | 67 +++++-- examples/gno.land/p/demo/json/escape.gno | 59 ++++--- examples/gno.land/p/demo/json/parser.gno | 124 ++++--------- .../gno.land/p/demo/json/state_machine.gno | 147 ++++++++++++++-- .../p/demo/json/state_machine_test.gno | 165 ++++++++++++++++++ examples/gno.land/p/demo/json/struct.gno | 42 ++++- examples/gno.land/p/demo/json/struct_test.gno | 23 --- examples/gno.land/p/demo/json/utils.gno | 65 +++++-- gnovm/tests/files/json1.gno | 1 + gnovm/tests/files/json2.gno | 1 + gnovm/tests/files/json4.gno | 23 +++ 11 files changed, 534 insertions(+), 183 deletions(-) create mode 100644 gnovm/tests/files/json4.gno diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index f484441e972..740740cd7c3 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -3,21 +3,54 @@ package json import "errors" const ( - KeyPathNotFoundError error = errors.New("key path not found") - ArrayIndexNotFound = errors.New("array index not found") - TokenNotFound = errors.New("token not found") - KeyLevelNotMatched = errors.New("key level not matched") - Overflow = errors.New("overflow") - EmptyBytes = errors.New("empty bytes") - InvalidArrayIndex = errors.New("invalid array index") - InvalidExponents = errors.New("invalid exponents") - NonDigitCharacters = errors.New("non-digit characters") - MultipleDecimalPoints = errors.New("multiple decimal points") - MalformedType = errors.New("malformed type") - MalformedString = errors.New("malformed string") - MalformedValue = errors.New("malformed value") - MalformedObject = errors.New("malformed object") - MalformedArray = errors.New("malformed array") - MalformedJson = errors.New("malformed json array") - UnknownValueType = errors.New("unknown value type") + // KeyPathNotFoundError occurs when a specified key path does not exist in the JSON structure. + KeyPathNotFoundError error = errors.New("JSON Error: key path not found in the JSON structure") + + // ArrayIndexNotFound occurs when the specified index is beyond the range of the array. + ArrayIndexNotFound = errors.New("JSON Error: array index not found or out of range") + + // TokenNotFound occurs when a particular token (expected as part of the structure) is not found. + TokenNotFound = errors.New("JSON Error: expected token not found in the JSON input") + + // KeyLevelNotMatched occurs when the key levels do not match the expected structure or depth. + KeyLevelNotMatched = errors.New("JSON Error: key level does not match the expected structure or depth") + + // Overflow occurs when a number in the JSON exceeds the range that can be handled. + Overflow = errors.New("JSON Error: numeric value exceeds the range limit") + + // EmptyBytes occurs when the JSON input is empty or has no content. + EmptyBytes = errors.New("JSON Error: empty bytes: the JSON input is empty or has no content") + + // InvalidArrayIndex occurs when the index used for an array is not an integer or out of valid range. + InvalidArrayIndex = errors.New("JSON Error: invalid array index: index should be an integer and within the valid range") + + // InvalidExponents occurs when there's an error related to the format or range of exponents in numbers. + InvalidExponents = errors.New("JSON Error: invalid format or range of exponents in a numeric value") + + // NonDigitCharacters occurs when there are non-digit characters where a number is expected. + NonDigitCharacters = errors.New("JSON Error: non-digit characters found where a number is expected") + + // MultipleDecimalPoints occurs when a number has more than one decimal point. + MultipleDecimalPoints = errors.New("JSON Error: multiple decimal points found in a number") + + // MalformedType occurs when the type of a value does not match the expected type. + MalformedType = errors.New("JSON Error: malformed type: the type of the value does not match the expected type") + + // MalformedString occurs when a string is improperly formatted, like unescaped characters or incorrect quotes. + MalformedString = errors.New("JSON Error: malformed string: improperly formatted string, check for unescaped characters or incorrect quotes") + + // MalformedValue occurs when a value does not conform to the expected format or structure. + MalformedValue = errors.New("JSON Error: malformed value: the value does not conform to the expected format or structure") + + // MalformedObject occurs when a JSON object is improperly formatted. + MalformedObject = errors.New("JSON Error: malformed object: the JSON object is improperly formatted or structured") + + // MalformedArray occurs when a JSON array is improperly formatted. + MalformedArray = errors.New("JSON Error: malformed array: the JSON array is improperly formatted or structured") + + // MalformedJson occurs when the entire JSON structure is improperly formatted or structured. + MalformedJson = errors.New("JSON Error: malformed JSON: the entire JSON structure is improperly formatted or structured") + + // UnknownValueType occurs when the JSON contains a value of an unrecognized or unsupported type. + UnknownValueType = errors.New("JSON Error: unknown value type: the value type is unrecognized or unsupported") ) diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 36fa0cd5260..cc9bd73898f 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -6,15 +6,22 @@ import ( "unicode/utf8" ) +// unescape takes an input byte slice, processes it to unescape certain characters, +// and writes the result into an output byte slice. +// +// it returns the processed slice and any error encountered during the unescape operation. func unescape(input, output []byte) ([]byte, error) { + // find the index of the first backslash in the input slice. firstBackslash := bytes.IndexByte(input, BackSlashToken) if firstBackslash == -1 { return input, nil } + // ensure the output slice has enough capacity to hold the result. if cap(output) < len(input) { output = make([]byte, len(input)) } else { + // if the capacity is sufficient, slice the output to the length of the input. output = output[0:len(input)] } @@ -23,15 +30,16 @@ func unescape(input, output []byte) ([]byte, error) { buf := output[firstBackslash:] for len(input) > 0 { - inLen, bufLen := processEscapedUTF8(input, buf) - if inLen == -1 { - return nil, errors.New("Encountered an invalid escape sequence in a string") + // inLen is the number of bytes consumed in the input + // bufLen is the number of bytes written to buf. + if inLen, bufLen := processEscapedUTF8(input, buf); inLen == -1 { + return nil, errors.New("JSON Error: Encountered an invalid escape sequence in a string.") + } else { + input = input[inLen:] + buf = buf[bufLen:] } - input = input[inLen:] - buf = buf[bufLen:] - - // copy everything until the next backslash + // find the next backslash in the remaining input. nextBackslash := bytes.IndexByte(input, BackSlashToken) if nextBackslash == -1 { copy(buf, input) @@ -47,17 +55,6 @@ func unescape(input, output []byte) ([]byte, error) { return output[:len(output)-len(buf)], nil } -var escapeMap = map[byte]byte{ - '"': DoublyQuoteToken, - '\\': BackSlashToken, - '/': SlashToken, - 'b': BackSpaceToken, - 'f': FormFeedToken, - 'n': NewLineToken, - 'r': CarriageReturnToken, - 't': TabToken, -} - // isSurrogatePair returns true if the rune is a surrogate pair. // // A surrogate pairs are used in UTF-16 encoding to encode characters @@ -118,6 +115,17 @@ func decodeUnicodeEscape(b []byte) (rune, int) { return combineSurrogates(r, r2), 12 } +var escapeByteSet = [256]byte{ + '"': DoublyQuoteToken, + '\\': BackSlashToken, + '/': SlashToken, + 'b': BackSpaceToken, + 'f': FormFeedToken, + 'n': NewLineToken, + 'r': CarriageReturnToken, + 't': TabToken, +} + // processEscapedUTF8 processes the escape sequence in the given byte slice and // and converts them to UTF-8 characters. The function returns the length of the processed input and output. // @@ -135,16 +143,19 @@ func processEscapedUTF8(in, out []byte) (inLen int, outLen int) { return -1, -1 } - if val, ok := escapeMap[in[1]]; ok { - out[0] = val - return 2, 1 - } - - if in[1] == 'u' { + escapeSeqLen := 2 + escapeChar := in[1] + if escapeChar == 'u' { if r, size := decodeUnicodeEscape(in); size != -1 { outLen = utf8.EncodeRune(out, r) return size, outLen } + } else { + val := escapeByteSet[escapeChar] + if val != 0 { + out[0] = val + return escapeSeqLen, 1 + } } return -1, -1 diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index cd6001945eb..2db31f32131 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -31,13 +31,13 @@ func nextToken(data []byte) (int, error) { return -1, TokenNotFound } -func stringEnd(data []byte) (lastCharIdx int, escaped bool) { +func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { escaped = false for lastCharIdx, c := range data { switch c { case DoublyQuoteToken: if !escaped { - return lastCharIdx + 1, false + return lastCharIdx + 1, false, nil } prevCharIdx := lastCharIdx - 1 @@ -46,42 +46,42 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool) { } if (lastCharIdx-prevCharIdx)%2 == 0 { - return lastCharIdx + 1, true + return lastCharIdx + 1, true, nil } - return lastCharIdx + 1, false + return lastCharIdx + 1, false, nil case BackSlashToken: escaped = true } } - return -1, escaped + return -1, escaped, MalformedString } // Find end of the data structure, array or object. // For array openSym and closeSym will be '[' and ']', for object '{' and '}' -func blockEnd(data []byte, openSym byte, closeSym byte) int { +func blockEnd(data []byte, openSym byte, closeSym byte) (int, error) { level := 0 for i := 0; i < len(data); i++ { switch data[i] { case DoublyQuoteToken: - se, _ := stringEnd(data[i+1:]) - if se == -1 { - return -1 + if se, _, err := stringEnd(data[i+1:]); err != nil { + return -1, err + } else { + i += se } - i += se case openSym: level += 1 case closeSym: level -= 1 if level == 0 { - return i + 1 + return i + 1, nil } } } - return -1 + return -1, errors.New("JSON Error: malformed data structure found while parsing container value") } func keyMatched( @@ -127,7 +127,7 @@ func ParseBoolLiteral(data []byte) (bool, error) { case bytes.Equal(data, []byte("false")): return false, nil default: - return false, MalformedValue + return false, errors.New("JSON Error: malformed boolean value found while parsing boolean value") } } @@ -145,7 +145,7 @@ func ParseFloatLiteral(bytes []byte) (value float64, err error) { var exponentPart []byte for i, c := range bytes { - if c == 'e' || c == 'E' { + if lower(c) == 'e' { exponentPart = bytes[i+1:] bytes = bytes[:i] break @@ -216,14 +216,13 @@ func ParseIntLiteral(bytes []byte) (v int64, err error) { return int64(n), nil } -// lower is a lower-case letter if and only if -// c is either that lower-case letter or the equivalent upper-case letter. -// Instead of writing c == 'x' || c == 'X' one can write lower(c) == 'x'. -// Note that lower of non-letters can produce other non-letters. -func lower(c byte) byte { - return c | ('x' - 'X') -} - +// extractMantissaAndExp10 parses a byte slice representing a decimal number and extracts the mantissa and the exponent of its base-10 representation. +// It iterates through the bytes, constructing the mantissa by treating each byte as a digit. +// If a decimal point is encountered, the function keeps track of the position of the decimal point to calculate the exponent. +// The function ensures that: +// - The number contains at most one decimal point. +// - All characters in the byte slice are digits or a single decimal point. +// - The resulting mantissa does not overflow a uint64. func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { var man uint64 var exp10 int @@ -258,66 +257,16 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { return man, exp10, nil } -// underscoreOK reports whether the underscores in s are allowed. -// Checking them in this one function lets all the parsers skip over them simply. -// Underscore must appear only between digits or between a base prefix and a digit. -func underscoreOK(s string) bool { - // saw tracks the last character (class) we saw: - // ^ for beginning of number, - // 0 for a digit or base prefix, - // _ for an underscore, - // ! for none of the above. - saw := '^' - i := 0 - - // Optional sign. - if len(s) >= 1 && (s[0] == MinusToken || s[0] == PlusToken) { - s = s[1:] - } - - // Optional base prefix. - hex := false - if len(s) >= 2 && s[0] == '0' && (lower(s[1]) == 'b' || lower(s[1]) == 'o' || lower(s[1]) == 'x') { - i = 2 - saw = '0' // base prefix counts as a digit for "underscore as digit separator" - hex = lower(s[1]) == 'x' - } - - for ; i < len(s); i++ { - currentChar := s[i] - isDigit := '0' <= currentChar && currentChar <= '9' - isHexChar := hex && 'a' <= lower(currentChar) && lower(currentChar) <= 'f' - - if isDigit || isHexChar { - saw = '0' - } else if currentChar == UnderScoreToken { - if saw != '0' { - return false - } - saw = UnderScoreToken - } else { - if saw == UnderScoreToken { - return false - } - saw = BangToken - } - } - - return saw != UnderScoreToken -} - // getTypeFromByteSlice is a function that takes a byte slice and an offset as input parameters. +// // It returns a byte slice representing the data type, a ValueType indicating the type of data, // an integer representing the end offset, and an error if any. -// If the input byte slice is empty, it returns an error indicating that no JSON data was provided. -// Otherwise, it calls the parseValue function to parse the value and returns the parsed data type, -// the end offset, and any error encountered during parsing. func getTypeFromByteSlice(data []byte, offset int) ([]byte, ValueType, int, error) { if len(data) == 0 { - return nil, Unknown, offset, errors.New("no JSON data provided") + return nil, Unknown, offset, errors.New("JSON Error: JSON data not provided.") } - dataType, endOffset, err := parseValue(data, offset) + dataType, endOffset, err := ParseValue(data, offset) if err != nil { return nil, dataType, offset, err } @@ -347,9 +296,9 @@ func getTypeFromValue(v interface{}) ValueType { } } -// parseValue parses a JSON value from the given data starting at the specified offset. +// ParseValue parses a JSON value from the given data starting at the specified offset. // It returns the parsed value, the new offset after parsing, and an error if any. -func parseValue(data []byte, offset int) (ValueType, int, error) { +func ParseValue(data []byte, offset int) (ValueType, int, error) { switch data[offset] { case DoublyQuoteToken: return parseString(data, offset) @@ -362,10 +311,11 @@ func parseValue(data []byte, offset int) (ValueType, int, error) { // parseString parses a JSON string and returns its type and the end position. func parseString(data []byte, offset int) (ValueType, int, error) { - if idx, _ := stringEnd(data[offset+1:]); idx != -1 { + if idx, _, err := stringEnd(data[offset+1:]); err == nil { return String, offset + idx + 1, nil } - return String, offset, errors.New("malformed string") + + return String, offset, errors.New("JSON Error: Invalid string input found while parsing string value") } // parseContainer parses a JSON array or object and returns its type and the end position. @@ -381,9 +331,9 @@ func parseContainer(data []byte, offset int) (ValueType, int, error) { closing = CurlyCloseToken } - endOffset := blockEnd(data[offset:], data[offset], closing) + endOffset, err := blockEnd(data[offset:], data[offset], closing) if endOffset == -1 { - return containerType, offset, MalformedType + return containerType, offset, err } return containerType, offset + endOffset, nil @@ -392,13 +342,13 @@ func parseContainer(data []byte, offset int) (ValueType, int, error) { func parsePrimitive(data []byte, offset int) (ValueType, int, error) { end := tokenEnd(data[offset:]) if end == -1 { - return Unknown, offset, errors.New("malformed value") + return Unknown, offset, errors.New("JSON Error: malformed value found while parsing primitive value") } value := data[offset : offset+end] switch data[offset] { case 't', 'f': - if _boolValue, err := ParseBoolLiteral(value); err != nil { + if _b, err := ParseBoolLiteral(value); err != nil { return Unknown, offset, err } return Boolean, offset + end, nil @@ -406,20 +356,20 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { if bytes.Equal(value, nullLiteral) { return Null, offset + end, nil } - return Unknown, offset, errors.New("unknown null type") + return Unknown, offset, UnknownValueType case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': if !bytes.ContainsAny(value, ".eE") { - if _intValue, err := ParseIntLiteral(value); err != nil { + if _i, err := ParseIntLiteral(value); err != nil { return Number, offset + end, err } } - if _floatValue, err := ParseFloatLiteral(value); err != nil { + if _f, err := ParseFloatLiteral(value); err != nil { return Number, offset + end, err } return Number, offset + end, nil } - return Unknown, offset, errors.New("unknown value type") + return Unknown, offset, errors.New("JSON Error: unknown value type found while parsing primitive value") } diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 42c3358281a..b03a777b200 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -3,6 +3,7 @@ package json import ( "bytes" "errors" + "fmt" "regexp" "strconv" ) @@ -47,10 +48,11 @@ func findValueIndex(data []byte, keys ...string) (int, error) { i += 1 keyBegin := i - strEnd, keyEscaped := stringEnd(data[i:]) - if strEnd == -1 { - return -1, MalformedJson + strEnd, keyEscaped, err := stringEnd(data[i:]) + if err != nil { + return -1, err } + i += strEnd keyEnd := i - 1 @@ -87,9 +89,9 @@ func findValueIndex(data []byte, keys ...string) (int, error) { // in case parent key is matched then only we will increase the level otherwise can directly // can move to the end of this block if !lastMatched { - end := blockEnd(data[i:], CurlyOpenToken, CurlyCloseToken) + end, err := blockEnd(data[i:], CurlyOpenToken, CurlyCloseToken) if end == -1 { - return -1, MalformedJson + return -1, err } i += end - 1 } else { @@ -138,13 +140,13 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } } else { // Do not search for keys inside arrays - if arraySkip := blockEnd(data[i:], '[', ']'); arraySkip == -1 { - return -1, MalformedJson + if arraySkip, err := blockEnd(data[i:], SquareOpenToken, SquareCloseToken); arraySkip == -1 { + return -1, err } else { i += arraySkip - 1 } } - case ColonToken: // If encountered, JSON data is malformed + case ColonToken: return -1, MalformedJson } } @@ -160,6 +162,7 @@ func decreaseLevel(level, keyLevel int) (int, int) { return level, keyLevel } +// findKeyStart finds the start of a specific key in a given byte array. func findKeyStart(data []byte, key string) (int, error) { i, _ := nextToken(data) if i == -1 { @@ -182,7 +185,7 @@ func findKeyStart(data []byte, key string) (int, error) { i++ keyBegin := i - strEnd, keyEscaped := stringEnd(data[i:]) + strEnd, keyEscaped, _ := stringEnd(data[i:]) if strEnd == -1 { break } @@ -213,12 +216,12 @@ func findKeyStart(data []byte, key string) (int, error) { } case SquareOpenToken: - if end := blockEnd(data[i:], data[i], SquareCloseToken); end != -1 { + if end, _ := blockEnd(data[i:], data[i], SquareCloseToken); end != -1 { i = i + end } case CurlyOpenToken: - if end := blockEnd(data[i:], data[i], CurlyCloseToken); end != -1 { + if end, _ := blockEnd(data[i:], data[i], CurlyCloseToken); end != -1 { i = i + end } } @@ -290,7 +293,11 @@ func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset return a, b, d, e } -func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { +func ArrayEach( + data []byte, + cb func(value []byte, dataType ValueType, offset int, err error), + keys ...string, +) (offset int, err error) { if len(data) == 0 { return -1, MalformedObject } @@ -369,3 +376,119 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int return offset, nil } + +func ObjectEach( + data []byte, + cb func(key, value []byte, dataType ValueType, offset int) error, + keys ...string, +) (err error) { + offset := 0 + + // descent to desired key + if len(keys) > 0 { + if off, err := findValueIndex(data, keys...); err != nil { + errors.New("json.ObjectEach: key path not found") + } else { + offset += off + } + } + + // validate and skip past opening token + if off, err := nextToken(data[offset:]); err != nil { + return MalformedObject + } else if offset += off; data[offset] != CurlyOpenToken { + return errors.New(fmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) + } else { + offset += 1 + } + + // skip to the 1st token insize the object + if off, err := nextToken(data[offset:]); err != nil { + return errors.New(fmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) + } else if offset += off; data[offset] == CurlyCloseToken { + // find the end of the object. should return here + return nil + } + + for offset < len(data) { + // find the next key + var key []byte + + // check the next token kind + switch data[offset] { + case DoublyQuoteToken: + offset += 1 // consider as a string type and skip the first quote + case CurlyCloseToken: + return nil // end of the object + default: + return MalformedObject // invalid token found + } + + // find the end of the key + var escaped bool + if off, ok, err := stringEnd(data[offset:]); err != nil { + return errors.New(fmt.Sprintf("json.ObjectEach: invalid string at offset %d", offset)) + } else { + key, escaped = data[offset:offset+off-1], ok + offset += off + } + + // unescape the key if necessary + if escaped { + var stackbuf [unescapeStackBufSize]byte + if ku, err := unescape(key, stackbuf[:]); err != nil { + return errors.New("json.ObjectEach: string is not valid UTF-8") + } else { + key = ku + } + } + + // skip colon + if off, err := nextToken(data[offset:]); err != nil { + return errors.New(fmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) + } else if offset += off; data[offset] != ColonToken { + return errors.New(fmt.Sprintf("json.ObjectEach: cannot find colon at expected offset %d", offset)) + } else { + offset += 1 + } + + // find nearby value, call callback + if value, typ, off, err := Get(data[offset:]); err != nil { + return err + } else if err := cb(key, value, typ, offset+off); err != nil { + return err + } else { + offset += off + } + + // skip the next comma to the following token + if off, err := nextToken(data[offset:]); err != nil { + return errors.New(fmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) + } else { + offset += off + + switch data[offset] { + case CurlyCloseToken: + return nil // end of the object. should stop here. + case CommaToken: + offset += 1 // skip the comma and continue + default: + return errors.New( + fmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset), + ) + } + } + + // skip the next token if comma is found + if off, err := nextToken(data[offset:]); err != nil { + return errors.New(fmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) + } else { + offset += off // skip the next token + } + + // reset the key + key = nil + } + + return errors.New(fmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset)) +} diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index 38fa4878218..de6cda4e14e 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -1,6 +1,7 @@ package json import ( + "bytes" "errors" "fmt" "testing" @@ -586,3 +587,167 @@ func TestArrayEach2(t *testing.T) { } }, "a") } + +func TestObjectEach(t *testing.T) { + tests := []struct { + name string + input []byte + expected map[string]string + root string + isErr bool + }{ + { + name: "empty object", + input: []byte(`{}`), + expected: map[string]string{}, + root: "", + }, + { + name: "single key", + input: []byte(`{"a": 1}`), + expected: map[string]string{"a": "1"}, + root: "", + }, + { + name: "two keys", + input: []byte(`{"a": {"b": 1, "c": 2}}`), + expected: map[string]string{"b": "1", "c": "2"}, + root: "a", + }, + { + name: "two keys 2", + input: []byte(`{"a": {"b": 1, "c": 2}, "d": 4}`), + expected: map[string]string{"b": "1", "c": "2"}, + root: "a", + }, + { + name: "another set of keys", + input: []byte(`{"x": {"y": 3, "z": 4}}`), + expected: map[string]string{"y": "3", "z": "4"}, + root: "x", + }, + { + name: "multiple key-value object with many value types", + input: []byte(`{ + "key1": null, + "key2": true, + "key3": 1.23, + "key4": "string value", + "key5": [1,2,3], + "key6": {"a":"b"} + }`), + expected: map[string]string{ + "key1": "null", + "key2": "true", + "key3": "1.23", + "key4": "string value", + "key5": "[1,2,3]", + "key6": `{"a":"b"}`, + }, + }, + { + name: "so many white spaces", + input: []byte(`{"a" : 1}`), + expected: map[string]string{"a": "1"}, + }, + /* Error Cases */ + { + name: "unmatched brace", + input: []byte(`{`), + isErr: true, + }, + { + name: "unmatched brace 2", + input: []byte(`{"a": 1`), + root: "a", + isErr: true, + }, + { + name: "unmatched brace 3", + input: []byte(`}`), + isErr: true, + }, + { + name: "unmatched brace 3", + input: []byte(`{{}}{`), + isErr: true, + }, + { + name: "no object present", + input: []byte(`\t\n\r`), + isErr: true, + }, + { + name: "malformed key", + input: []byte(`{"foo: 1}`), + isErr: true, + }, + { + name: "malformed key 2", + input: []byte(`{"foo": 1, "bar: "11"}`), + root: "foo", + isErr: true, + }, + { + name: "malformed key in nested object", + input: []byte(`{"foo": {"bar: 1}}`), + root: "foo", + isErr: true, + }, + { + name: "bad value", + input: []byte(`{"foo": bar}`), + isErr: true, + }, + { + name: "no colon", + input: []byte(`{"foo" "bar"}`), + isErr: true, + }, + { + name: "no colon 2", + input: []byte(`{"foo""bar"}`), + isErr: true, + }, + { + name: "no colon 3", + input: []byte(`{"foo"; "bar"}`), + isErr: true, + }, + { + name: "invalid colon", + input: []byte(`{"foo":: "bar"}`), + isErr: true, + }, + { + name: "no trailing comma", + input: []byte(`{"foo": "bar" "baz": "qux"}`), + root: "foo", + isErr: true, + }, + { + name: "key with unicode escape", + input: []byte(`{"b": 10, "c": {"a\uD83D\uDE03b":1, "aa": false}}`), + root: "b", + isErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := ObjectEach(tc.input, func(key, value []byte, dataType ValueType, offset int) error { + expectedValue, ok := tc.expected[string(key)] + if !ok { + t.Errorf("Unexpected key: %s", string(key)) + } else if string(value) != expectedValue { + t.Errorf("Wrong value for key %s: got %s, want %s", string(key), string(value), expectedValue) + } + return nil + }, tc.root) + + if (err != nil) != tc.isErr { + t.Errorf("ObjectEach(%s) error = %v, wantErr %v", tc.name, err, tc.isErr) + } + }) + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index a64759d077d..449100e427a 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -38,6 +38,10 @@ func NewStruct() *Struct { return &Struct{} } +func (s *Struct) Fields() []Field { + return s.fields +} + // addField adds a field to the struct. The order of the fields is preserved. func (s *Struct) addField(name string, fieldType ValueType, value interface{}) *Struct { field := Field{name: name, typ: fieldType, value: value} @@ -76,15 +80,15 @@ func (s *Struct) AddObjectField(name string, inner *Struct) *Struct { return s } -func (s *Struct) AddArrayField(name string, array []interface{}) *Struct { - f := Field{ - name: name, - typ: Array, - arrayValues: array, - } - s.fields = append(s.fields, f) +func (s *Struct) AddArrayField(name string, arrayStruct *Struct) *Struct { + f := Field{ + name: name, + typ: Array, + arrayValues: arrayStruct.Fields(), + } + s.fields = append(s.fields, f) - return s + return s } /* Encoder */ @@ -249,7 +253,27 @@ func Unmarshal(data []byte, s *Struct) error { } case Null: s.AddNullField(key) - // TODO: Support Array and Object types + // case Object: + // nestedStruct := NewStruct() + // if err := ObjectEach(val, func(innerKey, innerValue []byte, innerDataType ValueType, offset int) error { + // return Unmarshal(innerValue, nestedStruct) + // }); err != nil { + // return err + // } + // s.AddObjectField(key, nestedStruct) + // case Array: + // arrayStruct := NewStruct() + // _, err := ArrayEach(val, func(value []byte, dataType ValueType, offset int, err error) { + // if err != nil { + // return + // } + // // unmarshal here + // }) + + // if err != nil { + // return err + // } + // s.AddArrayField(key, arrayStruct) default: return errors.New("json.Unmarshal: unknown data type") } diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno index 31a98791846..6e68c306591 100644 --- a/examples/gno.land/p/demo/json/struct_test.gno +++ b/examples/gno.land/p/demo/json/struct_test.gno @@ -186,26 +186,3 @@ func TestUpdateField(t *testing.T) { }) } } - -func fieldsMatch(fs []Field, names []string) bool { - if len(fs) != len(names) { - return false - } - - for i, f := range fs { - if f.name != names[i] { - return false - } - } - - return true -} - -func fieldNames(fs []Field) []string { - names := make([]string, len(fs)) - for i, f := range fs { - names[i] = f.name - } - - return names -} diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index 4a61280a40b..c2351b51f3b 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -1,20 +1,38 @@ package json func notDigit(c byte) bool { - return c < '0' || c > '9' + return (c & 0xF0) != 0x30 } -func h2i(c byte) int { - switch { - case c >= '0' && c <= '9': - return int(c - '0') - case c >= 'A' && c <= 'F': - return int(c - 'A' + 10) - case c >= 'a' && c <= 'f': - return int(c - 'a' + 10) - } +// lower converts a byte to lower case if it is an uppercase letter. +// +// In ASCII, the lowercase letters have the 6th bit set to 1, which is not set in their uppercase counterparts. +// This function sets the 6th bit of the byte, effectively converting uppercase letters to lowercase. +// It has no effect on bytes that are not uppercase letters. +func lower(c byte) byte { + return c | 0x20 +} - return BadHex +const hexLookupTable = [256]int{ + '0': 0x0, '1': 0x1, '2': 0x2, '3': 0x3, '4': 0x4, + '5': 0x5, '6': 0x6, '7': 0x7, '8': 0x8, '9': 0x9, + 'A': 0xA, 'B': 0xB, 'C': 0xC, 'D': 0xD, 'E': 0xE, 'F': 0xF, + 'a': 0xA, 'b': 0xB, 'c': 0xC, 'd': 0xD, 'e': 0xE, 'f': 0xF, + // Fill unspecified index-value pairs with key and value of -1 + 'G': -1, 'H': -1, 'I': -1, 'J': -1, + 'K': -1, 'L': -1, 'M': -1, 'N': -1, + 'O': -1, 'P': -1, 'Q': -1, 'R': -1, + 'S': -1, 'T': -1, 'U': -1, 'V': -1, + 'W': -1, 'X': -1, 'Y': -1, 'Z': -1, + 'g': -1, 'h': -1, 'i': -1, 'j': -1, + 'k': -1, 'l': -1, 'm': -1, 'n': -1, + 'o': -1, 'p': -1, 'q': -1, 'r': -1, + 's': -1, 't': -1, 'u': -1, 'v': -1, + 'w': -1, 'x': -1, 'y': -1, 'z': -1, +} + +func h2i(c byte) int { + return hexLookupTable[c] } func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { @@ -25,6 +43,8 @@ func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { return false, bytes } +/* Testing Helper Function */ + func isEqualMap(a, b map[string]interface{}) bool { if len(a) != len(b) { return false @@ -68,3 +88,26 @@ func isEqualSlice(a, b []interface{}) bool { return true } + +func fieldsMatch(fs []Field, names []string) bool { + if len(fs) != len(names) { + return false + } + + for i, f := range fs { + if f.name != names[i] { + return false + } + } + + return true +} + +func fieldNames(fs []Field) []string { + names := make([]string, len(fs)) + for i, f := range fs { + names[i] = f.name + } + + return names +} diff --git a/gnovm/tests/files/json1.gno b/gnovm/tests/files/json1.gno index 0a013cfef45..a8b504ebee5 100644 --- a/gnovm/tests/files/json1.gno +++ b/gnovm/tests/files/json1.gno @@ -6,6 +6,7 @@ import ( "gno.land/p/demo/json" ) +// Marshaling example func main() { contactStruct := json.NewStruct(). AddStringField("email", "john@example.com"). diff --git a/gnovm/tests/files/json2.gno b/gnovm/tests/files/json2.gno index bba3a12bf1a..603d08cc3c2 100644 --- a/gnovm/tests/files/json2.gno +++ b/gnovm/tests/files/json2.gno @@ -6,6 +6,7 @@ import ( "gno.land/p/demo/json" ) +// Unmarshal JSON data into a struct. func main() { data := []byte(`{"name": "Foo", "age": 20, "score": 4.2, "subscribed": true, "option": null}`) diff --git a/gnovm/tests/files/json4.gno b/gnovm/tests/files/json4.gno new file mode 100644 index 00000000000..19895ce9ec1 --- /dev/null +++ b/gnovm/tests/files/json4.gno @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + + "gno.land/p/demo/json" +) + +// dynamic struct creation and marshal to json +func main() { + s := json.NewStruct() + + for i:=0; i < 10; i++ { + s.AddIntField(fmt.Sprintf("field%d", i), i) // <- add field in runtime + } + + str, err := s.Marshal() + if err != nil { + panic(err) + } + + fmt.Println(string(str)) +} \ No newline at end of file From a49efbb245cbe0a364abef38ece1c20df35d8c95 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Sun, 11 Feb 2024 14:18:16 +0900 Subject: [PATCH 45/72] change fmt package into p/demo/ufmt --- .../gno.land/p/demo/json/state_machine.gno | 21 ++++++++++--------- examples/gno.land/p/demo/json/struct.gno | 19 ++++++++--------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index b03a777b200..12e09ca0604 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -3,9 +3,10 @@ package json import ( "bytes" "errors" - "fmt" "regexp" "strconv" + + "gno.land/p/demo/ufmt" ) func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { @@ -397,14 +398,14 @@ func ObjectEach( if off, err := nextToken(data[offset:]); err != nil { return MalformedObject } else if offset += off; data[offset] != CurlyOpenToken { - return errors.New(fmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) } else { offset += 1 } // skip to the 1st token insize the object if off, err := nextToken(data[offset:]); err != nil { - return errors.New(fmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) } else if offset += off; data[offset] == CurlyCloseToken { // find the end of the object. should return here return nil @@ -427,7 +428,7 @@ func ObjectEach( // find the end of the key var escaped bool if off, ok, err := stringEnd(data[offset:]); err != nil { - return errors.New(fmt.Sprintf("json.ObjectEach: invalid string at offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: invalid string at offset %d", offset)) } else { key, escaped = data[offset:offset+off-1], ok offset += off @@ -445,9 +446,9 @@ func ObjectEach( // skip colon if off, err := nextToken(data[offset:]); err != nil { - return errors.New(fmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) } else if offset += off; data[offset] != ColonToken { - return errors.New(fmt.Sprintf("json.ObjectEach: cannot find colon at expected offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: cannot find colon at expected offset %d", offset)) } else { offset += 1 } @@ -463,7 +464,7 @@ func ObjectEach( // skip the next comma to the following token if off, err := nextToken(data[offset:]); err != nil { - return errors.New(fmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) } else { offset += off @@ -474,14 +475,14 @@ func ObjectEach( offset += 1 // skip the comma and continue default: return errors.New( - fmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset), + ufmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset), ) } } // skip the next token if comma is found if off, err := nextToken(data[offset:]); err != nil { - return errors.New(fmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) } else { offset += off // skip the next token } @@ -490,5 +491,5 @@ func ObjectEach( key = nil } - return errors.New(fmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset)) + return errors.New(ufmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset)) } diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 449100e427a..fdb959f788b 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -3,7 +3,6 @@ package json import ( "bytes" "errors" - "fmt" "regexp" "strconv" "strings" @@ -201,17 +200,17 @@ func marshalValue(value interface{}) ([]byte, error) { quoted := strconv.Quote(v) buf.WriteString(quoted) case int, int8, int16, int32, int64: - buf.WriteString(fmt.Sprintf("%d", v)) + buf.WriteString(ufmt.Sprintf("%d", v)) case uint, uint8, uint16, uint32, uint64: - buf.WriteString(fmt.Sprintf("%d", v)) + buf.WriteString(ufmt.Sprintf("%d", v)) case float32, float64: - buf.WriteString(fmt.Sprintf("%g", v)) + buf.WriteString(ufmt.Sprintf("%g", v)) case bool: - buf.WriteString(fmt.Sprintf("%t", v)) + buf.WriteString(ufmt.Sprintf("%t", v)) case nil: buf.WriteString("null") default: - return nil, errors.New(fmt.Sprintf("json.Marshal: unsupported type %s", value)) + return nil, errors.New(ufmt.Sprintf("json.Marshal: unsupported type %s", value)) } return buf.Bytes(), nil @@ -331,7 +330,7 @@ func (s *Struct) Search(name string) (*Field, error) { } } - return nil, errors.New(fmt.Sprintf("json.Search: Field name %s not found", name)) + return nil, errors.New(ufmt.Sprintf("json.Search: Field name %s not found", name)) } // Remove removes a field from the struct instance. @@ -344,7 +343,7 @@ func (s *Struct) Remove(name string) error { } } - return errors.New(fmt.Sprintf("json.Search: Field name %s not found", name)) + return errors.New(ufmt.Sprintf("json.Search: Field name %s not found", name)) } // Update updates the value of the field. @@ -355,14 +354,14 @@ func (s *Struct) Update(name string, newVal interface{}) error { // search field by name f, err := s.Search(name) if err != nil { - return errors.New(fmt.Sprintf("json.Update: Field name %s not found", name)) + return errors.New(ufmt.Sprintf("json.Update: Field name %s not found", name)) } // type assertion newValType := getTypeFromValue(newVal) if f.typ != newValType { return errors.New( - fmt.Sprintf("json.Update: Field type mismatch. Expected %s, got %s", + ufmt.Sprintf("json.Update: Field type mismatch. Expected %s, got %s", f.typ.String(), newValType.String(), )) From 95e41ed844deb347b3e815096adfbdc921abdd6b Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Sun, 11 Feb 2024 15:06:09 +0900 Subject: [PATCH 46/72] regex as global --- examples/gno.land/p/demo/json/const.gno | 8 +++ .../gno.land/p/demo/json/state_machine.gno | 52 ++++++++++--------- examples/gno.land/p/demo/json/struct.gno | 15 ++---- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno index 7aa714e8aa2..23f5407326f 100644 --- a/examples/gno.land/p/demo/json/const.gno +++ b/examples/gno.land/p/demo/json/const.gno @@ -1,5 +1,7 @@ package json +import "regexp" + const ( SupplementalPlanesOffset = 0x10000 HighSurrogateOffset = 0xD800 @@ -20,3 +22,9 @@ const ( intSize = 32 << (^uint(0) >> 63) IntSize = intSize ) + +var ( + tabRegex = regexp.MustCompile(`(?m)^\t+`) + lastLineSpaceRegex = regexp.MustCompile(`(?m)^\s+\}`) + jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) +) \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 12e09ca0604..b92b7b43369 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -3,13 +3,17 @@ package json import ( "bytes" "errors" - "regexp" "strconv" "gno.land/p/demo/ufmt" ) +// extractValueTypeFromToken recoginizes the type of JSON value by peeking at the first byte of the value. func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { + if len(b) == 0 { + return Unknown, 0, errors.New("json: empty value, cannot determine type.") + } + switch b { case 't', 'f': if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { @@ -84,7 +88,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } if keyLevel == len(keys) { - return i + 1, nil + return i+1, nil } case CurlyOpenToken: // in case parent key is matched then only we will increase the level otherwise can directly @@ -94,6 +98,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { if end == -1 { return -1, err } + i += end - 1 } else { level += 1 @@ -107,6 +112,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { if keyLen < 3 || keys[level][0] != SquareOpenToken || keys[level][keyLen-1] != SquareCloseToken { return -1, InvalidArrayIndex } + aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) if err != nil { return -1, InvalidArrayIndex @@ -117,28 +123,33 @@ func findValueIndex(data []byte, keys ...string) (int, error) { valueFound []byte valueOffset int ) + curI := i + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { if curIdx == aIdx { valueFound = value valueOffset = offset + if dataType == String { valueOffset = valueOffset - 2 valueFound = data[curI+valueOffset : curI+valueOffset+len(value)+2] } } + curIdx += 1 }) if valueFound == nil { return -1, ArrayIndexNotFound - } else { - subIndex, err := findValueIndex(valueFound, keys[level+1:]...) - if err != nil { - return -1, KeyPathNotFoundError - } - return i + valueOffset + subIndex, nil } + + subIndex, err := findValueIndex(valueFound, keys[level+1:]...) + if err != nil { + return -1, KeyPathNotFoundError + } + + return i + valueOffset + subIndex, nil } else { // Do not search for keys inside arrays if arraySkip, err := blockEnd(data[i:], SquareOpenToken, SquareCloseToken); arraySkip == -1 { @@ -190,11 +201,12 @@ func findKeyStart(data []byte, key string) (int, error) { if strEnd == -1 { break } + i += strEnd keyEnd := i - 1 - valueOffset, _ := nextToken(data[i:]) - if valueOffset == -1 { + valueOffset, err := nextToken(data[i:]) + if err != nil { break } @@ -202,6 +214,7 @@ func findKeyStart(data []byte, key string) (int, error) { // if string is a key, and key level match k := data[keyBegin:keyEnd] + // for unescape: if there are no escape sequences, this is cheap; if there are, it is a // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize if keyEscaped { @@ -216,15 +229,10 @@ func findKeyStart(data []byte, key string) (int, error) { return keyBegin - 1, nil } - case SquareOpenToken: + case SquareOpenToken, CurlyOpenToken: if end, _ := blockEnd(data[i:], data[i], SquareCloseToken); end != -1 { i = i + end } - - case CurlyOpenToken: - if end, _ := blockEnd(data[i:], data[i], CurlyCloseToken); end != -1 { - i = i + end - } } } @@ -266,14 +274,8 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, // adjustIndentation is a kind of hack to remove the extra tabs and spaces from the Object values. // TODO: storing depth information while parsing step, may allows us to avoid regex. func adjustIndentation(value []byte) []byte { - re := regexp.MustCompile(`(?m)^\t+`) - - // remove the rest, leaving only one of the tabs at the beginning of each line - value = re.ReplaceAll(value, []byte("\t")) - - // remove the last line's spaces - reLastLineSpaces := regexp.MustCompile(`(?m)^\s+\}`) - value = reLastLineSpaces.ReplaceAll(value, []byte("}")) + value = tabRegex.ReplaceAll(value, []byte("\t")) + value = lastLineSpaceRegex.ReplaceAll(value, []byte("}")) return value } @@ -291,6 +293,7 @@ func adjustIndentation(value []byte) []byte { // It returns the value `Prichard` with data type `String`, the offset `37`, and `nil` for the error. func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { a, b, _, d, e := internalGet(data, keys...) + return a, b, d, e } @@ -437,6 +440,7 @@ func ObjectEach( // unescape the key if necessary if escaped { var stackbuf [unescapeStackBufSize]byte + if ku, err := unescape(key, stackbuf[:]); err != nil { return errors.New("json.ObjectEach: string is not valid UTF-8") } else { diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index fdb959f788b..86e8b93a041 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -282,27 +282,22 @@ func Unmarshal(data []byte, s *Struct) error { } func extractKeysFromJSON(data []byte) ([]string, error) { - // parse the JSON key names. - // TODO: reduce overhead. regex is easy to use but slow. - re := regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) - - // check if the JSON is valid. - if !re.Match(data) { + if !jsonKeyRegex.Match(data) { return nil, errors.New("Invalid JSON format") } var keys []string - matches := re.FindAllSubmatch(data, -1) + matches := jsonKeyRegex.FindAllSubmatch(data, -1) for _, match := range matches { - key := string(match[0]) + // extract the key, trimming the trailing colon and any whitespace + key := strings.TrimSuffix(string(match[0]), ":") + key = strings.TrimSpace(key) // reject empty keys if key == "" { return nil, errors.New("json: key cannot be an empty string") } - key = strings.TrimSuffix(key, ":") - key = strings.TrimSpace(key) keys = append(keys, key) } From ee9eec9d2e96cf6c6490086a25274fbb0c203328 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Sun, 11 Feb 2024 15:31:02 +0900 Subject: [PATCH 47/72] fmt --- examples/gno.land/p/demo/json/const.gno | 18 +- .../gno.land/p/demo/json/eisel_lemire.gno | 17 +- examples/gno.land/p/demo/json/escape.gno | 10 +- examples/gno.land/p/demo/json/escape_test.gno | 6 +- .../gno.land/p/demo/json/state_machine.gno | 39 ++- .../p/demo/json/state_machine_test.gno | 291 ++++-------------- examples/gno.land/p/demo/json/struct.gno | 2 - .../p/demo/json/{utils.gno => utils.go} | 2 +- 8 files changed, 126 insertions(+), 259 deletions(-) rename examples/gno.land/p/demo/json/{utils.gno => utils.go} (97%) diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno index 23f5407326f..c7c2d531523 100644 --- a/examples/gno.land/p/demo/json/const.gno +++ b/examples/gno.land/p/demo/json/const.gno @@ -3,16 +3,18 @@ package json import "regexp" const ( - SupplementalPlanesOffset = 0x10000 - HighSurrogateOffset = 0xD800 - LowSurrogateOffset = 0xDC00 + supplementalPlanesOffset = 0x10000 + highSurrogateOffset = 0xD800 + lowSurrogateOffset = 0xDC00 - SurrogateEnd = 0xDFFF - BasicMultilingualPlaneOffset = 0xFFFF + surrogateEnd = 0xDFFF + basicMultilingualPlaneOffset = 0xFFFF - BadHex = -1 + badHex = -1 unescapeStackBufSize = 64 + float32ExponentBias = 127 + float64ExponentBias = 1023 ) const ( @@ -26,5 +28,5 @@ const ( var ( tabRegex = regexp.MustCompile(`(?m)^\t+`) lastLineSpaceRegex = regexp.MustCompile(`(?m)^\s+\}`) - jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) -) \ No newline at end of file + jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) +) diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno index 4e16e2d6124..8647c0e739d 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -59,8 +59,10 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { if neg { f = math.Float64frombits(0x80000000_00000000) // Negative zero. } + return f, true } + if exp10 < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < exp10 { return 0, false } @@ -68,7 +70,6 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { // Normalization. clz := bits.LeadingZeros64(man) man <<= uint(clz) - const float64ExponentBias = 1023 retExp2 := uint64(217706*exp10>>16+64+float64ExponentBias) - uint64(clz) // Multiplication. @@ -81,9 +82,11 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { if mergedLo < xLo { mergedHi++ } + if mergedHi&0x1FF == 0x1FF && mergedLo+1 == 0 && yLo+man < man { return 0, false } + xHi, xLo = mergedHi, mergedLo } @@ -104,6 +107,7 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { retMantissa >>= 1 retExp2 += 1 } + // retExp2 is a uint64. Zero or underflow means that we're in subnormal // float64 space. 0x7FF or above means that we're in Inf/NaN float64 space. // @@ -112,10 +116,12 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { if retExp2-1 >= 0x7FF-1 { return 0, false } + retBits := retExp2<<52 | retMantissa&0x000FFFFF_FFFFFFFF if neg { retBits |= 0x80000000_00000000 } + return math.Float64frombits(retBits), true } @@ -134,8 +140,10 @@ func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { if neg { f = math.Float32frombits(0x80000000) // Negative zero. } + return f, true } + if exp10 < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < exp10 { return 0, false } @@ -143,7 +151,7 @@ func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { // Normalization. clz := bits.LeadingZeros64(man) man <<= uint(clz) - const float32ExponentBias = 127 + retExp2 := uint64(217706*exp10>>16+64+float32ExponentBias) - uint64(clz) // Multiplication. @@ -156,9 +164,11 @@ func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { if mergedLo < xLo { mergedHi++ } + if mergedHi&0x3F_FFFFFFFF == 0x3F_FFFFFFFF && mergedLo+1 == 0 && yLo+man < man { return 0, false } + xHi, xLo = mergedHi, mergedLo } @@ -179,6 +189,7 @@ func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { retMantissa >>= 1 retExp2 += 1 } + // retExp2 is a uint64. Zero or underflow means that we're in subnormal // float32 space. 0xFF or above means that we're in Inf/NaN float32 space. // @@ -187,10 +198,12 @@ func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { if retExp2-1 >= 0xFF-1 { return 0, false } + retBits := retExp2<<23 | retMantissa&0x007FFFFF if neg { retBits |= 0x80000000 } + return math.Float32frombits(uint32(retBits)), true } diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index cc9bd73898f..dff25d3add5 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -60,7 +60,7 @@ func unescape(input, output []byte) ([]byte, error) { // A surrogate pairs are used in UTF-16 encoding to encode characters // outside the Basic Multilingual Plane (BMP). func isSurrogatePair(r rune) bool { - return HighSurrogateOffset <= r && r <= SurrogateEnd + return highSurrogateOffset <= r && r <= surrogateEnd } // combineSurrogates reconstruct the original unicode code points in the @@ -72,7 +72,7 @@ func isSurrogatePair(r rune) bool { // The formula to combine the surrogates is: // (high - 0xD800) * 0x400 + (low - 0xDC00) + 0x10000 func combineSurrogates(high, low rune) rune { - return ((high - HighSurrogateOffset) << 10) + (low - LowSurrogateOffset) + SupplementalPlanesOffset + return ((high - highSurrogateOffset) << 10) + (low - lowSurrogateOffset) + supplementalPlanesOffset } // deocdeSingleUnicodeEscape decodes a unicode escape sequence (e.g., \uXXXX) into a rune. @@ -83,7 +83,7 @@ func decodeSingleUnicodeEscape(b []byte) (rune, bool) { // convert hex to decimal h1, h2, h3, h4 := h2i(b[2]), h2i(b[3]), h2i(b[4]), h2i(b[5]) - if h1 == BadHex || h2 == BadHex || h3 == BadHex || h4 == BadHex { + if h1 == badHex || h2 == badHex || h3 == badHex || h4 == badHex { return utf8.RuneError, false } @@ -98,7 +98,7 @@ func decodeUnicodeEscape(b []byte) (rune, int) { } // determine valid unicode escapes within the BMP - if r <= BasicMultilingualPlaneOffset && !isSurrogatePair(r) { + if r <= basicMultilingualPlaneOffset && !isSurrogatePair(r) { return r, 6 } @@ -108,7 +108,7 @@ func decodeUnicodeEscape(b []byte) (rune, int) { return utf8.RuneError, -1 } - if r2 < LowSurrogateOffset { + if r2 < lowSurrogateOffset { return utf8.RuneError, -1 } diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index 4d740e04bff..a48d3dfc570 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -18,9 +18,9 @@ func TestHexToInt(t *testing.T) { {"Uppercase F", 'F', 15}, {"Lowercase a", 'a', 10}, {"Lowercase f", 'f', 15}, - {"Invalid character1", 'g', BadHex}, - {"Invalid character2", 'G', BadHex}, - {"Invalid character3", 'z', BadHex}, + {"Invalid character1", 'g', badHex}, + {"Invalid character2", 'G', badHex}, + {"Invalid character3", 'z', badHex}, } for _, tt := range tests { diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index b92b7b43369..76da71b608b 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -88,7 +88,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } if keyLevel == len(keys) { - return i+1, nil + return i + 1, nil } case CurlyOpenToken: // in case parent key is matched then only we will increase the level otherwise can directly @@ -260,7 +260,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, // strip quotes from string values if dataType == String { - value = value[1 : len(value)-1] + value = value[1:len(value)-1] } // remove unnecessary whitespace characters to make more readable @@ -303,36 +303,41 @@ func ArrayEach( keys ...string, ) (offset int, err error) { if len(data) == 0 { - return -1, MalformedObject + return -1, errors.New("json.ArrayEach: empty data") } if next, err := nextToken(data); err != nil { - return -1, MalformedObject + return -1, errors.New("json.ArrayEach: invalid token") } else { offset += next } if len(keys) > 0 { if offset, err = findValueIndex(data, keys...); err != nil { - return -1, KeyPathNotFoundError + return -1, errors.New(ufmt.Sprintf("json.ArrayEach: key path not found. keys: %v", keys)) } // go to closest value if n0, err := nextToken(data[offset:]); err != nil { - return -1, MalformedObject + return -1, errors.New( + ufmt.Sprintf( + "json.ArrayEach: invalid token found. offset: %d. got: %s", + offset, + data[offset:offset+1], + )) } else { offset += n0 } if data[offset] != SquareOpenToken { - return -1, MalformedObject + return -1, errors.New("json.ArrayEach: invalid array. must start with `[`") } else { offset += 1 } } if n0, err := nextToken(data[offset:]); err != nil { - return -1, MalformedObject + return -1, errors.New("json.ArrayEach: invalid token. value not found") } else { offset += n0 } @@ -362,7 +367,12 @@ func ArrayEach( offset += o if skipToToken, err := nextToken(data[offset:]); err != nil { - return offset, MalformedArray + return offset, errors.New( + ufmt.Sprintf( + "json.ArrayEach: invalid token at offset %d. got: %s", + offset, + data[offset:offset+1], + )) } else { offset += skipToToken } @@ -372,7 +382,12 @@ func ArrayEach( } if data[offset] != CommaToken { - return offset, MalformedArray + return offset, errors.New( + ufmt.Sprintf( + "json.ArrayEach: cannot find comma at offset %d. got: %s", + offset, + data[offset:offset+1], + )) } else { offset += 1 } @@ -459,9 +474,9 @@ func ObjectEach( // find nearby value, call callback if value, typ, off, err := Get(data[offset:]); err != nil { - return err + return errors.New(ufmt.Sprintf("json.ObjectEach: %s", err)) } else if err := cb(key, value, typ, offset+off); err != nil { - return err + return errors.New(ufmt.Sprintf("json.ObjectEach: %s", err)) } else { offset += off } diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index de6cda4e14e..6e9f72b36dc 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -294,77 +294,6 @@ var getTests = []GetTest{ isFound: true, data: `1`, }, - - // Not found key tests ** current: throw reflect error. should be fixed - // { - // desc: `empty input`, - // json: ``, - // path: []string{"a"}, - // isFound: false, - // }, - // { - // desc: "non-existent key 1", - // json: `{"a":"b"}`, - // path: []string{"c"}, - // isFound: false, - // }, - // { - // desc: "non-existent key 2", - // json: `{"a":"b"}`, - // path: []string{"b"}, - // isFound: false, - // }, - // { - // desc: "non-existent key 3", - // json: `{"aa":"b"}`, - // path: []string{"a"}, - // isFound: false, - // }, - // { - // desc: "apply scope of parent when search for nested key", - // json: `{"a": { "b": 1}, "c": 2 }`, - // path: []string{"a", "b", "c"}, - // isFound: false, - // }, - // { - // desc: `apply scope to key level`, - // json: `{"a": { "b": 1}, "c": 2 }`, - // path: []string{"b"}, - // isFound: false, - // }, - // { - // desc: `handle escaped quote in key name in JSON`, - // json: `{"key\"key": 1}`, - // path: []string{"key"}, - // isFound: false, - // }, - // { - // desc: "handling multiple keys with different name", - // json: `{"a":{"a":1},"b":{"a":3,"c":[1,2]}}`, - // path: []string{"a", "c"}, - // isFound: false, - // }, - // { - // desc: "handling nested json", - // json: `{"a":{"b":{"c":1},"d":4}}`, - // path: []string{"a", "d"}, - // isFound: true, - // data: `4`, - // }, - // { - // desc: `missing key in different key same level`, - // json: `{"s":"s","ic":2,"r":{"o":"invalid"}}`, - // path: []string{"ic", "o"}, - // isFound: false, - // }, - - // Error/invalid tests - // { - // desc: `handle escaped quote in key name in JSON`, - // json: `{"key\"key": 1}`, - // path: []string{"key"}, - // isFound: false, - // }, { desc: `missing closing brace, but can still find key`, json: `{"a":"b"`, @@ -415,24 +344,6 @@ var getTests = []GetTest{ isFound: false, isErr: true, }, - // { // Issue #81 - // desc: `missing key in object in array`, - // json: `{"p":{"a":[{"u":"abc","t":"th"}]}}`, - // path: []string{"p", "a", "[0]", "x"}, - // }, - // { // Issue #81 counter test - // desc: `existing key in object in array`, - // json: `{"p":{"a":[{"u":"abc","t":"th"}]}}`, - // path: []string{"p", "a", "[0]", "u"}, - // isFound: true, - // data: "abc", - // }, - // { // This test returns not found instead of a parse error, as checking for the malformed JSON would reduce performance - // desc: "malformed key (followed by comma followed by colon)", - // json: `{"a",:1}`, - // path: []string{"a"}, - // isFound: false, - // }, { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance (this is not ideal) desc: "malformed 'colon chain', lookup first string", json: `{"a":"b":"c"}`, @@ -447,78 +358,6 @@ var getTests = []GetTest{ isFound: true, data: "c", }, - // Array index paths - // { - // desc: "last key in path is index", - // json: `{"a":[{"b":1},{"b":"2"}, 3],"c":{"c":[1,2]}}`, - // path: []string{"a", "[1]"}, - // isFound: true, - // data: `{"b":"2"}`, - // }, - // { - // desc: "get string from array", - // json: `{"a":[{"b":1},"foo", 3],"c":{"c":[1,2]}}`, - // path: []string{"a", "[1]"}, - // isFound: true, - // data: "foo", - // }, - // { - // desc: "key in path is index", - // json: `{"a":[{"b":"1"},{"b":"2"},3],"c":{"c":[1,2]}}`, - // path: []string{"a", "[0]", "b"}, - // isFound: true, - // data: `1`, - // }, - // { - // desc: "last key in path is an index to value in array (formatted json)", - // json: `{ - // "a": [ - // { - // "b": 1 - // }, - // {"b":"2"}, - // 3 - // ], - // "c": { - // "c": [ - // 1, - // 2 - // ] - // } - // }`, - // path: []string{"a", "[1]"}, - // isFound: true, - // data: `{"b":"2"}`, - // }, - // { - // desc: "key in path is index (formatted json)", - // json: `{ - // "a": [ - // {"b": 1}, - // {"b": "2"}, - // 3 - // ], - // "c": { - // "c": [1, 2] - // } - // }`, - // path: []string{"a", "[0]", "b"}, - // isFound: true, - // data: `1`, - // }, - // { - // // Issue #178: Crash in searchKeys - // desc: `invalid json`, - // json: `{{{"":`, - // path: []string{"a", "b"}, - // isFound: false, - // }, - // { - // desc: `opening brace instead of closing and without key`, - // json: `{"a":1{`, - // path: []string{"b"}, - // isFound: false, - // }, } func TestGet(t *testing.T) { @@ -589,43 +428,43 @@ func TestArrayEach2(t *testing.T) { } func TestObjectEach(t *testing.T) { - tests := []struct { - name string - input []byte - expected map[string]string - root string + tests := []struct { + name string + input []byte + expected map[string]string + root string isErr bool - }{ + }{ { - name: "empty object", - input: []byte(`{}`), + name: "empty object", + input: []byte(`{}`), expected: map[string]string{}, - root: "", + root: "", }, { - name: "single key", - input: []byte(`{"a": 1}`), + name: "single key", + input: []byte(`{"a": 1}`), expected: map[string]string{"a": "1"}, - root: "", + root: "", }, { - name: "two keys", - input: []byte(`{"a": {"b": 1, "c": 2}}`), - expected: map[string]string{"b": "1", "c": "2"}, - root: "a", - }, + name: "two keys", + input: []byte(`{"a": {"b": 1, "c": 2}}`), + expected: map[string]string{"b": "1", "c": "2"}, + root: "a", + }, { - name: "two keys 2", - input: []byte(`{"a": {"b": 1, "c": 2}, "d": 4}`), + name: "two keys 2", + input: []byte(`{"a": {"b": 1, "c": 2}, "d": 4}`), expected: map[string]string{"b": "1", "c": "2"}, - root: "a", + root: "a", + }, + { + name: "another set of keys", + input: []byte(`{"x": {"y": 3, "z": 4}}`), + expected: map[string]string{"y": "3", "z": "4"}, + root: "x", }, - { - name: "another set of keys", - input: []byte(`{"x": {"y": 3, "z": 4}}`), - expected: map[string]string{"y": "3", "z": "4"}, - root: "x", - }, { name: "multiple key-value object with many value types", input: []byte(`{ @@ -646,108 +485,108 @@ func TestObjectEach(t *testing.T) { }, }, { - name: "so many white spaces", - input: []byte(`{"a" : 1}`), + name: "so many white spaces", + input: []byte(`{"a" : 1}`), expected: map[string]string{"a": "1"}, }, /* Error Cases */ { - name: "unmatched brace", + name: "unmatched brace", input: []byte(`{`), isErr: true, }, { - name: "unmatched brace 2", + name: "unmatched brace 2", input: []byte(`{"a": 1`), - root: "a", + root: "a", isErr: true, }, { - name: "unmatched brace 3", + name: "unmatched brace 3", input: []byte(`}`), isErr: true, }, { - name: "unmatched brace 3", + name: "unmatched brace 3", input: []byte(`{{}}{`), isErr: true, }, { - name: "no object present", + name: "no object present", input: []byte(`\t\n\r`), isErr: true, }, { - name: "malformed key", + name: "malformed key", input: []byte(`{"foo: 1}`), isErr: true, }, { - name: "malformed key 2", + name: "malformed key 2", input: []byte(`{"foo": 1, "bar: "11"}`), - root: "foo", + root: "foo", isErr: true, }, { - name: "malformed key in nested object", + name: "malformed key in nested object", input: []byte(`{"foo": {"bar: 1}}`), - root: "foo", + root: "foo", isErr: true, }, { - name: "bad value", + name: "bad value", input: []byte(`{"foo": bar}`), isErr: true, }, { - name: "no colon", + name: "no colon", input: []byte(`{"foo" "bar"}`), isErr: true, }, { - name: "no colon 2", + name: "no colon 2", input: []byte(`{"foo""bar"}`), isErr: true, }, { - name: "no colon 3", + name: "no colon 3", input: []byte(`{"foo"; "bar"}`), isErr: true, }, { - name: "invalid colon", + name: "invalid colon", input: []byte(`{"foo":: "bar"}`), isErr: true, }, { - name: "no trailing comma", + name: "no trailing comma", input: []byte(`{"foo": "bar" "baz": "qux"}`), - root: "foo", + root: "foo", isErr: true, }, { - name: "key with unicode escape", + name: "key with unicode escape", input: []byte(`{"b": 10, "c": {"a\uD83D\uDE03b":1, "aa": false}}`), - root: "b", + root: "b", isErr: true, }, - } + } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := ObjectEach(tc.input, func(key, value []byte, dataType ValueType, offset int) error { - expectedValue, ok := tc.expected[string(key)] - if !ok { - t.Errorf("Unexpected key: %s", string(key)) - } else if string(value) != expectedValue { - t.Errorf("Wrong value for key %s: got %s, want %s", string(key), string(value), expectedValue) - } - return nil - }, tc.root) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := ObjectEach(tc.input, func(key, value []byte, dataType ValueType, offset int) error { + expectedValue, ok := tc.expected[string(key)] + if !ok { + t.Errorf("Unexpected key: %s", string(key)) + } else if string(value) != expectedValue { + t.Errorf("Wrong value for key %s: got %s, want %s", string(key), string(value), expectedValue) + } + return nil + }, tc.root) if (err != nil) != tc.isErr { - t.Errorf("ObjectEach(%s) error = %v, wantErr %v", tc.name, err, tc.isErr) - } - }) - } -} \ No newline at end of file + t.Errorf("ObjectEach(%s) error = %v, wantErr %v", tc.name, err, tc.isErr) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 86e8b93a041..19e79705b59 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -28,8 +28,6 @@ type Field struct { type Struct struct { // fields holds the fields of the struct. fields []Field - // TODO: may use map to speed up the search. - // fmap map[string]*Field } // NewStruct create new empty struct placeholder. diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.go similarity index 97% rename from examples/gno.land/p/demo/json/utils.gno rename to examples/gno.land/p/demo/json/utils.go index c2351b51f3b..b2bcf88198b 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.go @@ -15,7 +15,7 @@ func lower(c byte) byte { const hexLookupTable = [256]int{ '0': 0x0, '1': 0x1, '2': 0x2, '3': 0x3, '4': 0x4, - '5': 0x5, '6': 0x6, '7': 0x7, '8': 0x8, '9': 0x9, + '5': 0x5, '6': 0x6, '7': 0x7, '8': 0x8, '9': 0x9, 'A': 0xA, 'B': 0xB, 'C': 0xC, 'D': 0xD, 'E': 0xE, 'F': 0xF, 'a': 0xA, 'b': 0xB, 'c': 0xC, 'd': 0xD, 'e': 0xE, 'f': 0xF, // Fill unspecified index-value pairs with key and value of -1 From db0c8ab3b5fec01992db4b4dcefb959f4f865148 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Sun, 11 Feb 2024 15:31:21 +0900 Subject: [PATCH 48/72] fixup --- examples/gno.land/p/demo/json/{utils.go => utils.gno} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/gno.land/p/demo/json/{utils.go => utils.gno} (100%) diff --git a/examples/gno.land/p/demo/json/utils.go b/examples/gno.land/p/demo/json/utils.gno similarity index 100% rename from examples/gno.land/p/demo/json/utils.go rename to examples/gno.land/p/demo/json/utils.gno From 48049c29d388159350a960eebde943c867613852 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 12 Feb 2024 14:52:21 +0900 Subject: [PATCH 49/72] create EachKey --- examples/gno.land/p/demo/json/const.gno | 32 --- .../gno.land/p/demo/json/eisel_lemire.gno | 5 + examples/gno.land/p/demo/json/errors.gno | 34 +-- examples/gno.land/p/demo/json/escape.gno | 41 ++- examples/gno.land/p/demo/json/parser.gno | 6 + .../gno.land/p/demo/json/state_machine.gno | 272 ++++++++++++++++-- .../p/demo/json/state_machine_test.gno | 172 +++++++++++ examples/gno.land/p/demo/json/struct.gno | 2 + examples/gno.land/p/demo/json/utils.gno | 4 + gnovm/tests/files/json0.gno | 9 +- 10 files changed, 492 insertions(+), 85 deletions(-) delete mode 100644 examples/gno.land/p/demo/json/const.gno diff --git a/examples/gno.land/p/demo/json/const.gno b/examples/gno.land/p/demo/json/const.gno deleted file mode 100644 index c7c2d531523..00000000000 --- a/examples/gno.land/p/demo/json/const.gno +++ /dev/null @@ -1,32 +0,0 @@ -package json - -import "regexp" - -const ( - supplementalPlanesOffset = 0x10000 - highSurrogateOffset = 0xD800 - lowSurrogateOffset = 0xDC00 - - surrogateEnd = 0xDFFF - basicMultilingualPlaneOffset = 0xFFFF - - badHex = -1 - - unescapeStackBufSize = 64 - float32ExponentBias = 127 - float64ExponentBias = 1023 -) - -const ( - absMinInt64 = 1 << 63 - maxInt64 = 1<<63 - 1 - maxUint64 = 1<<64 - 1 - intSize = 32 << (^uint(0) >> 63) - IntSize = intSize -) - -var ( - tabRegex = regexp.MustCompile(`(?m)^\t+`) - lastLineSpaceRegex = regexp.MustCompile(`(?m)^\s+\}`) - jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) -) diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno index 8647c0e739d..20e82314dfb 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -18,6 +18,11 @@ import ( "math/bits" ) +const ( + float32ExponentBias = 127 + float64ExponentBias = 1023 +) + // The Eisel-Lemire algorithm accomplishes its task through the following methods: // // 1. Extracting Mantissa and Exponent diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index 740740cd7c3..84a1abf2a82 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -4,53 +4,53 @@ import "errors" const ( // KeyPathNotFoundError occurs when a specified key path does not exist in the JSON structure. - KeyPathNotFoundError error = errors.New("JSON Error: key path not found in the JSON structure") + KeyPathNotFoundError error = errors.New("key path not found in the JSON structure") // ArrayIndexNotFound occurs when the specified index is beyond the range of the array. - ArrayIndexNotFound = errors.New("JSON Error: array index not found or out of range") + ArrayIndexNotFound = errors.New("array index not found or out of range") // TokenNotFound occurs when a particular token (expected as part of the structure) is not found. - TokenNotFound = errors.New("JSON Error: expected token not found in the JSON input") + TokenNotFound = errors.New("expected token not found in the JSON input") // KeyLevelNotMatched occurs when the key levels do not match the expected structure or depth. - KeyLevelNotMatched = errors.New("JSON Error: key level does not match the expected structure or depth") + KeyLevelNotMatched = errors.New("key level does not match the expected structure or depth") // Overflow occurs when a number in the JSON exceeds the range that can be handled. - Overflow = errors.New("JSON Error: numeric value exceeds the range limit") + Overflow = errors.New("numeric value exceeds the range limit") // EmptyBytes occurs when the JSON input is empty or has no content. - EmptyBytes = errors.New("JSON Error: empty bytes: the JSON input is empty or has no content") + EmptyBytes = errors.New("empty bytes: the JSON input is empty or has no content") // InvalidArrayIndex occurs when the index used for an array is not an integer or out of valid range. - InvalidArrayIndex = errors.New("JSON Error: invalid array index: index should be an integer and within the valid range") + InvalidArrayIndex = errors.New("invalid array index: index should be an integer and within the valid range") // InvalidExponents occurs when there's an error related to the format or range of exponents in numbers. - InvalidExponents = errors.New("JSON Error: invalid format or range of exponents in a numeric value") + InvalidExponents = errors.New("invalid format or range of exponents in a numeric value") // NonDigitCharacters occurs when there are non-digit characters where a number is expected. - NonDigitCharacters = errors.New("JSON Error: non-digit characters found where a number is expected") + NonDigitCharacters = errors.New("non-digit characters found where a number is expected") // MultipleDecimalPoints occurs when a number has more than one decimal point. - MultipleDecimalPoints = errors.New("JSON Error: multiple decimal points found in a number") + MultipleDecimalPoints = errors.New("multiple decimal points found in a number") // MalformedType occurs when the type of a value does not match the expected type. - MalformedType = errors.New("JSON Error: malformed type: the type of the value does not match the expected type") + MalformedType = errors.New("malformed type: the type of the value does not match the expected type") // MalformedString occurs when a string is improperly formatted, like unescaped characters or incorrect quotes. - MalformedString = errors.New("JSON Error: malformed string: improperly formatted string, check for unescaped characters or incorrect quotes") + MalformedString = errors.New("malformed string: improperly formatted string, check for unescaped characters or incorrect quotes") // MalformedValue occurs when a value does not conform to the expected format or structure. - MalformedValue = errors.New("JSON Error: malformed value: the value does not conform to the expected format or structure") + MalformedValue = errors.New("malformed value: the value does not conform to the expected format or structure") // MalformedObject occurs when a JSON object is improperly formatted. - MalformedObject = errors.New("JSON Error: malformed object: the JSON object is improperly formatted or structured") + MalformedObject = errors.New("malformed object: the JSON object is improperly formatted or structured") // MalformedArray occurs when a JSON array is improperly formatted. - MalformedArray = errors.New("JSON Error: malformed array: the JSON array is improperly formatted or structured") + MalformedArray = errors.New("malformed array: the JSON array is improperly formatted or structured") // MalformedJson occurs when the entire JSON structure is improperly formatted or structured. - MalformedJson = errors.New("JSON Error: malformed JSON: the entire JSON structure is improperly formatted or structured") + MalformedJson = errors.New("malformed JSON: the entire JSON structure is improperly formatted or structured") // UnknownValueType occurs when the JSON contains a value of an unrecognized or unsupported type. - UnknownValueType = errors.New("JSON Error: unknown value type: the value type is unrecognized or unsupported") + UnknownValueType = errors.New("unknown value type: the value type is unrecognized or unsupported") ) diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index dff25d3add5..09e1c9548fa 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -6,6 +6,17 @@ import ( "unicode/utf8" ) +const ( + supplementalPlanesOffset = 0x10000 + highSurrogateOffset = 0xD800 + lowSurrogateOffset = 0xDC00 + + surrogateEnd = 0xDFFF + basicMultilingualPlaneOffset = 0xFFFF + + badHex = -1 +) + // unescape takes an input byte slice, processes it to unescape certain characters, // and writes the result into an output byte slice. // @@ -18,11 +29,12 @@ func unescape(input, output []byte) ([]byte, error) { } // ensure the output slice has enough capacity to hold the result. - if cap(output) < len(input) { - output = make([]byte, len(input)) + inputLen := len(input) + if cap(output) < inputLen { + output = make([]byte, inputLen) } else { // if the capacity is sufficient, slice the output to the length of the input. - output = output[0:len(input)] + output = output[:inputLen] } copy(output, input[:firstBackslash]) @@ -30,26 +42,27 @@ func unescape(input, output []byte) ([]byte, error) { buf := output[firstBackslash:] for len(input) > 0 { - // inLen is the number of bytes consumed in the input - // bufLen is the number of bytes written to buf. - if inLen, bufLen := processEscapedUTF8(input, buf); inLen == -1 { + inLen, bufLen := processEscapedUTF8(input, buf) + if inLen == -1 { return nil, errors.New("JSON Error: Encountered an invalid escape sequence in a string.") - } else { - input = input[inLen:] - buf = buf[bufLen:] } - // find the next backslash in the remaining input. + input = input[inLen:] // the number of bytes consumed in the input + buf = buf[bufLen:] // the number of bytes written to buf + + // find the next backslash in the remaining input nextBackslash := bytes.IndexByte(input, BackSlashToken) if nextBackslash == -1 { copy(buf, input) buf = buf[len(input):] + break - } else { - copy(buf, input[:nextBackslash]) - buf = buf[nextBackslash:] - input = input[nextBackslash:] } + + copy(buf, input[:nextBackslash]) + + input = input[nextBackslash:] + buf = buf[nextBackslash:] } return output[:len(output)-len(buf)], nil diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 2db31f32131..ec5cb42e353 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -7,6 +7,12 @@ import ( "strconv" ) +const ( + absMinInt64 = 1<<63 + maxInt64 = absMinInt64 - 1 + maxUint64 = 1<<64 - 1 +) + func tokenEnd(data []byte) int { for i, tok := range data { switch tok { diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 76da71b608b..abe29b7ec3e 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -3,11 +3,21 @@ package json import ( "bytes" "errors" + "regexp" "strconv" "gno.land/p/demo/ufmt" ) +const ( + unescapeStackBufSize = 64 +) + +var ( + tabRegex = regexp.MustCompile(`(?m)^\t+`) + lastLineSpaceRegex = regexp.MustCompile(`(?m)^\s+\}`) +) + // extractValueTypeFromToken recoginizes the type of JSON value by peeking at the first byte of the value. func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { if len(b) == 0 { @@ -50,6 +60,10 @@ func findValueIndex(data []byte, keys ...string) (int, error) { for i := 0; i < len(data); i++ { switch data[i] { case DoublyQuoteToken: + if level < 1 { + return -1, MalformedJson + } + i += 1 keyBegin := i @@ -72,10 +86,6 @@ func findValueIndex(data []byte, keys ...string) (int, error) { continue } - if level < 1 { - return -1, MalformedJson - } - key := data[keyBegin:keyEnd] keyUnesc, err := keyMatched(key, keyEscaped, stackbuf, keys, level) if err != nil { @@ -167,7 +177,9 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } func decreaseLevel(level, keyLevel int) (int, int) { - if level -= 1; level == keyLevel { + level -= 1 + + if level - 1 == keyLevel { keyLevel -= 1 } @@ -176,9 +188,9 @@ func decreaseLevel(level, keyLevel int) (int, int) { // findKeyStart finds the start of a specific key in a given byte array. func findKeyStart(data []byte, key string) (int, error) { - i, _ := nextToken(data) - if i == -1 { - return i, KeyPathNotFoundError + i, err := nextToken(data) + if err != nil { + return -1, err } ln := len(data) @@ -260,7 +272,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, // strip quotes from string values if dataType == String { - value = value[1:len(value)-1] + value = value[1 : len(value)-1] } // remove unnecessary whitespace characters to make more readable @@ -349,7 +361,7 @@ func ArrayEach( for true { v, t, o, e := Get(data[offset:]) - if e != nil { + if e != nil{ return -1, e } @@ -361,10 +373,6 @@ func ArrayEach( cb(v, t, offset+o-len(v), e) } - if e != nil { - break - } - offset += o if skipToToken, err := nextToken(data[offset:]); err != nil { return offset, errors.New( @@ -377,11 +385,12 @@ func ArrayEach( offset += skipToToken } - if data[offset] == SquareCloseToken { + currToken := data[offset] + if currToken == SquareCloseToken { break } - if data[offset] != CommaToken { + if currToken != CommaToken { return offset, errors.New( ufmt.Sprintf( "json.ArrayEach: cannot find comma at offset %d. got: %s", @@ -476,7 +485,7 @@ func ObjectEach( if value, typ, off, err := Get(data[offset:]); err != nil { return errors.New(ufmt.Sprintf("json.ObjectEach: %s", err)) } else if err := cb(key, value, typ, offset+off); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: %s", err)) + return errors.New(ufmt.Sprintf("json.ObjectEach: %s", err)) } else { offset += off } @@ -512,3 +521,232 @@ func ObjectEach( return errors.New(ufmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset)) } + +func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) (int, error) { + var ( + x struct{} + level int + pathsMatched int + i int + ) + + pathFlags := initPathFlags(paths) + pathsBuf, maxPath := initPathBuf(paths) + + ln := len(data) + for i < ln { + switch data[i] { + case DoublyQuoteToken: + i++ + keyBegin := i + + strEnd, keyEscaped, err := stringEnd(data[i:]) + if err != nil { + return -1, errors.New("json.EachKey: invalid string") + } + + i += strEnd + keyEnd := i - 1 + + valueOffset, err := nextToken(data[i:]) + if err != nil { + return -1, errors.New("json.EachKey: invalid token found") + } + + i += valueOffset + + // if string is a key, and key level match + if data[i] == ColonToken { + match := -1 + if maxPath > 0 && maxPath >= level { + keyUnesc, err := processKey(data[keyBegin:keyEnd], keyEscaped) + if err != nil { + return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + } + + pathsBuf[level-1] = string(keyUnesc) + for pi, path := range paths { + if !isPathValid(path, level, pathsBuf) || pathFlags[pi] { + continue + } + + match = pi + + pathsMatched += 1 + pathFlags[pi] = true + + val, dataTyp, _, err := Get(data[i+1:]) + cb(pi, val, dataTyp, err) + + if pathsMatched == len(paths) { + break + } + } + + if pathsMatched == len(paths) { + return i, nil + } + } + + if match == -1 { + tokenOffset, err := nextToken(data[i+1:]) + i += tokenOffset + + if data[i] == CurlyOpenToken { + blockSkip, err := blockEnd(data[i:], CurlyOpenToken, CurlyCloseToken) + if blockSkip == -1 { + return -1, err + } + + i += blockSkip + 1 + } + } + + if i < ln { + switch data[i] { + case CurlyOpenToken, CurlyCloseToken, SquareOpenToken, DoublyQuoteToken: + i -= 1 + } + } + } else { + i -= 1 + } + case CurlyOpenToken: + level += 1 + case CurlyCloseToken: + level -= 1 + case SquareOpenToken: + var ok bool + + arrIdxFlags := make(map[int]struct{}) + pIdxFlags := initPathIdxFlags(paths) + + if level < 0 { + err := MalformedJson + cb(-1, nil, Unknown, err) + + return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + } + + for p, path := range paths { + if !isPathValid(path, level, pathsBuf) || pathFlags[p] { + continue + } + + currPos := path[level] + if len(currPos) >= 2 { + arrIdx, err := strconv.Atoi(currPos[1 : len(currPos)-1]) + if err != nil { + return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + } + + pIdxFlags[p] = true + arrIdxFlags[arrIdx] = x + } + } + + if len(arrIdxFlags) > 0 { + level += 1 + + var currIdx int + arrOff, err := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + if _, ok := arrIdxFlags[currIdx]; ok { + for i, path := range paths { + if pIdxFlags[i] { + currLevelPath := path[level-1] + arrIdx, _ := strconv.Atoi(currLevelPath[1 : len(currLevelPath)-1]) + + if currIdx != arrIdx { + continue + } + + off, _ := findValueIndex(value, path[level:]...) + + if off != -1 { + val, dataTyp, _, err := Get(value[off:]) + cb(i, val, dataTyp, err) + } + + pathsMatched += 1 + pathFlags[i] = true + } + } + } + + currIdx += 1 + }) + + if pathsMatched == len(paths) { + return i, nil + } + + i += arrOff - 1 + } else { + // do not search for keys inside arrays + if arraySkip, err := blockEnd(data[i:], SquareOpenToken, SquareCloseToken); arraySkip == -1 { + return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + } else { + i += arraySkip + } + } + + case SquareCloseToken: + level -= 1 + } + + i += 1 + } + + return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", KeyPathNotFoundError)) +} + +func initPathFlags(paths [][]string) []bool { + return make([]bool, len(paths)) +} + +func initPathBuf(paths [][]string) ([]string, int) { + var maxPath int + + for _, p := range paths { + if len(p) > maxPath { + maxPath = len(p) + } + } + + return make([]string, maxPath), maxPath +} + +func initPathIdxFlags(paths [][]string) []bool { + return make([]bool, len(paths)) +} + +func sameTree(p1, p2 []string) bool { + minLen := len(p1) + p2Len := len(p2) + + if p2Len < minLen { + minLen = p2Len + } + + for pi_1, p_1 := range p1[:minLen] { + if p2[pi_1] != p_1 { + return false + } + } + + return true +} + +func processKey(key []byte, keyEscaped bool) ([]byte, error) { + if !keyEscaped { + return key, nil + } + + var stackBuf [unescapeStackBufSize]byte + + return unescape(key, stackBuf[:]) +} + +func isPathValid(path []string, level int, pathsBuf []string) bool { + return len(path) == level && sameTree(path, pathsBuf[:level]) +} diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno index 6e9f72b36dc..1e5967c0023 100644 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ b/examples/gno.land/p/demo/json/state_machine_test.gno @@ -5,6 +5,8 @@ import ( "errors" "fmt" "testing" + + "gno.land/p/demo/ufmt" ) func TestSearchKeys(t *testing.T) { @@ -590,3 +592,173 @@ func TestObjectEach(t *testing.T) { }) } } + +var testJson = []byte(`{ + "name": "Name", + "order": "Order", + "sum": 100, + "len": 12, + "isPaid": true, + "nested": {"a":"test", "b":2, "nested3":{"a":"test3","b":4}, "c": "unknown"}, + "nested2": { + "a":"test2", + "b":3 + }, + "arr": [ + { + "a":"zxc", + "b": 1 + }, + { + "a":"123", + "b":2 + } + ], + "arrInt": [1,2,3,4], + "intPtr": 10, +}`) + +func TestEachKey(t *testing.T) { + paths := [][]string{ + {"name"}, + {"order"}, + {"nested", "a"}, + {"nested", "b"}, + {"nested2", "a"}, + {"nested", "nested3", "b"}, + {"arr", "[1]", "b"}, + {"arrInt", "[3]"}, + {"arrInt", "[5]"}, + {"nested"}, + {"arr", "["}, + {"a\n", "b\n"}, + {"nested", "b"}, + } + + keysFound := 0 + EachKey(testJson, func(idx int, value []byte, vt ValueType, err error) { + keysFound++ + + expectedValues := []string{"Name", "Order", "test", "2", "test2", "4", "2", "4", "", `{"a":"test", "b":2, "nested3":{"a":"test3","b":4}, "c": "unknown"}`, "", "99", "2"} + if idx < len(expectedValues) { + if string(value) != expectedValues[idx] { + t.Errorf("Invalid key #%v. Expected: %v, Got: %v", idx, expectedValues[idx], string(value)) + } + } else { + t.Errorf("Should find only %v keys, but got an extra key with index %v and value %v", len(expectedValues), idx, string(value)) + } + }, paths...) + + expectedKeysFound := 8 + if keysFound != expectedKeysFound { + t.Errorf("Mismatch in number of keys found. Expected: %v, Got: %v", expectedKeysFound, keysFound) + } +} + +func TestIsPathValid(t *testing.T) { + tests := []struct { + name string + path []string + level int + pathsBuf []string + want bool + }{ + { + name: "Exact match", + path: []string{"users", "id"}, + level: 2, + pathsBuf: []string{"users", "id"}, + want: true, + }, + { + name: "Path shorter than level", + path: []string{"users"}, + level: 2, + pathsBuf: []string{"users", "id"}, + want: false, + }, + { + name: "Path length matches level but paths differ", + path: []string{"users", "name"}, + level: 2, + pathsBuf: []string{"users", "id"}, + want: false, + }, + { + name: "Path longer than level but matches up to level", + path: []string{"users", "id", "name"}, + level: 2, + pathsBuf: []string{"users", "id"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isPathValid(tt.path, tt.level, tt.pathsBuf) + if got != tt.want { + t.Errorf("isPathValid(%s) = %v, want %v", tt.name, got, tt.want) + } + }) + } +} + +func TestSameTree(t *testing.T) { + tests := []struct { + name string + p1 []string + p2 []string + want bool + }{ + { + name: "Completely identical", + p1: []string{"a", "b", "c"}, + p2: []string{"a", "b", "c"}, + want: true, + }, + { + name: "Partially identical", + p1: []string{"a", "b"}, + p2: []string{"a", "b", "c"}, + want: true, + }, + { + name: "Not identical", + p1: []string{"a", "x"}, + p2: []string{"a", "b", "c"}, + want: false, + }, + { + name: "One path longer than the other", + p1: []string{"a", "b", "c", "d"}, + p2: []string{"a", "b", "c"}, + want: true, + }, + { + name: "One path (p1) is empty", + p1: []string{}, + p2: []string{"a", "b", "c"}, + want: true, + }, + { + name: "One path (p2) is empty", + p1: []string{"a", "b", "c"}, + p2: []string{}, + want: true, + }, + { + name: "Both paths are empty", + p1: []string{}, + p2: []string{}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sameTree(tt.p1, tt.p2); got != tt.want { + t.Errorf("sameTree(%v, %v) = %v, want %v", tt.p1, tt.p2, got, tt.want) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 19e79705b59..13cfc229ebd 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -10,6 +10,8 @@ import ( "gno.land/p/demo/ufmt" ) +var jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) + // Field stores each field's information. type Field struct { // name holds the field name. diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index b2bcf88198b..7528acf7fce 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -43,6 +43,10 @@ func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { return false, bytes } +func equalStr(b *[]byte, s string) bool { + return string(*b) == s +} + /* Testing Helper Function */ func isEqualMap(a, b map[string]interface{}) bool { diff --git a/gnovm/tests/files/json0.gno b/gnovm/tests/files/json0.gno index 0219b8d2aef..64743df0667 100644 --- a/gnovm/tests/files/json0.gno +++ b/gnovm/tests/files/json0.gno @@ -1,8 +1,7 @@ package main import ( - "fmt" - + "gno.land/p/demo/ufmt" "gno.land/p/demo/json" ) @@ -18,11 +17,11 @@ func main() { jsonStr, err := s.Marshal() if err != nil { - fmt.Println("Error marshaling:", err) + ufmt.Println("Error marshaling:", err) return } - println("Marshalled:") - println(string(jsonStr)) + ufmt.Println("Marshalled:") + ufmt.Println(string(jsonStr)) } // Output: From d8c64981f12fa89ad2b9232ff41865615243697d Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 15 Feb 2024 16:06:40 +0900 Subject: [PATCH 50/72] fixup --- examples/gno.land/p/demo/json/gno.mod | 2 + .../gno.land/p/demo/json/state_machine.gno | 2 +- examples/gno.land/p/demo/json/struct.gno | 2 +- gnovm/pkg/gnolang/precompile.go | 1 + gnovm/tests/files/json0.gno | 29 ------ gnovm/tests/files/json1.gno | 40 -------- gnovm/tests/files/json2.gno | 24 ----- gnovm/tests/files/json3.gno | 93 ------------------- gnovm/tests/files/json4.gno | 23 ----- 9 files changed, 5 insertions(+), 211 deletions(-) delete mode 100644 gnovm/tests/files/json0.gno delete mode 100644 gnovm/tests/files/json1.gno delete mode 100644 gnovm/tests/files/json2.gno delete mode 100644 gnovm/tests/files/json3.gno delete mode 100644 gnovm/tests/files/json4.gno diff --git a/examples/gno.land/p/demo/json/gno.mod b/examples/gno.land/p/demo/json/gno.mod index 831fa56c0f9..ef794458c56 100644 --- a/examples/gno.land/p/demo/json/gno.mod +++ b/examples/gno.land/p/demo/json/gno.mod @@ -1 +1,3 @@ module gno.land/p/demo/json + +require gno.land/p/demo/ufmt v0.0.0-latest diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index abe29b7ec3e..717c3304b54 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -15,7 +15,7 @@ const ( var ( tabRegex = regexp.MustCompile(`(?m)^\t+`) - lastLineSpaceRegex = regexp.MustCompile(`(?m)^\s+\}`) + lastLineSpaceRegex = regexp.MustCompile(`(?m)^\\s+\\}`) ) // extractValueTypeFromToken recoginizes the type of JSON value by peeking at the first byte of the value. diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 13cfc229ebd..e720949d1fc 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -10,7 +10,7 @@ import ( "gno.land/p/demo/ufmt" ) -var jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\s*:`) +var jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\\s*:`) // Field stores each field's information. type Field struct { diff --git a/gnovm/pkg/gnolang/precompile.go b/gnovm/pkg/gnolang/precompile.go index 42748145de8..b8320845ea6 100644 --- a/gnovm/pkg/gnolang/precompile.go +++ b/gnovm/pkg/gnolang/precompile.go @@ -57,6 +57,7 @@ var stdlibWhitelist = []string{ "text/template", "time", "unicode/utf8", + "unicode/utf16", // gno "std", diff --git a/gnovm/tests/files/json0.gno b/gnovm/tests/files/json0.gno deleted file mode 100644 index 64743df0667..00000000000 --- a/gnovm/tests/files/json0.gno +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "gno.land/p/demo/ufmt" - "gno.land/p/demo/json" -) - - -func main() { - s := json.NewStruct(). - AddStringField("Name", "John Doe"). - AddIntField("Age", 30). - AddBoolField("IsActive", true). - AddFloatField("Score", 4.5). - AddNullField("Option"). - AddArrayField("arr", []interface{}{"one", 2, true, nil}) - - jsonStr, err := s.Marshal() - if err != nil { - ufmt.Println("Error marshaling:", err) - return - } - ufmt.Println("Marshalled:") - ufmt.Println(string(jsonStr)) -} - -// Output: -// Marshalled: -// {"Name": "John Doe", "Age": 30, "IsActive": true, "Score": 4.5, "Option": null, "arr": ["one", 2, true, null]} \ No newline at end of file diff --git a/gnovm/tests/files/json1.gno b/gnovm/tests/files/json1.gno deleted file mode 100644 index a8b504ebee5..00000000000 --- a/gnovm/tests/files/json1.gno +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "fmt" - - "gno.land/p/demo/json" -) - -// Marshaling example -func main() { - contactStruct := json.NewStruct(). - AddStringField("email", "john@example.com"). - AddStringField("phone", "123-456-7890") - - addressStruct := json.NewStruct(). - AddStringField("street", "123 Main St"). - AddStringField("city", "Anytown"). - AddStringField("state", "CA"). - AddIntField("zip", 12345) - - userStruct := json.NewStruct(). - AddStringField("name", "John Doe"). - AddObjectField("contact", contactStruct). - AddObjectField("address", addressStruct). - AddArrayField("roles", []interface{}{"admin", "user"}) - - s := json.NewStruct(). - AddObjectField("user", userStruct) - - jsonStr, err := s.Marshal() - if err != nil { - fmt.Println("Error marshaling:", err) - return - } - - fmt.Println(string(jsonStr)) -} - -// Ouput: -// {"user": {"name": "John Doe", "contact": {"email": "john@example.com", "phone": "123-456-7890"}, "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": 12345}, "roles": ["admin", "user"]}} \ No newline at end of file diff --git a/gnovm/tests/files/json2.gno b/gnovm/tests/files/json2.gno deleted file mode 100644 index 603d08cc3c2..00000000000 --- a/gnovm/tests/files/json2.gno +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "fmt" - - "gno.land/p/demo/json" -) - -// Unmarshal JSON data into a struct. -func main() { - data := []byte(`{"name": "Foo", "age": 20, "score": 4.2, "subscribed": true, "option": null}`) - - s := json.NewStruct() - err := json.Unmarshal(data, s) - if err != nil { - panic(err) - } - - // `String()` method use Marshal internally to generate human readable JSON. - fmt.Println(s.String()) -} - -// Output: -// {"name": "Foo", "age": 20, "score": 4.2, "subscribed": true, "option": null} \ No newline at end of file diff --git a/gnovm/tests/files/json3.gno b/gnovm/tests/files/json3.gno deleted file mode 100644 index e88794c798d..00000000000 --- a/gnovm/tests/files/json3.gno +++ /dev/null @@ -1,93 +0,0 @@ -package main - -import ( - "fmt" - - "gno.land/p/demo/json" -) - -// demo for value retrieval from JSON data -func main() { - data := []byte(` - { - "character": { - "race": "Tenebrine", - "species": "Raccoon", - "class": "Thief", - "name": "Whisker" - "stats": { - "strength": 8, - "dexterity": 18, - "constitution": 12, - "intelligence": 11, - "wisdom": 12, - "charisma": 14 - }, - }, - "skills": { - "acrobatics": 4, - "animal_handling": 1, - "arcana": 1, - "athletics": 2, - "deception": 5, - "history": 1, - "insight": 3, - "intimidation": 2, - "investigation": 4, - "medicine": 1, - "nature": 1, - "perception": 4, - "performance": 3, - "persuasion": 3, - "religion": 1, - "sleight_of_hand": 6, - "stealth": 5, - "survival": 3 - }, - "equipment": { - "weapons": ["Daggers", "Short sword"], - "armor": "Leather armor", - "items": ["Thieves' tools", "Backpack", "Rations (5 days)", "Lock picks", "Disguise kit"], - "gold": 200 - }, - "backstory": "Raised on the streets of the shadowy Tenebrine city, ${player} learned to survive by his wits and agility.\nSkilled in the art of stealth and deception, he mastered the craft of thievery to claim what the streets denied him.\nKnown only by his alias 'Shadowpaw', he now seeks greater challenges and wealth beyond the alleyways of his upbringing." - } - `) - - // `json.Get` function takes a JSON and a path of keys (must enter each key name separately) to retrieve the value. - if val, typ, _, err := json.Get(data, "character", "name"); err != nil { - panic(err) - } else { - fmt.Printf("name: %s type: %v\n", val, typ.String()) - } - - if ab, typ, _, err := json.Get(data, "character", "stats"); err != nil { - panic(err) - } else { - fmt.Printf("stats: %s, type: %v\n\n", ab, typ.String()) - } - - if stealth, typ, _, err := json.Get(data, "skills", "stealth"); err != nil { - panic(err) - } else { - fmt.Printf("stealth: %s, type: %v\n\n", stealth, typ.String()) - } - - if weapons, typ, _, err := json.Get(data, "equipment", "weapons"); err != nil { - panic(err) - } else { - fmt.Printf("weapons: %s, type: %v\n\n", weapons, typ.String()) - } - - if armor, typ, _, err := json.Get(data, "equipment", "armor"); err != nil { - panic(err) - } else { - fmt.Printf("armor: %s, type: %v\n\n", armor, typ.String()) - } - - if backstory, typ, _, err := json.Get(data, "backstory"); err != nil { - panic(err) - } else { - fmt.Printf("backstory: %s, type: %v\n\n", backstory, typ.String()) - } -} \ No newline at end of file diff --git a/gnovm/tests/files/json4.gno b/gnovm/tests/files/json4.gno deleted file mode 100644 index 19895ce9ec1..00000000000 --- a/gnovm/tests/files/json4.gno +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "fmt" - - "gno.land/p/demo/json" -) - -// dynamic struct creation and marshal to json -func main() { - s := json.NewStruct() - - for i:=0; i < 10; i++ { - s.AddIntField(fmt.Sprintf("field%d", i), i) // <- add field in runtime - } - - str, err := s.Marshal() - if err != nil { - panic(err) - } - - fmt.Println(string(str)) -} \ No newline at end of file From ac77d7113a18b6624dce8b3d1ee3d02daf5287a1 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 15 Feb 2024 16:18:12 +0900 Subject: [PATCH 51/72] fmt --- examples/gno.land/p/demo/json/escape.gno | 4 ++-- examples/gno.land/p/demo/json/parser.gno | 2 +- .../gno.land/p/demo/json/state_machine.gno | 4 ++-- examples/gno.land/p/demo/json/struct.gno | 20 +++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 09e1c9548fa..e1cfae0987f 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -47,8 +47,8 @@ func unescape(input, output []byte) ([]byte, error) { return nil, errors.New("JSON Error: Encountered an invalid escape sequence in a string.") } - input = input[inLen:] // the number of bytes consumed in the input - buf = buf[bufLen:] // the number of bytes written to buf + input = input[inLen:] // the number of bytes consumed in the input + buf = buf[bufLen:] // the number of bytes written to buf // find the next backslash in the remaining input nextBackslash := bytes.IndexByte(input, BackSlashToken) diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index ec5cb42e353..47e9625d235 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -8,7 +8,7 @@ import ( ) const ( - absMinInt64 = 1<<63 + absMinInt64 = 1 << 63 maxInt64 = absMinInt64 - 1 maxUint64 = 1<<64 - 1 ) diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 717c3304b54..0705b8c08ef 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -179,7 +179,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { func decreaseLevel(level, keyLevel int) (int, int) { level -= 1 - if level - 1 == keyLevel { + if level-1 == keyLevel { keyLevel -= 1 } @@ -361,7 +361,7 @@ func ArrayEach( for true { v, t, o, e := Get(data[offset:]) - if e != nil{ + if e != nil { return -1, e } diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index e720949d1fc..0c8b70c2d72 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -38,7 +38,7 @@ func NewStruct() *Struct { } func (s *Struct) Fields() []Field { - return s.fields + return s.fields } // addField adds a field to the struct. The order of the fields is preserved. @@ -80,14 +80,14 @@ func (s *Struct) AddObjectField(name string, inner *Struct) *Struct { } func (s *Struct) AddArrayField(name string, arrayStruct *Struct) *Struct { - f := Field{ - name: name, - typ: Array, - arrayValues: arrayStruct.Fields(), - } - s.fields = append(s.fields, f) - - return s + f := Field{ + name: name, + typ: Array, + arrayValues: arrayStruct.Fields(), + } + s.fields = append(s.fields, f) + + return s } /* Encoder */ @@ -268,7 +268,7 @@ func Unmarshal(data []byte, s *Struct) error { // } // // unmarshal here // }) - + // if err != nil { // return err // } From 680f152fafabbb68fe2eab4c1a93b6493224a5ca Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 19 Feb 2024 14:01:13 +0900 Subject: [PATCH 52/72] create error code --- examples/gno.land/p/demo/json/errors.gno | 94 +++++++++---------- examples/gno.land/p/demo/json/gno.mod | 4 +- examples/gno.land/p/demo/json/parser.gno | 21 +++-- .../gno.land/p/demo/json/state_machine.gno | 32 +++---- 4 files changed, 75 insertions(+), 76 deletions(-) diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index 84a1abf2a82..35552fecce9 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -1,56 +1,52 @@ package json -import "errors" +import ( + "errors" -const ( - // KeyPathNotFoundError occurs when a specified key path does not exist in the JSON structure. - KeyPathNotFoundError error = errors.New("key path not found in the JSON structure") - - // ArrayIndexNotFound occurs when the specified index is beyond the range of the array. - ArrayIndexNotFound = errors.New("array index not found or out of range") - - // TokenNotFound occurs when a particular token (expected as part of the structure) is not found. - TokenNotFound = errors.New("expected token not found in the JSON input") - - // KeyLevelNotMatched occurs when the key levels do not match the expected structure or depth. - KeyLevelNotMatched = errors.New("key level does not match the expected structure or depth") - - // Overflow occurs when a number in the JSON exceeds the range that can be handled. - Overflow = errors.New("numeric value exceeds the range limit") - - // EmptyBytes occurs when the JSON input is empty or has no content. - EmptyBytes = errors.New("empty bytes: the JSON input is empty or has no content") - - // InvalidArrayIndex occurs when the index used for an array is not an integer or out of valid range. - InvalidArrayIndex = errors.New("invalid array index: index should be an integer and within the valid range") - - // InvalidExponents occurs when there's an error related to the format or range of exponents in numbers. - InvalidExponents = errors.New("invalid format or range of exponents in a numeric value") - - // NonDigitCharacters occurs when there are non-digit characters where a number is expected. - NonDigitCharacters = errors.New("non-digit characters found where a number is expected") - - // MultipleDecimalPoints occurs when a number has more than one decimal point. - MultipleDecimalPoints = errors.New("multiple decimal points found in a number") - - // MalformedType occurs when the type of a value does not match the expected type. - MalformedType = errors.New("malformed type: the type of the value does not match the expected type") - - // MalformedString occurs when a string is improperly formatted, like unescaped characters or incorrect quotes. - MalformedString = errors.New("malformed string: improperly formatted string, check for unescaped characters or incorrect quotes") - - // MalformedValue occurs when a value does not conform to the expected format or structure. - MalformedValue = errors.New("malformed value: the value does not conform to the expected format or structure") - - // MalformedObject occurs when a JSON object is improperly formatted. - MalformedObject = errors.New("malformed object: the JSON object is improperly formatted or structured") + "gno.land/p/demo/ufmt" +) - // MalformedArray occurs when a JSON array is improperly formatted. - MalformedArray = errors.New("malformed array: the JSON array is improperly formatted or structured") +type jsonError struct { + code int + msg string +} - // MalformedJson occurs when the entire JSON structure is improperly formatted or structured. - MalformedJson = errors.New("malformed JSON: the entire JSON structure is improperly formatted or structured") +func (e *jsonError) Error() string { + return ufmt.Sprintf("Error %d: %s", e.code, e.msg) +} - // UnknownValueType occurs when the JSON contains a value of an unrecognized or unsupported type. - UnknownValueType = errors.New("unknown value type: the value type is unrecognized or unsupported") +// error code +const ( + // parser errors (1xx) + KeyPathNotFound = 100 + iota + ArrayIndexNotFound + TokenNotFound + KeyLevelNotMatched + NextValueNotFound + + // syntax errors (2xx) + Overflow = 200 + iota + EmptyBytes + InvalidNull + InvalidBoolean + InvalidArrayIndex + InvalidExponents + NotNumber + MultipleDecimalPoints + + // type errors (3xx) + InvalidType = 300 + iota + InvalidString + InvalidValue + InvalidObject + InvalidArray + InvalidJSONFormat + UnknownValueType + + // buffer errors (4xx) + EofErrorCode = 400 + iota ) + +func newJSONError(code int, msg string) error { + return &jsonError{code, msg} +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/gno.mod b/examples/gno.land/p/demo/json/gno.mod index ef794458c56..f7e1076002f 100644 --- a/examples/gno.land/p/demo/json/gno.mod +++ b/examples/gno.land/p/demo/json/gno.mod @@ -1,3 +1,5 @@ module gno.land/p/demo/json -require gno.land/p/demo/ufmt v0.0.0-latest +require ( + gno.land/p/demo/ufmt v0.0.0-latest +) \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 47e9625d235..475b8553fc5 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -61,7 +61,7 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { } } - return -1, escaped, MalformedString + return -1, escaped, newJSONError(InvalidString, "Invalid string input found while parsing string value") } // Find end of the data structure, array or object. @@ -119,7 +119,7 @@ func ParseStringLiteral(data []byte) (string, error) { bf, err := unescape(data, buf[:]) if err != nil { - return "", MalformedString + return "", newJSONError(InvalidString, "Invalid string input found while parsing string value") } return string(bf), nil @@ -190,18 +190,18 @@ func ParseIntLiteral(bytes []byte) (v int64, err error) { var n uint64 = 0 for _, c := range bytes { if notDigit(c) { - return 0, NonDigitCharacters + return 0, newJSONError(NotNumber, "non-digit characters found while parsing integer value") } if n > maxUint64/10 { - return 0, Overflow + return 0, newJSONError(Overflow, "numeric value exceeds the range limit") } n *= 10 n1 := n + uint64(c-'0') if n1 < n { - return 0, Overflow + return 0, newJSONError(Overflow, "numeric value exceeds the range limit") } n = n1 @@ -212,7 +212,7 @@ func ParseIntLiteral(bytes []byte) (v int64, err error) { return -absMinInt64, nil } - return 0, Overflow + return 0, newJSONError(Overflow, "numeric value exceeds the range limit") } if neg { @@ -244,13 +244,13 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { } if notDigit(c) { - return 0, 0, NonDigitCharacters + return 0, 0, newJSONError(NotNumber, "non-digit characters found while parsing integer value") } digit := uint64(c - '0') if man > (maxUint64-digit)/10 { - return 0, 0, Overflow + return 0, 0, newJSONError(Overflow, "numeric value exceeds the range limit") } man = man*10 + digit @@ -362,7 +362,7 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { if bytes.Equal(value, nullLiteral) { return Null, offset + end, nil } - return Unknown, offset, UnknownValueType + return Unknown, offset, newJSONError(InvalidNull, "Invalid null value found while parsing null value") case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': if !bytes.ContainsAny(value, ".eE") { if _i, err := ParseIntLiteral(value); err != nil { @@ -377,5 +377,6 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { return Number, offset + end, nil } - return Unknown, offset, errors.New("JSON Error: unknown value type found while parsing primitive value") + // return Unknown, offset, errors.New("JSON Error: unknown value type found while parsing primitive value") + return Unknown, offset, newJSONError(UnknownValueType, "unknown value type found while parsing primitive value") } diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 0705b8c08ef..2e50facc9d9 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -29,16 +29,16 @@ func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { dataType = Boolean } - return Unknown, offset, UnknownValueType + return Unknown, offset, newJSONError(InvalidBoolean, "invalid boolean value. expected: true or false") case 'u', 'n': if bytes.Equal(value, nullLiteral) { dataType = Null } - return Unknown, offset, UnknownValueType + return Unknown, offset, newJSONError(InvalidNull, "invalid null value. expected: null") case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': dataType = Number default: - return Unknown, offset, UnknownValueType + return Unknown, offset, newJSONError(UnknownValueType, "unsupported value type or invalid JSON type") } } @@ -61,7 +61,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { switch data[i] { case DoublyQuoteToken: if level < 1 { - return -1, MalformedJson + return -1, newJSONError(InvalidJSONFormat, "invalid JSON format") } i += 1 @@ -156,7 +156,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { subIndex, err := findValueIndex(valueFound, keys[level+1:]...) if err != nil { - return -1, KeyPathNotFoundError + return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") } return i + valueOffset + subIndex, nil @@ -169,11 +169,11 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } } case ColonToken: - return -1, MalformedJson + return -1, newJSONError(InvalidJSONFormat, "invalid JSON format") } } - return -1, KeyPathNotFoundError + return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") } func decreaseLevel(level, keyLevel int) (int, int) { @@ -248,7 +248,7 @@ func findKeyStart(data []byte, key string) (int, error) { } } - return -1, KeyPathNotFoundError + return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") } func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { @@ -262,7 +262,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, // Go to closest value nO, err := nextToken(data[offset:]) if err != nil { - return nil, NotExist, offset, -1, MalformedJson + return nil, NotExist, offset, -1, newJSONError(InvalidJSONFormat, "invalid JSON format") } offset += nO @@ -342,14 +342,14 @@ func ArrayEach( } if data[offset] != SquareOpenToken { - return -1, errors.New("json.ArrayEach: invalid array. must start with `[`") + return -1, newJSONError(InvalidArray, "invalid array. must start with `[`") } else { offset += 1 } } if n0, err := nextToken(data[offset:]); err != nil { - return -1, errors.New("json.ArrayEach: invalid token. value not found") + return -1, newJSONError(NextValueNotFound, "next value not found") } else { offset += n0 } @@ -415,7 +415,7 @@ func ObjectEach( // descent to desired key if len(keys) > 0 { if off, err := findValueIndex(data, keys...); err != nil { - errors.New("json.ObjectEach: key path not found") + newJSONError(KeyPathNotFound, "key path not found in the JSON structure") } else { offset += off } @@ -423,7 +423,7 @@ func ObjectEach( // validate and skip past opening token if off, err := nextToken(data[offset:]); err != nil { - return MalformedObject + return newJSONError(InvalidObject, "Object is not properly formed") } else if offset += off; data[offset] != CurlyOpenToken { return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) } else { @@ -449,7 +449,7 @@ func ObjectEach( case CurlyCloseToken: return nil // end of the object default: - return MalformedObject // invalid token found + return newJSONError(InvalidObject, "Object is not properly formed") } // find the end of the key @@ -622,7 +622,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str pIdxFlags := initPathIdxFlags(paths) if level < 0 { - err := MalformedJson + err := newJSONError(InvalidJSONFormat, "invalid JSON format") cb(-1, nil, Unknown, err) return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) @@ -697,7 +697,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str i += 1 } - return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", KeyPathNotFoundError)) + return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") } func initPathFlags(paths [][]string) []bool { From e8ab3cf355ad1041b5488f1b69368762ad8a22da Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 19 Feb 2024 14:01:27 +0900 Subject: [PATCH 53/72] basic buffer --- examples/gno.land/p/demo/json/buffer.gno | 48 +++++ examples/gno.land/p/demo/json/internal.gno | 195 +++++++++++++++++++++ examples/gno.land/p/demo/json/token.gno | 4 + 3 files changed, 247 insertions(+) create mode 100644 examples/gno.land/p/demo/json/buffer.gno create mode 100644 examples/gno.land/p/demo/json/internal.gno diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno new file mode 100644 index 00000000000..d83cec5fdc6 --- /dev/null +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -0,0 +1,48 @@ +package json + +import ( + "errors" +) + +type buffer struct { + data []byte + length uint + index uint + + last States + state States + class Classes +} + +type ( + rpn []string + tokens []string +) + +// newBuffer creates a new buffer with the given data +func NewBuffer(data []byte) *buffer { + return &buffer{ + data: data, + length: uint(len(data)), + last: GO, + state: GO, + } +} + +func (b *buffer) current() (byte, error) { + if b.index >= b.length { + return 0, newJSONError(EofErrorCode, "EOF") + } + + return b.data[b.index], nil +} + +func (b *buffer) step() error { + if b.index+1 >= b.length { + return newJSONError(EofErrorCode, "EOF") + } + + b.index++ + + return nil +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/internal.gno b/examples/gno.land/p/demo/json/internal.gno new file mode 100644 index 00000000000..1fa7cf30f4e --- /dev/null +++ b/examples/gno.land/p/demo/json/internal.gno @@ -0,0 +1,195 @@ +package json + +// CREDIT: https://github.com/freddierice/php_source/blob/467ed5d6edff72219afd3e644516f131118ef48e/ext/json/JSON_parser.c +// Copyright (c) 2005 JSON.org + +type ( + States int8 + Classes int8 +) + +const __ = -1 + +// enum classes +const ( + C_SPACE Classes = iota /* space */ + C_WHITE /* other whitespace */ + C_LCURB /* { */ + C_RCURB /* } */ + C_LSQRB /* [ */ + C_RSQRB /* ] */ + C_COLON /* : */ + C_COMMA /* , */ + C_QUOTE /* " */ + C_BACKS /* \ */ + C_SLASH /* / */ + C_PLUS /* + */ + C_MINUS /* - */ + C_POINT /* . */ + C_ZERO /* 0 */ + C_DIGIT /* 123456789 */ + C_LOW_A /* a */ + C_LOW_B /* b */ + C_LOW_C /* c */ + C_LOW_D /* d */ + C_LOW_E /* e */ + C_LOW_F /* f */ + C_LOW_L /* l */ + C_LOW_N /* n */ + C_LOW_R /* r */ + C_LOW_S /* s */ + C_LOW_T /* t */ + C_LOW_U /* u */ + C_ABCDF /* ABCDF */ + C_E /* E */ + C_ETC /* everything else */ +) + +// AsciiClasses array maps the 128 ASCII characters into character classes. +var AsciiClasses = [128]Classes{ + /* + This array maps the 128 ASCII characters into character classes. + The remaining Unicode characters should be mapped to C_ETC. + Non-whitespace control characters are errors. + */ + __, __, __, __, __, __, __, __, + __, C_WHITE, C_WHITE, __, __, C_WHITE, __, __, + __, __, __, __, __, __, __, __, + __, __, __, __, __, __, __, __, + + C_SPACE, C_ETC, C_QUOTE, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, + C_ETC, C_ETC, C_ETC, C_PLUS, C_COMMA, C_MINUS, C_POINT, C_SLASH, + C_ZERO, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, + C_DIGIT, C_DIGIT, C_COLON, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, + + C_ETC, C_ABCDF, C_ABCDF, C_ABCDF, C_ABCDF, C_E, C_ABCDF, C_ETC, + C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, + C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, + C_ETC, C_ETC, C_ETC, C_LSQRB, C_BACKS, C_RSQRB, C_ETC, C_ETC, + + C_ETC, C_LOW_A, C_LOW_B, C_LOW_C, C_LOW_D, C_LOW_E, C_LOW_F, C_ETC, + C_ETC, C_ETC, C_ETC, C_ETC, C_LOW_L, C_ETC, C_LOW_N, C_ETC, + C_ETC, C_ETC, C_LOW_R, C_LOW_S, C_LOW_T, C_LOW_U, C_ETC, C_ETC, + C_ETC, C_ETC, C_ETC, C_LCURB, C_ETC, C_RCURB, C_ETC, C_ETC, +} + +// QuoteAsciiClasses is a HACK for single quote from AsciiClasses +var QuoteAsciiClasses = [128]Classes{ + /* + This array maps the 128 ASCII characters into character classes. + The remaining Unicode characters should be mapped to C_ETC. + Non-whitespace control characters are errors. + */ + __, __, __, __, __, __, __, __, + __, C_WHITE, C_WHITE, __, __, C_WHITE, __, __, + __, __, __, __, __, __, __, __, + __, __, __, __, __, __, __, __, + + C_SPACE, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_QUOTE, + C_ETC, C_ETC, C_ETC, C_PLUS, C_COMMA, C_MINUS, C_POINT, C_SLASH, + C_ZERO, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, + C_DIGIT, C_DIGIT, C_COLON, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, + + C_ETC, C_ABCDF, C_ABCDF, C_ABCDF, C_ABCDF, C_E, C_ABCDF, C_ETC, + C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, + C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, C_ETC, + C_ETC, C_ETC, C_ETC, C_LSQRB, C_BACKS, C_RSQRB, C_ETC, C_ETC, + + C_ETC, C_LOW_A, C_LOW_B, C_LOW_C, C_LOW_D, C_LOW_E, C_LOW_F, C_ETC, + C_ETC, C_ETC, C_ETC, C_ETC, C_LOW_L, C_ETC, C_LOW_N, C_ETC, + C_ETC, C_ETC, C_LOW_R, C_LOW_S, C_LOW_T, C_LOW_U, C_ETC, C_ETC, + C_ETC, C_ETC, C_ETC, C_LCURB, C_ETC, C_RCURB, C_ETC, C_ETC, +} + +/* +The state codes. +*/ +const ( + GO States = iota /* start */ + OK /* ok */ + OB /* object */ + KE /* key */ + CO /* colon */ + VA /* value */ + AR /* array */ + ST /* string */ + ES /* escape */ + U1 /* u1 */ + U2 /* u2 */ + U3 /* u3 */ + U4 /* u4 */ + MI /* minus */ + ZE /* zero */ + IN /* integer */ + DT /* dot */ + FR /* fraction */ + E1 /* e */ + E2 /* ex */ + E3 /* exp */ + T1 /* tr */ + T2 /* tru */ + T3 /* true */ + F1 /* fa */ + F2 /* fal */ + F3 /* fals */ + F4 /* false */ + N1 /* nu */ + N2 /* nul */ + N3 /* null */ +) + +// List of action codes +const ( + cl States = -2 /* colon */ + cm States = -3 /* comma */ + qt States = -4 /* quote */ + bo States = -5 /* bracket open */ + co States = -6 /* curly br. open */ + bc States = -7 /* bracket close */ + cc States = -8 /* curly br. close */ + ec States = -9 /* curly br. empty */ +) + +// StateTransitionTable is the state transition table takes the current state and the current symbol, and returns either +// a new state or an action. An action is represented as a negative number. A JSON text is accepted if at the end of the +// text the state is OK and if the mode is DONE. +var StateTransitionTable = [31][31]States{ + /* + The state transition table takes the current state and the current symbol, + and returns either a new state or an action. An action is represented as a + negative number. A JSON text is accepted if at the end of the text the + state is OK and if the mode is DONE. + white 1-9 ABCDF etc + space | { } [ ] : , " \ / + - . 0 | a b c d e f l n r s t u | E |*/ + /*start GO*/ {GO, GO, co, __, bo, __, __, __, ST, __, __, __, MI, __, ZE, IN, __, __, __, __, __, F1, __, N1, __, __, T1, __, __, __, __}, + /*ok OK*/ {OK, OK, __, cc, __, bc, __, cm, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*object OB*/ {OB, OB, __, ec, __, __, __, __, ST, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*key KE*/ {KE, KE, __, __, __, __, __, __, ST, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*colon CO*/ {CO, CO, __, __, __, __, cl, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*value VA*/ {VA, VA, co, __, bo, __, __, __, ST, __, __, __, MI, __, ZE, IN, __, __, __, __, __, F1, __, N1, __, __, T1, __, __, __, __}, + /*array AR*/ {AR, AR, co, __, bo, bc, __, __, ST, __, __, __, MI, __, ZE, IN, __, __, __, __, __, F1, __, N1, __, __, T1, __, __, __, __}, + /*string ST*/ {ST, __, ST, ST, ST, ST, ST, ST, qt, ES, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST}, + /*escape ES*/ {__, __, __, __, __, __, __, __, ST, ST, ST, __, __, __, __, __, __, ST, __, __, __, ST, __, ST, ST, __, ST, U1, __, __, __}, + /*u1 U1*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, U2, U2, U2, U2, U2, U2, U2, U2, __, __, __, __, __, __, U2, U2, __}, + /*u2 U2*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, U3, U3, U3, U3, U3, U3, U3, U3, __, __, __, __, __, __, U3, U3, __}, + /*u3 U3*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, U4, U4, U4, U4, U4, U4, U4, U4, __, __, __, __, __, __, U4, U4, __}, + /*u4 U4*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, ST, ST, ST, ST, ST, ST, ST, ST, __, __, __, __, __, __, ST, ST, __}, + /*minus MI*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, ZE, IN, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*zero ZE*/ {OK, OK, __, cc, __, bc, __, cm, __, __, __, __, __, DT, __, __, __, __, __, __, E1, __, __, __, __, __, __, __, __, E1, __}, + /*int IN*/ {OK, OK, __, cc, __, bc, __, cm, __, __, __, __, __, DT, IN, IN, __, __, __, __, E1, __, __, __, __, __, __, __, __, E1, __}, + /*dot DT*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, FR, FR, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*frac FR*/ {OK, OK, __, cc, __, bc, __, cm, __, __, __, __, __, __, FR, FR, __, __, __, __, E1, __, __, __, __, __, __, __, __, E1, __}, + /*e E1*/ {__, __, __, __, __, __, __, __, __, __, __, E2, E2, __, E3, E3, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*ex E2*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, E3, E3, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*exp E3*/ {OK, OK, __, cc, __, bc, __, cm, __, __, __, __, __, __, E3, E3, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*tr T1*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, T2, __, __, __, __, __, __}, + /*tru T2*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, T3, __, __, __}, + /*true T3*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, OK, __, __, __, __, __, __, __, __, __, __}, + /*fa F1*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, F2, __, __, __, __, __, __, __, __, __, __, __, __, __, __}, + /*fal F2*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, F3, __, __, __, __, __, __, __, __}, + /*fals F3*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, F4, __, __, __, __, __}, + /*false F4*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, OK, __, __, __, __, __, __, __, __, __, __}, + /*nu N1*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, N2, __, __, __}, + /*nul N2*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, N3, __, __, __, __, __, __, __, __}, + /*null N3*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, OK, __, __, __, __, __, __, __, __}, +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 70aef23da0e..23d00442911 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -15,7 +15,9 @@ const ( WhiteSpaceToken = ' ' PlusToken = '+' MinusToken = '-' + AesteriskToken = '*' BangToken = '!' + QuestionToken = '?' NewLineToken = '\n' TabToken = '\t' CarriageReturnToken = '\r' @@ -24,6 +26,8 @@ const ( SlashToken = '/' BackSlashToken = '\\' UnderScoreToken = '_' + DollarToken = '$' + AtToken = '@' ) const ( From 2ab0eedc4d47cbdb2e774b308ee7eec825121832 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Mon, 19 Feb 2024 14:26:19 +0900 Subject: [PATCH 54/72] License --- examples/gno.land/p/demo/json/README.md | 13 ++++++++++++ .../gno.land/p/demo/json/eisel_lemire.gno | 17 ++++----------- examples/gno.land/p/demo/json/internal.gno | 2 +- .../gno.land/p/demo/json/license/LICENSE.txt | 21 +++++++++++++++++++ 4 files changed, 39 insertions(+), 14 deletions(-) create mode 100644 examples/gno.land/p/demo/json/README.md create mode 100644 examples/gno.land/p/demo/json/license/LICENSE.txt diff --git a/examples/gno.land/p/demo/json/README.md b/examples/gno.land/p/demo/json/README.md new file mode 100644 index 00000000000..ef9fce85617 --- /dev/null +++ b/examples/gno.land/p/demo/json/README.md @@ -0,0 +1,13 @@ +## Source + +### JSON Parser + +The code for this project/package can be found at [here](https://github.com/buger/jsonparser). This code is provided for actual project development purposes and has been slightly modified from its original version for specific use within this project. + +For more detailed information on the modifications and their purposes, please refer to the documentation within the project repository. + +#### License + +This project is licensed under the MIT License. This permits the use, copying, modification, merging, publishing, distribution, sublicensing, and/or selling of copies of the software, and allows those to whom the software is furnished to do so as well, under the same terms. + +The full text of the license can be found in the 'LICENSE' file located in the package's root directory. This license specifies the terms under which you agree to use this project. For more detailed information on the license and its terms, please refer to the original license documentation in the [project's repository](https://github.com/buger/jsonparser) or the 'LICENSE.txt' file. diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno index 20e82314dfb..c80aed34655 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -1,3 +1,7 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package json // This file implements the Eisel-Lemire ParseFloat algorithm, published in @@ -23,19 +27,6 @@ const ( float64ExponentBias = 1023 ) -// The Eisel-Lemire algorithm accomplishes its task through the following methods: -// -// 1. Extracting Mantissa and Exponent -// It involves separating the mantissa, or the significant part of the number, -// from the exponent when converting a string into a floating-point number. -// -// 2. Normalization and Range checking -// This step normalizes the mantissa and ensures the exponent falls within an acceptable range. -// -// 3. Conversion to a Floating-Point Number -// Finally, the algorithm uses the normalized mantissa and exponent to convert the data -// into an actual floating-point number. - // eiselLemire64 parses a floating-point number from its mantissa and exponent representation. // This implementation is based on the Eisel-Lemire ParseFloat algorithm, which is efficient // and precise for converting strings to floating-point numbers. diff --git a/examples/gno.land/p/demo/json/internal.gno b/examples/gno.land/p/demo/json/internal.gno index 1fa7cf30f4e..53f616a0add 100644 --- a/examples/gno.land/p/demo/json/internal.gno +++ b/examples/gno.land/p/demo/json/internal.gno @@ -1,6 +1,6 @@ package json -// CREDIT: https://github.com/freddierice/php_source/blob/467ed5d6edff72219afd3e644516f131118ef48e/ext/json/JSON_parser.c +// Reference: https://github.com/freddierice/php_source/blob/467ed5d6edff72219afd3e644516f131118ef48e/ext/json/JSON_parser.c // Copyright (c) 2005 JSON.org type ( diff --git a/examples/gno.land/p/demo/json/license/LICENSE.txt b/examples/gno.land/p/demo/json/license/LICENSE.txt new file mode 100644 index 00000000000..7dad6acda0f --- /dev/null +++ b/examples/gno.land/p/demo/json/license/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Leonid Bugaev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file From e3298e37d678cdb670ae8a151a5e0ee3f45979bd Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 20 Feb 2024 00:35:11 +0900 Subject: [PATCH 55/72] json path parser --- examples/gno.land/p/demo/json/buffer.gno | 190 ++++++++- examples/gno.land/p/demo/json/buffer_test.gno | 383 ++++++++++++++++++ examples/gno.land/p/demo/json/errors.gno | 9 +- examples/gno.land/p/demo/json/gno.mod | 4 +- examples/gno.land/p/demo/json/internal.gno | 2 + examples/gno.land/p/demo/json/parser.gno | 4 +- examples/gno.land/p/demo/json/path.gno | 74 ++++ examples/gno.land/p/demo/json/path_test.gno | 50 +++ examples/gno.land/p/demo/json/utils.gno | 36 +- 9 files changed, 696 insertions(+), 56 deletions(-) create mode 100644 examples/gno.land/p/demo/json/buffer_test.gno create mode 100644 examples/gno.land/p/demo/json/path.gno create mode 100644 examples/gno.land/p/demo/json/path_test.gno diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index d83cec5fdc6..7c61413d88e 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -1,48 +1,208 @@ package json import ( - "errors" + "strings" + + "gno.land/p/demo/ufmt" ) type buffer struct { - data []byte + data []byte length uint index uint - last States + last States state States class Classes } -type ( - rpn []string - tokens []string -) +type tokens []string // newBuffer creates a new buffer with the given data func NewBuffer(data []byte) *buffer { return &buffer{ - data: data, + data: data, length: uint(len(data)), - last: GO, - state: GO, + last: GO, + state: GO, } } +// current returns the byte of the current index. func (b *buffer) current() (byte, error) { if b.index >= b.length { - return 0, newJSONError(EofErrorCode, "EOF") + return 0, newJSONError(EofError, "EOF") } return b.data[b.index], nil } +// next moves to the next byte and returns it. +func (b *buffer) next() (byte, error) { + b.index++ + return b.current() +} + +// step just moves to the next position. func (b *buffer) step() error { - if b.index+1 >= b.length { - return newJSONError(EofErrorCode, "EOF") + _, err := b.next() + return err +} + +// move moves the index by the given position. +func (b *buffer) move(pos uint) error { + newIndex := b.index + pos + + if newIndex > b.length { + return newJSONError(EofError, "EOF") } - b.index++ + b.index = newIndex return nil -} \ No newline at end of file +} + +// slice returns the slice from the current index to the given position. +func (b *buffer) slice(pos uint) ([]byte, error) { + end := b.index + pos + + if end > b.length { + return nil, newJSONError(EofError, "EOF") + } + + return b.data[b.index:end], nil +} + +// sliceFromIndices returns a slice of the buffer's data starting from 'start' up to (but not including) 'stop'. +func (b *buffer) sliceFromIndices(start, stop uint) []byte { + if start > b.length { + start = b.length + } + + if stop > b.length { + stop = b.length + } + + return b.data[start:stop] +} + +// skip moves the index to skip the given byte. +func (b *buffer) skip(bs byte) error { + for b.index < b.length { + if b.data[b.index] == bs && !b.backslash() { + return nil + } + + b.index++ + } + + return newJSONError(EofError, "EOF") +} + +// skipAny moves the index until it encounters one of the given set of bytes. +func (b *buffer) skipAny(endTokens map[byte]bool) error { + for b.index < b.length { + if _, exists := endTokens[b.data[b.index]]; exists { + return nil + } + + b.index++ + } + + // build error message + var tokens []string + for token := range endTokens { + tokens = append(tokens, string(token)) + } + + return newJSONError( + EofError, + ufmt.Sprintf( + "EOF reached before encountering one of the expected tokens: %s", + strings.Join(tokens, ", "), + )) +} + +// skipAndReturnIndex moves the buffer index forward by one and returns the new index. +func (b *buffer) skipAndReturnIndex() (uint, error) { + err := b.step() + if err != nil { + return 0, err + } + + return b.index, nil +} + +// skipUntil moves the buffer index forward until it encounters a byte contained in the endTokens set. +func (b *buffer) skipUntil(endTokens map[byte]bool) (uint, error) { + for b.index < b.length { + currentByte, err := b.current() + if err != nil { + return b.index, newJSONError(EofError, "Reached end of buffer without finding token") + } + + // Check if the current byte is in the set of end tokens. + if _, exists := endTokens[currentByte]; exists { + return b.index, nil + } + + b.index++ + } + + return b.index, newJSONError(EofError, "End of file reached before token found") +} + + +// significantTokens is a map where the keys are the significant characters in a JSON path. +// The values in the map are all true, which allows us to use the map as a set for quick lookups. +var significantTokens = map[byte]bool{ + DotToken: true, // access properties of an object + DollarToken: true, // root object + AtToken: true, // current object in a filter expression + SquareOpenToken: true, // start of an array index or filter expression + SquareCloseToken: true, // end of an array index or filter expression +} + +// skipToNextSignificantToken advances the buffer index to the next significant character. +// Significant characters are defined based on the JSON path syntax. +func (b *buffer) skipToNextSignificantToken() { + for b.index < b.length { + current := b.data[b.index] + + if _, ok := significantTokens[current]; ok { + break + } + + b.index++ + } +} + +// backslash checks to see if the number of backslashes before the current index is odd. +// +// This is used to check if the current character is escaped. However, unlike the "unescape" function, +// "backslash" only serves to check the number of backslashes. +func (b *buffer) backslash() bool { + if b.index == 0 { + return false + } + + count := 0 + for i := b.index - 1; ; i-- { + if i >= b.length || b.data[i] != '\\' { + break + } + + count++ + + if i == 0 { + break + } + } + + return count%2 != 0 +} + +// reset initializes the state of the buffer. +func (b *buffer) reset() { + b.last = GO +} diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno new file mode 100644 index 00000000000..33bee3f0e56 --- /dev/null +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -0,0 +1,383 @@ +package json + +import ( + "testing" +) + +func TestBufferCurrent(t *testing.T) { + tests := []struct { + name string + buffer *buffer + expected byte + wantErr bool + }{ + { + name: "Valid current byte", + buffer: &buffer{ + data: []byte("test"), + length: 4, + index: 1, + }, + expected: 'e', + wantErr: false, + }, + { + name: "EOF", + buffer: &buffer{ + data: []byte("test"), + length: 4, + index: 4, + }, + expected: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func (t *testing.T) { + got, err := tt.buffer.current() + if (err != nil) != tt.wantErr { + t.Errorf("buffer.current() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.expected { + t.Errorf("buffer.current() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestBufferStep(t *testing.T) { + tests := []struct { + name string + buffer *buffer + wantErr bool + }{ + { + name: "Valid step", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + wantErr: false, + }, + { + name: "EOF error", + buffer: &buffer{data: []byte("test"), length: 4, index: 3}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.buffer.step() + if (err != nil) != tt.wantErr { + t.Errorf("buffer.step() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestBufferNext(t *testing.T) { + tests := []struct { + name string + buffer *buffer + want byte + wantErr bool + }{ + { + name: "Valid next byte", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + want: 'e', + wantErr: false, + }, + { + name: "EOF error", + buffer: &buffer{data: []byte("test"), length: 4, index: 3}, + want: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.buffer.next() + if (err != nil) != tt.wantErr { + t.Errorf("buffer.next() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("buffer.next() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestBufferSlice(t *testing.T) { + tests := []struct { + name string + buffer *buffer + pos uint + want []byte + wantErr bool + }{ + { + name: "Valid slice -- 0 characters", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + pos: 0, + want: nil, + wantErr: false, + }, + { + name: "Valid slice -- 1 character", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + pos: 1, + want: []byte("t"), + wantErr: false, + }, + { + name: "Valid slice -- 2 characters", + buffer: &buffer{data: []byte("test"), length: 4, index: 1}, + pos: 2, + want: []byte("es"), + wantErr: false, + }, + { + name: "Valid slice -- 3 characters", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + pos: 3, + want: []byte("tes"), + wantErr: false, + }, + { + name: "Valid slice -- 4 characters", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + pos: 4, + want: []byte("test"), + wantErr: false, + }, + { + name: "EOF error", + buffer: &buffer{data: []byte("test"), length: 4, index: 3}, + pos: 2, + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.buffer.slice(tt.pos) + if (err != nil) != tt.wantErr { + t.Errorf("buffer.slice() error = %v, wantErr %v", err, tt.wantErr) + return + } + if string(got) != string(tt.want) { + t.Errorf("buffer.slice() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestBufferMove(t *testing.T) { + tests := []struct { + name string + buffer *buffer + pos uint + wantErr bool + wantIdx uint + }{ + { + name: "Valid move", + buffer: &buffer{data: []byte("test"), length: 4, index: 1}, + pos: 2, + wantErr: false, + wantIdx: 3, + }, + { + name: "Move beyond length", + buffer: &buffer{data: []byte("test"), length: 4, index: 1}, + pos: 4, + wantErr: true, + wantIdx: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.buffer.move(tt.pos) + if (err != nil) != tt.wantErr { + t.Errorf("buffer.move() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.buffer.index != tt.wantIdx { + t.Errorf("buffer.move() index = %v, want %v", tt.buffer.index, tt.wantIdx) + } + }) + } +} + +func TestBufferSkip(t *testing.T) { + tests := []struct { + name string + buffer *buffer + b byte + wantErr bool + }{ + { + name: "Skip byte", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + b: 'e', + wantErr: false, + }, + { + name: "Skip to EOF", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + b: 'x', + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.buffer.skip(tt.b) + if (err != nil) != tt.wantErr { + t.Errorf("buffer.skip() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestBufferSkipAny(t *testing.T) { + tests := []struct { + name string + buffer *buffer + s map[byte]bool + wantErr bool + }{ + { + name: "Skip any valid byte", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + s: map[byte]bool{'e': true, 'o': true}, + wantErr: false, + }, + { + name: "Skip any to EOF", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + s: map[byte]bool{'x': true, 'y': true}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.buffer.skipAny(tt.s) + if (err != nil) != tt.wantErr { + t.Errorf("buffer.skipAny() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestSkipToNextSignificantToken(t *testing.T) { + tests := []struct { + name string + input []byte + expected uint + }{ + {"No significant chars", []byte("abc"), 3}, + {"One significant char at start", []byte(".abc"), 0}, + {"Significant char in middle", []byte("ab.c"), 2}, + {"Multiple significant chars", []byte("a$.c"), 1}, + {"Significant char at end", []byte("abc$"), 3}, + {"Only significant chars", []byte("$."), 0}, + {"Empty string", []byte(""), 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := NewBuffer(tt.input) + b.skipToNextSignificantToken() + if b.index != tt.expected { + t.Errorf("after skipToNextSignificantToken(), got index = %v, want %v", b.index, tt.expected) + } + }) + } +} + +const testString = "abcdefg" + +func mockBuffer(s string) *buffer { + return NewBuffer([]byte(s)) +} + +func TestSkipAndReturnIndex(t *testing.T) { + tests := []struct { + name string + input string + expected uint + }{ + {"StartOfString", "", 0}, + {"MiddleOfString", "abcdef", 1}, + {"EndOfString", "abc", 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := mockBuffer(tt.input) + got, err := buf.skipAndReturnIndex() + if err != nil && tt.input != "" { // Expect no error unless input is empty + t.Errorf("skipAndReturnIndex() error = %v", err) + } + if got != tt.expected { + t.Errorf("skipAndReturnIndex() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestSkipUntil(t *testing.T) { + tests := []struct { + name string + input string + tokens map[byte]bool + expected uint + }{ + {"SkipToToken", "abcdefg", map[byte]bool{'c': true}, 2}, + {"SkipToEnd", "abcdefg", map[byte]bool{'h': true}, 7}, + {"SkipNone", "abcdefg", map[byte]bool{'a': true}, 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := mockBuffer(tt.input) + got, err := buf.skipUntil(tt.tokens) + if err != nil && got != uint(len(tt.input)) { // Expect error only if reached end without finding token + t.Errorf("skipUntil() error = %v", err) + } + if got != tt.expected { + t.Errorf("skipUntil() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestSliceFromIndices(t *testing.T) { + tests := []struct { + name string + input string + start uint + end uint + expected string + }{ + {"FullString", "abcdefg", 0, 7, "abcdefg"}, + {"Substring", "abcdefg", 2, 5, "cde"}, + {"OutOfBounds", "abcdefg", 5, 10, "fg"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := mockBuffer(tt.input) + got := buf.sliceFromIndices(tt.start, tt.end) + if string(got) != tt.expected { + t.Errorf("sliceFromIndices() = %v, want %v", string(got), tt.expected) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno index 35552fecce9..33751583c77 100644 --- a/examples/gno.land/p/demo/json/errors.gno +++ b/examples/gno.land/p/demo/json/errors.gno @@ -8,7 +8,7 @@ import ( type jsonError struct { code int - msg string + msg string } func (e *jsonError) Error() string { @@ -44,9 +44,12 @@ const ( UnknownValueType // buffer errors (4xx) - EofErrorCode = 400 + iota + EofError = 400 + iota + + // JSON path error (5xx) + InvalidPath = 500 + iota ) func newJSONError(code int, msg string) error { return &jsonError{code, msg} -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/gno.mod b/examples/gno.land/p/demo/json/gno.mod index f7e1076002f..ef794458c56 100644 --- a/examples/gno.land/p/demo/json/gno.mod +++ b/examples/gno.land/p/demo/json/gno.mod @@ -1,5 +1,3 @@ module gno.land/p/demo/json -require ( - gno.land/p/demo/ufmt v0.0.0-latest -) \ No newline at end of file +require gno.land/p/demo/ufmt v0.0.0-latest diff --git a/examples/gno.land/p/demo/json/internal.gno b/examples/gno.land/p/demo/json/internal.gno index 53f616a0add..97d17e5f66a 100644 --- a/examples/gno.land/p/demo/json/internal.gno +++ b/examples/gno.land/p/demo/json/internal.gno @@ -3,6 +3,8 @@ package json // Reference: https://github.com/freddierice/php_source/blob/467ed5d6edff72219afd3e644516f131118ef48e/ext/json/JSON_parser.c // Copyright (c) 2005 JSON.org +// Go implementation is taken from: https://github.com/spyzhov/ajson/blob/master/internal/state.go + type ( States int8 Classes int8 diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 475b8553fc5..82a69b50152 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -128,9 +128,9 @@ func ParseStringLiteral(data []byte) (string, error) { // ParseBoolLiteral parses a boolean value from the given byte slice. func ParseBoolLiteral(data []byte) (bool, error) { switch { - case bytes.Equal(data, []byte("true")): + case bytes.Equal(data, trueLiteral): return true, nil - case bytes.Equal(data, []byte("false")): + case bytes.Equal(data, falseLiteral): return false, nil default: return false, errors.New("JSON Error: malformed boolean value found while parsing boolean value") diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno new file mode 100644 index 00000000000..6ebf6e0a64f --- /dev/null +++ b/examples/gno.land/p/demo/json/path.gno @@ -0,0 +1,74 @@ +package json + +// ParsePath takes a JSONPath string and returns a slice of strings representing the path segments. +func ParsePath(path string) ([]string, error) { + buf := NewBuffer([]byte(path)) + result := make([]string, 0) + + for { + b, err := buf.current() + if err != nil { + break + } + + switch { + case b == DollarToken || b == AtToken: + result = append(result, string(b)) + buf.step() + + case b == DotToken: + buf.step() + + if next, _ := buf.current(); next == DotToken { + buf.step() + result = append(result, "..") + + extractNextSegment(buf, &result) + } else { + extractNextSegment(buf, &result) + } + + case b == SquareOpenToken: + start := buf.index + buf.step() + + for { + if buf.index >= buf.length || buf.data[buf.index] == SquareCloseToken { + break + } + + buf.step() + } + + if buf.index >= buf.length { + return nil, newJSONError(EofError, "unexpected end of path") + } + + segment := string(buf.sliceFromIndices(start+1, buf.index)) + result = append(result, segment) + + buf.step() + + default: + buf.step() + } + } + + return result, nil +} + +// extractNextSegment extracts the segment from the current index +// to the next significant character and adds it to the resulting slice. +func extractNextSegment(buf *buffer, result *[]string) { + start := buf.index + buf.skipToNextSignificantToken() + + if buf.index <= start { + return + } + + segment := string(buf.sliceFromIndices(start, buf.index)) + if segment != "" { + *result = append(*result, segment) + } +} diff --git a/examples/gno.land/p/demo/json/path_test.gno b/examples/gno.land/p/demo/json/path_test.gno new file mode 100644 index 00000000000..7caff6f3b18 --- /dev/null +++ b/examples/gno.land/p/demo/json/path_test.gno @@ -0,0 +1,50 @@ +package json + +import ( + "testing" +) + +func TestParseJSONPath(t *testing.T) { + tests := []struct { + name string + path string + expected []string + }{ + {name: "Empty string path", path: "", expected: []string{}}, + {name: "Root only path", path: "$", expected: []string{"$"}}, + {name: "Root with dot path", path: "$.", expected: []string{"$"}}, + {name: "All objects in path", path: "$..", expected: []string{"$", ".."}}, + {name: "Only children in path", path: "$.*", expected: []string{"$", "*"}}, + {name: "All objects' children in path", path: "$..*", expected: []string{"$", "..", "*"}}, + {name: "Simple dot notation path", path: "$.root.element", expected: []string{"$", "root", "element"}}, + {name: "Complex dot notation path with wildcard", path: "$.root.*.element", expected: []string{"$", "root", "*", "element"}}, + {name: "Path with array wildcard", path: "$.phoneNumbers[*].type", expected: []string{"$", "phoneNumbers", "*", "type"}}, + {name: "Path with filter expression", path: "$.store.book[?(@.price < 10)].title", expected: []string{"$", "store", "book", "?(@.price < 10)", "title"}}, + {name: "Path with formula", path: "$..phoneNumbers..('ty' + 'pe')", expected: []string{"$", "..", "phoneNumbers", "..", "('ty' + 'pe')"}}, + {name: "Simple bracket notation path", path: "$['root']['element']", expected: []string{"$", "'root'", "'element'"}}, + {name: "Complex bracket notation path with wildcard", path: "$['root'][*]['element']", expected: []string{"$", "'root'", "*", "'element'"}}, + {name: "Bracket notation path with integer index", path: "$['store']['book'][0]['title']", expected: []string{"$", "'store'", "'book'", "0", "'title'"}}, + {name: "Complex path with wildcard in bracket notation", path: "$['root'].*['element']", expected: []string{"$", "'root'", "*", "'element'"}}, + {name: "Mixed notation path with dot after bracket", path: "$.['root'].*.['element']", expected: []string{"$", "'root'", "*", "'element'"}}, + {name: "Mixed notation path with dot before bracket", path: "$['root'].*.['element']", expected: []string{"$", "'root'", "*", "'element'"}}, + {name: "Single character path with root", path: "$.a", expected: []string{"$", "a"}}, + {name: "Multiple characters path with root", path: "$.abc", expected: []string{"$", "abc"}}, + {name: "Multiple segments path with root", path: "$.a.b.c", expected: []string{"$", "a", "b", "c"}}, + {name: "Multiple segments path with wildcard and root", path: "$.a.*.c", expected: []string{"$", "a", "*", "c"}}, + {name: "Multiple segments path with filter and root", path: "$.a[?(@.b == 'c')].d", expected: []string{"$", "a", "?(@.b == 'c')", "d"}}, + {name: "Complex path with multiple filters", path: "$.a[?(@.b == 'c')].d[?(@.e == 'f')].g", expected: []string{"$", "a", "?(@.b == 'c')", "d", "?(@.e == 'f')", "g"}}, + {name: "Complex path with multiple filters and wildcards", path: "$.a[?(@.b == 'c')].*.d[?(@.e == 'f')].g", expected: []string{"$", "a", "?(@.b == 'c')", "*", "d", "?(@.e == 'f')", "g"}}, + {name: "Path with array index and root", path: "$.a[0].b", expected: []string{"$", "a", "0", "b"}}, + {name: "Path with multiple array indices and root", path: "$.a[0].b[1].c", expected: []string{"$", "a", "0", "b", "1", "c"}}, + {name: "Path with array index, wildcard and root", path: "$.a[0].*.c", expected: []string{"$", "a", "0", "*", "c"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reult, _ := ParsePath(tt.path) + if !isEqualSlice(reult, tt.expected) { + t.Errorf("ParsePath(%s) expected: %v, got: %v", tt.path, tt.expected, reult) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index 7528acf7fce..09e8f4072a9 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -49,43 +49,13 @@ func equalStr(b *[]byte, s string) bool { /* Testing Helper Function */ -func isEqualMap(a, b map[string]interface{}) bool { +func isEqualSlice(a, b []string) bool { if len(a) != len(b) { return false } - for key, valueA := range a { - valueB, ok := b[key] - if !ok { - return false - } - - switch valueA := valueA.(type) { - case []interface{}: - if valueB, ok := valueB.([]interface{}); ok { - if !isEqualSlice(valueA, valueB) { - return false - } - } else { - return false - } - default: - if valueA != valueB { - return false - } - } - } - - return true -} - -func isEqualSlice(a, b []interface{}) bool { - if len(a) != len(b) { - return false - } - - for i := range a { - if a[i] != b[i] { + for i, v := range a { + if v != b[i] { return false } } From ecaab8a83bbdc671b798c67e7fc0e7051dfe5492 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 21 Feb 2024 22:02:06 +0900 Subject: [PATCH 56/72] json path token handler --- examples/gno.land/p/demo/json/buffer.gno | 181 +++++++++++++++++- examples/gno.land/p/demo/json/buffer_test.gno | 139 ++++++++++++++ examples/gno.land/p/demo/json/errors.gno | 55 ------ examples/gno.land/p/demo/json/escape.gno | 8 +- examples/gno.land/p/demo/json/internal.gno | 7 +- examples/gno.land/p/demo/json/node.gno | 1 + examples/gno.land/p/demo/json/parser.gno | 40 ++-- examples/gno.land/p/demo/json/path.gno | 6 +- .../gno.land/p/demo/json/state_machine.gno | 69 +++---- examples/gno.land/p/demo/json/token.gno | 4 + examples/gno.land/p/demo/json/utils.gno | 6 +- gnovm/pkg/gnolang/precompile.go | 1 + 12 files changed, 391 insertions(+), 126 deletions(-) delete mode 100644 examples/gno.land/p/demo/json/errors.gno create mode 100644 examples/gno.land/p/demo/json/node.gno diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index 7c61413d88e..63276c72ece 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -1,6 +1,7 @@ package json import ( + "errors" "strings" "gno.land/p/demo/ufmt" @@ -31,7 +32,7 @@ func NewBuffer(data []byte) *buffer { // current returns the byte of the current index. func (b *buffer) current() (byte, error) { if b.index >= b.length { - return 0, newJSONError(EofError, "EOF") + return 0, errors.New("EOF") } return b.data[b.index], nil @@ -54,7 +55,7 @@ func (b *buffer) move(pos uint) error { newIndex := b.index + pos if newIndex > b.length { - return newJSONError(EofError, "EOF") + return errors.New("EOF") } b.index = newIndex @@ -67,7 +68,7 @@ func (b *buffer) slice(pos uint) ([]byte, error) { end := b.index + pos if end > b.length { - return nil, newJSONError(EofError, "EOF") + return nil, errors.New("EOF") } return b.data[b.index:end], nil @@ -96,7 +97,7 @@ func (b *buffer) skip(bs byte) error { b.index++ } - return newJSONError(EofError, "EOF") + return errors.New("EOF") } // skipAny moves the index until it encounters one of the given set of bytes. @@ -115,8 +116,7 @@ func (b *buffer) skipAny(endTokens map[byte]bool) error { tokens = append(tokens, string(token)) } - return newJSONError( - EofError, + return errors.New( ufmt.Sprintf( "EOF reached before encountering one of the expected tokens: %s", strings.Join(tokens, ", "), @@ -138,7 +138,7 @@ func (b *buffer) skipUntil(endTokens map[byte]bool) (uint, error) { for b.index < b.length { currentByte, err := b.current() if err != nil { - return b.index, newJSONError(EofError, "Reached end of buffer without finding token") + return b.index, err } // Check if the current byte is in the set of end tokens. @@ -149,7 +149,7 @@ func (b *buffer) skipUntil(endTokens map[byte]bool) (uint, error) { b.index++ } - return b.index, newJSONError(EofError, "End of file reached before token found") + return b.index, errors.New("EOF") } @@ -188,7 +188,7 @@ func (b *buffer) backslash() bool { count := 0 for i := b.index - 1; ; i-- { - if i >= b.length || b.data[i] != '\\' { + if i >= b.length || b.data[i] != BackSlashToken { break } @@ -202,7 +202,168 @@ func (b *buffer) backslash() bool { return count%2 != 0 } -// reset initializes the state of the buffer. func (b *buffer) reset() { b.last = GO } + +func (b *buffer) numeric(token bool) error { + if token { + b.last = GO + } + + for _; b.index < b.length; b.index++ { + b.class = b.getClasses(DoublyQuoteToken) + if b.class == __ { + return errors.New("invalid token found while parsing path") + } + + b.state = StateTransitionTable[b.last][b.class] + if b.state == __ { + if token { + break + } + + return errors.New("invalid token found while parsing path") + } + + if b.state < __ { + return nil + } + + if b.state < MI || b.state > E3 { + return nil + } + + b.last = b.state + } + + // zero, integer, fraction, exponent + if b.last != ZE && b.last != IN && b.last != FR && b.last != E3 { + return errors.New("invalid token found while parsing path") + } + + return nil +} + +func (b *buffer) getClasses(c byte) Classes { + if b.data[b.index] >= 128 { + return C_ETC + } + + if c == QuoteToken { + return QuoteAsciiClasses[b.data[b.index]] + } + + return AsciiClasses[b.data[b.index]] +} + +func (b *buffer) getState() States { + b.last = b.state + + b.class = b.getClasses(DoublyQuoteToken) + if b.class == __ { + return __ + } + + b.state = StateTransitionTable[b.last][b.class] + + return b.state +} + +func (b *buffer) pathToken() error { + var ( + c byte + stack []byte + first = b.index + start int + inToken bool = false + inNumber bool = false + ) + +tokenLoop: + for ; b.index < b.length; b.index++ { + c = b.data[b.index] + + switch { + case c == DoublyQuoteToken || c == QuoteToken: + inToken = true + if err := b.step(); err != nil { + return errors.New("error stepping through buffer") + } + + err := b.skip(c) + if err != nil { + return errors.New("unmatched quote in path") + } + + if b.index >= b.length { + return errors.New("unmatched quote in path") + } + + case c == SquareOpenToken || c == RoundOpenToken: + inToken = true + stack = append(stack, c) + + case c == SquareCloseToken || c == RoundCloseToken: + inToken = true + if len(stack) == 0 || (c == SquareCloseToken && stack[len(stack)-1] != SquareOpenToken) || (c == RoundCloseToken && stack[len(stack)-1] != RoundOpenToken) { + return errors.New("mismatched bracket or parenthesis") + } + stack = stack[:len(stack)-1] + + case c == DotToken || c == DollarToken || c == AtToken || c == AesteriskToken || ('A' <= c && c <= 'z') || ('0' <= c && c <= '9'): + inToken = true + // do nothing + + case c == PlusToken || c == MinusToken: + if inNumber || (b.index > 0 && (b.data[b.index-1] == 'e' || b.data[b.index-1] == 'E')) { + inToken = true + continue + } else if !inToken && (b.index+1 < b.length && isDigit(b.data[b.index+1])) { + inToken = true + inNumber = true + continue + } else if !inToken { + return errors.New("unexpected operator at start of token") + } + + case c == AndToken || c == OrToken || c == CommaToken: + if !inToken { + return errors.New( + ufmt.Sprintf( + "unexpected operator at start of token: %s", + string(c), + ), + ) + } + + // End token parsing after an operator if not within brackets. + if len(stack) == 0 { + break tokenLoop + } + + default: + if len(stack) != 0 || inToken { + // Continue parsing within brackets or if a token has been started. + inToken = true + continue + } + + break tokenLoop + } + } + + if len(stack) != 0 { + return errors.New("unclosed bracket or parenthesis at end of path") + } + + if first == b.index { + return errors.New("no token found") + } + + if inNumber && !isDigit(c) && c != '.' && c != 'e' && c != 'E' { + inNumber = false + } + + return nil +} diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno index 33bee3f0e56..d07dd64ec4f 100644 --- a/examples/gno.land/p/demo/json/buffer_test.gno +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -381,3 +381,142 @@ func TestSliceFromIndices(t *testing.T) { }) } } + +func TestBufferToken(t *testing.T) { + tests := []struct { + name string + path string + index uint + isErr bool + }{ + { + name: "Simple valid path", + path: "@.length", + index: 8, + isErr: false, + }, + { + name: "Path with array expr", + path: "@['foo'].0.bar", + index: 14, + isErr: false, + }, + { + name: "Path with array expr and simple fomula", + path: "@['foo'].[(@.length - 1)].*", + index: 27, + isErr: false, + }, + { + name: "Path with filter expr", + path: "@['foo'].[?(@.bar == 1 & @.baz < @.length)].*", + index: 45, + isErr: false, + }, + { + name: "addition of foo and bar", + path: "@.foo+@.bar", + index: 11, + isErr: false, + }, + { + name: "logical AND of foo and bar", + path: "@.foo & @.bar", + index: 6, + isErr: false, + }, + { + name: "accessing third element of foo", + path: "@.foo,3", + index: 5, + isErr: false, + }, + { + name: "accessing last element of array", + path: "@.length-1", + index: 10, + isErr: false, + }, + { + name: "number 1", + path: "1", + index: 1, + isErr: false, + }, + { + name: "float", + path: "3.1e4", + index: 5, + isErr: false, + }, + { + name: "float with minus", + path: "3.1e-4", + index: 6, + isErr: false, + }, + { + name: "float with plus", + path: "3.1e+4", + index: 6, + isErr: false, + }, + { + name: "negative number", + path: "-12345", + index: 6, + isErr: false, + }, + { + name: "negative float", + path: "-3.1e4", + index: 6, + isErr: false, + }, + { + name: "negative float with minus", + path: "-3.1e-4", + index: 7, + isErr: false, + }, + { + name: "negative float with plus", + path: "-3.1e+4", + index: 7, + isErr: false, + }, + { + name: "string number", + path: "'12345'", + index: 7, + isErr: false, + }, + { + name: "string with backslash", + path: "'foo \\'bar '", + index: 12, + isErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := NewBuffer([]byte(tt.path)) + + err := buf.pathToken() + if tt.isErr { + if err == nil { + t.Errorf("Expected an error for path %s, but got none", tt.path) + } + } + + if err == nil && tt.isErr { + t.Errorf("Expected an error for path %s, but got none", tt.path) + } + + if buf.index != tt.index { + t.Errorf("Expected final index %d, got %d (token: %s) for path %s", tt.index, buf.index, string(buf.data[buf.index]), tt.path) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/errors.gno b/examples/gno.land/p/demo/json/errors.gno deleted file mode 100644 index 33751583c77..00000000000 --- a/examples/gno.land/p/demo/json/errors.gno +++ /dev/null @@ -1,55 +0,0 @@ -package json - -import ( - "errors" - - "gno.land/p/demo/ufmt" -) - -type jsonError struct { - code int - msg string -} - -func (e *jsonError) Error() string { - return ufmt.Sprintf("Error %d: %s", e.code, e.msg) -} - -// error code -const ( - // parser errors (1xx) - KeyPathNotFound = 100 + iota - ArrayIndexNotFound - TokenNotFound - KeyLevelNotMatched - NextValueNotFound - - // syntax errors (2xx) - Overflow = 200 + iota - EmptyBytes - InvalidNull - InvalidBoolean - InvalidArrayIndex - InvalidExponents - NotNumber - MultipleDecimalPoints - - // type errors (3xx) - InvalidType = 300 + iota - InvalidString - InvalidValue - InvalidObject - InvalidArray - InvalidJSONFormat - UnknownValueType - - // buffer errors (4xx) - EofError = 400 + iota - - // JSON path error (5xx) - InvalidPath = 500 + iota -) - -func newJSONError(code int, msg string) error { - return &jsonError{code, msg} -} diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index e1cfae0987f..55d941326cd 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -32,19 +32,18 @@ func unescape(input, output []byte) ([]byte, error) { inputLen := len(input) if cap(output) < inputLen { output = make([]byte, inputLen) - } else { - // if the capacity is sufficient, slice the output to the length of the input. - output = output[:inputLen] } + output = output[:inputLen] copy(output, input[:firstBackslash]) + input = input[firstBackslash:] buf := output[firstBackslash:] for len(input) > 0 { inLen, bufLen := processEscapedUTF8(input, buf) if inLen == -1 { - return nil, errors.New("JSON Error: Encountered an invalid escape sequence in a string.") + return nil, errors.New("Invalid escape sequence in a string") } input = input[inLen:] // the number of bytes consumed in the input @@ -55,7 +54,6 @@ func unescape(input, output []byte) ([]byte, error) { if nextBackslash == -1 { copy(buf, input) buf = buf[len(input):] - break } diff --git a/examples/gno.land/p/demo/json/internal.gno b/examples/gno.land/p/demo/json/internal.gno index 97d17e5f66a..1b77e06509b 100644 --- a/examples/gno.land/p/demo/json/internal.gno +++ b/examples/gno.land/p/demo/json/internal.gno @@ -6,8 +6,8 @@ package json // Go implementation is taken from: https://github.com/spyzhov/ajson/blob/master/internal/state.go type ( - States int8 - Classes int8 + States int8 // possible states of the parser + Classes int8 // JSON string character types ) const __ = -1 @@ -140,7 +140,8 @@ const ( N3 /* null */ ) -// List of action codes +// List of action codes. +// these constants are defining an action that should be performed under certain conditions. const ( cl States = -2 /* colon */ cm States = -3 /* comma */ diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno new file mode 100644 index 00000000000..a5b981cc690 --- /dev/null +++ b/examples/gno.land/p/demo/json/node.gno @@ -0,0 +1 @@ +package json diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 82a69b50152..1d3e8f595bf 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -34,7 +34,7 @@ func nextToken(data []byte) (int, error) { } } - return -1, TokenNotFound + return -1, errors.New("JSON Error: no token found") } func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { @@ -61,7 +61,7 @@ func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { } } - return -1, escaped, newJSONError(InvalidString, "Invalid string input found while parsing string value") + return -1, escaped, errors.New("Invalid string input found while parsing string value") } // Find end of the data structure, array or object. @@ -107,7 +107,7 @@ func keyMatched( } if level > len(keys) { - return nil, KeyLevelNotMatched + return nil, errors.New("key level exceeds the length of the keys") } return keyUnesc, err @@ -119,7 +119,7 @@ func ParseStringLiteral(data []byte) (string, error) { bf, err := unescape(data, buf[:]) if err != nil { - return "", newJSONError(InvalidString, "Invalid string input found while parsing string value") + return "", errors.New("Invalid string input found while parsing string value") } return string(bf), nil @@ -144,7 +144,7 @@ func ParseBoolLiteral(data []byte) (bool, error) { // TODO: support for 32-bit floating-point format func ParseFloatLiteral(bytes []byte) (value float64, err error) { if len(bytes) == 0 { - return -1, EmptyBytes + return -1, errors.New("JSON Error: empty byte slice found while parsing float value") } neg, bytes := trimNegativeSign(bytes) @@ -166,7 +166,7 @@ func ParseFloatLiteral(bytes []byte) (value float64, err error) { if len(exponentPart) > 0 { exp, err := strconv.Atoi(string(exponentPart)) if err != nil { - return -1, InvalidExponents + return -1, errors.New("JSON Error: invalid exponent value found while parsing float value") } exp10 += exp } @@ -182,7 +182,7 @@ func ParseFloatLiteral(bytes []byte) (value float64, err error) { func ParseIntLiteral(bytes []byte) (v int64, err error) { if len(bytes) == 0 { - return 0, EmptyBytes + return 0, errors.New("JSON Error: empty byte slice found while parsing integer value") } neg, bytes := trimNegativeSign(bytes) @@ -190,18 +190,18 @@ func ParseIntLiteral(bytes []byte) (v int64, err error) { var n uint64 = 0 for _, c := range bytes { if notDigit(c) { - return 0, newJSONError(NotNumber, "non-digit characters found while parsing integer value") + return 0, errors.New("JSON Error: non-digit characters found while parsing integer value") } if n > maxUint64/10 { - return 0, newJSONError(Overflow, "numeric value exceeds the range limit") + return 0, errors.New("JSON Error: numeric value exceeds the range limit") } n *= 10 n1 := n + uint64(c-'0') if n1 < n { - return 0, newJSONError(Overflow, "numeric value exceeds the range limit") + return 0, errors.New("JSON Error: numeric value exceeds the range limit") } n = n1 @@ -212,7 +212,7 @@ func ParseIntLiteral(bytes []byte) (v int64, err error) { return -absMinInt64, nil } - return 0, newJSONError(Overflow, "numeric value exceeds the range limit") + return 0, errors.New("JSON Error: numeric value exceeds the range limit") } if neg { @@ -237,20 +237,20 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { for _, c := range bytes { if c == '.' { if decimalFound { - return 0, 0, MultipleDecimalPoints + return 0, 0, errors.New("JSON Error: multiple decimal points found while parsing float value") } decimalFound = true continue } if notDigit(c) { - return 0, 0, newJSONError(NotNumber, "non-digit characters found while parsing integer value") + return 0, 0, errors.New("JSON Error: non-digit characters found while parsing integer value") } digit := uint64(c - '0') if man > (maxUint64-digit)/10 { - return 0, 0, newJSONError(Overflow, "numeric value exceeds the range limit") + return 0, 0, errors.New("JSON Error: numeric value exceeds the range limit") } man = man*10 + digit @@ -269,7 +269,7 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { // an integer representing the end offset, and an error if any. func getTypeFromByteSlice(data []byte, offset int) ([]byte, ValueType, int, error) { if len(data) == 0 { - return nil, Unknown, offset, errors.New("JSON Error: JSON data not provided.") + return nil, Unknown, offset, errors.New("JSON Error: empty byte slice found while parsing value") } dataType, endOffset, err := ParseValue(data, offset) @@ -350,6 +350,7 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { if end == -1 { return Unknown, offset, errors.New("JSON Error: malformed value found while parsing primitive value") } + value := data[offset : offset+end] switch data[offset] { @@ -357,12 +358,16 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { if _b, err := ParseBoolLiteral(value); err != nil { return Unknown, offset, err } + return Boolean, offset + end, nil + case 'n': if bytes.Equal(value, nullLiteral) { return Null, offset + end, nil } - return Unknown, offset, newJSONError(InvalidNull, "Invalid null value found while parsing null value") + + return Unknown, offset, errors.New("JSON Error: invalid null value found while parsing null value") + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': if !bytes.ContainsAny(value, ".eE") { if _i, err := ParseIntLiteral(value); err != nil { @@ -377,6 +382,5 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { return Number, offset + end, nil } - // return Unknown, offset, errors.New("JSON Error: unknown value type found while parsing primitive value") - return Unknown, offset, newJSONError(UnknownValueType, "unknown value type found while parsing primitive value") + return Unknown, offset, errors.New("JSON Error: unknown value type found while parsing primitive value") } diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 6ebf6e0a64f..85fe98252f3 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -1,5 +1,9 @@ package json +import ( + "errors" +) + // ParsePath takes a JSONPath string and returns a slice of strings representing the path segments. func ParsePath(path string) ([]string, error) { buf := NewBuffer([]byte(path)) @@ -41,7 +45,7 @@ func ParsePath(path string) ([]string, error) { } if buf.index >= buf.length { - return nil, newJSONError(EofError, "unexpected end of path") + return nil, errors.New("unexpected end of path") } segment := string(buf.sliceFromIndices(start+1, buf.index)) diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno index 2e50facc9d9..356f2c3ba3a 100644 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ b/examples/gno.land/p/demo/json/state_machine.gno @@ -9,9 +9,7 @@ import ( "gno.land/p/demo/ufmt" ) -const ( - unescapeStackBufSize = 64 -) +const unescapeStackBufSize = 64 var ( tabRegex = regexp.MustCompile(`(?m)^\t+`) @@ -21,7 +19,7 @@ var ( // extractValueTypeFromToken recoginizes the type of JSON value by peeking at the first byte of the value. func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { if len(b) == 0 { - return Unknown, 0, errors.New("json: empty value, cannot determine type.") + return Unknown, 0, errors.New("cannot determine type of empty value") } switch b { @@ -29,16 +27,21 @@ func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { dataType = Boolean } - return Unknown, offset, newJSONError(InvalidBoolean, "invalid boolean value. expected: true or false") + + return Unknown, offset, errors.New("invalid boolean value. expected: true or false") + case 'u', 'n': if bytes.Equal(value, nullLiteral) { dataType = Null } - return Unknown, offset, newJSONError(InvalidNull, "invalid null value. expected: null") + + return Unknown, offset, errors.New("invalid null value. expected: null") + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': dataType = Number + default: - return Unknown, offset, newJSONError(UnknownValueType, "unsupported value type or invalid JSON type") + return Unknown, offset, errors.New("unsupported value type or invalid JSON type") } } @@ -61,7 +64,7 @@ func findValueIndex(data []byte, keys ...string) (int, error) { switch data[i] { case DoublyQuoteToken: if level < 1 { - return -1, newJSONError(InvalidJSONFormat, "invalid JSON format") + return -1, errors.New("invalid JSON format") } i += 1 @@ -120,12 +123,12 @@ func findValueIndex(data []byte, keys ...string) (int, error) { if keyLevel == level && keys[level][0] == SquareOpenToken { keyLen := len(keys[level]) if keyLen < 3 || keys[level][0] != SquareOpenToken || keys[level][keyLen-1] != SquareCloseToken { - return -1, InvalidArrayIndex + return -1, errors.New("invalid array index") } aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) if err != nil { - return -1, InvalidArrayIndex + return -1, errors.New("invalid array index") } var ( @@ -151,12 +154,12 @@ func findValueIndex(data []byte, keys ...string) (int, error) { }) if valueFound == nil { - return -1, ArrayIndexNotFound + return -1, errors.New("array index not found") } subIndex, err := findValueIndex(valueFound, keys[level+1:]...) if err != nil { - return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") + return -1, err } return i + valueOffset + subIndex, nil @@ -169,11 +172,11 @@ func findValueIndex(data []byte, keys ...string) (int, error) { } } case ColonToken: - return -1, newJSONError(InvalidJSONFormat, "invalid JSON format") + return -1, errors.New("invalid JSON format") } } - return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") + return -1, errors.New("key path not found in the JSON structure") } func decreaseLevel(level, keyLevel int) (int, int) { @@ -248,7 +251,7 @@ func findKeyStart(data []byte, key string) (int, error) { } } - return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") + return -1, errors.New("key path not found in the JSON structure") } func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { @@ -262,7 +265,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, // Go to closest value nO, err := nextToken(data[offset:]) if err != nil { - return nil, NotExist, offset, -1, newJSONError(InvalidJSONFormat, "invalid JSON format") + return nil, NotExist, offset, -1, errors.New("key path not found in the JSON structure") } offset += nO @@ -315,25 +318,25 @@ func ArrayEach( keys ...string, ) (offset int, err error) { if len(data) == 0 { - return -1, errors.New("json.ArrayEach: empty data") + return -1, errors.New("empty JSON data provided") } if next, err := nextToken(data); err != nil { - return -1, errors.New("json.ArrayEach: invalid token") + return -1, err } else { offset += next } if len(keys) > 0 { if offset, err = findValueIndex(data, keys...); err != nil { - return -1, errors.New(ufmt.Sprintf("json.ArrayEach: key path not found. keys: %v", keys)) + return -1, errors.New(ufmt.Sprintf("key path not found. keys: %v", keys)) } // go to closest value if n0, err := nextToken(data[offset:]); err != nil { return -1, errors.New( ufmt.Sprintf( - "json.ArrayEach: invalid token found. offset: %d. got: %s", + "invalid token found. offset: %d. got: %s", offset, data[offset:offset+1], )) @@ -342,14 +345,14 @@ func ArrayEach( } if data[offset] != SquareOpenToken { - return -1, newJSONError(InvalidArray, "invalid array. must start with `[`") + return -1, errors.New("invalid array. must start with `[`") } else { offset += 1 } } if n0, err := nextToken(data[offset:]); err != nil { - return -1, newJSONError(NextValueNotFound, "next value not found") + return -1, errors.New("next value not found") } else { offset += n0 } @@ -415,7 +418,7 @@ func ObjectEach( // descent to desired key if len(keys) > 0 { if off, err := findValueIndex(data, keys...); err != nil { - newJSONError(KeyPathNotFound, "key path not found in the JSON structure") + errors.New("key path not found in the JSON structure") } else { offset += off } @@ -423,7 +426,7 @@ func ObjectEach( // validate and skip past opening token if off, err := nextToken(data[offset:]); err != nil { - return newJSONError(InvalidObject, "Object is not properly formed") + return errors.New("Object is not properly formed") } else if offset += off; data[offset] != CurlyOpenToken { return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) } else { @@ -449,7 +452,7 @@ func ObjectEach( case CurlyCloseToken: return nil // end of the object default: - return newJSONError(InvalidObject, "Object is not properly formed") + return errors.New("Object is not properly formed") } // find the end of the key @@ -542,7 +545,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str strEnd, keyEscaped, err := stringEnd(data[i:]) if err != nil { - return -1, errors.New("json.EachKey: invalid string") + return -1, errors.New("invalid string") } i += strEnd @@ -550,7 +553,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str valueOffset, err := nextToken(data[i:]) if err != nil { - return -1, errors.New("json.EachKey: invalid token found") + return -1, errors.New("invalid token found") } i += valueOffset @@ -561,7 +564,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str if maxPath > 0 && maxPath >= level { keyUnesc, err := processKey(data[keyBegin:keyEnd], keyEscaped) if err != nil { - return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + return -1, err } pathsBuf[level-1] = string(keyUnesc) @@ -622,10 +625,10 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str pIdxFlags := initPathIdxFlags(paths) if level < 0 { - err := newJSONError(InvalidJSONFormat, "invalid JSON format") + err := errors.New("invalid JSON format") cb(-1, nil, Unknown, err) - return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + return -1, err } for p, path := range paths { @@ -637,7 +640,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str if len(currPos) >= 2 { arrIdx, err := strconv.Atoi(currPos[1 : len(currPos)-1]) if err != nil { - return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + return -1, err } pIdxFlags[p] = true @@ -684,7 +687,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str } else { // do not search for keys inside arrays if arraySkip, err := blockEnd(data[i:], SquareOpenToken, SquareCloseToken); arraySkip == -1 { - return -1, errors.New(ufmt.Sprintf("json.EachKey: %s", err)) + return -1, err } else { i += arraySkip } @@ -697,7 +700,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str i += 1 } - return -1, newJSONError(KeyPathNotFound, "key path not found in the JSON structure") + return -1, errors.New("key path not found in the JSON structure") } func initPathFlags(paths [][]string) []bool { diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 23d00442911..77dc6fa7c62 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -5,6 +5,8 @@ const ( SquareCloseToken = ']' CurlyOpenToken = '{' CurlyCloseToken = '}' + RoundOpenToken = '(' + RoundCloseToken = ')' CommaToken = ',' DotToken = '.' ColonToken = ':' @@ -28,6 +30,8 @@ const ( UnderScoreToken = '_' DollarToken = '$' AtToken = '@' + AndToken = '&' + OrToken = '|' ) const ( diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index 09e8f4072a9..47d79659dac 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -4,6 +4,10 @@ func notDigit(c byte) bool { return (c & 0xF0) != 0x30 } +func isDigit(c byte) bool { + return '0' <= c && c <= '9' +} + // lower converts a byte to lower case if it is an uppercase letter. // // In ASCII, the lowercase letters have the 6th bit set to 1, which is not set in their uppercase counterparts. @@ -36,7 +40,7 @@ func h2i(c byte) int { } func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { - if bytes[0] == '-' { + if bytes[0] == MinusToken { return true, bytes[1:] } diff --git a/gnovm/pkg/gnolang/precompile.go b/gnovm/pkg/gnolang/precompile.go index b8320845ea6..d84b06686b0 100644 --- a/gnovm/pkg/gnolang/precompile.go +++ b/gnovm/pkg/gnolang/precompile.go @@ -49,6 +49,7 @@ var stdlibWhitelist = []string{ "io/util", "math", "math/big", + "math/bits", "math/rand", "regexp", "sort", From 2faabe2822a495afe954a4a65d15444696e59387 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 22 Feb 2024 12:50:24 +0900 Subject: [PATCH 57/72] add test case --- examples/gno.land/p/demo/json/buffer.gno | 246 ++++++++---------- examples/gno.land/p/demo/json/buffer_test.gno | 72 ++++- examples/gno.land/p/demo/json/token.gno | 4 +- examples/gno.land/p/demo/json/utils.gno | 4 - 4 files changed, 180 insertions(+), 146 deletions(-) diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index 63276c72ece..7a0f5bc2b8b 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -1,6 +1,7 @@ package json import ( + "bytes" "errors" "strings" @@ -112,9 +113,9 @@ func (b *buffer) skipAny(endTokens map[byte]bool) error { // build error message var tokens []string - for token := range endTokens { - tokens = append(tokens, string(token)) - } + for token := range endTokens { + tokens = append(tokens, string(token)) + } return errors.New( ufmt.Sprintf( @@ -135,46 +136,45 @@ func (b *buffer) skipAndReturnIndex() (uint, error) { // skipUntil moves the buffer index forward until it encounters a byte contained in the endTokens set. func (b *buffer) skipUntil(endTokens map[byte]bool) (uint, error) { - for b.index < b.length { - currentByte, err := b.current() - if err != nil { + for b.index < b.length { + currentByte, err := b.current() + if err != nil { return b.index, err - } + } - // Check if the current byte is in the set of end tokens. - if _, exists := endTokens[currentByte]; exists { - return b.index, nil - } + // Check if the current byte is in the set of end tokens. + if _, exists := endTokens[currentByte]; exists { + return b.index, nil + } - b.index++ - } + b.index++ + } - return b.index, errors.New("EOF") + return b.index, errors.New("EOF") } - // significantTokens is a map where the keys are the significant characters in a JSON path. // The values in the map are all true, which allows us to use the map as a set for quick lookups. var significantTokens = map[byte]bool{ - DotToken: true, // access properties of an object - DollarToken: true, // root object - AtToken: true, // current object in a filter expression - SquareOpenToken: true, // start of an array index or filter expression - SquareCloseToken: true, // end of an array index or filter expression + DotToken: true, // access properties of an object + DollarToken: true, // root object + AtToken: true, // current object in a filter expression + SquareOpenToken: true, // start of an array index or filter expression + SquareCloseToken: true, // end of an array index or filter expression } // skipToNextSignificantToken advances the buffer index to the next significant character. // Significant characters are defined based on the JSON path syntax. func (b *buffer) skipToNextSignificantToken() { - for b.index < b.length { - current := b.data[b.index] + for b.index < b.length { + current := b.data[b.index] - if _, ok := significantTokens[current]; ok { - break - } + if _, ok := significantTokens[current]; ok { + break + } - b.index++ - } + b.index++ + } } // backslash checks to see if the number of backslashes before the current index is odd. @@ -188,16 +188,16 @@ func (b *buffer) backslash() bool { count := 0 for i := b.index - 1; ; i-- { - if i >= b.length || b.data[i] != BackSlashToken { - break - } + if i >= b.length || b.data[i] != BackSlashToken { + break + } - count++ + count++ - if i == 0 { - break - } - } + if i == 0 { + break + } + } return count%2 != 0 } @@ -206,6 +206,83 @@ func (b *buffer) reset() { b.last = GO } +func (b *buffer) pathToken() error { + var stack []byte + + inToken := false + inNumber := false + first := b.index + + for b.index < b.length { + c := b.data[b.index] + + switch { + case c == DoublyQuoteToken || c == QuoteToken: + inToken = true + if err := b.step(); err != nil { + return errors.New("error stepping through buffer") + } + + if err := b.skip(c); err != nil { + return errors.New("unmatched quote in path") + } + + if b.index >= b.length { + return errors.New("unmatched quote in path") + } + + case c == SquareOpenToken || c == RoundOpenToken: + inToken = true + stack = append(stack, c) + + case c == SquareCloseToken || c == RoundCloseToken: + inToken = true + if len(stack) == 0 || (c == SquareCloseToken && stack[len(stack)-1] != SquareOpenToken) || (c == RoundCloseToken && stack[len(stack)-1] != RoundOpenToken) { + return errors.New("mismatched bracket or parenthesis") + } + + stack = stack[:len(stack)-1] + + case bytes.ContainsAny([]byte{DotToken, CommaToken, DollarToken, AtToken, AesteriskToken, AndToken, OrToken}, string(c)) || ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || ('0' <= c && c <= '9'): + inToken = true + + case c == PlusToken || c == MinusToken: + if inNumber || (b.index > 0 && bytes.ContainsAny([]byte("eE"), string(b.data[b.index-1]))) { + inToken = true + } else if !inToken && (b.index+1 < b.length && bytes.IndexByte([]byte("0123456789"), b.data[b.index+1]) != -1) { + inToken = true + inNumber = true + } else if !inToken { + return errors.New("unexpected operator at start of token") + } + + default: + if len(stack) != 0 || inToken { + inToken = true + } else { + goto end + } + } + + b.index++ + } + +end: + if len(stack) != 0 { + return errors.New("unclosed bracket or parenthesis at end of path") + } + + if first == b.index { + return errors.New("no token found") + } + + if inNumber && !bytes.ContainsAny([]byte("0123456789.eE"), string(b.data[b.index-1])) { + inNumber = false + } + + return nil +} + func (b *buffer) numeric(token bool) error { if token { b.last = GO @@ -237,7 +314,6 @@ func (b *buffer) numeric(token bool) error { b.last = b.state } - // zero, integer, fraction, exponent if b.last != ZE && b.last != IN && b.last != FR && b.last != E3 { return errors.New("invalid token found while parsing path") } @@ -269,101 +345,3 @@ func (b *buffer) getState() States { return b.state } - -func (b *buffer) pathToken() error { - var ( - c byte - stack []byte - first = b.index - start int - inToken bool = false - inNumber bool = false - ) - -tokenLoop: - for ; b.index < b.length; b.index++ { - c = b.data[b.index] - - switch { - case c == DoublyQuoteToken || c == QuoteToken: - inToken = true - if err := b.step(); err != nil { - return errors.New("error stepping through buffer") - } - - err := b.skip(c) - if err != nil { - return errors.New("unmatched quote in path") - } - - if b.index >= b.length { - return errors.New("unmatched quote in path") - } - - case c == SquareOpenToken || c == RoundOpenToken: - inToken = true - stack = append(stack, c) - - case c == SquareCloseToken || c == RoundCloseToken: - inToken = true - if len(stack) == 0 || (c == SquareCloseToken && stack[len(stack)-1] != SquareOpenToken) || (c == RoundCloseToken && stack[len(stack)-1] != RoundOpenToken) { - return errors.New("mismatched bracket or parenthesis") - } - stack = stack[:len(stack)-1] - - case c == DotToken || c == DollarToken || c == AtToken || c == AesteriskToken || ('A' <= c && c <= 'z') || ('0' <= c && c <= '9'): - inToken = true - // do nothing - - case c == PlusToken || c == MinusToken: - if inNumber || (b.index > 0 && (b.data[b.index-1] == 'e' || b.data[b.index-1] == 'E')) { - inToken = true - continue - } else if !inToken && (b.index+1 < b.length && isDigit(b.data[b.index+1])) { - inToken = true - inNumber = true - continue - } else if !inToken { - return errors.New("unexpected operator at start of token") - } - - case c == AndToken || c == OrToken || c == CommaToken: - if !inToken { - return errors.New( - ufmt.Sprintf( - "unexpected operator at start of token: %s", - string(c), - ), - ) - } - - // End token parsing after an operator if not within brackets. - if len(stack) == 0 { - break tokenLoop - } - - default: - if len(stack) != 0 || inToken { - // Continue parsing within brackets or if a token has been started. - inToken = true - continue - } - - break tokenLoop - } - } - - if len(stack) != 0 { - return errors.New("unclosed bracket or parenthesis at end of path") - } - - if first == b.index { - return errors.New("no token found") - } - - if inNumber && !isDigit(c) && c != '.' && c != 'e' && c != 'E' { - inNumber = false - } - - return nil -} diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno index d07dd64ec4f..fc5ad1c16a8 100644 --- a/examples/gno.land/p/demo/json/buffer_test.gno +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -421,14 +421,20 @@ func TestBufferToken(t *testing.T) { }, { name: "logical AND of foo and bar", - path: "@.foo & @.bar", - index: 6, + path: "@.foo && @.bar", + index: 14, + isErr: false, + }, + { + name: "logical OR of foo and bar", + path: "@.foo || @.bar", + index: 14, isErr: false, }, { name: "accessing third element of foo", path: "@.foo,3", - index: 5, + index: 7, isErr: false, }, { @@ -497,6 +503,60 @@ func TestBufferToken(t *testing.T) { index: 12, isErr: false, }, + { + name: "string with inner double quotes", + path: "'foo \"bar \"'", + index: 12, + isErr: false, + }, + { + name: "parenthesis 1", + path: "(@abc)", + index: 6, + isErr: false, + }, + { + name: "parenthesis 2", + path: "[()]", + index: 4, + isErr: false, + }, + { + name: "parenthesis mismatch", + path: "[(])", + index: 2, + isErr: true, + }, + { + name: "parenthesis mismatch 2", + path: "(", + index: 1, + isErr: true, + }, + { + name: "parenthesis mismatch 3", + path: "())]", + index: 2, + isErr: true, + }, + { + name: "bracket mismatch", + path: "[()", + index: 3, + isErr: true, + }, + { + name: "bracket mismatch 2", + path: "()]", + index: 2, + isErr: true, + }, + { + name: "path does not close bracket", + path: "@.foo[)", + index: 6, + isErr: false, + }, } for _, tt := range tests { @@ -506,16 +566,16 @@ func TestBufferToken(t *testing.T) { err := buf.pathToken() if tt.isErr { if err == nil { - t.Errorf("Expected an error for path %s, but got none", tt.path) + t.Errorf("Expected an error for path `%s`, but got none", tt.path) } } if err == nil && tt.isErr { - t.Errorf("Expected an error for path %s, but got none", tt.path) + t.Errorf("Expected an error for path `%s`, but got none", tt.path) } if buf.index != tt.index { - t.Errorf("Expected final index %d, got %d (token: %s) for path %s", tt.index, buf.index, string(buf.data[buf.index]), tt.path) + t.Errorf("Expected final index %d, got %d (token: `%s`) for path `%s`", tt.index, buf.index, string(buf.data[buf.index]), tt.path) } }) } diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 77dc6fa7c62..0bd36a6f6e9 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -3,10 +3,10 @@ package json const ( SquareOpenToken = '[' SquareCloseToken = ']' - CurlyOpenToken = '{' - CurlyCloseToken = '}' RoundOpenToken = '(' RoundCloseToken = ')' + CurlyOpenToken = '{' + CurlyCloseToken = '}' CommaToken = ',' DotToken = '.' ColonToken = ':' diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno index 47d79659dac..14d595d0719 100644 --- a/examples/gno.land/p/demo/json/utils.gno +++ b/examples/gno.land/p/demo/json/utils.gno @@ -4,10 +4,6 @@ func notDigit(c byte) bool { return (c & 0xF0) != 0x30 } -func isDigit(c byte) bool { - return '0' <= c && c <= '9' -} - // lower converts a byte to lower case if it is an uppercase letter. // // In ASCII, the lowercase letters have the 6th bit set to 1, which is not set in their uppercase counterparts. From f7716c44dbd6199fb572f2a18be0ffe1930d690a Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 22 Feb 2024 19:27:58 +0900 Subject: [PATCH 58/72] save --- examples/gno.land/p/demo/json/escape.gno | 117 ++++++++++++ examples/gno.land/p/demo/json/escape_test.gno | 33 +++- examples/gno.land/p/demo/json/node.gno | 156 ++++++++++++++++ examples/gno.land/p/demo/json/node_test.gno | 175 ++++++++++++++++++ examples/gno.land/p/demo/json/parser.gno | 4 +- examples/gno.land/p/demo/json/struct.gno | 2 +- gnovm/pkg/gnolang/precompile.go | 1 + gnovm/stdlibs/unicode/utf16/utf16_test.gno | 1 + 8 files changed, 485 insertions(+), 4 deletions(-) create mode 100644 examples/gno.land/p/demo/json/node_test.gno diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 55d941326cd..48a5e2fcf1a 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -3,7 +3,9 @@ package json import ( "bytes" "errors" + "unicode" "unicode/utf8" + "unicode/utf16" ) const ( @@ -126,6 +128,121 @@ func decodeUnicodeEscape(b []byte) (rune, int) { return combineSurrogates(r, r2), 12 } +func unquote(bs []byte, border byte) (string, error) { + res, err := unquoteBytes(bs, border) + if err != nil { + return "", err + } + + return string(res), nil +} + +func unquoteBytes(s []byte, border byte) ([]byte, error) { + if len(s) < 2 || s[0] != border || s[len(s)-1] != border { + return nil, errors.New("Invalid JSON string") + } + + s = s[1 : len(s)-1] + + r := 0 + for r < len(s) { + c := s[r] + + if c == BackSlashToken || c == border || c < 0x20 { + break + } else if c >= utf8.RuneSelf { + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + + r += size + continue + } + + r++ + } + + if r == len(s) { + return s, nil + } + + b := make([]byte, len(s) + utf8.UTFMax * 2) + w := copy(b, s[:r]) + + for r < len(s) { + if w >= len(b) - (utf8.UTFMax * 2) { + nb := make([]byte, (utf8.UTFMax + len(b)) * 2) + copy(nb, b) + b = nb + } + + c := s[r] + if c == BackSlashToken { + r++ + if r >= len(s) { + return nil, errors.New("Invalid JSON string") + } + + if s[r] == 'u' { + rr, res := decodeUnicodeEscape(s[r-1:]) + if res < 0 { + return nil, errors.New("Invalid JSON string") + } + + w += utf8.EncodeRune(b[w:], rr) + r += 5 + } else { + var decode byte + + switch s[r] { + case 'b': + decode = BackSpaceToken + + case 'f': + decode = FormFeedToken + + case 'n': + decode = NewLineToken + + case 'r': + decode = CarriageReturnToken + + case 't': + decode = TabToken + + case border, BackSlashToken, SlashToken, QuoteToken: + decode = s[r] + + default: + return nil, errors.New("Invalid JSON string") + } + + b[w] = decode + r++ + w++ + } + } else if c == border || c < 0x20 { + return nil, errors.New("Invalid JSON string") + } else if c < utf8.RuneSelf { + b[w] = c + r++ + w++ + } else { + rr, size := utf8.DecodeRune(s[r:]) + + if rr == utf8.RuneError && size == 1 { + return nil, errors.New("Invalid JSON string") + } + + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + + return b[:w], nil +} + var escapeByteSet = [256]byte{ '"': DoublyQuoteToken, '\\': BackSlashToken, diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index a48d3dfc570..7318c02f97c 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -172,7 +172,7 @@ func TestUnescape(t *testing.T) { {"SingleEscape", []byte("hello\\nworld"), []byte("hello\nworld")}, {"MultipleEscapes", []byte("line1\\nline2\\r\\nline3"), []byte("line1\nline2\r\nline3")}, {"UnicodeEscape", []byte("snowman:\\u2603"), []byte("snowman:\u2603")}, - {"Complex", []byte("test\\n\\u2603\\r\\nend"), []byte("test\n\u2603\r\nend")}, + {"Complex", []byte("tc\\n\\u2603\\r\\nend"), []byte("tc\n\u2603\r\nend")}, } for _, tc := range testCases { @@ -184,3 +184,34 @@ func TestUnescape(t *testing.T) { }) } } + +func TestUnquoteBytes(t *testing.T) { + tests := []struct { + input []byte + border byte + expected []byte + ok bool + }{ + {[]byte("\"hello\""), '"', []byte("hello"), true}, + {[]byte("'hello'"), '\'', []byte("hello"), true}, + {[]byte("\"hello"), '"', nil, false}, + {[]byte("hello\""), '"', nil, false}, + {[]byte("\"he\\\"llo\""), '"', []byte("he\"llo"), true}, + {[]byte("\"he\\nllo\""), '"', []byte("he\nllo"), true}, + {[]byte("\"\""), '"', []byte(""), true}, + {[]byte("''"), '\'', []byte(""), true}, + {[]byte("\"\\u0041\""), '"', []byte("A"), true}, + } + + for _, tc := range tests { + result, err := unquoteBytes(tc.input, tc.border) + + if tc.ok != (err == nil) { + t.Errorf("unquoteBytes(%q) = %q; want %q", tc.input, result, tc.expected) + } + + if !bytes.Equal(result, tc.expected) { + t.Errorf("unquoteBytes(%q) = %q; want %q", tc.input, result, tc.expected) + } + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index a5b981cc690..6c353cb50c0 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -1 +1,157 @@ package json + +import ( + "errors" + "strconv" + + "gno.land/p/demo/ufmt" +) + +type NodeValue struct { + value interface{} + typ ValueType +} + +func newNodeValue(value interface{}) *NodeValue { + return &NodeValue{ + value: value, + typ: typeOf(value), + } +} + +func (nv *NodeValue) Store(value interface{}) { + nv.value = value + nv.typ = typeOf(value) +} + +func (nv *NodeValue) Load() interface{} { + return nv.value +} + +type Node struct { + prev *Node + next map[string]*Node + key string + data []byte + value *NodeValue + index int + borders [2]int // start, end + modified bool +} + + +func (n *Node) Key() string { + if n == nil || n.key == "" { + return "" + } + + return n.key +} + +func (n *Node) Type() string { + return n.value.typ.String() +} + +func (n *Node) Value() interface{} { + return n.value.Load() +} + +func (n *Node) Size() uint { + if n == nil { + return 0 + } + + return uint(len(n.next)) +} + +func (n *Node) Keys() []string { + if n == nil { + return nil + } + + result := make([]string, 0, len(n.next)) + for key := range n.next { + result = append(result, key) + } + + return result +} + +func NullNode(key string) *Node { + return &Node{ + key: key, + value: &NodeValue{value: nil, typ: Null}, + modified: true, + } +} + +func NumberNode(key string, value float64) *Node { + return &Node{ + key: key, + value: &NodeValue{ + value: value, + typ: Number, + }, + modified: true, + } +} + +func StringNode(key string, value string) *Node { + val := newNodeValue(value) + + return &Node{ + key: key, + value: val, + modified: true, + } +} + +func BoolNode(key string, value bool) *Node { + val := newNodeValue(value) + + return &Node{ + key: key, + value: val, + modified: true, + } +} + +func ArrayNode(key string, value []*Node) *Node { + val := newNodeValue(value) + curr := &Node{ + key: key, + value: val, + modified: true, + } + + curr.next = make(map[string]*Node, len(value)) + if value != nil { + curr.value.Store(value) + + for i, v := range value { + idx := i + curr.next[strconv.Itoa(i)] = v + + v.prev = curr + v.index = idx + } + } + + return curr +} + +func (n *Node) ready() bool { + return n.borders[1] != 0 +} + +func (n *Node) Source() []byte { + if n == nil { + return nil + } + + if n.ready() && !n.modified && n.data != nil { + return (n.data)[n.borders[0]:n.borders[1]] + } + + return nil +} diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno new file mode 100644 index 00000000000..197c7f9b886 --- /dev/null +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -0,0 +1,175 @@ +package json + +import ( + "strconv" + "testing" +) + +func TestNodeValue(t *testing.T) { + tests := []struct { + name string + value interface{} + expectType ValueType + }{ + {"string", "test value", String}, + {"integer", 123, Number}, + {"float", 1.23, Float}, + {"boolean", true, Boolean}, + {"null", nil, Null}, + {"array", []interface{}{1, 2, 3}, Array}, + {"object", map[string]interface{}{"key": "value"}, Object}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nv := &NodeValue{} + nv.Store(tt.value) + + if nv.value != tt.value { + t.Errorf("Store: %s want %v got %v", tt.name, tt.value, nv.value) + } + + typ := typeOf(tt.value) + if typ != tt.expectType { + t.Errorf("Store: %s want type %v got %v", tt.name, tt.expectType.String(), typ.String()) + } + + loadedValue := nv.Load() + if loadedValue != tt.value { + t.Errorf("Load: %s want %v got %v", tt.name, tt.value, loadedValue) + } + }) + } +} + +func TestNodeCreation(t *testing.T) { + tests := []struct { + name string + testFunc func() *Node + expected Node + }{ + { + name: "NullNode", + testFunc: func() *Node { + return NullNode("testNull") + }, + expected: Node{ + key: "testNull", + value: &NodeValue{value: nil, typ: Null}, + modified: true, + }, + }, + { + name: "NumberNode", + testFunc: func() *Node { + return NumberNode("testNumber", 42.0) + }, + expected: Node{ + key: "testNumber", + value: &NodeValue{value: 42.0, typ: Number}, + modified: true, + }, + }, + { + name: "StringNode", + testFunc: func() *Node { + return StringNode("testString", "Hello") + }, + expected: Node{ + key: "testString", + value: &NodeValue{value: "Hello", typ: String}, + modified: true, + }, + }, + { + name: "BoolNode", + testFunc: func() *Node { + return BoolNode("testBool", true) + }, + expected: Node{ + key: "testBool", + value: &NodeValue{value: true, typ: Boolean}, + modified: true, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual := tc.testFunc() + if actual.key != tc.expected.key { + t.Errorf("expected key %s, got %s", tc.expected.key, actual.key) + } + if actual.value.typ != tc.expected.value.typ { + t.Errorf("expected type %v, got %v", tc.expected.value.typ.String(), actual.value.typ.String()) + } + if actual.modified != tc.expected.modified { + t.Errorf("expected modified %t, got %t", tc.expected.modified, actual.modified) + } + // Special handling for comparing slices and maps + if tc.name == "ArrayNode" { + if len(actual.next) != len(tc.expected.next) { + t.Errorf("expected next length %d, got %d", len(tc.expected.next), len(actual.next)) + } + for k, v := range tc.expected.next { + if _, ok := actual.next[k]; !ok { + t.Errorf("expected next to have key %s", k) + } + if actual.next[k].key != v.key { + t.Errorf("expected next node with key %s, got %s", v.key, actual.next[k].key) + } + } + } + }) + } +} + +func TestNodeSource(t *testing.T) { + tests := []struct { + name string + node *Node + expected []byte + }{ + { + name: "nil node", + node: nil, + expected: nil, + }, + { + name: "ready unmodified node with data", + node: &Node{ + data: []byte("test data"), + borders: [2]int{0, 9}, + modified: false, + }, + expected: []byte("test data"), + }, + { + name: "ready modified node with data", + node: &Node{ + data: []byte("test data"), + borders: [2]int{0, 9}, + modified: true, + }, + expected: nil, + }, + { + name: "not ready node", + node: &Node{ + data: nil, + borders: [2]int{0, 0}, + modified: false, + }, + expected: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual := tc.node.Source() + if string(actual) != string(tc.expected) { + t.Errorf("expected %v, got %v", tc.expected, actual) + } + }) + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 1d3e8f595bf..2e4a1ba9140 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -280,8 +280,8 @@ func getTypeFromByteSlice(data []byte, offset int) ([]byte, ValueType, int, erro return data[offset:endOffset], dataType, endOffset, nil } -// getTypeFromValue check the type of the given value and returns the corresponding ValueType. -func getTypeFromValue(v interface{}) ValueType { +// typeOf check the type of the given value and returns the corresponding ValueType. +func typeOf(v interface{}) ValueType { switch v.(type) { case string: return String diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 0c8b70c2d72..47055fc072c 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -353,7 +353,7 @@ func (s *Struct) Update(name string, newVal interface{}) error { } // type assertion - newValType := getTypeFromValue(newVal) + newValType := typeOf(newVal) if f.typ != newValType { return errors.New( ufmt.Sprintf("json.Update: Field type mismatch. Expected %s, got %s", diff --git a/gnovm/pkg/gnolang/precompile.go b/gnovm/pkg/gnolang/precompile.go index d84b06686b0..a0360bb89f0 100644 --- a/gnovm/pkg/gnolang/precompile.go +++ b/gnovm/pkg/gnolang/precompile.go @@ -57,6 +57,7 @@ var stdlibWhitelist = []string{ "strings", "text/template", "time", + "unicode", "unicode/utf8", "unicode/utf16", diff --git a/gnovm/stdlibs/unicode/utf16/utf16_test.gno b/gnovm/stdlibs/unicode/utf16/utf16_test.gno index bc2f39622ae..75b3af4ebbe 100644 --- a/gnovm/stdlibs/unicode/utf16/utf16_test.gno +++ b/gnovm/stdlibs/unicode/utf16/utf16_test.gno @@ -7,6 +7,7 @@ package utf16 import ( "testing" "unicode" + "unicode/utf16" ) // Validate the constants redefined from unicode. From de817baeddc6fcd8753f1d312097d1f803371fd3 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Sat, 2 Mar 2024 13:12:42 +0900 Subject: [PATCH 59/72] fmt and mv license --- .../json/{license/LICENSE.txt => LICENSE} | 0 examples/gno.land/p/demo/json/buffer_test.gno | 338 +++++++++--------- examples/gno.land/p/demo/json/escape.gno | 8 +- examples/gno.land/p/demo/json/escape_test.gno | 24 +- examples/gno.land/p/demo/json/internal.gno | 6 +- examples/gno.land/p/demo/json/node.gno | 21 +- examples/gno.land/p/demo/json/node_test.gno | 52 +-- examples/gno.land/p/demo/json/path.gno | 102 +++--- examples/gno.land/p/demo/json/struct.gno | 3 +- examples/gno.land/p/demo/json/token.gno | 4 +- 10 files changed, 279 insertions(+), 279 deletions(-) rename examples/gno.land/p/demo/json/{license/LICENSE.txt => LICENSE} (100%) diff --git a/examples/gno.land/p/demo/json/license/LICENSE.txt b/examples/gno.land/p/demo/json/LICENSE similarity index 100% rename from examples/gno.land/p/demo/json/license/LICENSE.txt rename to examples/gno.land/p/demo/json/LICENSE diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno index fc5ad1c16a8..396cf5229fb 100644 --- a/examples/gno.land/p/demo/json/buffer_test.gno +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -6,35 +6,35 @@ import ( func TestBufferCurrent(t *testing.T) { tests := []struct { - name string - buffer *buffer + name string + buffer *buffer expected byte - wantErr bool + wantErr bool }{ { name: "Valid current byte", buffer: &buffer{ - data: []byte("test"), + data: []byte("test"), length: 4, - index: 1, + index: 1, }, expected: 'e', - wantErr: false, + wantErr: false, }, { name: "EOF", buffer: &buffer{ - data: []byte("test"), + data: []byte("test"), length: 4, - index: 4, + index: 4, }, expected: 0, - wantErr: true, + wantErr: true, }, } for _, tt := range tests { - t.Run(tt.name, func (t *testing.T) { + t.Run(tt.name, func(t *testing.T) { got, err := tt.buffer.current() if (err != nil) != tt.wantErr { t.Errorf("buffer.current() error = %v, wantErr %v", err, tt.wantErr) @@ -119,17 +119,17 @@ func TestBufferSlice(t *testing.T) { wantErr bool }{ { - name: "Valid slice -- 0 characters", - buffer: &buffer{data: []byte("test"), length: 4, index: 0}, - pos: 0, - want: nil, + name: "Valid slice -- 0 characters", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + pos: 0, + want: nil, wantErr: false, }, { - name: "Valid slice -- 1 character", - buffer: &buffer{data: []byte("test"), length: 4, index: 0}, - pos: 1, - want: []byte("t"), + name: "Valid slice -- 1 character", + buffer: &buffer{data: []byte("test"), length: 4, index: 0}, + pos: 1, + want: []byte("t"), wantErr: false, }, { @@ -276,284 +276,284 @@ func TestBufferSkipAny(t *testing.T) { } func TestSkipToNextSignificantToken(t *testing.T) { - tests := []struct { - name string - input []byte - expected uint - }{ - {"No significant chars", []byte("abc"), 3}, - {"One significant char at start", []byte(".abc"), 0}, - {"Significant char in middle", []byte("ab.c"), 2}, - {"Multiple significant chars", []byte("a$.c"), 1}, - {"Significant char at end", []byte("abc$"), 3}, - {"Only significant chars", []byte("$."), 0}, - {"Empty string", []byte(""), 0}, - } + tests := []struct { + name string + input []byte + expected uint + }{ + {"No significant chars", []byte("abc"), 3}, + {"One significant char at start", []byte(".abc"), 0}, + {"Significant char in middle", []byte("ab.c"), 2}, + {"Multiple significant chars", []byte("a$.c"), 1}, + {"Significant char at end", []byte("abc$"), 3}, + {"Only significant chars", []byte("$."), 0}, + {"Empty string", []byte(""), 0}, + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := NewBuffer(tt.input) - b.skipToNextSignificantToken() - if b.index != tt.expected { - t.Errorf("after skipToNextSignificantToken(), got index = %v, want %v", b.index, tt.expected) - } - }) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := NewBuffer(tt.input) + b.skipToNextSignificantToken() + if b.index != tt.expected { + t.Errorf("after skipToNextSignificantToken(), got index = %v, want %v", b.index, tt.expected) + } + }) + } } const testString = "abcdefg" func mockBuffer(s string) *buffer { - return NewBuffer([]byte(s)) + return NewBuffer([]byte(s)) } func TestSkipAndReturnIndex(t *testing.T) { - tests := []struct { - name string - input string - expected uint - }{ - {"StartOfString", "", 0}, - {"MiddleOfString", "abcdef", 1}, - {"EndOfString", "abc", 1}, - } + tests := []struct { + name string + input string + expected uint + }{ + {"StartOfString", "", 0}, + {"MiddleOfString", "abcdef", 1}, + {"EndOfString", "abc", 1}, + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buf := mockBuffer(tt.input) - got, err := buf.skipAndReturnIndex() - if err != nil && tt.input != "" { // Expect no error unless input is empty - t.Errorf("skipAndReturnIndex() error = %v", err) - } - if got != tt.expected { - t.Errorf("skipAndReturnIndex() = %v, want %v", got, tt.expected) - } - }) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := mockBuffer(tt.input) + got, err := buf.skipAndReturnIndex() + if err != nil && tt.input != "" { // Expect no error unless input is empty + t.Errorf("skipAndReturnIndex() error = %v", err) + } + if got != tt.expected { + t.Errorf("skipAndReturnIndex() = %v, want %v", got, tt.expected) + } + }) + } } func TestSkipUntil(t *testing.T) { - tests := []struct { - name string - input string - tokens map[byte]bool - expected uint - }{ - {"SkipToToken", "abcdefg", map[byte]bool{'c': true}, 2}, - {"SkipToEnd", "abcdefg", map[byte]bool{'h': true}, 7}, - {"SkipNone", "abcdefg", map[byte]bool{'a': true}, 0}, - } + tests := []struct { + name string + input string + tokens map[byte]bool + expected uint + }{ + {"SkipToToken", "abcdefg", map[byte]bool{'c': true}, 2}, + {"SkipToEnd", "abcdefg", map[byte]bool{'h': true}, 7}, + {"SkipNone", "abcdefg", map[byte]bool{'a': true}, 0}, + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buf := mockBuffer(tt.input) - got, err := buf.skipUntil(tt.tokens) - if err != nil && got != uint(len(tt.input)) { // Expect error only if reached end without finding token - t.Errorf("skipUntil() error = %v", err) - } - if got != tt.expected { - t.Errorf("skipUntil() = %v, want %v", got, tt.expected) - } - }) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := mockBuffer(tt.input) + got, err := buf.skipUntil(tt.tokens) + if err != nil && got != uint(len(tt.input)) { // Expect error only if reached end without finding token + t.Errorf("skipUntil() error = %v", err) + } + if got != tt.expected { + t.Errorf("skipUntil() = %v, want %v", got, tt.expected) + } + }) + } } func TestSliceFromIndices(t *testing.T) { - tests := []struct { - name string - input string - start uint - end uint - expected string - }{ - {"FullString", "abcdefg", 0, 7, "abcdefg"}, - {"Substring", "abcdefg", 2, 5, "cde"}, - {"OutOfBounds", "abcdefg", 5, 10, "fg"}, - } + tests := []struct { + name string + input string + start uint + end uint + expected string + }{ + {"FullString", "abcdefg", 0, 7, "abcdefg"}, + {"Substring", "abcdefg", 2, 5, "cde"}, + {"OutOfBounds", "abcdefg", 5, 10, "fg"}, + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buf := mockBuffer(tt.input) - got := buf.sliceFromIndices(tt.start, tt.end) - if string(got) != tt.expected { - t.Errorf("sliceFromIndices() = %v, want %v", string(got), tt.expected) - } - }) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := mockBuffer(tt.input) + got := buf.sliceFromIndices(tt.start, tt.end) + if string(got) != tt.expected { + t.Errorf("sliceFromIndices() = %v, want %v", string(got), tt.expected) + } + }) + } } func TestBufferToken(t *testing.T) { tests := []struct { - name string - path string + name string + path string index uint isErr bool }{ { - name: "Simple valid path", - path: "@.length", + name: "Simple valid path", + path: "@.length", index: 8, isErr: false, }, { - name: "Path with array expr", - path: "@['foo'].0.bar", + name: "Path with array expr", + path: "@['foo'].0.bar", index: 14, isErr: false, }, { - name: "Path with array expr and simple fomula", - path: "@['foo'].[(@.length - 1)].*", + name: "Path with array expr and simple fomula", + path: "@['foo'].[(@.length - 1)].*", index: 27, isErr: false, }, { - name: "Path with filter expr", - path: "@['foo'].[?(@.bar == 1 & @.baz < @.length)].*", + name: "Path with filter expr", + path: "@['foo'].[?(@.bar == 1 & @.baz < @.length)].*", index: 45, isErr: false, }, { - name: "addition of foo and bar", - path: "@.foo+@.bar", + name: "addition of foo and bar", + path: "@.foo+@.bar", index: 11, isErr: false, }, { - name: "logical AND of foo and bar", - path: "@.foo && @.bar", + name: "logical AND of foo and bar", + path: "@.foo && @.bar", index: 14, isErr: false, }, { - name: "logical OR of foo and bar", - path: "@.foo || @.bar", + name: "logical OR of foo and bar", + path: "@.foo || @.bar", index: 14, isErr: false, }, { - name: "accessing third element of foo", - path: "@.foo,3", + name: "accessing third element of foo", + path: "@.foo,3", index: 7, isErr: false, }, { - name: "accessing last element of array", - path: "@.length-1", + name: "accessing last element of array", + path: "@.length-1", index: 10, isErr: false, }, { - name: "number 1", - path: "1", + name: "number 1", + path: "1", index: 1, isErr: false, }, { - name: "float", - path: "3.1e4", + name: "float", + path: "3.1e4", index: 5, isErr: false, }, { - name: "float with minus", - path: "3.1e-4", + name: "float with minus", + path: "3.1e-4", index: 6, isErr: false, }, { - name: "float with plus", - path: "3.1e+4", + name: "float with plus", + path: "3.1e+4", index: 6, isErr: false, }, { - name: "negative number", - path: "-12345", + name: "negative number", + path: "-12345", index: 6, isErr: false, }, { - name: "negative float", - path: "-3.1e4", + name: "negative float", + path: "-3.1e4", index: 6, isErr: false, }, { - name: "negative float with minus", - path: "-3.1e-4", + name: "negative float with minus", + path: "-3.1e-4", index: 7, isErr: false, }, { - name: "negative float with plus", - path: "-3.1e+4", + name: "negative float with plus", + path: "-3.1e+4", index: 7, isErr: false, }, { - name: "string number", - path: "'12345'", + name: "string number", + path: "'12345'", index: 7, isErr: false, }, { - name: "string with backslash", - path: "'foo \\'bar '", + name: "string with backslash", + path: "'foo \\'bar '", index: 12, isErr: false, }, { - name: "string with inner double quotes", - path: "'foo \"bar \"'", + name: "string with inner double quotes", + path: "'foo \"bar \"'", index: 12, isErr: false, }, { - name: "parenthesis 1", - path: "(@abc)", + name: "parenthesis 1", + path: "(@abc)", index: 6, isErr: false, }, { - name: "parenthesis 2", - path: "[()]", + name: "parenthesis 2", + path: "[()]", index: 4, isErr: false, }, { - name: "parenthesis mismatch", - path: "[(])", + name: "parenthesis mismatch", + path: "[(])", index: 2, isErr: true, }, { - name: "parenthesis mismatch 2", - path: "(", + name: "parenthesis mismatch 2", + path: "(", index: 1, isErr: true, }, { - name: "parenthesis mismatch 3", - path: "())]", + name: "parenthesis mismatch 3", + path: "())]", index: 2, isErr: true, }, { - name: "bracket mismatch", - path: "[()", + name: "bracket mismatch", + path: "[()", index: 3, isErr: true, }, { - name: "bracket mismatch 2", - path: "()]", + name: "bracket mismatch 2", + path: "()]", index: 2, isErr: true, }, { - name: "path does not close bracket", - path: "@.foo[)", + name: "path does not close bracket", + path: "@.foo[)", index: 6, isErr: false, }, @@ -565,18 +565,18 @@ func TestBufferToken(t *testing.T) { err := buf.pathToken() if tt.isErr { - if err == nil { - t.Errorf("Expected an error for path `%s`, but got none", tt.path) - } - } + if err == nil { + t.Errorf("Expected an error for path `%s`, but got none", tt.path) + } + } if err == nil && tt.isErr { t.Errorf("Expected an error for path `%s`, but got none", tt.path) } - if buf.index != tt.index { - t.Errorf("Expected final index %d, got %d (token: `%s`) for path `%s`", tt.index, buf.index, string(buf.data[buf.index]), tt.path) - } + if buf.index != tt.index { + t.Errorf("Expected final index %d, got %d (token: `%s`) for path `%s`", tt.index, buf.index, string(buf.data[buf.index]), tt.path) + } }) } } diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 48a5e2fcf1a..c3398e1b2db 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -4,8 +4,8 @@ import ( "bytes" "errors" "unicode" - "unicode/utf8" "unicode/utf16" + "unicode/utf8" ) const ( @@ -167,12 +167,12 @@ func unquoteBytes(s []byte, border byte) ([]byte, error) { return s, nil } - b := make([]byte, len(s) + utf8.UTFMax * 2) + b := make([]byte, len(s)+utf8.UTFMax*2) w := copy(b, s[:r]) for r < len(s) { - if w >= len(b) - (utf8.UTFMax * 2) { - nb := make([]byte, (utf8.UTFMax + len(b)) * 2) + if w >= len(b)-(utf8.UTFMax*2) { + nb := make([]byte, (utf8.UTFMax+len(b))*2) copy(nb, b) b = nb } diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index 7318c02f97c..fbc4910b1af 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -187,20 +187,20 @@ func TestUnescape(t *testing.T) { func TestUnquoteBytes(t *testing.T) { tests := []struct { - input []byte - border byte + input []byte + border byte expected []byte - ok bool + ok bool }{ {[]byte("\"hello\""), '"', []byte("hello"), true}, - {[]byte("'hello'"), '\'', []byte("hello"), true}, - {[]byte("\"hello"), '"', nil, false}, - {[]byte("hello\""), '"', nil, false}, - {[]byte("\"he\\\"llo\""), '"', []byte("he\"llo"), true}, - {[]byte("\"he\\nllo\""), '"', []byte("he\nllo"), true}, - {[]byte("\"\""), '"', []byte(""), true}, - {[]byte("''"), '\'', []byte(""), true}, - {[]byte("\"\\u0041\""), '"', []byte("A"), true}, + {[]byte("'hello'"), '\'', []byte("hello"), true}, + {[]byte("\"hello"), '"', nil, false}, + {[]byte("hello\""), '"', nil, false}, + {[]byte("\"he\\\"llo\""), '"', []byte("he\"llo"), true}, + {[]byte("\"he\\nllo\""), '"', []byte("he\nllo"), true}, + {[]byte("\"\""), '"', []byte(""), true}, + {[]byte("''"), '\'', []byte(""), true}, + {[]byte("\"\\u0041\""), '"', []byte("A"), true}, } for _, tc := range tests { @@ -214,4 +214,4 @@ func TestUnquoteBytes(t *testing.T) { t.Errorf("unquoteBytes(%q) = %q; want %q", tc.input, result, tc.expected) } } -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/internal.gno b/examples/gno.land/p/demo/json/internal.gno index 1b77e06509b..990f6cb0bf5 100644 --- a/examples/gno.land/p/demo/json/internal.gno +++ b/examples/gno.land/p/demo/json/internal.gno @@ -6,8 +6,8 @@ package json // Go implementation is taken from: https://github.com/spyzhov/ajson/blob/master/internal/state.go type ( - States int8 // possible states of the parser - Classes int8 // JSON string character types + States int8 // possible states of the parser + Classes int8 // JSON string character types ) const __ = -1 @@ -195,4 +195,4 @@ var StateTransitionTable = [31][31]States{ /*nu N1*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, N2, __, __, __}, /*nul N2*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, N3, __, __, __, __, __, __, __, __}, /*null N3*/ {__, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, OK, __, __, __, __, __, __, __, __}, -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index 6c353cb50c0..74ea9f2f1b4 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -35,11 +35,10 @@ type Node struct { data []byte value *NodeValue index int - borders [2]int // start, end + borders [2]int // start, end modified bool } - func (n *Node) Key() string { if n == nil || n.key == "" { return "" @@ -79,15 +78,15 @@ func (n *Node) Keys() []string { func NullNode(key string) *Node { return &Node{ - key: key, - value: &NodeValue{value: nil, typ: Null}, + key: key, + value: &NodeValue{value: nil, typ: Null}, modified: true, } } func NumberNode(key string, value float64) *Node { return &Node{ - key: key, + key: key, value: &NodeValue{ value: value, typ: Number, @@ -100,8 +99,8 @@ func StringNode(key string, value string) *Node { val := newNodeValue(value) return &Node{ - key: key, - value: val, + key: key, + value: val, modified: true, } } @@ -110,8 +109,8 @@ func BoolNode(key string, value bool) *Node { val := newNodeValue(value) return &Node{ - key: key, - value: val, + key: key, + value: val, modified: true, } } @@ -119,8 +118,8 @@ func BoolNode(key string, value bool) *Node { func ArrayNode(key string, value []*Node) *Node { val := newNodeValue(value) curr := &Node{ - key: key, - value: val, + key: key, + value: val, modified: true, } diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index 197c7f9b886..bc86d21f84b 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -6,40 +6,40 @@ import ( ) func TestNodeValue(t *testing.T) { - tests := []struct { - name string - value interface{} + tests := []struct { + name string + value interface{} expectType ValueType - }{ - {"string", "test value", String}, - {"integer", 123, Number}, - {"float", 1.23, Float}, - {"boolean", true, Boolean}, + }{ + {"string", "test value", String}, + {"integer", 123, Number}, + {"float", 1.23, Float}, + {"boolean", true, Boolean}, {"null", nil, Null}, {"array", []interface{}{1, 2, 3}, Array}, {"object", map[string]interface{}{"key": "value"}, Object}, - } + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - nv := &NodeValue{} - nv.Store(tt.value) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nv := &NodeValue{} + nv.Store(tt.value) - if nv.value != tt.value { - t.Errorf("Store: %s want %v got %v", tt.name, tt.value, nv.value) - } + if nv.value != tt.value { + t.Errorf("Store: %s want %v got %v", tt.name, tt.value, nv.value) + } - typ := typeOf(tt.value) + typ := typeOf(tt.value) if typ != tt.expectType { t.Errorf("Store: %s want type %v got %v", tt.name, tt.expectType.String(), typ.String()) } - loadedValue := nv.Load() - if loadedValue != tt.value { - t.Errorf("Load: %s want %v got %v", tt.name, tt.value, loadedValue) - } - }) - } + loadedValue := nv.Load() + if loadedValue != tt.value { + t.Errorf("Load: %s want %v got %v", tt.name, tt.value, loadedValue) + } + }) + } } func TestNodeCreation(t *testing.T) { @@ -131,8 +131,8 @@ func TestNodeSource(t *testing.T) { expected []byte }{ { - name: "nil node", - node: nil, + name: "nil node", + node: nil, expected: nil, }, { @@ -172,4 +172,4 @@ func TestNodeSource(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 85fe98252f3..d1fbe3fbc9d 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -1,78 +1,78 @@ package json import ( - "errors" + "errors" ) // ParsePath takes a JSONPath string and returns a slice of strings representing the path segments. func ParsePath(path string) ([]string, error) { - buf := NewBuffer([]byte(path)) - result := make([]string, 0) - - for { - b, err := buf.current() - if err != nil { - break - } - - switch { - case b == DollarToken || b == AtToken: - result = append(result, string(b)) + buf := NewBuffer([]byte(path)) + result := make([]string, 0) + + for { + b, err := buf.current() + if err != nil { + break + } + + switch { + case b == DollarToken || b == AtToken: + result = append(result, string(b)) buf.step() - case b == DotToken: - buf.step() + case b == DotToken: + buf.step() - if next, _ := buf.current(); next == DotToken { - buf.step() - result = append(result, "..") + if next, _ := buf.current(); next == DotToken { + buf.step() + result = append(result, "..") extractNextSegment(buf, &result) } else { extractNextSegment(buf, &result) - } + } - case b == SquareOpenToken: - start := buf.index - buf.step() + case b == SquareOpenToken: + start := buf.index + buf.step() - for { - if buf.index >= buf.length || buf.data[buf.index] == SquareCloseToken { - break - } + for { + if buf.index >= buf.length || buf.data[buf.index] == SquareCloseToken { + break + } - buf.step() - } + buf.step() + } - if buf.index >= buf.length { - return nil, errors.New("unexpected end of path") - } + if buf.index >= buf.length { + return nil, errors.New("unexpected end of path") + } - segment := string(buf.sliceFromIndices(start+1, buf.index)) - result = append(result, segment) + segment := string(buf.sliceFromIndices(start+1, buf.index)) + result = append(result, segment) - buf.step() + buf.step() - default: - buf.step() - } - } + default: + buf.step() + } + } - return result, nil + return result, nil } -// extractNextSegment extracts the segment from the current index +// extractNextSegment extracts the segment from the current index // to the next significant character and adds it to the resulting slice. func extractNextSegment(buf *buffer, result *[]string) { - start := buf.index - buf.skipToNextSignificantToken() - - if buf.index <= start { - return - } - - segment := string(buf.sliceFromIndices(start, buf.index)) - if segment != "" { - *result = append(*result, segment) - } + start := buf.index + buf.skipToNextSignificantToken() + + if buf.index <= start { + return + } + + segment := string(buf.sliceFromIndices(start, buf.index)) + if segment != "" { + *result = append(*result, segment) + } } diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno index 47055fc072c..eb00065eb7b 100644 --- a/examples/gno.land/p/demo/json/struct.gno +++ b/examples/gno.land/p/demo/json/struct.gno @@ -334,7 +334,8 @@ func (s *Struct) Remove(name string) error { if field.name == name { // arranges the slice to remove the field. s.fields = append(s.fields[:i], s.fields[i+1:]...) - break + + return nil } } diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 0bd36a6f6e9..63e2c5a01dd 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -30,8 +30,8 @@ const ( UnderScoreToken = '_' DollarToken = '$' AtToken = '@' - AndToken = '&' - OrToken = '|' + AndToken = '&' + OrToken = '|' ) const ( From 29224d9367290eb5f36b3da754edb0feef921427 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 5 Mar 2024 16:22:11 +0900 Subject: [PATCH 60/72] state machine decoder and some node getters --- examples/gno.land/p/demo/json/buffer.gno | 77 ++- examples/gno.land/p/demo/json/buffer_test.gno | 45 ++ examples/gno.land/p/demo/json/decode.gno | 253 +++++++++ examples/gno.land/p/demo/json/decode_test.gno | 511 ++++++++++++++++++ examples/gno.land/p/demo/json/escape.gno | 4 +- examples/gno.land/p/demo/json/internal.gno | 6 +- examples/gno.land/p/demo/json/node.gno | 345 +++++++++++- examples/gno.land/p/demo/json/node_test.gno | 88 +-- examples/gno.land/p/demo/json/parser.gno | 21 +- 9 files changed, 1234 insertions(+), 116 deletions(-) create mode 100644 examples/gno.land/p/demo/json/decode.gno create mode 100644 examples/gno.land/p/demo/json/decode_test.gno diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index 7a0f5bc2b8b..8d61c6b5913 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -4,6 +4,7 @@ import ( "bytes" "errors" "strings" + "io" "gno.land/p/demo/ufmt" ) @@ -30,10 +31,23 @@ func NewBuffer(data []byte) *buffer { } } +// first retrieves the first non-whitespace (or other escaped) character in the buffer. +func (b *buffer) first() (byte, error) { + for _; b.index < b.length; b.index++ { + c := b.data[b.index] + + if !(c == WhiteSpaceToken || c == CarriageReturnToken || c == NewLineToken || c == TabToken) { + return c, nil + } + } + + return 0, io.EOF +} + // current returns the byte of the current index. func (b *buffer) current() (byte, error) { if b.index >= b.length { - return 0, errors.New("EOF") + return 0, io.EOF } return b.data[b.index], nil @@ -56,7 +70,7 @@ func (b *buffer) move(pos uint) error { newIndex := b.index + pos if newIndex > b.length { - return errors.New("EOF") + return io.EOF } b.index = newIndex @@ -69,7 +83,7 @@ func (b *buffer) slice(pos uint) ([]byte, error) { end := b.index + pos if end > b.length { - return nil, errors.New("EOF") + return nil, io.EOF } return b.data[b.index:end], nil @@ -98,7 +112,7 @@ func (b *buffer) skip(bs byte) error { b.index++ } - return errors.New("EOF") + return io.EOF } // skipAny moves the index until it encounters one of the given set of bytes. @@ -150,7 +164,7 @@ func (b *buffer) skipUntil(endTokens map[byte]bool) (uint, error) { b.index++ } - return b.index, errors.New("EOF") + return b.index, io.EOF } // significantTokens is a map where the keys are the significant characters in a JSON path. @@ -345,3 +359,56 @@ func (b *buffer) getState() States { return b.state } + +func (b *buffer) string(search byte, token bool) error { + if token { + b.last = GO + } + + for _; b.index < b.length; b.index++ { + b.class = b.getClasses(search) + + if b.class == __ { + return errors.New("invalid token found while parsing path") + } + + b.state = StateTransitionTable[b.last][b.class] + if b.state == __ { + return errors.New("invalid token found while parsing path") + } + + if b.state < __ { + break + } + + b.last = b.state + } + + return nil +} + +func (b *buffer) word(bs []byte) error { + var c byte + + max := len(bs) + index := 0 + + for _; b.index < b.length; b.index++ { + c = b.data[b.index] + + if c != bs[index] { + return errors.New("invalid token found while parsing path") + } + + index++ + if index >= max { + break + } + } + + if index != max { + return errors.New("invalid token found while parsing path") + } + + return nil +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno index 396cf5229fb..b86574f03b3 100644 --- a/examples/gno.land/p/demo/json/buffer_test.gno +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -1,6 +1,7 @@ package json import ( + "errors" "testing" ) @@ -580,3 +581,47 @@ func TestBufferToken(t *testing.T) { }) } } + +func TestBufferFirst(t *testing.T) { + tests := []struct { + name string + data []byte + expected byte + }{ + { + name: "Valid first byte", + data: []byte("test"), + expected: 't', + }, + { + name: "Empty buffer", + data: []byte(""), + expected: 0, + }, + { + name: "Whitespace buffer", + data: []byte(" "), + expected: 0, + }, + { + name: "whitespace in middle", + data: []byte("hello world"), + expected: 'h', + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := NewBuffer(tt.data) + + got, err := b.first() + if err != nil && tt.expected != 0 { + t.Errorf("Unexpected error: %v", err) + } + + if got != tt.expected { + t.Errorf("Expected first byte to be %q, got %q", tt.expected, got) + } + }) + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/decode.gno b/examples/gno.land/p/demo/json/decode.gno new file mode 100644 index 00000000000..1f2009dbffb --- /dev/null +++ b/examples/gno.land/p/demo/json/decode.gno @@ -0,0 +1,253 @@ +package json + +import ( + "errors" + "io" + + "gno.land/p/demo/ufmt" +) + +func unmarshal2(data []byte) (*Node, error) { + buf := NewBuffer(data) + + var ( + state States + key string + current *Node + hasKey bool + ) + + _, err := buf.first() + if err != nil { + return nil, io.EOF + } + + for { + state = buf.getState() + if state == __ { + return nil, unexpectedTokenError(buf.data, buf.index) + } + + if state >= GO { + switch buf.state { + case ST: + if current != nil && current.isObject() && !hasKey { + // key detected + key, err = getString(buf) + hasKey = true + buf.state = CO + } else { + // string detected + err = createOrUpdateNode(¤t, buf, String, &hasKey, &key) + if err != nil { + return nil, err + } + + if err = buf.string(DoublyQuoteToken, false); err != nil { + return nil, err + } + + current.borders[1] = buf.index + 1 + buf.state = OK + + updateCurrentNode(¤t) + } + + case MI, ZE, IN: + if err = createOrUpdateNode(¤t, buf, Number, &hasKey, &key); err != nil { + return nil, err + } + + if err = buf.numeric(false); err != nil { + return nil, err + } + + current.borders[1] = buf.index + buf.index -= 1 + buf.state = OK + + updateCurrentNode(¤t) + + case T1, F1: + if err = createOrUpdateNode(¤t, buf, Boolean, &hasKey, &key); err != nil { + return nil, err + } + + if buf.state == T1 { + err = buf.word(trueLiteral) + } else { + err = buf.word(falseLiteral) + } + + if err != nil { + return nil, err + } + + current.borders[1] = buf.index + 1 + buf.state = OK + + updateCurrentNode(¤t) + + case N1: + if err = createOrUpdateNode(¤t, buf, Null, &hasKey, &key); err != nil { + return nil, err + } + + if err = buf.word(nullLiteral); err != nil { + return nil, err + } + + current.borders[1] = buf.index + 1 + buf.state = OK + + updateCurrentNode(¤t) + } + } else { + switch state { + case ec, cc: + if current != nil && current.isObject() && !current.ready() { + current.borders[1] = buf.index + 1 + updateCurrentNode(¤t) + } else { + if err = unexpectedTokenError(buf.data, buf.index); err != nil { + return nil, err + } + } + + buf.state = OK + + case bc: + if current != nil && current.isArray() && !current.ready() { + current.borders[1] = buf.index + 1 + updateCurrentNode(¤t) + } else { + if err = unexpectedTokenError(buf.data, buf.index); err != nil { + return nil, err + } + } + + buf.state = OK + + case co: + if err = createOrUpdateNode(¤t, buf, Object, &hasKey, &key); err != nil { + return nil, err + } + + buf.state = OB + + case bo: + if err = createOrUpdateNode(¤t, buf, Array, &hasKey, &key); err != nil { + return nil, err + } + + buf.state = AR + + case cm: + if current == nil { + return nil, unexpectedTokenError(buf.data, buf.index) + } + + if current.isObject() { + buf.state = KE // key expected + } else if current.isArray() { + buf.state = VA // value expected + } else { + if err = unexpectedTokenError(buf.data, buf.index); err != nil { + return nil, err + } + } + + case cl: + if current == nil || !current.isObject() || !hasKey { + if err = unexpectedTokenError(buf.data, buf.index); err != nil { + return nil, err + } + } + + buf.state = VA + + default: + if err = unexpectedTokenError(buf.data, buf.index); err != nil { + return nil, err + } + } + } + + if buf.step() != nil { + break + } + + if _, err = buf.first(); err != nil { + err = nil + break + } + } + + return checkRootNode(current, buf) +} + +func UnmarshalSafe(data []byte) (*Node, error) { + var safe []byte + safe = append(safe, data...) + return unmarshal2(safe) +} + +func Must(root *Node, expect error) *Node { + if expect != nil { + panic(expect) + } + + return root +} + +// TODO: improve this function to avoid pointer dereferencing +func createOrUpdateNode(curr **Node, buf *buffer, t ValueType, hasKey *bool, key *string) error { + var err error + + if *hasKey { + *curr, err = NewNode(*curr, buf, t, key) + *hasKey = false + } else { + *curr, err = NewNode(*curr, buf, t, nil) + } + + return err +} + +// getString extracts a string from the buffer and advances the buffer index past the string. +func getString(b *buffer) (string, error) { + if err := b.string(DoublyQuoteToken, false); err != nil { + return "", err + } + + start := b.index + value, err := unquote(b.data[start:start+1], DoublyQuoteToken) + if err != nil { + return "", errors.New(ufmt.Sprintf("token %s is not a valid string", b.data[start:start+1])) + } + + return value, nil +} + +func updateCurrentNode(current **Node) { + if (*current).prev != nil { + *current = (*current).prev + } +} + +func checkRootNode(current *Node, buf *buffer) (*Node, error) { + if current == nil || buf.state != OK { + return nil, io.EOF + } + + root := current.root() + if !root.ready() { + return nil, io.EOF + } + + return root, nil +} + +func unexpectedTokenError(data []byte, index uint) error { + return errors.New(ufmt.Sprintf("unexpected token %s", data[index:index+1])) +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/decode_test.gno b/examples/gno.land/p/demo/json/decode_test.gno new file mode 100644 index 00000000000..d301bb26f88 --- /dev/null +++ b/examples/gno.land/p/demo/json/decode_test.gno @@ -0,0 +1,511 @@ +package json + +import ( + "bytes" + "testing" +) + +type testCase struct { + name string + input []byte + _type ValueType + value []byte +} + +func simpleValid(test *testCase, t *testing.T) { + root, err := unmarshal2(test.input) + if err != nil { + t.Errorf("Error on Unmarshal(%s): %s", test.name, err.Error()) + } else if root == nil { + t.Errorf("Error on Unmarshal(%s): root is nil", test.name) + } else if root.value.typ != test._type { + t.Errorf("Error on Unmarshal(%s): wrong type", test.name) + } else if !bytes.Equal(root.Source(), test.value) { + t.Errorf("Error on Unmarshal(%s): %s != %s", test.name, root.Source(), test.value) + } +} + +func simpleInvalid(test *testCase, t *testing.T) { + root, err := unmarshal2(test.input) + if err == nil { + t.Errorf("Error on Unmarshal(%s): error expected, got '%s'", test.name, root.Source()) + } else if root != nil { + t.Errorf("Error on Unmarshal(%s): root is not nil", test.name) + } +} + +func simpleCorrupted(name string) *testCase { + return &testCase{name: name, input: []byte(name)} +} + +func TestUnmarshal_StringSimpleSuccess(t *testing.T) { + tests := []*testCase{ + {name: "blank", input: []byte("\"\""), _type: String, value: []byte("\"\"")}, + {name: "char", input: []byte("\"c\""), _type: String, value: []byte("\"c\"")}, + {name: "word", input: []byte("\"cat\""), _type: String, value: []byte("\"cat\"")}, + {name: "spaces", input: []byte(" \"good cat or dog\"\r\n "), _type: String, value: []byte("\"good cat or dog\"")}, + {name: "backslash", input: []byte("\"good \\\"cat\\\"\""), _type: String, value: []byte("\"good \\\"cat\\\"\"")}, + {name: "backslash 2", input: []byte("\"good \\\\\\\"cat\\\"\""), _type: String, value: []byte("\"good \\\\\\\"cat\\\"\"")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleValid(test, t) + }) + } +} + +func TestUnmarshal_NumericSimpleSuccess(t *testing.T) { + tests := []*testCase{ + {name: "1", input: []byte("1"), _type: Number, value: []byte("1")}, + {name: "-1", input: []byte("-1"), _type: Number, value: []byte("-1")}, + + {name: "1234567890", input: []byte("1234567890"), _type: Number, value: []byte("1234567890")}, + {name: "-123", input: []byte("-123"), _type: Number, value: []byte("-123")}, + + {name: "123.456", input: []byte("123.456"), _type: Number, value: []byte("123.456")}, + {name: "-123.456", input: []byte("-123.456"), _type: Number, value: []byte("-123.456")}, + + {name: "1e3", input: []byte("1e3"), _type: Number, value: []byte("1e3")}, + {name: "1e+3", input: []byte("1e+3"), _type: Number, value: []byte("1e+3")}, + {name: "1e-3", input: []byte("1e-3"), _type: Number, value: []byte("1e-3")}, + {name: "-1e3", input: []byte("-1e3"), _type: Number, value: []byte("-1e3")}, + {name: "-1e-3", input: []byte("-1e-3"), _type: Number, value: []byte("-1e-3")}, + + {name: "1.123e3456", input: []byte("1.123e3456"), _type: Number, value: []byte("1.123e3456")}, + {name: "1.123e-3456", input: []byte("1.123e-3456"), _type: Number, value: []byte("1.123e-3456")}, + {name: "-1.123e3456", input: []byte("-1.123e3456"), _type: Number, value: []byte("-1.123e3456")}, + {name: "-1.123e-3456", input: []byte("-1.123e-3456"), _type: Number, value: []byte("-1.123e-3456")}, + + {name: "1E3", input: []byte("1E3"), _type: Number, value: []byte("1E3")}, + {name: "1E-3", input: []byte("1E-3"), _type: Number, value: []byte("1E-3")}, + {name: "-1E3", input: []byte("-1E3"), _type: Number, value: []byte("-1E3")}, + {name: "-1E-3", input: []byte("-1E-3"), _type: Number, value: []byte("-1E-3")}, + + {name: "1.123E3456", input: []byte("1.123E3456"), _type: Number, value: []byte("1.123E3456")}, + {name: "1.123E-3456", input: []byte("1.123E-3456"), _type: Number, value: []byte("1.123E-3456")}, + {name: "-1.123E3456", input: []byte("-1.123E3456"), _type: Number, value: []byte("-1.123E3456")}, + {name: "-1.123E-3456", input: []byte("-1.123E-3456"), _type: Number, value: []byte("-1.123E-3456")}, + + {name: "-1.123E-3456 with spaces", input: []byte(" \r -1.123E-3456 \t\n"), _type: Number, value: []byte("-1.123E-3456")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleValid(test, t) + }) + } +} + +func TestUnmarshal_StringSimpleCorrupted(t *testing.T) { + tests := []*testCase{ + // {name: "one quote", input: []byte("\"")}, + // {name: "one quote char", input: []byte("\"c")}, + {name: "white NL", input: []byte("\"foo\nbar\"")}, + {name: "white R", input: []byte("\"foo\rbar\"")}, + {name: "white Tab", input: []byte("\"foo\tbar\"")}, + {name: "wrong quotes", input: []byte("'cat'")}, + {name: "double string", input: []byte("\"Hello\" \"World\"")}, + {name: "quotes in quotes", input: []byte("\"good \"cat\"\"")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleInvalid(test, t) + }) + } +} + +func TestUnmarshal_ObjectSimpleSuccess(t *testing.T) { + tests := []*testCase{ + {name: "{}", input: []byte("{}"), _type: Object, value: []byte("{}")}, + {name: `{ \r\n }`, input: []byte("{ \r\n }"), _type: Object, value: []byte("{ \r\n }")}, + {name: `{"key":1}`, input: []byte(`{"key":1}`), _type: Object, value: []byte(`{"key":1}`)}, + {name: `{"key":true}`, input: []byte(`{"key":true}`), _type: Object, value: []byte(`{"key":true}`)}, + {name: `{"key":"value"}`, input: []byte(`{"key":"value"}`), _type: Object, value: []byte(`{"key":"value"}`)}, + {name: `{"foo":"bar","baz":"foo"}`, input: []byte(`{"foo":"bar", "baz":"foo"}`), _type: Object, value: []byte(`{"foo":"bar", "baz":"foo"}`)}, + {name: "spaces", input: []byte(` { "foo" : "bar" , "baz" : "foo" } `), _type: Object, value: []byte(`{ "foo" : "bar" , "baz" : "foo" }`)}, + {name: "nested", input: []byte(`{"foo":{"bar":{"baz":{}}}}`), _type: Object, value: []byte(`{"foo":{"bar":{"baz":{}}}}`)}, + {name: "array", input: []byte(`{"array":[{},{},{"foo":[{"bar":["baz"]}]}]}`), _type: Object, value: []byte(`{"array":[{},{},{"foo":[{"bar":["baz"]}]}]}`)}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleValid(test, t) + }) + } +} + +func TestUnmarshal_ObjectSimpleCorrupted(t *testing.T) { + tests := []*testCase{ + simpleCorrupted("{,}"), + simpleCorrupted("{:}"), + simpleCorrupted(`{"foo"}`), + simpleCorrupted(`{"foo":}`), + simpleCorrupted(`{:"foo"}`), + simpleCorrupted(`{"foo":bar}`), + simpleCorrupted(`{"foo":"bar",}`), + simpleCorrupted(`{}{}`), + simpleCorrupted(`{},{}`), + simpleCorrupted(`{[},{]}`), + simpleCorrupted(`{[,]}`), + simpleCorrupted(`{[]}`), + simpleCorrupted(`{}1`), + simpleCorrupted(`1{}`), + simpleCorrupted(`{"x"::1}`), + simpleCorrupted(`{null:null}`), + simpleCorrupted(`{"foo:"bar"}`), + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleInvalid(test, t) + }) + } +} + +func TestUnmarshal_NullSimpleCorrupted(t *testing.T) { + tests := []*testCase{ + {name: "nul", input: []byte("nul")}, + {name: "NILL", input: []byte("NILL")}, + {name: "Null", input: []byte("Null")}, + {name: "NULL", input: []byte("NULL")}, + {name: "spaces", input: []byte("Nu ll")}, + {name: "null1", input: []byte("null1")}, + {name: "double", input: []byte("null null")}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleInvalid(test, t) + }) + } +} + +func TestUnmarshal_BoolSimpleSuccess(t *testing.T) { + tests := []*testCase{ + {name: "lower true", input: []byte("true"), _type: Boolean, value: []byte("true")}, + {name: "lower false", input: []byte("false"), _type: Boolean, value: []byte("false")}, + {name: "spaces true", input: []byte(" true\r\n "), _type: Boolean, value: []byte("true")}, + {name: "spaces false", input: []byte(" false\r\n "), _type: Boolean, value: []byte("false")}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleValid(test, t) + }) + } +} + +func TestUnmarshal_BoolSimpleCorrupted(t *testing.T) { + tests := []*testCase{ + simpleCorrupted("tru"), + simpleCorrupted("fals"), + simpleCorrupted("tre"), + simpleCorrupted("fal se"), + simpleCorrupted("true false"), + simpleCorrupted("True"), + simpleCorrupted("TRUE"), + simpleCorrupted("False"), + simpleCorrupted("FALSE"), + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleInvalid(test, t) + }) + } +} + +func TestUnmarshal_ArraySimpleSuccess(t *testing.T) { + tests := []*testCase{ + {name: "[]", input: []byte("[]"), _type: Array, value: []byte("[]")}, + {name: "[1]", input: []byte("[1]"), _type: Array, value: []byte("[1]")}, + {name: "[1,2,3]", input: []byte("[1,2,3]"), _type: Array, value: []byte("[1,2,3]")}, + {name: "[1, 2, 3]", input: []byte("[1, 2, 3]"), _type: Array, value: []byte("[1, 2, 3]")}, + {name: "[1,[2],3]", input: []byte("[1,[2],3]"), _type: Array, value: []byte("[1,[2],3]")}, + {name: "[[],[],[]]", input: []byte("[[],[],[]]"), _type: Array, value: []byte("[[],[],[]]")}, + {name: "[[[[[]]]]]", input: []byte("[[[[[]]]]]"), _type: Array, value: []byte("[[[[[]]]]]")}, + {name: "[true,null,1,\"foo\",[]]", input: []byte("[true,null,1,\"foo\",[]]"), _type: Array, value: []byte("[true,null,1,\"foo\",[]]")}, + {name: "spaces", input: []byte("\n\r [\n1\n ]\r\n"), _type: Array, value: []byte("[\n1\n ]")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleValid(test, t) + }) + } +} + +func TestUnmarshal_ArraySimpleCorrupted(t *testing.T) { + tests := []*testCase{ + simpleCorrupted("[,]"), + simpleCorrupted("[]\\"), + simpleCorrupted("[1,]"), + simpleCorrupted("[[]"), + simpleCorrupted("[]]"), + simpleCorrupted("1[]"), + simpleCorrupted("[]1"), + simpleCorrupted("[[]1]"), + // simpleCorrupted("‌[],[]"), // invisible char + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + simpleInvalid(test, t) + }) + } +} + +// Examples from https://json.org/example.html +func TestUnmarshal(t *testing.T) { + tests := []struct { + name string + value string + }{ + { + name: "glossary", + value: `{ + "glossary": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } +}`, + }, + { + name: "menu", + value: `{"menu": { + "id": "file", + "value": "File", + "popup": { + "menuitem": [ + {"value": "New", "onclick": "CreateNewDoc()"}, + {"value": "Open", "onclick": "OpenDoc()"}, + {"value": "Close", "onclick": "CloseDoc()"} + ] + } +}}`, + }, + { + name: "widget", + value: `{"widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "name": "sun1", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "name": "text1", + "hOffset": 250, + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } +}} `, + }, + { + name: "web-app", + value: `{"web-app": { + "servlet": [ + { + "servlet-name": "cofaxCDS", + "servlet-class": "org.cofax.cds.CDSServlet", + "init-param": { + "configGlossary:installationAt": "Philadelphia, PA", + "configGlossary:adminEmail": "ksm@pobox.com", + "configGlossary:poweredBy": "Cofax", + "configGlossary:poweredByIcon": "/images/cofax.gif", + "configGlossary:staticPath": "/content/static", + "templateProcessorClass": "org.cofax.WysiwygTemplate", + "templateLoaderClass": "org.cofax.FilesTemplateLoader", + "templatePath": "templates", + "templateOverridePath": "", + "defaultListTemplate": "listTemplate.htm", + "defaultFileTemplate": "articleTemplate.htm", + "useJSP": false, + "jspListTemplate": "listTemplate.jsp", + "jspFileTemplate": "articleTemplate.jsp", + "cachePackageTagsTrack": 200, + "cachePackageTagsStore": 200, + "cachePackageTagsRefresh": 60, + "cacheTemplatesTrack": 100, + "cacheTemplatesStore": 50, + "cacheTemplatesRefresh": 15, + "cachePagesTrack": 200, + "cachePagesStore": 100, + "cachePagesRefresh": 10, + "cachePagesDirtyRead": 10, + "searchEngineListTemplate": "forSearchEnginesList.htm", + "searchEngineFileTemplate": "forSearchEngines.htm", + "searchEngineRobotsDb": "WEB-INF/robots.db", + "useDataStore": true, + "dataStoreClass": "org.cofax.SqlDataStore", + "redirectionClass": "org.cofax.SqlRedirection", + "dataStoreName": "cofax", + "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", + "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", + "dataStoreUser": "sa", + "dataStorePassword": "dataStoreTestQuery", + "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", + "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", + "dataStoreInitConns": 10, + "dataStoreMaxConns": 100, + "dataStoreConnUsageLimit": 100, + "dataStoreLogLevel": "debug", + "maxUrlLength": 500}}, + { + "servlet-name": "cofaxEmail", + "servlet-class": "org.cofax.cds.EmailServlet", + "init-param": { + "mailHost": "mail1", + "mailHostOverride": "mail2"}}, + { + "servlet-name": "cofaxAdmin", + "servlet-class": "org.cofax.cds.AdminServlet"}, + + { + "servlet-name": "fileServlet", + "servlet-class": "org.cofax.cds.FileServlet"}, + { + "servlet-name": "cofaxTools", + "servlet-class": "org.cofax.cms.CofaxToolsServlet", + "init-param": { + "templatePath": "toolstemplates/", + "log": 1, + "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", + "logMaxSize": "", + "dataLog": 1, + "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", + "dataLogMaxSize": "", + "removePageCache": "/content/admin/remove?cache=pages&id=", + "removeTemplateCache": "/content/admin/remove?cache=templates&id=", + "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", + "lookInContext": 1, + "adminGroupID": 4, + "betaServer": true}}], + "servlet-mapping": { + "cofaxCDS": "/", + "cofaxEmail": "/cofaxutil/aemail/*", + "cofaxAdmin": "/admin/*", + "fileServlet": "/static/*", + "cofaxTools": "/tools/*"}, + + "taglib": { + "taglib-uri": "cofax.tld", + "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}`, + }, + { + name: "SVG Viewer", + value: `{"menu": { + "header": "SVG Viewer", + "items": [ + {"id": "Open"}, + {"id": "OpenNew", "label": "Open New"}, + null, + {"id": "ZoomIn", "label": "Zoom In"}, + {"id": "ZoomOut", "label": "Zoom Out"}, + {"id": "OriginalView", "label": "Original View"}, + null, + {"id": "Quality"}, + {"id": "Pause"}, + {"id": "Mute"}, + null, + {"id": "Find", "label": "Find..."}, + {"id": "FindAgain", "label": "Find Again"}, + {"id": "Copy"}, + {"id": "CopyAgain", "label": "Copy Again"}, + {"id": "CopySVG", "label": "Copy SVG"}, + {"id": "ViewSVG", "label": "View SVG"}, + {"id": "ViewSource", "label": "View Source"}, + {"id": "SaveAs", "label": "Save As"}, + null, + {"id": "Help"}, + {"id": "About", "label": "About Adobe CVG Viewer..."} + ] +}}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := unmarshal2([]byte(test.value)) + if err != nil { + t.Errorf("Error on Unmarshal: %s", err.Error()) + } + }) + } +} + +var ( + jsonExample = []byte(`{ "store": { + "book": [ + { "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, + { "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + } +}`) +) + +func TestUnmarshalSafe(t *testing.T) { + safe, err := UnmarshalSafe(jsonExample) + if err != nil { + t.Errorf("Error on Unmarshal: %s", err.Error()) + } else if safe == nil { + t.Errorf("Error on Unmarshal: safe is nil") + } else { + root, err := unmarshal2(jsonExample) + if err != nil { + t.Errorf("Error on Unmarshal: %s", err.Error()) + } else if root == nil { + t.Errorf("Error on Unmarshal: root is nil") + } else if !bytes.Equal(root.Source(), safe.Source()) { + t.Errorf("Error on UnmarshalSafe: values not same") + } + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index c3398e1b2db..1cbb5e39f86 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -128,8 +128,8 @@ func decodeUnicodeEscape(b []byte) (rune, int) { return combineSurrogates(r, r2), 12 } -func unquote(bs []byte, border byte) (string, error) { - res, err := unquoteBytes(bs, border) +func unquote(bs []byte, unquoteToken byte) (string, error) { + res, err := unquoteBytes(bs, unquoteToken) if err != nil { return "", err } diff --git a/examples/gno.land/p/demo/json/internal.gno b/examples/gno.land/p/demo/json/internal.gno index 990f6cb0bf5..cae7fbaba7a 100644 --- a/examples/gno.land/p/demo/json/internal.gno +++ b/examples/gno.land/p/demo/json/internal.gno @@ -147,10 +147,10 @@ const ( cm States = -3 /* comma */ qt States = -4 /* quote */ bo States = -5 /* bracket open */ - co States = -6 /* curly br. open */ + co States = -6 /* curly bracket open */ bc States = -7 /* bracket close */ - cc States = -8 /* curly br. close */ - ec States = -9 /* curly br. empty */ + cc States = -8 /* curly bracket close */ + ec States = -9 /* curly bracket empty */ ) // StateTransitionTable is the state transition table takes the current state and the current symbol, and returns either diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index 74ea9f2f1b4..3ed7028b34b 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -31,28 +31,121 @@ func (nv *NodeValue) Load() interface{} { type Node struct { prev *Node next map[string]*Node - key string + key *string data []byte value *NodeValue index int - borders [2]int // start, end + borders [2]uint // start, end modified bool } -func (n *Node) Key() string { - if n == nil || n.key == "" { - return "" +func NewNode(prev *Node, b *buffer, typ ValueType, key *string) (*Node, error) { + curr := &Node{ + prev: prev, + data: b.data, + borders: [2]uint{b.index, 0}, + key: key, + value: &NodeValue{ + value: nil, + typ: typ, + }, + modified: false, + } + + if typ == Object || typ == Array { + curr.next = make(map[string]*Node) + } + + if prev != nil { + if prev.isArray() { + size := len(prev.next) + curr.index = size + + prev.next[strconv.Itoa(size)] = curr + } else if prev.isObject() { + if key == nil { + return nil, errors.New("key is required for object") + } + + prev.next[*key] = curr + } else { + return nil, errors.New("invalid parent type") + } + } + + return curr, nil +} + +func (n *Node) Key() *string { + if n == nil || n.key == nil { + return nil } return n.key } -func (n *Node) Type() string { - return n.value.typ.String() +func (n *Node) Type() ValueType { + return n.value.typ } -func (n *Node) Value() interface{} { - return n.value.Load() +// TODO: may replace parsePrimitive (parser.gno) +func (n *Node) Value() (value interface{}, err error) { + value = n.value.Load() + + if value == nil { + switch n.value.typ { + case Null: + return nil, nil + + case Number: + value, err = ParseFloatLiteral(n.Source()) + if err != nil { + return nil, err + } + + n.value.Store(value) + + case String: + value, err = unquote(n.Source(), DoublyQuoteToken) + if err != nil { + return nil, err + } + + n.value.Store(value) + + case Boolean: + if len(n.Source()) == 0 { + return nil, errors.New("empty boolean value") + } + + b := n.Source()[0] + value = b == 't' || b == 'T' + + n.value.Store(value) + + case Array: + elems := make([]*Node, len(n.next)) + + for _, e := range n.next { + elems[e.index] = e + } + + value = elems + n.value.Store(value) + + case Object: + obj := make(map[string]*Node, len(n.next)) + + for k, v := range n.next { + obj[k] = v + } + + value = obj + n.value.Store(value) + } + } + + return value, nil } func (n *Node) Size() uint { @@ -76,7 +169,7 @@ func (n *Node) Keys() []string { return result } -func NullNode(key string) *Node { +func NullNode(key *string) *Node { return &Node{ key: key, value: &NodeValue{value: nil, typ: Null}, @@ -84,18 +177,18 @@ func NullNode(key string) *Node { } } -func NumberNode(key string, value float64) *Node { +func NumberNode(key *string, value float64) *Node { return &Node{ key: key, value: &NodeValue{ value: value, - typ: Number, + typ: Number, // treat Float and Number as Number type }, modified: true, } } -func StringNode(key string, value string) *Node { +func StringNode(key *string, value string) *Node { val := newNodeValue(value) return &Node{ @@ -105,7 +198,7 @@ func StringNode(key string, value string) *Node { } } -func BoolNode(key string, value bool) *Node { +func BoolNode(key *string, value bool) *Node { val := newNodeValue(value) return &Node{ @@ -115,7 +208,7 @@ func BoolNode(key string, value bool) *Node { } } -func ArrayNode(key string, value []*Node) *Node { +func ArrayNode(key *string, value []*Node) *Node { val := newNodeValue(value) curr := &Node{ key: key, @@ -139,6 +232,39 @@ func ArrayNode(key string, value []*Node) *Node { return curr } +func ObjectNode(key *string, value map[string]*Node) *Node { + curr := &Node{ + key: key, + value: &NodeValue{ + value: value, + typ: Object, + }, + next: value, + modified: true, + } + + if value != nil { + curr.value.Store(value) + + for k, v := range value { + v.prev = curr + v.key = &k + } + } else { + curr.next = make(map[string]*Node) + } + + return curr +} + +func (n *Node) isArray() bool { + return n.value.typ == Array +} + +func (n *Node) isObject() bool { + return n.value.typ == Object +} + func (n *Node) ready() bool { return n.borders[1] != 0 } @@ -154,3 +280,192 @@ func (n *Node) Source() []byte { return nil } + +func (n *Node) root() *Node { + if n == nil { + return nil + } + + curr := n + for curr.prev != nil { + curr = curr.prev + } + + return curr +} + +func (n *Node) GetNull() (interface{}, error) { + if n == nil { + return nil, errors.New("node is nil") + } + + if n.value.typ != Null { + return nil, errors.New("node is not null") + } + + return nil, nil +} + +func (n *Node) GetNumeric() (float64, error) { + if n == nil { + return 0, errors.New("node is nil") + } + + if n.value.typ != Number { + return 0, errors.New("node is not number") + } + + val, err := n.Value() + if err != nil { + return 0, err + } + + v, ok := val.(float64) + if !ok { + return 0, errors.New("node is not number") + } + + return v, nil +} + +func (n *Node) GetString() (string, error) { + if n == nil { + return "", errors.New("string node is empty") + } + + if n.value.typ != String { + return "", errors.New("node type is not string") + } + + val, err := n.Value() + if err != nil { + return "", err + } + + v, ok := val.(string) + if !ok { + return "", errors.New("node is not string") + } + + return v, nil +} + +func (n *Node) getBool() (bool, error) { + if n == nil { + return false, errors.New("node is nil") + } + + if n.value.typ != Boolean { + return false, errors.New("node is not boolean") + } + + val, err := n.Value() + if err != nil { + return false, err + } + + v, ok := val.(bool) + if !ok { + return false, errors.New("node is not boolean") + } + + return v, nil +} + +func (n *Node) GetArray() ([]*Node, error) { + if n == nil { + return nil, errors.New("node is nil") + } + + if n.value.typ != Array { + return nil, errors.New("node is not array") + } + + val, err := n.Value() + if err != nil { + return nil, err + } + + v, ok := val.([]*Node) + if !ok { + return nil, errors.New("node is not array") + } + + return v, nil +} + +func (n *Node) GetObject() (map[string]*Node, error) { + if n == nil { + return nil, errors.New("node is nil") + } + + if n.value.typ != Object { + return nil, errors.New("node is not object") + } + + val, err := n.Value() + if err != nil { + return nil, err + } + + v, ok := val.(map[string]*Node) + if !ok { + return nil, errors.New("node is not object") + } + + return v, nil +} + +func (n *Node) MustNull() interface{} { + v, err := n.GetNull() + if err != nil { + panic(err) + } + + return v +} + +func (n *Node) MustNumeric() float64 { + v, err := n.GetNumeric() + if err != nil { + panic(err) + } + + return v +} + +func (n *Node) MustString() string { + v, err := n.GetString() + if err != nil { + panic(err) + } + + return v +} + +func (n *Node) MustBool() bool { + v, err := n.getBool() + if err != nil { + panic(err) + } + + return v +} + +func (n *Node) MustArray() []*Node { + v, err := n.GetArray() + if err != nil { + panic(err) + } + + return v +} + +func (n *Node) MustObject() map[string]*Node { + v, err := n.GetObject() + if err != nil { + panic(err) + } + + return v +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index bc86d21f84b..4595cef31ed 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -42,88 +42,6 @@ func TestNodeValue(t *testing.T) { } } -func TestNodeCreation(t *testing.T) { - tests := []struct { - name string - testFunc func() *Node - expected Node - }{ - { - name: "NullNode", - testFunc: func() *Node { - return NullNode("testNull") - }, - expected: Node{ - key: "testNull", - value: &NodeValue{value: nil, typ: Null}, - modified: true, - }, - }, - { - name: "NumberNode", - testFunc: func() *Node { - return NumberNode("testNumber", 42.0) - }, - expected: Node{ - key: "testNumber", - value: &NodeValue{value: 42.0, typ: Number}, - modified: true, - }, - }, - { - name: "StringNode", - testFunc: func() *Node { - return StringNode("testString", "Hello") - }, - expected: Node{ - key: "testString", - value: &NodeValue{value: "Hello", typ: String}, - modified: true, - }, - }, - { - name: "BoolNode", - testFunc: func() *Node { - return BoolNode("testBool", true) - }, - expected: Node{ - key: "testBool", - value: &NodeValue{value: true, typ: Boolean}, - modified: true, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - actual := tc.testFunc() - if actual.key != tc.expected.key { - t.Errorf("expected key %s, got %s", tc.expected.key, actual.key) - } - if actual.value.typ != tc.expected.value.typ { - t.Errorf("expected type %v, got %v", tc.expected.value.typ.String(), actual.value.typ.String()) - } - if actual.modified != tc.expected.modified { - t.Errorf("expected modified %t, got %t", tc.expected.modified, actual.modified) - } - // Special handling for comparing slices and maps - if tc.name == "ArrayNode" { - if len(actual.next) != len(tc.expected.next) { - t.Errorf("expected next length %d, got %d", len(tc.expected.next), len(actual.next)) - } - for k, v := range tc.expected.next { - if _, ok := actual.next[k]; !ok { - t.Errorf("expected next to have key %s", k) - } - if actual.next[k].key != v.key { - t.Errorf("expected next node with key %s, got %s", v.key, actual.next[k].key) - } - } - } - }) - } -} - func TestNodeSource(t *testing.T) { tests := []struct { name string @@ -139,7 +57,7 @@ func TestNodeSource(t *testing.T) { name: "ready unmodified node with data", node: &Node{ data: []byte("test data"), - borders: [2]int{0, 9}, + borders: [2]uint{0, 9}, modified: false, }, expected: []byte("test data"), @@ -148,7 +66,7 @@ func TestNodeSource(t *testing.T) { name: "ready modified node with data", node: &Node{ data: []byte("test data"), - borders: [2]int{0, 9}, + borders: [2]uint{0, 9}, modified: true, }, expected: nil, @@ -157,7 +75,7 @@ func TestNodeSource(t *testing.T) { name: "not ready node", node: &Node{ data: nil, - borders: [2]int{0, 0}, + borders: [2]uint{0, 0}, modified: false, }, expected: nil, diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 2e4a1ba9140..f4683541a1a 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -230,12 +230,14 @@ func ParseIntLiteral(bytes []byte) (v int64, err error) { // - All characters in the byte slice are digits or a single decimal point. // - The resulting mantissa does not overflow a uint64. func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { - var man uint64 - var exp10 int - decimalFound := false + var ( + man uint64 + exp10 int + decimalFound bool + ) for _, c := range bytes { - if c == '.' { + if c == DotToken { if decimalFound { return 0, 0, errors.New("JSON Error: multiple decimal points found while parsing float value") } @@ -326,8 +328,10 @@ func parseString(data []byte, offset int) (ValueType, int, error) { // parseContainer parses a JSON array or object and returns its type and the end position. func parseContainer(data []byte, offset int) (ValueType, int, error) { - var containerType ValueType - var closing byte + var ( + containerType ValueType + closing byte + ) if data[offset] == SquareOpenToken { containerType = Array @@ -384,3 +388,8 @@ func parsePrimitive(data []byte, offset int) (ValueType, int, error) { return Unknown, offset, errors.New("JSON Error: unknown value type found while parsing primitive value") } + +// TODO: may integrate this code with parsePrimitive in parser.go +func isNumeric(c byte) bool { + return bytes.Contains([]byte("0123456789+-.eE"), []byte{c}) +} From 42b4a150a69791811d55235901d712f6de56a12a Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Mar 2024 18:49:43 +0900 Subject: [PATCH 61/72] rewrite JSON code to avoid struct --- examples/gno.land/p/demo/json/buffer.gno | 159 +- examples/gno.land/p/demo/json/buffer_test.gno | 37 +- examples/gno.land/p/demo/json/decode.gno | 263 +-- examples/gno.land/p/demo/json/decode_test.gno | 396 +++-- .../gno.land/p/demo/json/eisel_lemire.gno | 82 - examples/gno.land/p/demo/json/encode.gno | 119 ++ examples/gno.land/p/demo/json/encode_test.gno | 256 +++ examples/gno.land/p/demo/json/escape.gno | 165 +- examples/gno.land/p/demo/json/escape_test.gno | 39 +- examples/gno.land/p/demo/json/indent.gno | 144 ++ examples/gno.land/p/demo/json/indent_test.gno | 84 + examples/gno.land/p/demo/json/node.gno | 1118 ++++++++++-- examples/gno.land/p/demo/json/node_test.gno | 1581 ++++++++++++++++- examples/gno.land/p/demo/json/parser.gno | 238 +-- examples/gno.land/p/demo/json/parser_test.gno | 168 +- examples/gno.land/p/demo/json/path.gno | 14 +- examples/gno.land/p/demo/json/path_test.gno | 14 + .../gno.land/p/demo/json/state_machine.gno | 755 -------- .../p/demo/json/state_machine_test.gno | 764 -------- examples/gno.land/p/demo/json/struct.gno | 369 ---- examples/gno.land/p/demo/json/struct_test.gno | 188 -- examples/gno.land/p/demo/json/token.gno | 74 +- examples/gno.land/p/demo/json/utils.gno | 87 - 23 files changed, 3934 insertions(+), 3180 deletions(-) create mode 100644 examples/gno.land/p/demo/json/encode.gno create mode 100644 examples/gno.land/p/demo/json/encode_test.gno create mode 100644 examples/gno.land/p/demo/json/indent.gno create mode 100644 examples/gno.land/p/demo/json/indent_test.gno delete mode 100644 examples/gno.land/p/demo/json/state_machine.gno delete mode 100644 examples/gno.land/p/demo/json/state_machine_test.gno delete mode 100644 examples/gno.land/p/demo/json/struct.gno delete mode 100644 examples/gno.land/p/demo/json/struct_test.gno delete mode 100644 examples/gno.land/p/demo/json/utils.gno diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index 8d61c6b5913..10faffeb35c 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -1,31 +1,28 @@ package json import ( - "bytes" "errors" - "strings" "io" + "strings" "gno.land/p/demo/ufmt" ) type buffer struct { data []byte - length uint - index uint + length int + index int last States state States class Classes } -type tokens []string - // newBuffer creates a new buffer with the given data -func NewBuffer(data []byte) *buffer { +func newBuffer(data []byte) *buffer { return &buffer{ data: data, - length: uint(len(data)), + length: len(data), last: GO, state: GO, } @@ -33,10 +30,10 @@ func NewBuffer(data []byte) *buffer { // first retrieves the first non-whitespace (or other escaped) character in the buffer. func (b *buffer) first() (byte, error) { - for _; b.index < b.length; b.index++ { + for ; b.index < b.length; b.index++ { c := b.data[b.index] - if !(c == WhiteSpaceToken || c == CarriageReturnToken || c == NewLineToken || c == TabToken) { + if !(c == whiteSpace || c == carriageReturn || c == newLine || c == tab) { return c, nil } } @@ -66,7 +63,7 @@ func (b *buffer) step() error { } // move moves the index by the given position. -func (b *buffer) move(pos uint) error { +func (b *buffer) move(pos int) error { newIndex := b.index + pos if newIndex > b.length { @@ -79,7 +76,7 @@ func (b *buffer) move(pos uint) error { } // slice returns the slice from the current index to the given position. -func (b *buffer) slice(pos uint) ([]byte, error) { +func (b *buffer) slice(pos int) ([]byte, error) { end := b.index + pos if end > b.length { @@ -90,7 +87,7 @@ func (b *buffer) slice(pos uint) ([]byte, error) { } // sliceFromIndices returns a slice of the buffer's data starting from 'start' up to (but not including) 'stop'. -func (b *buffer) sliceFromIndices(start, stop uint) []byte { +func (b *buffer) sliceFromIndices(start, stop int) []byte { if start > b.length { start = b.length } @@ -131,15 +128,14 @@ func (b *buffer) skipAny(endTokens map[byte]bool) error { tokens = append(tokens, string(token)) } - return errors.New( - ufmt.Sprintf( - "EOF reached before encountering one of the expected tokens: %s", - strings.Join(tokens, ", "), - )) + return errors.New(ufmt.Sprintf( + "EOF reached before encountering one of the expected tokens: %s", + strings.Join(tokens, ", "), + )) } // skipAndReturnIndex moves the buffer index forward by one and returns the new index. -func (b *buffer) skipAndReturnIndex() (uint, error) { +func (b *buffer) skipAndReturnIndex() (int, error) { err := b.step() if err != nil { return 0, err @@ -149,7 +145,7 @@ func (b *buffer) skipAndReturnIndex() (uint, error) { } // skipUntil moves the buffer index forward until it encounters a byte contained in the endTokens set. -func (b *buffer) skipUntil(endTokens map[byte]bool) (uint, error) { +func (b *buffer) skipUntil(endTokens map[byte]bool) (int, error) { for b.index < b.length { currentByte, err := b.current() if err != nil { @@ -170,11 +166,18 @@ func (b *buffer) skipUntil(endTokens map[byte]bool) (uint, error) { // significantTokens is a map where the keys are the significant characters in a JSON path. // The values in the map are all true, which allows us to use the map as a set for quick lookups. var significantTokens = map[byte]bool{ - DotToken: true, // access properties of an object - DollarToken: true, // root object - AtToken: true, // current object in a filter expression - SquareOpenToken: true, // start of an array index or filter expression - SquareCloseToken: true, // end of an array index or filter expression + dot: true, // access properties of an object + dollarSign: true, // root object + atSign: true, // current object + bracketOpen: true, // start of an array index or filter expression + bracketClose: true, // end of an array index or filter expression +} + +// filterTokens stores the filter expression tokens. +var filterTokens = map[byte]bool{ + aesterisk: true, // wildcard + andSign: true, + orSign: true, } // skipToNextSignificantToken advances the buffer index to the next significant character. @@ -202,7 +205,7 @@ func (b *buffer) backslash() bool { count := 0 for i := b.index - 1; ; i-- { - if i >= b.length || b.data[i] != BackSlashToken { + if i >= b.length || b.data[i] != backSlash { break } @@ -216,10 +219,24 @@ func (b *buffer) backslash() bool { return count%2 != 0 } -func (b *buffer) reset() { - b.last = GO +// numIndex holds a map of valid numeric characters +var numIndex = map[byte]bool{ + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + '.': true, + 'e': true, + 'E': true, } +// pathToken checks if the current token is a valid JSON path token. func (b *buffer) pathToken() error { var stack []byte @@ -231,7 +248,7 @@ func (b *buffer) pathToken() error { c := b.data[b.index] switch { - case c == DoublyQuoteToken || c == QuoteToken: + case c == doubleQuote || c == singleQuote: inToken = true if err := b.step(); err != nil { return errors.New("error stepping through buffer") @@ -245,25 +262,25 @@ func (b *buffer) pathToken() error { return errors.New("unmatched quote in path") } - case c == SquareOpenToken || c == RoundOpenToken: + case c == bracketOpen || c == parenOpen: inToken = true stack = append(stack, c) - case c == SquareCloseToken || c == RoundCloseToken: + case c == bracketClose || c == parenClose: inToken = true - if len(stack) == 0 || (c == SquareCloseToken && stack[len(stack)-1] != SquareOpenToken) || (c == RoundCloseToken && stack[len(stack)-1] != RoundOpenToken) { + if len(stack) == 0 || (c == bracketClose && stack[len(stack)-1] != bracketOpen) || (c == parenClose && stack[len(stack)-1] != parenOpen) { return errors.New("mismatched bracket or parenthesis") } stack = stack[:len(stack)-1] - case bytes.ContainsAny([]byte{DotToken, CommaToken, DollarToken, AtToken, AesteriskToken, AndToken, OrToken}, string(c)) || ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || ('0' <= c && c <= '9'): + case pathStateContainsValidPathToken(c): inToken = true - case c == PlusToken || c == MinusToken: - if inNumber || (b.index > 0 && bytes.ContainsAny([]byte("eE"), string(b.data[b.index-1]))) { + case c == plus || c == minus: + if inNumber || (b.index > 0 && numIndex[b.data[b.index-1]]) { inToken = true - } else if !inToken && (b.index+1 < b.length && bytes.IndexByte([]byte("0123456789"), b.data[b.index+1]) != -1) { + } else if !inToken && (b.index+1 < b.length && numIndex[b.data[b.index+1]]) { inToken = true inNumber = true } else if !inToken { @@ -290,20 +307,40 @@ end: return errors.New("no token found") } - if inNumber && !bytes.ContainsAny([]byte("0123456789.eE"), string(b.data[b.index-1])) { + if inNumber && !numIndex[b.data[b.index-1]] { inNumber = false } return nil } +func pathStateContainsValidPathToken(c byte) bool { + if _, ok := significantTokens[c]; ok { + return true + } + + if _, ok := filterTokens[c]; ok { + return true + } + + if _, ok := numIndex[c]; ok { + return true + } + + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' { + return true + } + + return false +} + func (b *buffer) numeric(token bool) error { if token { b.last = GO } - for _; b.index < b.length; b.index++ { - b.class = b.getClasses(DoublyQuoteToken) + for ; b.index < b.length; b.index++ { + b.class = b.getClasses(doubleQuote) if b.class == __ { return errors.New("invalid token found while parsing path") } @@ -340,7 +377,7 @@ func (b *buffer) getClasses(c byte) Classes { return C_ETC } - if c == QuoteToken { + if c == singleQuote { return QuoteAsciiClasses[b.data[b.index]] } @@ -350,7 +387,7 @@ func (b *buffer) getClasses(c byte) Classes { func (b *buffer) getState() States { b.last = b.state - b.class = b.getClasses(DoublyQuoteToken) + b.class = b.getClasses(doubleQuote) if b.class == __ { return __ } @@ -360,12 +397,13 @@ func (b *buffer) getState() States { return b.state } +// string parses a string token from the buffer. func (b *buffer) string(search byte, token bool) error { if token { b.last = GO } - for _; b.index < b.length; b.index++ { + for ; b.index < b.length; b.index++ { b.class = b.getClasses(search) if b.class == __ { @@ -393,7 +431,7 @@ func (b *buffer) word(bs []byte) error { max := len(bs) index := 0 - for _; b.index < b.length; b.index++ { + for ; b.index < b.length; b.index++ { c = b.data[b.index] if c != bs[index] { @@ -411,4 +449,37 @@ func (b *buffer) word(bs []byte) error { } return nil -} \ No newline at end of file +} + +func numberKind2f64(value interface{}) (result float64, err error) { + switch typed := value.(type) { + case float64: + result = typed + case float32: + result = float64(typed) + case int: + result = float64(typed) + case int8: + result = float64(typed) + case int16: + result = float64(typed) + case int32: + result = float64(typed) + case int64: + result = float64(typed) + case uint: + result = float64(typed) + case uint8: + result = float64(typed) + case uint16: + result = float64(typed) + case uint32: + result = float64(typed) + case uint64: + result = float64(typed) + default: + err = errors.New(ufmt.Sprintf("invalid number type: %T", value)) + } + + return +} diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno index b86574f03b3..a1acce4eba0 100644 --- a/examples/gno.land/p/demo/json/buffer_test.gno +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -1,7 +1,6 @@ package json import ( - "errors" "testing" ) @@ -115,7 +114,7 @@ func TestBufferSlice(t *testing.T) { tests := []struct { name string buffer *buffer - pos uint + pos int want []byte wantErr bool }{ @@ -181,9 +180,9 @@ func TestBufferMove(t *testing.T) { tests := []struct { name string buffer *buffer - pos uint + pos int wantErr bool - wantIdx uint + wantIdx int }{ { name: "Valid move", @@ -280,7 +279,7 @@ func TestSkipToNextSignificantToken(t *testing.T) { tests := []struct { name string input []byte - expected uint + expected int }{ {"No significant chars", []byte("abc"), 3}, {"One significant char at start", []byte(".abc"), 0}, @@ -293,7 +292,7 @@ func TestSkipToNextSignificantToken(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := NewBuffer(tt.input) + b := newBuffer(tt.input) b.skipToNextSignificantToken() if b.index != tt.expected { t.Errorf("after skipToNextSignificantToken(), got index = %v, want %v", b.index, tt.expected) @@ -302,17 +301,15 @@ func TestSkipToNextSignificantToken(t *testing.T) { } } -const testString = "abcdefg" - func mockBuffer(s string) *buffer { - return NewBuffer([]byte(s)) + return newBuffer([]byte(s)) } func TestSkipAndReturnIndex(t *testing.T) { tests := []struct { name string input string - expected uint + expected int }{ {"StartOfString", "", 0}, {"MiddleOfString", "abcdef", 1}, @@ -338,7 +335,7 @@ func TestSkipUntil(t *testing.T) { name string input string tokens map[byte]bool - expected uint + expected int }{ {"SkipToToken", "abcdefg", map[byte]bool{'c': true}, 2}, {"SkipToEnd", "abcdefg", map[byte]bool{'h': true}, 7}, @@ -349,7 +346,7 @@ func TestSkipUntil(t *testing.T) { t.Run(tt.name, func(t *testing.T) { buf := mockBuffer(tt.input) got, err := buf.skipUntil(tt.tokens) - if err != nil && got != uint(len(tt.input)) { // Expect error only if reached end without finding token + if err != nil && got != len(tt.input) { // Expect error only if reached end without finding token t.Errorf("skipUntil() error = %v", err) } if got != tt.expected { @@ -363,8 +360,8 @@ func TestSliceFromIndices(t *testing.T) { tests := []struct { name string input string - start uint - end uint + start int + end int expected string }{ {"FullString", "abcdefg", 0, 7, "abcdefg"}, @@ -387,7 +384,7 @@ func TestBufferToken(t *testing.T) { tests := []struct { name string path string - index uint + index int isErr bool }{ { @@ -562,7 +559,7 @@ func TestBufferToken(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - buf := NewBuffer([]byte(tt.path)) + buf := newBuffer([]byte(tt.path)) err := buf.pathToken() if tt.isErr { @@ -604,15 +601,15 @@ func TestBufferFirst(t *testing.T) { expected: 0, }, { - name: "whitespace in middle", - data: []byte("hello world"), + name: "whitespace in middle", + data: []byte("hello world"), expected: 'h', }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := NewBuffer(tt.data) + b := newBuffer(tt.data) got, err := b.first() if err != nil && tt.expected != 0 { @@ -624,4 +621,4 @@ func TestBufferFirst(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/examples/gno.land/p/demo/json/decode.gno b/examples/gno.land/p/demo/json/decode.gno index 1f2009dbffb..79f287433f7 100644 --- a/examples/gno.land/p/demo/json/decode.gno +++ b/examples/gno.land/p/demo/json/decode.gno @@ -7,18 +7,40 @@ import ( "gno.land/p/demo/ufmt" ) -func unmarshal2(data []byte) (*Node, error) { - buf := NewBuffer(data) +// This limits the max nesting depth to prevent stack overflow. +// This is permitted by https://tools.ietf.org/html/rfc7159#section-9 +const maxNestingDepth = 10000 + +// State machine transition logic and grammar references +// [1] https://github.com/spyzhov/ajson/blob/master/decode.go +// [2] https://cs.opensource.google/go/go/+/refs/tags/go1.22.1:src/encoding/json/scanner.go + +// Unmarshal parses the JSON-encoded data and returns a Node. +// The data must be a valid JSON-encoded value. +// +// Usage: +// node, err := json.Unmarshal([]byte(`{"key": "value"}`)) +// if err != nil { +// ufmt.Println(err) +// } +// println(node) // {"key": "value"} +func Unmarshal(data []byte) (*Node, error) { + buf := newBuffer(data) var ( state States - key string + key *string current *Node - hasKey bool + nesting int + useKey = func() **string { + tmp := cptrs(key) + key = nil + return &tmp + } + err error ) - _, err := buf.first() - if err != nil { + if _, err = buf.first(); err != nil { return nil, io.EOF } @@ -28,33 +50,37 @@ func unmarshal2(data []byte) (*Node, error) { return nil, unexpectedTokenError(buf.data, buf.index) } + // region state machine if state >= GO { switch buf.state { - case ST: - if current != nil && current.isObject() && !hasKey { + case ST: // string + if current != nil && current.IsObject() && key == nil { // key detected - key, err = getString(buf) - hasKey = true + if key, err = getString(buf); err != nil { + return nil, err + } + buf.state = CO } else { - // string detected - err = createOrUpdateNode(¤t, buf, String, &hasKey, &key) + if nesting, err = checkNestingDepth(nesting); err != nil { + return nil, err + } + + current, err = createNode(current, buf, String, useKey()) if err != nil { return nil, err } - if err = buf.string(DoublyQuoteToken, false); err != nil { + if err = buf.string(doubleQuote, false); err != nil { return nil, err } - current.borders[1] = buf.index + 1 + current, nesting = updateNode(current, buf, nesting, true) buf.state = OK - - updateCurrentNode(¤t) } - case MI, ZE, IN: - if err = createOrUpdateNode(¤t, buf, Number, &hasKey, &key); err != nil { + case MI, ZE, IN: // number + if current, err = createNode(current, buf, Number, useKey()); err != nil { return nil, err } @@ -63,33 +89,34 @@ func unmarshal2(data []byte) (*Node, error) { } current.borders[1] = buf.index + if current.prev != nil { + current = current.prev + } + buf.index -= 1 buf.state = OK - updateCurrentNode(¤t) - - case T1, F1: - if err = createOrUpdateNode(¤t, buf, Boolean, &hasKey, &key); err != nil { + case T1, F1: // boolean + if current, err = createNode(current, buf, Boolean, useKey()); err != nil { return nil, err } + var literal []byte if buf.state == T1 { - err = buf.word(trueLiteral) + literal = trueLiteral } else { - err = buf.word(falseLiteral) + literal = falseLiteral } - if err != nil { + if err = buf.word(literal); err != nil { return nil, err } - current.borders[1] = buf.index + 1 + current, nesting = updateNode(current, buf, nesting, false) buf.state = OK - updateCurrentNode(¤t) - - case N1: - if err = createOrUpdateNode(¤t, buf, Null, &hasKey, &key); err != nil { + case N1: // null + if current, err = createNode(current, buf, Null, useKey()); err != nil { return nil, err } @@ -97,79 +124,76 @@ func unmarshal2(data []byte) (*Node, error) { return nil, err } - current.borders[1] = buf.index + 1 + current, nesting = updateNode(current, buf, nesting, false) buf.state = OK - - updateCurrentNode(¤t) } } else { + // region action switch state { - case ec, cc: - if current != nil && current.isObject() && !current.ready() { - current.borders[1] = buf.index + 1 - updateCurrentNode(¤t) - } else { - if err = unexpectedTokenError(buf.data, buf.index); err != nil { - return nil, err - } + case ec, cc: // } + if key != nil { + return nil, unexpectedTokenError(buf.data, buf.index) + } + + if !isValidContainerType(current, Object) { + return nil, unexpectedTokenError(buf.data, buf.index) } + current, nesting = updateNode(current, buf, nesting, true) buf.state = OK - case bc: - if current != nil && current.isArray() && !current.ready() { - current.borders[1] = buf.index + 1 - updateCurrentNode(¤t) - } else { - if err = unexpectedTokenError(buf.data, buf.index); err != nil { - return nil, err - } + case bc: // ] + if !isValidContainerType(current, Array) { + return nil, unexpectedTokenError(buf.data, buf.index) } + current, nesting = updateNode(current, buf, nesting, true) buf.state = OK - case co: - if err = createOrUpdateNode(¤t, buf, Object, &hasKey, &key); err != nil { + case co: // { + if nesting, err = checkNestingDepth(nesting); err != nil { + return nil, err + } + + if current, err = createNode(current, buf, Object, useKey()); err != nil { return nil, err } buf.state = OB - case bo: - if err = createOrUpdateNode(¤t, buf, Array, &hasKey, &key); err != nil { + case bo: // [ + if nesting, err = checkNestingDepth(nesting); err != nil { + return nil, err + } + + if current, err = createNode(current, buf, Array, useKey()); err != nil { return nil, err } buf.state = AR - case cm: + case cm: // , if current == nil { return nil, unexpectedTokenError(buf.data, buf.index) } - if current.isObject() { + if current.IsObject() { buf.state = KE // key expected - } else if current.isArray() { + } else if current.IsArray() { buf.state = VA // value expected } else { - if err = unexpectedTokenError(buf.data, buf.index); err != nil { - return nil, err - } + return nil, unexpectedTokenError(buf.data, buf.index) } - case cl: - if current == nil || !current.isObject() || !hasKey { - if err = unexpectedTokenError(buf.data, buf.index); err != nil { - return nil, err - } + case cl: // : + if current == nil || !current.IsObject() || key == nil { + return nil, unexpectedTokenError(buf.data, buf.index) } buf.state = VA default: - if err = unexpectedTokenError(buf.data, buf.index); err != nil { - return nil, err - } + return nil, unexpectedTokenError(buf.data, buf.index) } } @@ -183,71 +207,94 @@ func unmarshal2(data []byte) (*Node, error) { } } - return checkRootNode(current, buf) + if current == nil || buf.state != OK { + return nil, io.EOF + } + + root := current.root() + if !root.ready() { + return nil, io.EOF + } + + return root, err } -func UnmarshalSafe(data []byte) (*Node, error) { - var safe []byte - safe = append(safe, data...) - return unmarshal2(safe) +func isValidContainerType(current *Node, nodeType ValueType) bool { + switch nodeType { + case Object: + return current != nil && current.IsObject() && !current.ready() + case Array: + return current != nil && current.IsArray() && !current.ready() + default: + return false + } } -func Must(root *Node, expect error) *Node { - if expect != nil { - panic(expect) +// getString extracts a string from the buffer and advances the buffer index past the string. +func getString(b *buffer) (*string, error) { + start := b.index + if err := b.string(doubleQuote, false); err != nil { + return nil, err + } + + value, ok := Unquote(b.data[start:b.index+1], doubleQuote) + if !ok { + return nil, unexpectedTokenError(b.data, start) } - return root + return &value, nil } -// TODO: improve this function to avoid pointer dereferencing -func createOrUpdateNode(curr **Node, buf *buffer, t ValueType, hasKey *bool, key *string) error { - var err error +func unexpectedTokenError(data []byte, index int) error { + return errors.New(ufmt.Sprintf("unexpected token at index %d. data %b", index, data)) +} - if *hasKey { - *curr, err = NewNode(*curr, buf, t, key) - *hasKey = false - } else { - *curr, err = NewNode(*curr, buf, t, nil) +func createNode(current *Node, buf *buffer, nodeType ValueType, key **string) (*Node, error) { + var err error + current, err = NewNode(current, buf, nodeType, key) + if err != nil { + return nil, err } - return err + return current, nil } -// getString extracts a string from the buffer and advances the buffer index past the string. -func getString(b *buffer) (string, error) { - if err := b.string(DoublyQuoteToken, false); err != nil { - return "", err +func updateNode(current *Node, buf *buffer, nesting int, decreaseLevel bool) (*Node, int) { + current.borders[1] = buf.index + 1 + + prev := current.prev + if prev == nil { + return current, nesting } - start := b.index - value, err := unquote(b.data[start:start+1], DoublyQuoteToken) - if err != nil { - return "", errors.New(ufmt.Sprintf("token %s is not a valid string", b.data[start:start+1])) + current = prev + if decreaseLevel { + nesting-- } - return value, nil + return current, nesting } -func updateCurrentNode(current **Node) { - if (*current).prev != nil { - *current = (*current).prev +func checkNestingDepth(nesting int) (int, error) { + if nesting >= maxNestingDepth { + return nesting, errors.New("maximum nesting depth exceeded") } + + return nesting + 1, nil } -func checkRootNode(current *Node, buf *buffer) (*Node, error) { - if current == nil || buf.state != OK { - return nil, io.EOF +func cptrs(cpy *string) *string { + if cpy == nil { + return nil } - root := current.root() - if !root.ready() { - return nil, io.EOF - } + val := *cpy - return root, nil + return &val } -func unexpectedTokenError(data []byte, index uint) error { - return errors.New(ufmt.Sprintf("unexpected token %s", data[index:index+1])) -} \ No newline at end of file +func UnmarshalSafe(data []byte) (*Node, error) { + var safe []byte + safe = append(safe, data...) + return Unmarshal(safe) +} diff --git a/examples/gno.land/p/demo/json/decode_test.gno b/examples/gno.land/p/demo/json/decode_test.gno index d301bb26f88..c731164feb1 100644 --- a/examples/gno.land/p/demo/json/decode_test.gno +++ b/examples/gno.land/p/demo/json/decode_test.gno @@ -2,44 +2,45 @@ package json import ( "bytes" + "encoding/json" "testing" ) -type testCase struct { +type testNode struct { name string input []byte _type ValueType value []byte } -func simpleValid(test *testCase, t *testing.T) { - root, err := unmarshal2(test.input) +func simpleValid(test *testNode, t *testing.T) { + root, err := Unmarshal(test.input) if err != nil { - t.Errorf("Error on Unmarshal(%s): %s", test.name, err.Error()) + t.Errorf("Error on Unmarshal(%s): %s", test.input, err.Error()) } else if root == nil { t.Errorf("Error on Unmarshal(%s): root is nil", test.name) - } else if root.value.typ != test._type { + } else if root.nodeType != test._type { t.Errorf("Error on Unmarshal(%s): wrong type", test.name) - } else if !bytes.Equal(root.Source(), test.value) { - t.Errorf("Error on Unmarshal(%s): %s != %s", test.name, root.Source(), test.value) + } else if !bytes.Equal(root.source(), test.value) { + t.Errorf("Error on Unmarshal(%s): %s != %s", test.name, root.source(), test.value) } } -func simpleInvalid(test *testCase, t *testing.T) { - root, err := unmarshal2(test.input) +func simpleInvalid(test *testNode, t *testing.T) { + root, err := Unmarshal(test.input) if err == nil { - t.Errorf("Error on Unmarshal(%s): error expected, got '%s'", test.name, root.Source()) + t.Errorf("Error on Unmarshal(%s): error expected, got '%s'", test.name, root.source()) } else if root != nil { t.Errorf("Error on Unmarshal(%s): root is not nil", test.name) } } -func simpleCorrupted(name string) *testCase { - return &testCase{name: name, input: []byte(name)} +func simpleCorrupted(name string) *testNode { + return &testNode{name: name, input: []byte(name)} } func TestUnmarshal_StringSimpleSuccess(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ {name: "blank", input: []byte("\"\""), _type: String, value: []byte("\"\"")}, {name: "char", input: []byte("\"c\""), _type: String, value: []byte("\"c\"")}, {name: "word", input: []byte("\"cat\""), _type: String, value: []byte("\"cat\"")}, @@ -55,7 +56,7 @@ func TestUnmarshal_StringSimpleSuccess(t *testing.T) { } func TestUnmarshal_NumericSimpleSuccess(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ {name: "1", input: []byte("1"), _type: Number, value: []byte("1")}, {name: "-1", input: []byte("-1"), _type: Number, value: []byte("-1")}, @@ -90,15 +91,22 @@ func TestUnmarshal_NumericSimpleSuccess(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - simpleValid(test, t) + root, err := Unmarshal(test.input) + if err != nil { + t.Errorf("Error on Unmarshal(%s): %s", test.name, err.Error()) + } else if root == nil { + t.Errorf("Error on Unmarshal(%s): root is nil", test.name) + } else if root.nodeType != test._type { + t.Errorf("Error on Unmarshal(%s): wrong type", test.name) + } else if !bytes.Equal(root.source(), test.value) { + t.Errorf("Error on Unmarshal(%s): %s != %s", test.name, root.source(), test.value) + } }) } } func TestUnmarshal_StringSimpleCorrupted(t *testing.T) { - tests := []*testCase{ - // {name: "one quote", input: []byte("\"")}, - // {name: "one quote char", input: []byte("\"c")}, + tests := []*testNode{ {name: "white NL", input: []byte("\"foo\nbar\"")}, {name: "white R", input: []byte("\"foo\rbar\"")}, {name: "white Tab", input: []byte("\"foo\tbar\"")}, @@ -114,7 +122,7 @@ func TestUnmarshal_StringSimpleCorrupted(t *testing.T) { } func TestUnmarshal_ObjectSimpleSuccess(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ {name: "{}", input: []byte("{}"), _type: Object, value: []byte("{}")}, {name: `{ \r\n }`, input: []byte("{ \r\n }"), _type: Object, value: []byte("{ \r\n }")}, {name: `{"key":1}`, input: []byte(`{"key":1}`), _type: Object, value: []byte(`{"key":1}`)}, @@ -134,9 +142,17 @@ func TestUnmarshal_ObjectSimpleSuccess(t *testing.T) { } func TestUnmarshal_ObjectSimpleCorrupted(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ + simpleCorrupted("{{{\"key\": \"foo\"{{{{"), + simpleCorrupted("}"), + simpleCorrupted("{ }}}}}}}"), + simpleCorrupted(" }"), simpleCorrupted("{,}"), simpleCorrupted("{:}"), + simpleCorrupted("{100000}"), + simpleCorrupted("{1:1}"), + simpleCorrupted("{'1:2,3:4'}"), + simpleCorrupted(`{"d"}`), simpleCorrupted(`{"foo"}`), simpleCorrupted(`{"foo":}`), simpleCorrupted(`{:"foo"}`), @@ -162,8 +178,10 @@ func TestUnmarshal_ObjectSimpleCorrupted(t *testing.T) { } func TestUnmarshal_NullSimpleCorrupted(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ {name: "nul", input: []byte("nul")}, + {name: "nil", input: []byte("nil")}, + {name: "nill", input: []byte("nill")}, {name: "NILL", input: []byte("NILL")}, {name: "Null", input: []byte("Null")}, {name: "NULL", input: []byte("NULL")}, @@ -180,7 +198,7 @@ func TestUnmarshal_NullSimpleCorrupted(t *testing.T) { } func TestUnmarshal_BoolSimpleSuccess(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ {name: "lower true", input: []byte("true"), _type: Boolean, value: []byte("true")}, {name: "lower false", input: []byte("false"), _type: Boolean, value: []byte("false")}, {name: "spaces true", input: []byte(" true\r\n "), _type: Boolean, value: []byte("true")}, @@ -195,7 +213,7 @@ func TestUnmarshal_BoolSimpleSuccess(t *testing.T) { } func TestUnmarshal_BoolSimpleCorrupted(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ simpleCorrupted("tru"), simpleCorrupted("fals"), simpleCorrupted("tre"), @@ -215,7 +233,7 @@ func TestUnmarshal_BoolSimpleCorrupted(t *testing.T) { } func TestUnmarshal_ArraySimpleSuccess(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ {name: "[]", input: []byte("[]"), _type: Array, value: []byte("[]")}, {name: "[1]", input: []byte("[1]"), _type: Array, value: []byte("[1]")}, {name: "[1,2,3]", input: []byte("[1,2,3]"), _type: Array, value: []byte("[1,2,3]")}, @@ -226,6 +244,7 @@ func TestUnmarshal_ArraySimpleSuccess(t *testing.T) { {name: "[true,null,1,\"foo\",[]]", input: []byte("[true,null,1,\"foo\",[]]"), _type: Array, value: []byte("[true,null,1,\"foo\",[]]")}, {name: "spaces", input: []byte("\n\r [\n1\n ]\r\n"), _type: Array, value: []byte("[\n1\n ]")}, } + for _, test := range tests { t.Run(test.name, func(t *testing.T) { simpleValid(test, t) @@ -234,7 +253,7 @@ func TestUnmarshal_ArraySimpleSuccess(t *testing.T) { } func TestUnmarshal_ArraySimpleCorrupted(t *testing.T) { - tests := []*testCase{ + tests := []*testNode{ simpleCorrupted("[,]"), simpleCorrupted("[]\\"), simpleCorrupted("[1,]"), @@ -243,8 +262,8 @@ func TestUnmarshal_ArraySimpleCorrupted(t *testing.T) { simpleCorrupted("1[]"), simpleCorrupted("[]1"), simpleCorrupted("[[]1]"), - // simpleCorrupted("‌[],[]"), // invisible char } + for _, test := range tests { t.Run(test.name, func(t *testing.T) { simpleInvalid(test, t) @@ -252,15 +271,7 @@ func TestUnmarshal_ArraySimpleCorrupted(t *testing.T) { } } -// Examples from https://json.org/example.html -func TestUnmarshal(t *testing.T) { - tests := []struct { - name string - value string - }{ - { - name: "glossary", - value: `{ +var glossary = `{ "glossary": { "title": "example glossary", "GlossDiv": { @@ -281,25 +292,21 @@ func TestUnmarshal(t *testing.T) { } } } -}`, - }, - { - name: "menu", - value: `{"menu": { - "id": "file", - "value": "File", - "popup": { - "menuitem": [ - {"value": "New", "onclick": "CreateNewDoc()"}, - {"value": "Open", "onclick": "OpenDoc()"}, - {"value": "Close", "onclick": "CloseDoc()"} - ] - } -}}`, - }, - { - name: "widget", - value: `{"widget": { +}` + +var file = `{"menu": { + "id": "file", + "value": "File", + "popup": { + "menuitem": [ + {"value": "New", "onclick": "CreateNewDoc()"}, + {"value": "Open", "onclick": "OpenDoc()"}, + {"value": "Close", "onclick": "CloseDoc()"} + ] + } +}}` + +var widget = `{"widget": { "debug": "on", "window": { "title": "Sample Konfabulator Widget", @@ -324,102 +331,98 @@ func TestUnmarshal(t *testing.T) { "alignment": "center", "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" } -}} `, - }, - { - name: "web-app", - value: `{"web-app": { - "servlet": [ - { - "servlet-name": "cofaxCDS", - "servlet-class": "org.cofax.cds.CDSServlet", - "init-param": { - "configGlossary:installationAt": "Philadelphia, PA", - "configGlossary:adminEmail": "ksm@pobox.com", - "configGlossary:poweredBy": "Cofax", - "configGlossary:poweredByIcon": "/images/cofax.gif", - "configGlossary:staticPath": "/content/static", - "templateProcessorClass": "org.cofax.WysiwygTemplate", - "templateLoaderClass": "org.cofax.FilesTemplateLoader", - "templatePath": "templates", - "templateOverridePath": "", - "defaultListTemplate": "listTemplate.htm", - "defaultFileTemplate": "articleTemplate.htm", - "useJSP": false, - "jspListTemplate": "listTemplate.jsp", - "jspFileTemplate": "articleTemplate.jsp", - "cachePackageTagsTrack": 200, - "cachePackageTagsStore": 200, - "cachePackageTagsRefresh": 60, - "cacheTemplatesTrack": 100, - "cacheTemplatesStore": 50, - "cacheTemplatesRefresh": 15, - "cachePagesTrack": 200, - "cachePagesStore": 100, - "cachePagesRefresh": 10, - "cachePagesDirtyRead": 10, - "searchEngineListTemplate": "forSearchEnginesList.htm", - "searchEngineFileTemplate": "forSearchEngines.htm", - "searchEngineRobotsDb": "WEB-INF/robots.db", - "useDataStore": true, - "dataStoreClass": "org.cofax.SqlDataStore", - "redirectionClass": "org.cofax.SqlRedirection", - "dataStoreName": "cofax", - "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", - "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", - "dataStoreUser": "sa", - "dataStorePassword": "dataStoreTestQuery", - "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", - "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", - "dataStoreInitConns": 10, - "dataStoreMaxConns": 100, - "dataStoreConnUsageLimit": 100, - "dataStoreLogLevel": "debug", - "maxUrlLength": 500}}, - { - "servlet-name": "cofaxEmail", - "servlet-class": "org.cofax.cds.EmailServlet", - "init-param": { - "mailHost": "mail1", - "mailHostOverride": "mail2"}}, - { - "servlet-name": "cofaxAdmin", - "servlet-class": "org.cofax.cds.AdminServlet"}, - - { - "servlet-name": "fileServlet", - "servlet-class": "org.cofax.cds.FileServlet"}, - { - "servlet-name": "cofaxTools", - "servlet-class": "org.cofax.cms.CofaxToolsServlet", - "init-param": { - "templatePath": "toolstemplates/", - "log": 1, - "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", - "logMaxSize": "", - "dataLog": 1, - "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", - "dataLogMaxSize": "", - "removePageCache": "/content/admin/remove?cache=pages&id=", - "removeTemplateCache": "/content/admin/remove?cache=templates&id=", - "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", - "lookInContext": 1, - "adminGroupID": 4, - "betaServer": true}}], - "servlet-mapping": { - "cofaxCDS": "/", - "cofaxEmail": "/cofaxutil/aemail/*", - "cofaxAdmin": "/admin/*", - "fileServlet": "/static/*", - "cofaxTools": "/tools/*"}, - - "taglib": { - "taglib-uri": "cofax.tld", - "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}`, - }, - { - name: "SVG Viewer", - value: `{"menu": { +}} ` + +var webApp = `{"web-app": { + "servlet": [ + { + "servlet-name": "cofaxCDS", + "servlet-class": "org.cofax.cds.CDSServlet", + "init-param": { + "configGlossary:installationAt": "Philadelphia, PA", + "configGlossary:adminEmail": "ksm@pobox.com", + "configGlossary:poweredBy": "Cofax", + "configGlossary:poweredByIcon": "/images/cofax.gif", + "configGlossary:staticPath": "/content/static", + "templateProcessorClass": "org.cofax.WysiwygTemplate", + "templateLoaderClass": "org.cofax.FilesTemplateLoader", + "templatePath": "templates", + "templateOverridePath": "", + "defaultListTemplate": "listTemplate.htm", + "defaultFileTemplate": "articleTemplate.htm", + "useJSP": false, + "jspListTemplate": "listTemplate.jsp", + "jspFileTemplate": "articleTemplate.jsp", + "cachePackageTagsTrack": 200, + "cachePackageTagsStore": 200, + "cachePackageTagsRefresh": 60, + "cacheTemplatesTrack": 100, + "cacheTemplatesStore": 50, + "cacheTemplatesRefresh": 15, + "cachePagesTrack": 200, + "cachePagesStore": 100, + "cachePagesRefresh": 10, + "cachePagesDirtyRead": 10, + "searchEngineListTemplate": "forSearchEnginesList.htm", + "searchEngineFileTemplate": "forSearchEngines.htm", + "searchEngineRobotsDb": "WEB-INF/robots.db", + "useDataStore": true, + "dataStoreClass": "org.cofax.SqlDataStore", + "redirectionClass": "org.cofax.SqlRedirection", + "dataStoreName": "cofax", + "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", + "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", + "dataStoreUser": "sa", + "dataStorePassword": "dataStoreTestQuery", + "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", + "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", + "dataStoreInitConns": 10, + "dataStoreMaxConns": 100, + "dataStoreConnUsageLimit": 100, + "dataStoreLogLevel": "debug", + "maxUrlLength": 500}}, + { + "servlet-name": "cofaxEmail", + "servlet-class": "org.cofax.cds.EmailServlet", + "init-param": { + "mailHost": "mail1", + "mailHostOverride": "mail2"}}, + { + "servlet-name": "cofaxAdmin", + "servlet-class": "org.cofax.cds.AdminServlet"}, + + { + "servlet-name": "fileServlet", + "servlet-class": "org.cofax.cds.FileServlet"}, + { + "servlet-name": "cofaxTools", + "servlet-class": "org.cofax.cms.CofaxToolsServlet", + "init-param": { + "templatePath": "toolstemplates/", + "log": 1, + "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", + "logMaxSize": "", + "dataLog": 1, + "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", + "dataLogMaxSize": "", + "removePageCache": "/content/admin/remove?cache=pages&id=", + "removeTemplateCache": "/content/admin/remove?cache=templates&id=", + "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", + "lookInContext": 1, + "adminGroupID": 4, + "betaServer": true}}], + "servlet-mapping": { + "cofaxCDS": "/", + "cofaxEmail": "/cofaxutil/aemail/*", + "cofaxAdmin": "/admin/*", + "fileServlet": "/static/*", + "cofaxTools": "/tools/*"}, + + "taglib": { + "taglib-uri": "cofax.tld", + "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}` + +var svgViewer = `{"menu": { "header": "SVG Viewer", "items": [ {"id": "Open"}, @@ -445,21 +448,9 @@ func TestUnmarshal(t *testing.T) { {"id": "Help"}, {"id": "About", "label": "About Adobe CVG Viewer..."} ] -}}`, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := unmarshal2([]byte(test.value)) - if err != nil { - t.Errorf("Error on Unmarshal: %s", err.Error()) - } - }) - } -} +}}` -var ( - jsonExample = []byte(`{ "store": { +var bookStore = `{ "store": { "book": [ { "category": "reference", "author": "Nigel Rees", @@ -489,23 +480,88 @@ var ( "price": 19.95 } } -}`) -) +}` + +// Examples from https://json.org/example.html +func TestUnmarshal(t *testing.T) { + tests := []struct { + name string + value string + }{ + { + name: "glossary", + value: glossary, + }, + { + name: "menu", + value: file, + }, + { + name: "widget", + value: widget, + }, + { + name: "web-app", + value: webApp, + }, + { + name: "SVG Viewer", + value: svgViewer, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := Unmarshal([]byte(test.value)) + if err != nil { + t.Errorf("Error on Unmarshal: %s", err.Error()) + } + }) + } +} func TestUnmarshalSafe(t *testing.T) { - safe, err := UnmarshalSafe(jsonExample) + json := []byte(bookStore) + safe, err := UnmarshalSafe(json) if err != nil { t.Errorf("Error on Unmarshal: %s", err.Error()) } else if safe == nil { t.Errorf("Error on Unmarshal: safe is nil") } else { - root, err := unmarshal2(jsonExample) + root, err := Unmarshal(json) if err != nil { t.Errorf("Error on Unmarshal: %s", err.Error()) } else if root == nil { t.Errorf("Error on Unmarshal: root is nil") - } else if !bytes.Equal(root.Source(), safe.Source()) { + } else if !bytes.Equal(root.source(), safe.source()) { t.Errorf("Error on UnmarshalSafe: values not same") } } -} \ No newline at end of file +} + +// BenchmarkGoStdUnmarshal-8 61698 19350 ns/op 288 B/op 6 allocs/op +// BenchmarkUnmarshal-8 45620 26165 ns/op 21889 B/op 367 allocs/op +// +// type bench struct { +// Name string `json:"name"` +// Value int `json:"value"` +// } + +// func BenchmarkGoStdUnmarshal(b *testing.B) { +// data := []byte(webApp) +// for i := 0; i < b.N; i++ { +// err := json.Unmarshal(data, &bench{}) +// if err != nil { +// b.Fatal(err) +// } +// } +// } + +// func BenchmarkUnmarshal(b *testing.B) { +// data := []byte(webApp) +// for i := 0; i < b.N; i++ { +// _, err := Unmarshal(data) +// if err != nil { +// b.Fatal(err) +// } +// } +// } diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire.gno index c80aed34655..ef95b76f9da 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire.gno @@ -121,88 +121,6 @@ func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { return math.Float64frombits(retBits), true } -func eiselLemire32(man uint64, exp10 int, neg bool) (f float32, ok bool) { - // The terse comments in this function body refer to sections of the - // https://nigeltao.github.io/blog/2020/eisel-lemire.html blog post. - // - // That blog post discusses the float64 flavor (11 exponent bits with a - // -1023 bias, 52 mantissa bits) of the algorithm, but the same approach - // applies to the float32 flavor (8 exponent bits with a -127 bias, 23 - // mantissa bits). The computation here happens with 64-bit values (e.g. - // man, xHi, retMantissa) before finally converting to a 32-bit float. - - // Exp10 Range. - if man == 0 { - if neg { - f = math.Float32frombits(0x80000000) // Negative zero. - } - - return f, true - } - - if exp10 < detailedPowersOfTenMinExp10 || detailedPowersOfTenMaxExp10 < exp10 { - return 0, false - } - - // Normalization. - clz := bits.LeadingZeros64(man) - man <<= uint(clz) - - retExp2 := uint64(217706*exp10>>16+64+float32ExponentBias) - uint64(clz) - - // Multiplication. - xHi, xLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][1]) - - // Wider Approximation. - if xHi&0x3F_FFFFFFFF == 0x3F_FFFFFFFF && xLo+man < man { - yHi, yLo := bits.Mul64(man, detailedPowersOfTen[exp10-detailedPowersOfTenMinExp10][0]) - mergedHi, mergedLo := xHi, xLo+yHi - if mergedLo < xLo { - mergedHi++ - } - - if mergedHi&0x3F_FFFFFFFF == 0x3F_FFFFFFFF && mergedLo+1 == 0 && yLo+man < man { - return 0, false - } - - xHi, xLo = mergedHi, mergedLo - } - - // Shifting to 54 Bits (and for float32, it's shifting to 25 bits). - msb := xHi >> 63 - retMantissa := xHi >> (msb + 38) - retExp2 -= 1 ^ msb - - // Half-way Ambiguity. - if xLo == 0 && xHi&0x3F_FFFFFFFF == 0 && retMantissa&3 == 1 { - return 0, false - } - - // From 54 to 53 Bits (and for float32, it's from 25 to 24 bits). - retMantissa += retMantissa & 1 - retMantissa >>= 1 - if retMantissa>>24 > 0 { - retMantissa >>= 1 - retExp2 += 1 - } - - // retExp2 is a uint64. Zero or underflow means that we're in subnormal - // float32 space. 0xFF or above means that we're in Inf/NaN float32 space. - // - // The if block is equivalent to (but has fewer branches than): - // if retExp2 <= 0 || retExp2 >= 0xFF { etc } - if retExp2-1 >= 0xFF-1 { - return 0, false - } - - retBits := retExp2<<23 | retMantissa&0x007FFFFF - if neg { - retBits |= 0x80000000 - } - - return math.Float32frombits(uint32(retBits)), true -} - // detailedPowersOfTen{Min,Max}Exp10 is the power of 10 represented by the // first and last rows of detailedPowersOfTen. Both bounds are inclusive. const ( diff --git a/examples/gno.land/p/demo/json/encode.gno b/examples/gno.land/p/demo/json/encode.gno new file mode 100644 index 00000000000..00506813f8e --- /dev/null +++ b/examples/gno.land/p/demo/json/encode.gno @@ -0,0 +1,119 @@ +package json + +import ( + "bytes" + "errors" + "strconv" + + "gno.land/p/demo/ufmt" +) + +// Marshal returns the JSON encoding of a Node. +func Marshal(node *Node) ([]byte, error) { + var buf bytes.Buffer + var ( + sVal string + bVal bool + nVal float64 + oVal []byte + err error + ) + + if node == nil { + return nil, errors.New("node is nil") + } else if node.modified { + switch node.nodeType { + case Null: + buf.Write(nullLiteral) + + case Number: + nVal, err = node.GetNumeric() + if err != nil { + return nil, err + } + + // ufmt does not support %g. by doing so, we need to check if the number is an integer + // after then, apply the correct format for each float and integer numbers. + if nVal == float64(int64(nVal)) { + num := ufmt.Sprintf("%d", int64(nVal)) + buf.WriteString(num) + } else { + // TODO: fix float formatter + num := ufmt.Sprintf("%f", nVal) + buf.WriteString(num) + } + + case String: + sVal, err = node.GetString() + if err != nil { + return nil, err + } + + quoted := ufmt.Sprintf("%s", strconv.Quote(sVal)) + buf.WriteString(quoted) + + case Boolean: + bVal, err = node.GetBool() + if err != nil { + return nil, err + } + + bStr := ufmt.Sprintf("%t", bVal) + buf.WriteString(bStr) + + case Array: + buf.WriteByte(bracketOpen) + + for i := 0; i < len(node.next); i++ { + if i != 0 { + buf.WriteByte(comma) + } + + elem, ok := node.next[strconv.Itoa(i)] + if !ok { + return nil, errors.New(ufmt.Sprintf("array element %d is not found", i)) + } + + oVal, err = Marshal(elem) + if err != nil { + return nil, err + } + + buf.Write(oVal) + } + + buf.WriteByte(bracketClose) + + case Object: + buf.WriteByte(curlyOpen) + + bVal = false + for k, v := range node.next { + if bVal { + buf.WriteByte(comma) + } else { + bVal = true + } + + key := ufmt.Sprintf("%s", strconv.Quote(k)) + buf.WriteString(key) + buf.WriteByte(colon) + + oVal, err = Marshal(v) + if err != nil { + return nil, err + } + + buf.Write(oVal) + } + + buf.WriteByte(curlyClose) + } + } else if node.ready() { + buf.Write(node.source()) + } else { + return nil, errors.New("node is not modified") + } + + return buf.Bytes(), nil +} diff --git a/examples/gno.land/p/demo/json/encode_test.gno b/examples/gno.land/p/demo/json/encode_test.gno new file mode 100644 index 00000000000..97081c4381b --- /dev/null +++ b/examples/gno.land/p/demo/json/encode_test.gno @@ -0,0 +1,256 @@ +package json + +import ( + "testing" +) + +func TestMarshal_Primitive(t *testing.T) { + tests := []struct { + name string + node *Node + }{ + { + name: "null", + node: NullNode(""), + }, + { + name: "true", + node: BoolNode("", true), + }, + { + name: "false", + node: BoolNode("", false), + }, + { + name: `"string"`, + node: StringNode("", "string"), + }, + { + name: `"one \"encoded\" string"`, + node: StringNode("", `one "encoded" string`), + }, + { + name: `{"foo":"bar"}`, + node: ObjectNode("", map[string]*Node{ + "foo": StringNode("foo", "bar"), + }), + }, + { + name: "42", + node: NumberNode("", 42), + }, + { + name: "100.5", + node: NumberNode("", 100.5), + }, + { + name: `[1,2,3]`, + node: ArrayNode("", []*Node{ + NumberNode("0", 1), + NumberNode("2", 2), + NumberNode("3", 3), + }), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + value, err := Marshal(test.node) + if err != nil { + t.Errorf("unexpected error: %s", err) + } else if string(value) != test.name { + t.Errorf("wrong result: '%s', expected '%s'", value, test.name) + } + }) + } +} + +func TestMarshal_Object(t *testing.T) { + node := ObjectNode("", map[string]*Node{ + "foo": StringNode("foo", "bar"), + "baz": NumberNode("baz", 100500), + "qux": NullNode("qux"), + }) + + mustKey := []string{"foo", "baz", "qux"} + + value, err := Marshal(node) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + // the order of keys in the map is not guaranteed + // so we need to unmarshal the result and check the keys + decoded, err := Unmarshal(value) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + for _, key := range mustKey { + if node, err := decoded.GetKey(key); err != nil { + t.Errorf("unexpected error: %s", err) + } else { + if node == nil { + t.Errorf("node is nil") + } else if node.key == nil { + t.Errorf("key is nil") + } else if *node.key != key { + t.Errorf("wrong key: '%s', expected '%s'", *node.key, key) + } + } + } +} + +func valueNode(prev *Node, key string, typ ValueType, val interface{}) *Node { + curr := &Node{ + prev: prev, + data: nil, + key: &key, + borders: [2]int{0, 0}, + value: val, + modified: true, + } + + if val != nil { + curr.nodeType = typ + } + + return curr +} + +func TestMarshal_Errors(t *testing.T) { + tests := []struct { + name string + node func() (node *Node) + }{ + { + name: "nil", + node: func() (node *Node) { + return + }, + }, + { + name: "broken", + node: func() (node *Node) { + node = Must(Unmarshal([]byte(`{}`))) + node.borders[1] = 0 + return + }, + }, + { + name: "Numeric", + node: func() (node *Node) { + return valueNode(nil, "", Number, false) + }, + }, + { + name: "String", + node: func() (node *Node) { + return valueNode(nil, "", String, false) + }, + }, + { + name: "Bool", + node: func() (node *Node) { + return valueNode(nil, "", Boolean, 1) + }, + }, + { + name: "Array_1", + node: func() (node *Node) { + node = ArrayNode("", nil) + node.next["1"] = NullNode("1") + return + }, + }, + { + name: "Array_2", + node: func() (node *Node) { + return ArrayNode("", []*Node{valueNode(nil, "", Boolean, 1)}) + }, + }, + { + name: "Object", + node: func() (node *Node) { + return ObjectNode("", map[string]*Node{"key": valueNode(nil, "key", Boolean, 1)}) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + value, err := Marshal(test.node()) + if err == nil { + t.Errorf("expected error") + } else if len(value) != 0 { + t.Errorf("wrong result") + } + }) + } +} + +func TestMarshal_Nil(t *testing.T) { + _, err := Marshal(nil) + if err == nil { + t.Error("Expected error for nil node, but got nil") + } +} + +func TestMarshal_NotModified(t *testing.T) { + node := &Node{} + _, err := Marshal(node) + if err == nil { + t.Error("Expected error for not modified node, but got nil") + } +} + +func TestMarshalCycleReference(t *testing.T) { + node1 := &Node{ + key: stringPtr("node1"), + nodeType: String, + next: map[string]*Node{ + "next": nil, + }, + } + + node2 := &Node{ + key: stringPtr("node2"), + nodeType: String, + prev: node1, + } + + node1.next["next"] = node2 + + _, err := Marshal(node1) + if err == nil { + t.Error("Expected error for cycle reference, but got nil") + } +} + +func TestMarshalNoCycleReference(t *testing.T) { + node1 := &Node{ + key: stringPtr("node1"), + nodeType: String, + value: "value1", + modified: true, + } + + node2 := &Node{ + key: stringPtr("node2"), + nodeType: String, + value: "value2", + modified: true, + } + + _, err := Marshal(node1) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + _, err = Marshal(node2) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } +} + +func stringPtr(s string) *string { + return &s +} diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 1cbb5e39f86..9f4186fffed 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -3,29 +3,47 @@ package json import ( "bytes" "errors" - "unicode" - "unicode/utf16" "unicode/utf8" ) const ( - supplementalPlanesOffset = 0x10000 - highSurrogateOffset = 0xD800 - lowSurrogateOffset = 0xDC00 - + supplementalPlanesOffset = 0x10000 + highSurrogateOffset = 0xD800 + lowSurrogateOffset = 0xDC00 surrogateEnd = 0xDFFF basicMultilingualPlaneOffset = 0xFFFF - - badHex = -1 + badHex = -1 ) -// unescape takes an input byte slice, processes it to unescape certain characters, +var hexLookupTable = [256]int{ + '0': 0x0, '1': 0x1, '2': 0x2, '3': 0x3, '4': 0x4, + '5': 0x5, '6': 0x6, '7': 0x7, '8': 0x8, '9': 0x9, + 'A': 0xA, 'B': 0xB, 'C': 0xC, 'D': 0xD, 'E': 0xE, 'F': 0xF, + 'a': 0xA, 'b': 0xB, 'c': 0xC, 'd': 0xD, 'e': 0xE, 'f': 0xF, + // Fill unspecified index-value pairs with key and value of -1 + 'G': -1, 'H': -1, 'I': -1, 'J': -1, + 'K': -1, 'L': -1, 'M': -1, 'N': -1, + 'O': -1, 'P': -1, 'Q': -1, 'R': -1, + 'S': -1, 'T': -1, 'U': -1, 'V': -1, + 'W': -1, 'X': -1, 'Y': -1, 'Z': -1, + 'g': -1, 'h': -1, 'i': -1, 'j': -1, + 'k': -1, 'l': -1, 'm': -1, 'n': -1, + 'o': -1, 'p': -1, 'q': -1, 'r': -1, + 's': -1, 't': -1, 'u': -1, 'v': -1, + 'w': -1, 'x': -1, 'y': -1, 'z': -1, +} + +func h2i(c byte) int { + return hexLookupTable[c] +} + +// Unescape takes an input byte slice, processes it to Unescape certain characters, // and writes the result into an output byte slice. // -// it returns the processed slice and any error encountered during the unescape operation. -func unescape(input, output []byte) ([]byte, error) { +// it returns the processed slice and any error encountered during the Unescape operation. +func Unescape(input, output []byte) ([]byte, error) { // find the index of the first backslash in the input slice. - firstBackslash := bytes.IndexByte(input, BackSlashToken) + firstBackslash := bytes.IndexByte(input, backSlash) if firstBackslash == -1 { return input, nil } @@ -43,16 +61,16 @@ func unescape(input, output []byte) ([]byte, error) { buf := output[firstBackslash:] for len(input) > 0 { - inLen, bufLen := processEscapedUTF8(input, buf) - if inLen == -1 { - return nil, errors.New("Invalid escape sequence in a string") + inLen, bufLen, err := processEscapedUTF8(input, buf) + if err != nil { + return nil, err } input = input[inLen:] // the number of bytes consumed in the input buf = buf[bufLen:] // the number of bytes written to buf // find the next backslash in the remaining input - nextBackslash := bytes.IndexByte(input, BackSlashToken) + nextBackslash := bytes.IndexByte(input, backSlash) if nextBackslash == -1 { copy(buf, input) buf = buf[len(input):] @@ -128,18 +146,27 @@ func decodeUnicodeEscape(b []byte) (rune, int) { return combineSurrogates(r, r2), 12 } -func unquote(bs []byte, unquoteToken byte) (string, error) { - res, err := unquoteBytes(bs, unquoteToken) - if err != nil { - return "", err - } +var escapeByteSet = [256]byte{ + '"': doubleQuote, + '\\': backSlash, + '/': slash, + 'b': backSpace, + 'f': formFeed, + 'n': newLine, + 'r': carriageReturn, + 't': tab, +} - return string(res), nil +// Unquote takes a byte slice and unquotes it by removing the surrounding quotes and unescaping the contents. +func Unquote(s []byte, border byte) (t string, ok bool) { + s, ok = unquoteBytes(s, border) + t = string(s) + return } -func unquoteBytes(s []byte, border byte) ([]byte, error) { +func unquoteBytes(s []byte, border byte) ([]byte, bool) { if len(s) < 2 || s[0] != border || s[len(s)-1] != border { - return nil, errors.New("Invalid JSON string") + return nil, false } s = s[1 : len(s)-1] @@ -148,74 +175,61 @@ func unquoteBytes(s []byte, border byte) ([]byte, error) { for r < len(s) { c := s[r] - if c == BackSlashToken || c == border || c < 0x20 { + if c == backSlash || c == border || c < 0x20 { break - } else if c >= utf8.RuneSelf { - rr, size := utf8.DecodeRune(s[r:]) - if rr == utf8.RuneError && size == 1 { - break - } + } - r += size + if c < utf8.RuneSelf { + r++ continue } - r++ + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + + r += size } if r == len(s) { - return s, nil + return s, true } - b := make([]byte, len(s)+utf8.UTFMax*2) - w := copy(b, s[:r]) + utfDoubleMax := utf8.UTFMax * 2 + b := make([]byte, len(s)+utfDoubleMax) + w := copy(b, s[0:r]) for r < len(s) { - if w >= len(b)-(utf8.UTFMax*2) { - nb := make([]byte, (utf8.UTFMax+len(b))*2) + if w >= len(b)-utf8.UTFMax { + nb := make([]byte, utfDoubleMax+(2*len(b))) copy(nb, b) b = nb } c := s[r] - if c == BackSlashToken { + if c == backSlash { r++ if r >= len(s) { - return nil, errors.New("Invalid JSON string") + return nil, false } if s[r] == 'u' { rr, res := decodeUnicodeEscape(s[r-1:]) if res < 0 { - return nil, errors.New("Invalid JSON string") + return nil, false } w += utf8.EncodeRune(b[w:], rr) r += 5 } else { - var decode byte - - switch s[r] { - case 'b': - decode = BackSpaceToken - - case 'f': - decode = FormFeedToken - - case 'n': - decode = NewLineToken - - case 'r': - decode = CarriageReturnToken - - case 't': - decode = TabToken + decode := escapeByteSet[s[r]] + if decode == 0 { + return nil, false + } - case border, BackSlashToken, SlashToken, QuoteToken: + if decode == doubleQuote || decode == backSlash || decode == slash { decode = s[r] - - default: - return nil, errors.New("Invalid JSON string") } b[w] = decode @@ -223,7 +237,7 @@ func unquoteBytes(s []byte, border byte) ([]byte, error) { w++ } } else if c == border || c < 0x20 { - return nil, errors.New("Invalid JSON string") + return nil, false } else if c < utf8.RuneSelf { b[w] = c r++ @@ -232,7 +246,7 @@ func unquoteBytes(s []byte, border byte) ([]byte, error) { rr, size := utf8.DecodeRune(s[r:]) if rr == utf8.RuneError && size == 1 { - return nil, errors.New("Invalid JSON string") + return nil, false } r += size @@ -240,18 +254,7 @@ func unquoteBytes(s []byte, border byte) ([]byte, error) { } } - return b[:w], nil -} - -var escapeByteSet = [256]byte{ - '"': DoublyQuoteToken, - '\\': BackSlashToken, - '/': SlashToken, - 'b': BackSpaceToken, - 'f': FormFeedToken, - 'n': NewLineToken, - 'r': CarriageReturnToken, - 't': TabToken, + return b[:w], true } // processEscapedUTF8 processes the escape sequence in the given byte slice and @@ -266,9 +269,9 @@ var escapeByteSet = [256]byte{ // // If the escape sequence is invalid, or if 'in' does not completely enclose the escape sequence, // function returns (-1, -1) to indicate an error. -func processEscapedUTF8(in, out []byte) (inLen int, outLen int) { - if len(in) < 2 || in[0] != BackSlashToken { - return -1, -1 +func processEscapedUTF8(in, out []byte) (inLen int, outLen int, err error) { + if len(in) < 2 || in[0] != backSlash { + return -1, -1, errors.New("invalid escape sequence") } escapeSeqLen := 2 @@ -276,15 +279,15 @@ func processEscapedUTF8(in, out []byte) (inLen int, outLen int) { if escapeChar == 'u' { if r, size := decodeUnicodeEscape(in); size != -1 { outLen = utf8.EncodeRune(out, r) - return size, outLen + return size, outLen, nil } } else { val := escapeByteSet[escapeChar] if val != 0 { out[0] = val - return escapeSeqLen, 1 + return escapeSeqLen, 1, nil } } - return -1, -1 + return -1, -1, errors.New("invalid escape sequence") } diff --git a/examples/gno.land/p/demo/json/escape_test.gno b/examples/gno.land/p/demo/json/escape_test.gno index fbc4910b1af..40c118d93ce 100644 --- a/examples/gno.land/p/demo/json/escape_test.gno +++ b/examples/gno.land/p/demo/json/escape_test.gno @@ -78,7 +78,6 @@ func TestDecodeSingleUnicodeEscape(t *testing.T) { input []byte expected rune isValid bool - len int }{ // valid unicode escape sequences {[]byte(`\u0041`), 'A', true}, @@ -133,28 +132,32 @@ func TestUnescapeToUTF8(t *testing.T) { input []byte expectedIn int expectedOut int + isError bool }{ // valid escape sequences - {[]byte(`\n`), 2, 1}, - {[]byte(`\t`), 2, 1}, - {[]byte(`\u0041`), 6, 1}, - {[]byte(`\u03B1`), 6, 2}, - {[]byte(`\uD830\uDE03`), 12, 4}, + {[]byte(`\n`), 2, 1, false}, + {[]byte(`\t`), 2, 1, false}, + {[]byte(`\u0041`), 6, 1, false}, + {[]byte(`\u03B1`), 6, 2, false}, + {[]byte(`\uD830\uDE03`), 12, 4, false}, // invalid escape sequences - {[]byte(`\`), -1, -1}, // incomplete escape sequence - {[]byte(`\x`), -1, -1}, // invalid escape character - {[]byte(`\u`), -1, -1}, // incomplete unicode escape sequence - {[]byte(`\u004`), -1, -1}, // invalid unicode escape sequence - {[]byte(`\uXYZW`), -1, -1}, // invalid unicode escape sequence - {[]byte(`\uD83D\u0041`), -1, -1}, // invalid unicode escape sequence + {[]byte(`\`), -1, -1, true}, // incomplete escape sequence + {[]byte(`\x`), -1, -1, true}, // invalid escape character + {[]byte(`\u`), -1, -1, true}, // incomplete unicode escape sequence + {[]byte(`\u004`), -1, -1, true}, // invalid unicode escape sequence + {[]byte(`\uXYZW`), -1, -1, true}, // invalid unicode escape sequence + {[]byte(`\uD83D\u0041`), -1, -1, true}, // invalid unicode escape sequence } for _, tc := range testCases { input := make([]byte, len(tc.input)) copy(input, tc.input) output := make([]byte, utf8.UTFMax) - inLen, outLen := processEscapedUTF8(input, output) + inLen, outLen, err := processEscapedUTF8(input, output) + if (err != nil) != tc.isError { + t.Errorf("processEscapedUTF8(%q) = %v; want %v", tc.input, err, tc.isError) + } if inLen != tc.expectedIn || outLen != tc.expectedOut { t.Errorf("processEscapedUTF8(%q) = (%d, %d); want (%d, %d)", tc.input, inLen, outLen, tc.expectedIn, tc.expectedOut) @@ -177,7 +180,7 @@ func TestUnescape(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - output, _ := unescape(tc.input, make([]byte, len(tc.input)+10)) + output, _ := Unescape(tc.input, make([]byte, len(tc.input)+10)) if !bytes.Equal(output, tc.expected) { t.Errorf("unescape(%q) = %q; want %q", tc.input, output, tc.expected) } @@ -201,13 +204,15 @@ func TestUnquoteBytes(t *testing.T) { {[]byte("\"\""), '"', []byte(""), true}, {[]byte("''"), '\'', []byte(""), true}, {[]byte("\"\\u0041\""), '"', []byte("A"), true}, + {[]byte(`"Hello, 世界"`), '"', []byte("Hello, 世界"), true}, + {[]byte(`"Hello, \x80"`), '"', nil, false}, } for _, tc := range tests { - result, err := unquoteBytes(tc.input, tc.border) + result, pass := unquoteBytes(tc.input, tc.border) - if tc.ok != (err == nil) { - t.Errorf("unquoteBytes(%q) = %q; want %q", tc.input, result, tc.expected) + if pass != tc.ok { + t.Errorf("unquoteBytes(%q) = %v; want %v", tc.input, pass, tc.ok) } if !bytes.Equal(result, tc.expected) { diff --git a/examples/gno.land/p/demo/json/indent.gno b/examples/gno.land/p/demo/json/indent.gno new file mode 100644 index 00000000000..cdcfd4524ee --- /dev/null +++ b/examples/gno.land/p/demo/json/indent.gno @@ -0,0 +1,144 @@ +package json + +import ( + "bytes" + "strings" +) + +// indentGrowthFactor specifies the growth factor of indenting JSON input. +// A factor no higher than 2 ensures that wasted space never exceeds 50%. +const indentGrowthFactor = 2 + +// IndentJSON takes a JSON byte slice and a string for indentation, +// then formats the JSON according to the specified indent string. +// This function applies indentation rules as follows: +// +// 1. For top-level arrays and objects, no additional indentation is applied. +// +// 2. For nested structures like arrays within arrays or objects, indentation increases. +// +// 3. Indentation is applied after opening brackets ('[' or '{') and before closing brackets (']' or '}'). +// +// 4. Commas and colons are handled appropriately to maintain valid JSON format. +// +// 5. Nested arrays within objects or arrays receive new lines and indentation based on their depth level. +// +// The function returns the formatted JSON as a byte slice and an error if any issues occurred during formatting. +func Indent(data []byte, indent string) ([]byte, error) { + var ( + out bytes.Buffer + level int + inArray bool + arrayDepth int + ) + + for i := 0; i < len(data); i++ { + c := data[i] // current character + + switch c { + case bracketOpen: + arrayDepth++ + if arrayDepth > 1 { + level++ // increase the level if it's nested array + inArray = true + + if err := out.WriteByte(c); err != nil { + return nil, err + } + + if err := writeNewlineAndIndent(&out, level, indent); err != nil { + return nil, err + } + } else { + // case of the top-level array + inArray = true + if err := out.WriteByte(c); err != nil { + return nil, err + } + } + + case bracketClose: + if inArray && arrayDepth > 1 { // nested array + level-- + if err := writeNewlineAndIndent(&out, level, indent); err != nil { + return nil, err + } + } + + arrayDepth-- + if arrayDepth == 0 { + inArray = false + } + + if err := out.WriteByte(c); err != nil { + return nil, err + } + + case curlyOpen: + // check if the empty object or array + // we don't need to apply the indent when it's empty containers. + if i+1 < len(data) && data[i+1] == curlyClose { + if err := out.WriteByte(c); err != nil { + return nil, err + } + + i++ // skip next character + if err := out.WriteByte(data[i]); err != nil { + return nil, err + } + } else { + if err := out.WriteByte(c); err != nil { + return nil, err + } + + level++ + if err := writeNewlineAndIndent(&out, level, indent); err != nil { + return nil, err + } + } + + case curlyClose: + level-- + if err := writeNewlineAndIndent(&out, level, indent); err != nil { + return nil, err + } + if err := out.WriteByte(c); err != nil { + return nil, err + } + + case comma, colon: + if err := out.WriteByte(c); err != nil { + return nil, err + } + if inArray && arrayDepth > 1 { // nested array + if err := writeNewlineAndIndent(&out, level, indent); err != nil { + return nil, err + } + } else if c == colon { + if err := out.WriteByte(' '); err != nil { + return nil, err + } + } + + default: + if err := out.WriteByte(c); err != nil { + return nil, err + } + } + } + + return out.Bytes(), nil +} + +func writeNewlineAndIndent(out *bytes.Buffer, level int, indent string) error { + if err := out.WriteByte('\n'); err != nil { + return err + } + + idt := strings.Repeat(indent, level*indentGrowthFactor) + if _, err := out.WriteString(idt); err != nil { + return err + } + + return nil +} diff --git a/examples/gno.land/p/demo/json/indent_test.gno b/examples/gno.land/p/demo/json/indent_test.gno new file mode 100644 index 00000000000..bc57449b12d --- /dev/null +++ b/examples/gno.land/p/demo/json/indent_test.gno @@ -0,0 +1,84 @@ +package json + +import ( + "bytes" + "testing" +) + +func TestIndentJSON(t *testing.T) { + tests := []struct { + name string + input []byte + indent string + expected []byte + }{ + { + name: "empty object", + input: []byte(`{}`), + indent: " ", + expected: []byte(`{}`), + }, + { + name: "empty array", + input: []byte(`[]`), + indent: " ", + expected: []byte(`[]`), + }, + { + name: "nested object", + input: []byte(`{{}}`), + indent: "\t", + expected: []byte("{\n\t\t{}\n}"), + }, + { + name: "nested array", + input: []byte(`[[[]]]`), + indent: "\t", + expected: []byte("[[\n\t\t[\n\t\t\t\t\n\t\t]\n]]"), + }, + { + name: "top-level array", + input: []byte(`["apple","banana","cherry"]`), + indent: "\t", + expected: []byte(`["apple","banana","cherry"]`), + }, + { + name: "array of arrays", + input: []byte(`["apple",["banana","cherry"],"date"]`), + indent: " ", + expected: []byte("[\"apple\",[\n \"banana\",\n \"cherry\"\n],\"date\"]"), + }, + + { + name: "nested array in object", + input: []byte(`{"fruits":["apple",["banana","cherry"],"date"]}`), + indent: " ", + expected: []byte("{\n \"fruits\": [\"apple\",[\n \"banana\",\n \"cherry\"\n ],\"date\"]\n}"), + }, + { + name: "complex nested structure", + input: []byte(`{"data":{"array":[1,2,3],"bool":true,"nestedArray":[["a","b"],"c"]}}`), + indent: " ", + expected: []byte("{\n \"data\": {\n \"array\": [1,2,3],\"bool\": true,\"nestedArray\": [[\n \"a\",\n \"b\"\n ],\"c\"]\n }\n}"), + }, + { + name: "custom ident character", + input: []byte(`{"fruits":["apple",["banana","cherry"],"date"]}`), + indent: "*", + expected: []byte("{\n**\"fruits\": [\"apple\",[\n****\"banana\",\n****\"cherry\"\n**],\"date\"]\n}"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := Indent(tt.input, tt.indent) + if err != nil { + t.Errorf("IndentJSON() error = %v", err) + return + } + if !bytes.Equal(actual, tt.expected) { + t.Errorf("IndentJSON() = %q, want %q", actual, tt.expected) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index 3ed7028b34b..0d70131be76 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -3,52 +3,32 @@ package json import ( "errors" "strconv" + "strings" "gno.land/p/demo/ufmt" ) -type NodeValue struct { - value interface{} - typ ValueType -} - -func newNodeValue(value interface{}) *NodeValue { - return &NodeValue{ - value: value, - typ: typeOf(value), - } -} - -func (nv *NodeValue) Store(value interface{}) { - nv.value = value - nv.typ = typeOf(value) -} - -func (nv *NodeValue) Load() interface{} { - return nv.value -} - +// Node represents a JSON node. type Node struct { - prev *Node - next map[string]*Node - key *string - data []byte - value *NodeValue - index int - borders [2]uint // start, end - modified bool + prev *Node // prev is the parent node of the current node. + next map[string]*Node // next is the child nodes of the current node. + key *string // key holds the key of the current node in the parent node. + data []byte // byte slice of JSON data + value interface{} // value holds the value of the current node. + nodeType ValueType // NodeType holds the type of the current node. (Object, Array, String, Number, Boolean, Null) + index *int // index holds the index of the current node in the parent array node. + borders [2]int // borders stores the start and end index of the current node in the data. + modified bool // modified indicates the current node is changed or not. } -func NewNode(prev *Node, b *buffer, typ ValueType, key *string) (*Node, error) { +// NewNode creates a new node instance with the given parent node, buffer, type, and key. +func NewNode(prev *Node, b *buffer, typ ValueType, key **string) (*Node, error) { curr := &Node{ - prev: prev, - data: b.data, - borders: [2]uint{b.index, 0}, - key: key, - value: &NodeValue{ - value: nil, - typ: typ, - }, + prev: prev, + data: b.data, + borders: [2]int{b.index, 0}, + key: *key, + nodeType: typ, modified: false, } @@ -57,17 +37,17 @@ func NewNode(prev *Node, b *buffer, typ ValueType, key *string) (*Node, error) { } if prev != nil { - if prev.isArray() { + if prev.IsArray() { size := len(prev.next) - curr.index = size + curr.index = &size prev.next[strconv.Itoa(size)] = curr - } else if prev.isObject() { + } else if prev.IsObject() { if key == nil { return nil, errors.New("key is required for object") } - prev.next[*key] = curr + prev.next[**key] = curr } else { return nil, errors.New("invalid parent type") } @@ -76,62 +56,161 @@ func NewNode(prev *Node, b *buffer, typ ValueType, key *string) (*Node, error) { return curr, nil } -func (n *Node) Key() *string { +// load retrieves the value of the current node. +func (n *Node) load() interface{} { + return n.value +} + +// Changed checks the current node is changed or not. +func (n *Node) Changed() bool { + return n.modified +} + +// Key returns the key of the current node. +func (n *Node) Key() string { if n == nil || n.key == nil { - return nil + return "" + } + + return *n.key +} + +// HasKey checks the current node has the given key or not. +func (n *Node) HasKey(key string) bool { + if n == nil { + return false + } + + _, ok := n.next[key] + return ok +} + +// GetKey returns the value of the given key from the current object node. +func (n *Node) GetKey(key string) (*Node, error) { + if n == nil { + return nil, errors.New("node is nil") + } + + if n.Type() != Object { + return nil, errors.New(ufmt.Sprintf("target node is not object type. got: %s", n.Type().String())) } - return n.key + value, ok := n.next[key] + if !ok { + return nil, errors.New(ufmt.Sprintf("key not found: %s", key)) + } + + return value, nil } +// MustKey returns the value of the given key from the current object node. +func (n *Node) MustKey(key string) *Node { + val, err := n.GetKey(key) + if err != nil { + panic(err) + } + + return val +} + +// UniqueKeys traverses the current JSON nodes and collects all the unique keys. +func (n *Node) UniqueKeys() []string { + var collectKeys func(*Node) []string + collectKeys = func(node *Node) []string { + if node == nil || !node.IsObject() { + return nil + } + + result := make(map[string]bool) + for key, childNode := range node.next { + result[key] = true + childKeys := collectKeys(childNode) + for _, childKey := range childKeys { + result[childKey] = true + } + } + + keys := make([]string, 0, len(result)) + for key := range result { + keys = append(keys, key) + } + return keys + } + + return collectKeys(n) +} + +// TODO: implement the EachKey method. this takes a callback function and executes it for each key in the object node. +// func (n *Node) EachKey(callback func(key string, value *Node)) { ... } + +// Empty returns true if the current node is empty. +func (n *Node) Empty() bool { + if n == nil { + return false + } + + return len(n.next) == 0 +} + +// Type returns the type (ValueType) of the current node. func (n *Node) Type() ValueType { - return n.value.typ + return n.nodeType } -// TODO: may replace parsePrimitive (parser.gno) +// Value returns the value of the current node. +// +// Usage: +// +// root := Unmarshal([]byte(`{"key": "value"}`)) +// val, err := root.MustKey("key").Value() +// if err != nil { +// t.Errorf("Value returns error: %v", err) +// } +// +// result: "value" func (n *Node) Value() (value interface{}, err error) { - value = n.value.Load() + value = n.load() if value == nil { - switch n.value.typ { + switch n.nodeType { case Null: return nil, nil case Number: - value, err = ParseFloatLiteral(n.Source()) + value, err = ParseFloatLiteral(n.source()) if err != nil { return nil, err } - n.value.Store(value) + n.value = value case String: - value, err = unquote(n.Source(), DoublyQuoteToken) - if err != nil { - return nil, err + var ok bool + value, ok = Unquote(n.source(), doubleQuote) + if !ok { + return "", errors.New("invalid string value") } - n.value.Store(value) + n.value = value case Boolean: - if len(n.Source()) == 0 { + if len(n.source()) == 0 { return nil, errors.New("empty boolean value") } - b := n.Source()[0] + b := n.source()[0] value = b == 't' || b == 'T' - - n.value.Store(value) + n.value = value case Array: elems := make([]*Node, len(n.next)) for _, e := range n.next { - elems[e.index] = e + elems[*e.index] = e } value = elems - n.value.Store(value) + n.value = value case Object: obj := make(map[string]*Node, len(n.next)) @@ -141,114 +220,255 @@ func (n *Node) Value() (value interface{}, err error) { } value = obj - n.value.Store(value) + n.value = value } } return value, nil } -func (n *Node) Size() uint { +// Delete removes the current node from the parent node. +// +// Usage: +// +// root := Unmarshal([]byte(`{"key": "value"}`)) +// if err := root.MustKey("key").Delete(); err != nil { +// t.Errorf("Delete returns error: %v", err) +// } +// +// result: {} (empty object) +func (n *Node) Delete() error { + if n == nil { + return errors.New("can't delete nil node") + } + + if n.prev == nil { + return nil + } + + return n.prev.remove(n) +} + +// Size returns the number of sub-nodes of the current Array node. +// +// Usage: +// +// root := ArrayNode("", []*Node{StringNode("", "foo"), NumberNode("", 1)}) +// if root == nil { +// t.Errorf("ArrayNode returns nil") +// } +// +// if root.Size() != 2 { +// t.Errorf("ArrayNode returns wrong size: %d", root.Size()) +// } +func (n *Node) Size() int { if n == nil { return 0 } - return uint(len(n.next)) + return len(n.next) +} + +// Index returns the index of the current node in the parent array node. +// +// Usage: +// +// root := ArrayNode("", []*Node{StringNode("", "foo"), NumberNode("", 1)}) +// if root == nil { +// t.Errorf("ArrayNode returns nil") +// } +// +// if root.MustIndex(1).Index() != 1 { +// t.Errorf("Index returns wrong index: %d", root.MustIndex(1).Index()) +// } +// +// We can also use the index to the byte slice of the JSON data directly. +// +// Example: +// +// root := Unmarshal([]byte(`["foo", 1]`)) +// if root == nil { +// t.Errorf("Unmarshal returns nil") +// } +// +// if string(root.MustIndex(1).source()) != "1" { +// t.Errorf("source returns wrong result: %s", root.MustIndex(1).source()) +// } +func (n *Node) Index() int { + if n == nil || n.index == nil { + return -1 + } + + return *n.index +} + +// MustIndex returns the array element at the given index. +// +// If the index is negative, it returns the index is from the end of the array. +// Also, it panics if the index is not found. +// +// check the Index method for detailed usage. +func (n *Node) MustIndex(expectIdx int) *Node { + val, err := n.GetIndex(expectIdx) + if err != nil { + panic(err) + } + + return val } -func (n *Node) Keys() []string { +// GetIndex returns the array element at the given index. +// +// if the index is negative, it returns the index is from the end of the array. +func (n *Node) GetIndex(idx int) (*Node, error) { if n == nil { - return nil + return nil, errors.New("node is nil") } - result := make([]string, 0, len(n.next)) - for key := range n.next { - result = append(result, key) + if !n.IsArray() { + return nil, errors.New("node is not array") + } + + if idx < 0 { + idx += len(n.next) + } + + child, ok := n.next[strconv.Itoa(idx)] + if !ok { + return nil, errors.New("index not found") } - return result + return child, nil } -func NullNode(key *string) *Node { +// DeleteIndex removes the array element at the given index. +func (n *Node) DeleteIndex(idx int) error { + node, err := n.GetIndex(idx) + if err != nil { + return err + } + + return node.remove(node) +} + +// NullNode creates a new null type node. +// +// Usage: +// +// _ := NullNode("") +func NullNode(key string) *Node { return &Node{ - key: key, - value: &NodeValue{value: nil, typ: Null}, + key: &key, + value: nil, + nodeType: Null, modified: true, } } -func NumberNode(key *string, value float64) *Node { +// NumberNode creates a new number type node. +// +// Usage: +// +// root := NumberNode("", 1) +// if root == nil { +// t.Errorf("NumberNode returns nil") +// } +func NumberNode(key string, value float64) *Node { return &Node{ - key: key, - value: &NodeValue{ - value: value, - typ: Number, // treat Float and Number as Number type - }, + key: &key, + value: value, + nodeType: Number, modified: true, } } -func StringNode(key *string, value string) *Node { - val := newNodeValue(value) - +// StringNode creates a new string type node. +// +// Usage: +// +// root := StringNode("", "foo") +// if root == nil { +// t.Errorf("StringNode returns nil") +// } +func StringNode(key string, value string) *Node { return &Node{ - key: key, - value: val, + key: &key, + value: value, + nodeType: String, modified: true, } } -func BoolNode(key *string, value bool) *Node { - val := newNodeValue(value) - +// BoolNode creates a new given boolean value node. +// +// Usage: +// +// root := BoolNode("", true) +// if root == nil { +// t.Errorf("BoolNode returns nil") +// } +func BoolNode(key string, value bool) *Node { return &Node{ - key: key, - value: val, + key: &key, + value: value, + nodeType: Boolean, modified: true, } } -func ArrayNode(key *string, value []*Node) *Node { - val := newNodeValue(value) +// ArrayNode creates a new array type node. +// +// If the given value is nil, it creates an empty array node. +// +// Usage: +// +// root := ArrayNode("", []*Node{StringNode("", "foo"), NumberNode("", 1)}) +// if root == nil { +// t.Errorf("ArrayNode returns nil") +// } +func ArrayNode(key string, value []*Node) *Node { curr := &Node{ - key: key, - value: val, + key: &key, + nodeType: Array, modified: true, } curr.next = make(map[string]*Node, len(value)) if value != nil { - curr.value.Store(value) + curr.value = value for i, v := range value { - idx := i + var idx = i curr.next[strconv.Itoa(i)] = v v.prev = curr - v.index = idx + v.index = &idx } } return curr } -func ObjectNode(key *string, value map[string]*Node) *Node { +// ObjectNode creates a new object type node. +// +// If the given value is nil, it creates an empty object node. +// +// next is a map of key and value pairs of the object. +func ObjectNode(key string, value map[string]*Node) *Node { curr := &Node{ - key: key, - value: &NodeValue{ - value: value, - typ: Object, - }, + nodeType: Object, + key: &key, next: value, modified: true, } if value != nil { - curr.value.Store(value) + curr.value = value - for k, v := range value { - v.prev = curr - v.key = &k + for key, val := range value { + vkey := key + val.prev = curr + val.key = &vkey } } else { curr.next = make(map[string]*Node) @@ -257,19 +477,42 @@ func ObjectNode(key *string, value map[string]*Node) *Node { return curr } -func (n *Node) isArray() bool { - return n.value.typ == Array +// IsArray returns true if the current node is array type. +func (n *Node) IsArray() bool { + return n.nodeType == Array } -func (n *Node) isObject() bool { - return n.value.typ == Object +// IsObject returns true if the current node is object type. +func (n *Node) IsObject() bool { + return n.nodeType == Object +} + +// IsNull returns true if the current node is null type. +func (n *Node) IsNull() bool { + return n.nodeType == Null +} + +// IsBool returns true if the current node is boolean type. +func (n *Node) IsBool() bool { + return n.nodeType == Boolean +} + +// IsString returns true if the current node is string type. +func (n *Node) IsString() bool { + return n.nodeType == String +} + +// IsNumber returns true if the current node is number type. +func (n *Node) IsNumber() bool { + return n.nodeType == Number } func (n *Node) ready() bool { return n.borders[1] != 0 } -func (n *Node) Source() []byte { +// source returns the source of the current node. +func (n *Node) source() []byte { if n == nil { return nil } @@ -281,6 +524,7 @@ func (n *Node) Source() []byte { return nil } +// root returns the root node of the current node. func (n *Node) root() *Node { if n == nil { return nil @@ -294,24 +538,58 @@ func (n *Node) root() *Node { return curr } +// GetNull returns the null value if current node is null type. +// +// Usage: +// +// root := Unmarshal([]byte("null")) +// val, err := root.GetNull() +// if err != nil { +// t.Errorf("GetNull returns error: %v", err) +// } +// if val != nil { +// t.Errorf("GetNull returns wrong result: %v", val) +// } func (n *Node) GetNull() (interface{}, error) { if n == nil { return nil, errors.New("node is nil") } - if n.value.typ != Null { + if !n.IsNull() { return nil, errors.New("node is not null") } return nil, nil } +// MustNull returns the null value if current node is null type. +// +// It panics if the current node is not null type. +func (n *Node) MustNull() interface{} { + v, err := n.GetNull() + if err != nil { + panic(err) + } + + return v +} + +// GetNumeric returns the numeric (int/float) value if current node is number type. +// +// Usage: +// +// root := Unmarshal([]byte("10.5")) +// val, err := root.GetNumeric() +// if err != nil { +// t.Errorf("GetNumeric returns error: %v", err) +// } +// println(val) // 10.5 func (n *Node) GetNumeric() (float64, error) { if n == nil { return 0, errors.New("node is nil") } - if n.value.typ != Number { + if n.nodeType != Number { return 0, errors.New("node is not number") } @@ -328,12 +606,116 @@ func (n *Node) GetNumeric() (float64, error) { return v, nil } +// GetInts traverses the current JSON nodes and collects all integer values. +// +// The Number type stores both int and float types together as float64, +// but the GetInts function only retrieves values of the int type +// because it fetches values only if there is no fractional part when compared with float64 values. +// +// Usage: +// +// root := Must(Unmarshal([]byte(`{"key": 10.5, "key2": 10, "key3": "foo"}`))) +// ints := root.GetInts() +// if len(ints) != 1 { +// t.Errorf("GetInts returns wrong result: %v", ints) +// } +func (n *Node) GetInts() []int { + var collectInts func(*Node) []int + collectInts = func(node *Node) []int { + if node == nil { + return nil + } + + result := []int{} + if node.IsNumber() { + numVal, err := node.GetNumeric() + if err == nil && numVal == float64(int(numVal)) { // doesn't have a decimal part + result = append(result, int(numVal)) + } + } + + for _, childNode := range node.next { + childInts := collectInts(childNode) + result = append(result, childInts...) + } + + return result + } + + return collectInts(n) +} + +// GetFloats traverses the current JSON nodes and collects all float values. +// +// The Number type combines int and float types into float64 for storage, +// but the GetFloats function only accurately retrieves float types because it checks whether the numbers have a fractional part. +// +// Usage: +// +// root := Must(Unmarshal([]byte(`{"key": 10.5, "key2": 10, "key3": "foo"}`))) +// floats := root.GetFloats() +// if len(floats) != 1 { +// t.Errorf("GetFloats returns wrong result: %v", floats) +// } +func (n *Node) GetFloats() []float64 { + var collectFloats func(*Node) []float64 + collectFloats = func(node *Node) []float64 { + if node == nil { + return nil + } + + result := []float64{} + if node.IsNumber() { + numVal, err := node.GetNumeric() + if err == nil && numVal != float64(int(numVal)) { // check if it's a float + result = append(result, numVal) + } + } + + for _, childNode := range node.next { + childFloats := collectFloats(childNode) + result = append(result, childFloats...) + } + + return result + } + + return collectFloats(n) +} + +// MustNumeric returns the numeric (int/float) value if current node is number type. +// +// It panics if the current node is not number type. +func (n *Node) MustNumeric() float64 { + v, err := n.GetNumeric() + if err != nil { + panic(err) + } + + return v +} + +// GetString returns the string value if current node is string type. +// +// Usage: +// +// root, err := Unmarshal([]byte("foo")) +// if err != nil { +// t.Errorf("Error on Unmarshal(): %s", err) +// } +// +// str, err := root.GetString() +// if err != nil { +// t.Errorf("should retrieve string value: %s", err) +// } +// +// println(str) // "foo" func (n *Node) GetString() (string, error) { if n == nil { return "", errors.New("string node is empty") } - if n.value.typ != String { + if !n.IsString() { return "", errors.New("node type is not string") } @@ -350,12 +732,103 @@ func (n *Node) GetString() (string, error) { return v, nil } -func (n *Node) getBool() (bool, error) { +// GetStrings traverses the current JSON nodes and collects all string values. +// +// Usage: +// +// root := Must(Unmarshal([]byte(`{"key": "value", "key2": 10, "key3": "foo"}`)) +// strs := root.GetStrings() +// if len(strs) != 2 { +// t.Errorf("GetStrings returns wrong result: %v", strs) +// } +func (n *Node) GetStrings() []string { + var collectStrings func(*Node) []string + collectStrings = func(node *Node) []string { + if node == nil { + return nil + } + + result := []string{} + if node.IsString() { + strVal, err := node.GetString() + if err == nil { + result = append(result, strVal) + } + } + + for _, childNode := range node.next { + childStrings := collectStrings(childNode) + result = append(result, childStrings...) + } + + return result + } + + return collectStrings(n) +} + +// MustString returns the string value if current node is string type. +// +// It panics if the current node is not string type. +func (n *Node) MustString() string { + v, err := n.GetString() + if err != nil { + panic(err) + } + + return v +} + +// GetBools traverses the current JSON nodes and collects all boolean values. +// +// Usage: +// +// root := Must(Unmarshal([]byte(`{"key": true, "key2": 10, "key3": "foo"}`))) +// bools := root.GetBools() +// if len(bools) != 1 { +// t.Errorf("GetBools returns wrong result: %v", bools) +// } +func (n *Node) GetBools() []bool { + var collectBools func(*Node) []bool + collectBools = func(node *Node) []bool { + if node == nil { + return nil + } + + result := []bool{} + if node.IsBool() { + if boolVal, err := node.GetBool(); err == nil { + result = append(result, boolVal) + } + } + + for _, childNode := range node.next { + childBools := collectBools(childNode) + result = append(result, childBools...) + } + + return result + } + + return collectBools(n) +} + +// GetBool returns the boolean value if current node is boolean type. +// +// Usage: +// +// root := Unmarshal([]byte("true")) +// val, err := root.GetBool() +// if err != nil { +// t.Errorf("GetBool returns error: %v", err) +// } +// println(val) // true +func (n *Node) GetBool() (bool, error) { if n == nil { return false, errors.New("node is nil") } - if n.value.typ != Boolean { + if n.nodeType != Boolean { return false, errors.New("node is not boolean") } @@ -372,12 +845,39 @@ func (n *Node) getBool() (bool, error) { return v, nil } +// MustBool returns the boolean value if current node is boolean type. +// +// It panics if the current node is not boolean type. +func (n *Node) MustBool() bool { + v, err := n.GetBool() + if err != nil { + panic(err) + } + + return v +} + +// GetArray returns the array value if current node is array type. +// +// Usage: +// +// root := Must(Unmarshal([]byte(`["foo", 1]`))) +// arr, err := root.GetArray() +// if err != nil { +// t.Errorf("GetArray returns error: %v", err) +// } +// +// for _, val := range arr { +// println(val) +// } +// +// result: "foo", 1 func (n *Node) GetArray() ([]*Node, error) { if n == nil { return nil, errors.New("node is nil") } - if n.value.typ != Array { + if n.nodeType != Array { return nil, errors.New("node is not array") } @@ -394,12 +894,94 @@ func (n *Node) GetArray() ([]*Node, error) { return v, nil } +// MustArray returns the array value if current node is array type. +// +// It panics if the current node is not array type. +func (n *Node) MustArray() []*Node { + v, err := n.GetArray() + if err != nil { + panic(err) + } + + return v +} + +// AppendArray appends the given values to the current array node. +// +// If the current node is not array type, it returns an error. +// +// Example 1: +// +// root := Must(Unmarshal([]byte(`[{"foo":"bar"}]`))) +// if err := root.AppendArray(NullNode("")); err != nil { +// t.Errorf("should not return error: %s", err) +// } +// +// result: [{"foo":"bar"}, null] +// +// Example 2: +// +// root := Must(Unmarshal([]byte(`["bar", "baz"]`))) +// err := root.AppendArray(NumberNode("", 1), StringNode("", "foo")) +// if err != nil { +// t.Errorf("AppendArray returns error: %v", err) +// } +// +// result: ["bar", "baz", 1, "foo"] +func (n *Node) AppendArray(value ...*Node) error { + if !n.IsArray() { + return errors.New("can't append value to non-array node") + } + + for _, val := range value { + if err := n.append(nil, val); err != nil { + return err + } + } + + n.mark() + return nil +} + +// ArrayEach executes the callback for each element in the JSON array. +// +// Usage: +// +// jsonArrayNode.ArrayEach(func(i int, valueNode *Node) { +// ufmt.Println(i, valueNode) +// }) +func (n *Node) ArrayEach(callback func(i int, target *Node)) { + if n == nil || !n.IsArray() { + return + } + + for idx := 0; idx < len(n.next); idx++ { + element, err := n.GetIndex(idx) + if err != nil { + continue + } + + callback(idx, element) + } +} + +// GetObject returns the object value if current node is object type. +// +// Usage: +// +// root := Must(Unmarshal([]byte(`{"key": "value"}`))) +// obj, err := root.GetObject() +// if err != nil { +// t.Errorf("GetObject returns error: %v", err) +// } +// +// result: map[string]*Node{"key": StringNode("key", "value")} func (n *Node) GetObject() (map[string]*Node, error) { if n == nil { return nil, errors.New("node is nil") } - if n.value.typ != Object { + if !n.IsObject() { return nil, errors.New("node is not object") } @@ -416,8 +998,11 @@ func (n *Node) GetObject() (map[string]*Node, error) { return v, nil } -func (n *Node) MustNull() interface{} { - v, err := n.GetNull() +// MustObject returns the object value if current node is object type. +// +// It panics if the current node is not object type. +func (n *Node) MustObject() map[string]*Node { + v, err := n.GetObject() if err != nil { panic(err) } @@ -425,47 +1010,290 @@ func (n *Node) MustNull() interface{} { return v } -func (n *Node) MustNumeric() float64 { - v, err := n.GetNumeric() - if err != nil { - panic(err) +// AppendObject appends the given key and value to the current object node. +// +// If the current node is not object type, it returns an error. +func (n *Node) AppendObject(key string, value *Node) error { + if !n.IsObject() { + return errors.New("can't append value to non-object node") } - return v + if err := n.append(&key, value); err != nil { + return err + } + + n.mark() + return nil } -func (n *Node) MustString() string { - v, err := n.GetString() - if err != nil { - panic(err) +// ObjectEach executes the callback for each key-value pair in the JSON object. +// +// Usage: +// +// jsonObjectNode.ObjectEach(func(key string, valueNode *Node) { +// ufmt.Println(key, valueNode) +// }) +func (n *Node) ObjectEach(callback func(key string, value *Node)) { + if n == nil || !n.IsObject() { + return } - return v + for key, child := range n.next { + callback(key, child) + } } -func (n *Node) MustBool() bool { - v, err := n.getBool() +// String converts the node to a string representation. +func (n *Node) String() string { + if n == nil { + return "" + } + + if n.ready() && !n.modified { + return string(n.source()) + } + + val, err := Marshal(n) if err != nil { - panic(err) + return "error: " + err.Error() } - return v + return string(val) } -func (n *Node) MustArray() []*Node { - v, err := n.GetArray() - if err != nil { - panic(err) +// Path builds the path of the current node. +// +// For example: +// +// { "key": { "sub": [ "val1", "val2" ] }} +// +// The path of "val2" is: $.key.sub[1] +func (n *Node) Path() string { + if n == nil { + return "" } - return v + var sb strings.Builder + + if n.prev == nil { + sb.WriteString("$") + } else { + sb.WriteString(n.prev.Path()) + + if n.key != nil { + sb.WriteString("['" + n.Key() + "']") + } else { + sb.WriteString("[" + strconv.Itoa(n.Index()) + "]") + } + } + + return sb.String() } -func (n *Node) MustObject() map[string]*Node { - v, err := n.GetObject() - if err != nil { - panic(err) +// mark marks the current node as modified. +func (n *Node) mark() { + node := n + for node != nil && !node.modified { + node.modified = true + node = node.prev } +} - return v -} \ No newline at end of file +// clear clears the current node's value +func (n *Node) clear() { + n.data = nil + n.borders[1] = 0 + + for key := range n.next { + n.next[key].prev = nil + } + + n.next = nil +} + +// validate checks the current node value is matching the given type. +func (n *Node) validate(t ValueType, v interface{}) error { + if n == nil { + return errors.New("node is nil") + } + + switch t { + case Null: + if v != nil { + return errors.New("invalid null value") + } + + // TODO: support uint256, int256 type later. + case Number: + switch v.(type) { + case float64, int, uint: + return nil + default: + return errors.New("invalid number value") + } + + case String: + if _, ok := v.(string); !ok { + return errors.New("invalid string value") + } + + case Boolean: + if _, ok := v.(bool); !ok { + return errors.New("invalid boolean value") + } + + case Array: + if _, ok := v.([]*Node); !ok { + return errors.New("invalid array value") + } + + case Object: + if _, ok := v.(map[string]*Node); !ok { + return errors.New("invalid object value") + } + } + + return nil +} + +// isContainer checks the current node type is array or object. +func (n *Node) isContainer() bool { + return n.IsArray() || n.IsObject() +} + +// remove removes the value from the current container type node. +func (n *Node) remove(v *Node) error { + if !n.isContainer() { + return errors.New(ufmt.Sprintf("can't remove value from non-array or non-object node. got=%s", n.Type().String())) + } + + if v.prev != n { + return errors.New("invalid parent node") + } + + n.mark() + if n.IsArray() { + delete(n.next, strconv.Itoa(*v.index)) + n.dropIndex(*v.index) + } else { + delete(n.next, *v.key) + } + + v.prev = nil + return nil +} + +// dropIndex rebase the index of current array node values. +func (n *Node) dropIndex(idx int) { + for i := idx + 1; i <= len(n.next); i++ { + prv := i - 1 + if curr, ok := n.next[strconv.Itoa(i)]; ok { + curr.index = &prv + n.next[strconv.Itoa(prv)] = curr + } + + delete(n.next, strconv.Itoa(i)) + } +} + +// append is a helper function to append the given value to the current container type node. +func (n *Node) append(key *string, val *Node) error { + if n.isSameOrParentNode(val) { + return errors.New("can't append same or parent node") + } + + if val.prev != nil { + if err := val.prev.remove(val); err != nil { + return err + } + } + + val.prev = n + val.key = key + + if key == nil { + size := len(n.next) + val.index = &size + n.next[strconv.Itoa(size)] = val + } else { + if old, ok := n.next[*key]; ok { + if err := n.remove(old); err != nil { + return err + } + } + n.next[*key] = val + } + + return nil +} + +func (n *Node) isSameOrParentNode(nd *Node) bool { + return n == nd || n.isParentNode(nd) +} + +func (n *Node) isParentNode(nd *Node) bool { + if n == nil { + return false + } + + for curr := nd.prev; curr != nil; curr = curr.prev { + if curr == n { + return true + } + } + + return false +} + +func (n *Node) setRef(prev *Node, key *string, idx *int) { + n.prev = prev + + if key == nil { + n.key = nil + } else { + tmp := *key + n.key = &tmp + } + + n.index = idx +} + +// Clone creates a new node instance with the same value of the current node. +func (n *Node) Clone() *Node { + node := n.clone() + node.setRef(nil, nil, nil) + + return node +} + +func (n *Node) clone() *Node { + node := &Node{ + prev: n.prev, + next: make(map[string]*Node, len(n.next)), + key: n.key, + data: n.data, + value: n.value, + nodeType: n.nodeType, + index: n.index, + borders: n.borders, + modified: n.modified, + } + + for k, v := range n.next { + node.next[k] = v + } + + return node +} + +// Must panics if the given node is not fulfilled the expectation. +// Usage: +// +// node := Must(Unmarshal([]byte(`{"key": "value"}`)) +func Must(root *Node, expect error) *Node { + if expect != nil { + panic(expect) + } + + return root +} diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index 4595cef31ed..81eb0c2f825 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -1,93 +1,1578 @@ package json import ( + "bytes" + "sort" "strconv" + "strings" "testing" + + "gno.land/p/demo/ufmt" +) + +var ( + nilKey *string + dummyKey = "key" ) -func TestNodeValue(t *testing.T) { +type _args struct { + prev *Node + buf *buffer + typ ValueType + key **string +} + +type simpleNode struct { + name string + node *Node +} + +func TestNode_CreateNewNode(t *testing.T) { + rel := &dummyKey + + tests := []struct { + name string + args _args + expectCurr *Node + expectErr bool + expectPanic bool + }{ + // { + // name: "object with empty key", + // args: _args{ + // prev: ObjectNode("", make(map[string]*Node)), + // buf: newBuffer(make([]byte, 10)), + // typ: Boolean, + // key: &nilKey, + // }, + // expectCurr: nil, + // expectPanic: true, + // }, + { + name: "child for non container type", + args: _args{ + prev: BoolNode("", true), + buf: newBuffer(make([]byte, 10)), + typ: Boolean, + key: &rel, + }, + expectCurr: nil, + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + if tt.expectPanic { + return + } + t.Errorf("%s panic occurred when not expected: %v", tt.name, r) + } else if tt.expectPanic { + t.Errorf("%s expected panic but didn't occur", tt.name) + } + }() + + got, err := NewNode(tt.args.prev, tt.args.buf, tt.args.typ, tt.args.key) + if (err != nil) != tt.expectErr { + t.Errorf("%s error = %v, expect error %v", tt.name, err, tt.expectErr) + return + } + + if tt.expectErr { + return + } + + if !compareNodes(got, tt.expectCurr) { + t.Errorf("%s got = %v, want %v", tt.name, got, tt.expectCurr) + } + }) + } +} + +func TestNode_Value(t *testing.T) { + tests := []struct { + name string + data []byte + _type ValueType + expected interface{} + errExpected bool + }{ + {name: "null", data: []byte("null"), _type: Null, expected: nil}, + {name: "1", data: []byte("1"), _type: Number, expected: float64(1)}, + {name: ".1", data: []byte(".1"), _type: Number, expected: float64(.1)}, + {name: "-.1e1", data: []byte("-.1e1"), _type: Number, expected: float64(-1)}, + {name: "string", data: []byte("\"foo\""), _type: String, expected: "foo"}, + {name: "space", data: []byte("\"foo bar\""), _type: String, expected: "foo bar"}, + {name: "true", data: []byte("true"), _type: Boolean, expected: true}, + {name: "invalid true", data: []byte("tru"), _type: Unknown, errExpected: true}, + {name: "invalid false", data: []byte("fals"), _type: Unknown, errExpected: true}, + {name: "false", data: []byte("false"), _type: Boolean, expected: false}, + {name: "e1", data: []byte("e1"), _type: Unknown, errExpected: true}, + {name: "1a", data: []byte("1a"), _type: Unknown, errExpected: true}, + {name: "string error", data: []byte("\"foo\nbar\""), _type: String, errExpected: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + curr := &Node{ + data: tt.data, + nodeType: tt._type, + borders: [2]int{0, len(tt.data)}, + } + + got, err := curr.Value() + if err != nil { + if !tt.errExpected { + t.Errorf("%s error = %v, expect error %v", tt.name, err, tt.errExpected) + } + return + } + + if got != tt.expected { + t.Errorf("%s got = %v, want %v", tt.name, got, tt.expected) + } + }) + } +} + +func TestNode_Delete(t *testing.T) { + root := Must(Unmarshal([]byte(`{"foo":"bar"}`))) + if err := root.Delete(); err != nil { + t.Errorf("Delete returns error: %v", err) + } + + if value, err := Marshal(root); err != nil { + t.Errorf("Marshal returns error: %v", err) + } else if string(value) != `{"foo":"bar"}` { + t.Errorf("Marshal returns wrong value: %s", string(value)) + } + + foo := root.MustKey("foo") + if err := foo.Delete(); err != nil { + t.Errorf("Delete returns error while handling foo: %v", err) + } + + if value, err := Marshal(root); err != nil { + t.Errorf("Marshal returns error: %v", err) + } else if string(value) != `{}` { + t.Errorf("Marshal returns wrong value: %s", string(value)) + } + + if value, err := Marshal(foo); err != nil { + t.Errorf("Marshal returns error: %v", err) + } else if string(value) != `"bar"` { + t.Errorf("Marshal returns wrong value: %s", string(value)) + } + + if foo.prev != nil { + t.Errorf("foo.prev should be nil") + } +} + +func TestNode_ObjectNode(t *testing.T) { + objs := map[string]*Node{ + "key1": NullNode("null"), + "key2": NumberNode("answer", 42), + "key3": StringNode("string", "foobar"), + "key4": BoolNode("bool", true), + } + + node := ObjectNode("test", objs) + + if len(node.next) != len(objs) { + t.Errorf("ObjectNode: want %v got %v", len(objs), len(node.next)) + } + + for k, v := range objs { + if node.next[k] == nil { + t.Errorf("ObjectNode: want %v got %v", v, node.next[k]) + } + } +} + +func TestNode_AppendObject(t *testing.T) { + if err := Must(Unmarshal([]byte(`{"foo":"bar","baz":null}`))).AppendObject("biz", NullNode("")); err != nil { + t.Errorf("AppendArray should return error") + } + + root := Must(Unmarshal([]byte(`{"foo":"bar"}`))) + if err := root.AppendObject("baz", NullNode("")); err != nil { + t.Errorf("AppendObject should not return error: %s", err) + } + + if value, err := Marshal(root); err != nil { + t.Errorf("Marshal returns error: %v", err) + } else if isSameObject(string(value), `"{"foo":"bar","baz":null}"`) { + t.Errorf("Marshal returns wrong value: %s", string(value)) + } + + // FIXME: this may fail if execute test in more than 3 times in a row. + if err := root.AppendObject("biz", NumberNode("", 42)); err != nil { + t.Errorf("AppendObject returns error: %v", err) + } + + val, err := Marshal(root) + + if err != nil { + t.Errorf("Marshal returns error: %v", err) + } + + // FIXME: this may fail if execute test in more than 3 times in a row. + if isSameObject(string(val), `"{"foo":"bar","baz":null,"biz":42}"`) { + t.Errorf("Marshal returns wrong value: %s", string(val)) + } +} + +func TestNode_ArrayNode(t *testing.T) { + arr := []*Node{ + NullNode("nil"), + NumberNode("num", 42), + StringNode("str", "foobar"), + BoolNode("bool", true), + } + + node := ArrayNode("test", arr) + + if len(node.next) != len(arr) { + t.Errorf("ArrayNode: want %v got %v", len(arr), len(node.next)) + } + + for i, v := range arr { + if node.next[strconv.Itoa(i)] == nil { + t.Errorf("ArrayNode: want %v got %v", v, node.next[strconv.Itoa(i)]) + } + } +} + +func TestNode_AppendArray(t *testing.T) { + if err := Must(Unmarshal([]byte(`[{"foo":"bar"}]`))).AppendArray(NullNode("")); err != nil { + t.Errorf("should return error") + } + + root := Must(Unmarshal([]byte(`[{"foo":"bar"}]`))) + if err := root.AppendArray(NullNode("")); err != nil { + t.Errorf("should not return error: %s", err) + } + + if value, err := Marshal(root); err != nil { + t.Errorf("Marshal returns error: %v", err) + } else if string(value) != `[{"foo":"bar"},null]` { + t.Errorf("Marshal returns wrong value: %s", string(value)) + } + + // TODO: Number is not handled properly. + // number does not make problem if they positioned in the array or object. + if err := root.AppendArray( + // NumberNode("", 1), + // StringNode("", "foo"), + Must(Unmarshal([]byte(`[0,1,null,true,"example"]`))), + Must(Unmarshal([]byte(`{"foo": true, "bar": null, "baz": 123}`))), + ); err != nil { + t.Errorf("AppendArray returns error: %v", err) + } + + // if value, err := Marshal(root); err != nil { + // t.Errorf("Marshal returns error: %v", err) + // } else if string(value) != `[{"foo":"bar"},null,1,"foo",[0,1,null,true,"example"],{"foo": true, "bar": null, "baz": 123}]` { + // t.Errorf("Marshal returns wrong value: %s", string(value)) + // } + + if value, err := Marshal(root); err != nil { + t.Errorf("Marshal returns error: %v", err) + } else if string(value) != `[{"foo":"bar"},null,[0,1,null,true,"example"],{"foo": true, "bar": null, "baz": 123}]` { + t.Errorf("Marshal returns wrong value: %s", string(value)) + } +} + +/******** value getter ********/ + +func TestNode_GetBool(t *testing.T) { + root, err := Unmarshal([]byte(`true`)) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + return + } + + value, err := root.GetBool() + if err != nil { + t.Errorf("Error on root.GetBool(): %s", err.Error()) + } + + if !value { + t.Errorf("root.GetBool() is corrupted") + } +} + +func TestNode_GetBool_Fail(t *testing.T) { + tests := []simpleNode{ + {"nil node", (*Node)(nil)}, + {"literally null node", NullNode("")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := tt.node.GetBool(); err == nil { + t.Errorf("%s should be an error", tt.name) + } + }) + } +} + +// countBooleans test function that counts the number of true and false values in a slice of booleans. +// because map doesn't maintain order, we can't use it to test the order of the values. +func countBooleans(bools []bool) (trueCount, falseCount int) { + for _, b := range bools { + if b { + trueCount++ + } else { + falseCount++ + } + } + return +} + +// TODO: resolve stack overflow +func TestNode_GetAllBools(t *testing.T) { + tests := []struct { + name string + json string + expectedTrues int + expectedFalses int + }{ + // { + // name: "no booleans", + // json: `{"foo": "bar", "number": 123}`, + // expectedTrues: 0, + // expectedFalses: 0, + // }, + // { + // name: "single boolean true", + // json: `{"isTrue": true}`, + // expectedTrues: 1, + // expectedFalses: 0, + // }, + // { + // name: "single boolean false", + // json: `{"isFalse": false}`, + // expectedTrues: 0, + // expectedFalses: 1, + // }, + // { + // name: "multiple booleans", + // json: `{"first": true, "info": {"second": false, "third": true}, "array": [false, true, false]}`, + // expectedTrues: 3, + // expectedFalses: 3, + // }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root, err := Unmarshal([]byte(tc.json)) + if err != nil { + t.Errorf("Failed to unmarshal JSON: %v", err) + return + } + + bools := root.GetBools() + trueCount, falseCount := countBooleans(bools) + if trueCount != tc.expectedTrues || falseCount != tc.expectedFalses { + t.Errorf("%s: expected %d true and %d false booleans, got %d true and %d false", tc.name, tc.expectedTrues, tc.expectedFalses, trueCount, falseCount) + } + }) + } +} + +func TestNode_IsBool(t *testing.T) { + tests := []simpleNode{ + {"true", BoolNode("", true)}, + {"false", BoolNode("", false)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !tt.node.IsBool() { + t.Errorf("%s should be a bool", tt.name) + } + }) + } +} + +func TestNode_IsBool_With_Unmarshal(t *testing.T) { + tests := []struct { + name string + json []byte + want bool + }{ + {"true", []byte("true"), true}, + {"false", []byte("false"), true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root, err := Unmarshal(tt.json) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + } + + if root.IsBool() != tt.want { + t.Errorf("%s should be a bool", tt.name) + } + }) + } +} + +var nullJson = []byte(`null`) + +func TestNode_GetNull(t *testing.T) { + root, err := Unmarshal(nullJson) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + } + + value, err := root.GetNull() + if err != nil { + t.Errorf("error occurred while getting null, %s", err) + } + + if value != nil { + t.Errorf("value is not matched. expected: nil, got: %v", value) + } +} + +func TestNode_GetNull_Fail(t *testing.T) { + tests := []simpleNode{ + {"nil node", (*Node)(nil)}, + {"number node is null", NumberNode("", 42)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := tt.node.GetNull(); err == nil { + t.Errorf("%s should be an error", tt.name) + } + }) + } +} + +func TestNode_MustNull(t *testing.T) { + root, err := Unmarshal(nullJson) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + } + + value := root.MustNull() + if value != nil { + t.Errorf("value is not matched. expected: nil, got: %v", value) + } +} + +func TestNode_GetNumeric_Float(t *testing.T) { + root, err := Unmarshal([]byte(`123.456`)) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + return + } + + value, err := root.GetNumeric() + if err != nil { + t.Errorf("Error on root.GetNumeric(): %s", err) + } + + if value != float64(123.456) { + t.Errorf(ufmt.Sprintf("value is not matched. expected: 123.456, got: %v", value)) + } +} + +func TestNode_GetNumeric_Scientific_Notation(t *testing.T) { + root, err := Unmarshal([]byte(`1e3`)) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + return + } + + value, err := root.GetNumeric() + if err != nil { + t.Errorf("Error on root.GetNumeric(): %s", err) + } + + if value != float64(1000) { + t.Errorf(ufmt.Sprintf("value is not matched. expected: 1000, got: %v", value)) + } +} + +func TestNode_GetNumeric_With_Unmarshal(t *testing.T) { + root, err := Unmarshal([]byte(`123`)) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + return + } + + value, err := root.GetNumeric() + if err != nil { + t.Errorf("Error on root.GetNumeric(): %s", err) + } + + if value != float64(123) { + t.Errorf(ufmt.Sprintf("value is not matched. expected: 123, got: %v", value)) + } +} + +func TestNode_GetNumeric_Fail(t *testing.T) { + tests := []simpleNode{ + {"nil node", (*Node)(nil)}, + {"null node", NullNode("")}, + {"string node", StringNode("", "123")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := tt.node.GetNumeric(); err == nil { + t.Errorf("%s should be an error", tt.name) + } + }) + } +} + +// TODO: resolve stack overflow +func TestNode_GetAllIntsFromNode(t *testing.T) { + tests := []struct { + name string + json string + expected []int + }{ + // { + // name: "no numbers", + // json: `{"foo": "bar", "baz": "qux"}`, + // expected: []int{}, + // }, + // { + // name: "mixed numbers", + // json: `{"int": 42, "float": 3.14, "nested": {"int2": 24, "array": [1, 2.34, 3]}}`, + // expected: []int{42, 24, 1, 3}, + // }, + // { + // name: "all floats", + // json: `{"float1": 0.1, "float2": 4.5}`, + // expected: []int{}, + // }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root, err := Unmarshal([]byte(tc.json)) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + got := root.GetInts() + for _, exp := range tc.expected { + if !containsInts(got, exp) { + t.Errorf("Expected integer %d is missing", exp) + } + } + }) + } +} + +// containsInts checks if a slice of integers contains a specific value. +func containsInts(slice []int, val int) bool { + for _, item := range slice { + if item == val { + return true + } + } + return false +} + +// TODO: resolve stack overflow +func TestNode_GetAllFloatsFromNode(t *testing.T) { + tests := []struct { + name string + json string + expected []float64 + }{ + // { + // name: "no numbers", + // json: `{"foo": "bar", "baz": "qux"}`, + // expected: []float64{}, + // }, + // { + // name: "mixed numbers", + // json: `{"int": 42, "float": 3.14, "nested": {"float2": 2.34, "array": [1, 2.34, 3]}}`, + // expected: []float64{3.14, 2.34, 2.34}, + // }, + // { + // name: "all ints", + // json: `{"int1": 1, "int2": 2}`, + // expected: []float64{}, + // }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root, err := Unmarshal([]byte(tc.json)) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + got := root.GetFloats() + for _, exp := range tc.expected { + if !containsFloats(got, exp) { + t.Errorf("Expected float %f is missing", exp) + } + } + }) + } +} + +// containsFloats checks if a slice of floats contains a specific value. +func containsFloats(slice []float64, val float64) bool { + for _, item := range slice { + if item == val { + return true + } + } + return false +} + +func TestNode_GetString(t *testing.T) { + root, err := Unmarshal([]byte(`"123foobar 3456"`)) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + } + + value, err := root.GetString() + if err != nil { + t.Errorf("Error on root.GetString(): %s", err) + } + + if value != "123foobar 3456" { + t.Errorf(ufmt.Sprintf("value is not matched. expected: 123, got: %s", value)) + } +} + +func TestNode_GetString_Fail(t *testing.T) { + tests := []simpleNode{ + {"nil node", (*Node)(nil)}, + {"null node", NullNode("")}, + {"number node", NumberNode("", 123)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := tt.node.GetString(); err == nil { + t.Errorf("%s should be an error", tt.name) + } + }) + } +} + +// TODO: resolve stack overflow +func TestNode_GetAllStringsFromNode(t *testing.T) { + tests := []struct { + name string + json string + expected []string + }{ + // { + // name: "no strings", + // json: `{"int": 42, "float": 3.14}`, + // expected: []string{}, + // }, + // { + // name: "multiple strings", + // json: `{"str": "hello", "nested": {"str2": "world"}, "array": ["foo", "bar"]}`, + // expected: []string{"hello", "world", "foo", "bar"}, + // }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root, err := Unmarshal([]byte(tc.json)) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + got := root.GetStrings() + if len(got) != len(tc.expected) { + t.Errorf("Expected %d strings, got %d", len(tc.expected), len(got)) + return + } + for _, exp := range tc.expected { + found := false + for _, str := range got { + if str == exp { + found = true + break + } + } + if !found { + t.Errorf("Expected string '%s' not found", exp) + } + } + }) + } +} + +func TestNode_MustString(t *testing.T) { + tests := []struct { + name string + data []byte + }{ + {"foo", []byte(`"foo"`)}, + {"foo bar", []byte(`"foo bar"`)}, + {"", []byte(`""`)}, + {"안녕하세요", []byte(`"안녕하세요"`)}, + {"こんにちは", []byte(`"こんにちは"`)}, + {"你好", []byte(`"你好"`)}, + {"one \"encoded\" string", []byte(`"one \"encoded\" string"`)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root, err := Unmarshal(tt.data) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + } + + value := root.MustString() + if value != tt.name { + t.Errorf("value is not matched. expected: %s, got: %s", tt.name, value) + } + }) + } +} + +func TestUnmarshal_Array(t *testing.T) { + root, err := Unmarshal([]byte(" [1,[\"1\",[1,[1,2,3]]]]\r\n")) + + if err != nil { + t.Errorf("Error on Unmarshal: %s", err.Error()) + } + + if root == nil { + t.Errorf("Error on Unmarshal: root is nil") + } + + if root.Type() != Array { + t.Errorf("Error on Unmarshal: wrong type") + } + + array, err := root.GetArray() + if err != nil { + t.Errorf("error occurred while getting array, %s", err) + } else if len(array) != 2 { + t.Errorf("expected 2 elements, got %d", len(array)) + } else if val, err := array[0].GetNumeric(); err != nil { + t.Errorf("value of array[0] is not numeric. got: %v", array[0].value) + } else if val != 1 { + t.Errorf("Error on array[0].GetNumeric(): expected to be '1', got: %v", val) + } else if val, err := array[1].GetArray(); err != nil { + t.Errorf("error occurred while getting array, %s", err.Error()) + } else if len(val) != 2 { + t.Errorf("Error on array[1].GetArray(): expected 2 elements, got %d", len(val)) + } else if el, err := val[0].GetString(); err != nil { + t.Errorf("error occurred while getting string, %s", err.Error()) + } else if el != "1" { + t.Errorf("Error on val[0].GetString(): expected to be '1', got: %s", el) + } +} + +var sampleArr = []byte(`[-1, 2, 3, 4, 5, 6]`) + +func TestNode_GetArray(t *testing.T) { + root, err := Unmarshal(sampleArr) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + return + } + + array, err := root.GetArray() + if err != nil { + t.Errorf("Error on root.GetArray(): %s", err) + } + + if len(array) != 6 { + t.Errorf(ufmt.Sprintf("length is not matched. expected: 3, got: %d", len(array))) + } + + for i, node := range array { + for j, val := range []int{-1, 2, 3, 4, 5, 6} { + if i == j { + if v, err := node.GetNumeric(); err != nil { + t.Errorf(ufmt.Sprintf("Error on node.GetNumeric(): %s", err)) + } else if v != float64(val) { + t.Errorf(ufmt.Sprintf("value is not matched. expected: %d, got: %v", val, v)) + } + } + } + } +} + +func TestNode_GetArray_Fail(t *testing.T) { + tests := []simpleNode{ + {"nil node", (*Node)(nil)}, + {"null node", NullNode("")}, + {"number node", NumberNode("", 123)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := tt.node.GetArray(); err == nil { + t.Errorf("%s should be an error", tt.name) + } + }) + } +} + +func TestNode_IsArray(t *testing.T) { + root, err := Unmarshal(sampleArr) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + return + } + + if root.Type() != Array { + t.Errorf(ufmt.Sprintf("Must be an array. got: %s", root.Type().String())) + } +} + +func TestNode_ArrayEach(t *testing.T) { + tests := []struct { + name string + json string + expected []int + }{ + { + name: "empty array", + json: `[]`, + expected: []int{}, + }, + { + name: "single element", + json: `[42]`, + expected: []int{42}, + }, + { + name: "multiple elements", + json: `[1, 2, 3, 4, 5]`, + expected: []int{1, 2, 3, 4, 5}, + }, + { + name: "multiple elements but all values are same", + json: `[1, 1, 1, 1, 1]`, + expected: []int{1, 1, 1, 1, 1}, + }, + { + name: "multiple elements with non-numeric values", + json: `["a", "b", "c", "d", "e"]`, + expected: []int{}, + }, + { + name: "non-array node", + json: `{"not": "an array"}`, + expected: []int{}, + }, + { + name: "array containing numeric and non-numeric elements", + json: `["1", 2, 3, "4", 5, "6"]`, + expected: []int{2, 3, 5}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root, err := Unmarshal([]byte(tc.json)) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + var result []int // callback result + root.ArrayEach(func(index int, element *Node) { + if val, err := strconv.Atoi(element.String()); err == nil { + result = append(result, val) + } + }) + + if len(result) != len(tc.expected) { + t.Errorf("%s: expected %d elements, got %d", tc.name, len(tc.expected), len(result)) + return + } + + for i, val := range result { + if val != tc.expected[i] { + t.Errorf("%s: expected value at index %d to be %d, got %d", tc.name, i, tc.expected[i], val) + } + } + }) + } +} + +func TestNode_Key(t *testing.T) { + root, err := Unmarshal([]byte(`{"foo": true, "bar": null, "baz": 123, "biz": [1,2,3]}`)) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + } + + obj := root.MustObject() + for key, node := range obj { + if key != node.Key() { + t.Errorf("Key() = %v, want %v", node.Key(), key) + } + } + + keys := []string{"foo", "bar", "baz", "biz"} + for _, key := range keys { + if obj[key].Key() != key { + t.Errorf("Key() = %v, want %v", obj[key].Key(), key) + } + } + + // TODO: resolve stack overflow + // if root.MustKey("foo").Clone().Key() != "" { + // t.Errorf("wrong key found for cloned key") + // } + + if (*Node)(nil).Key() != "" { + t.Errorf("wrong key found for nil node") + } +} + +func TestNode_Size(t *testing.T) { + root, err := Unmarshal(sampleArr) + if err != nil { + t.Errorf("error occurred while unmarshal") + } + + size := root.Size() + if size != 6 { + t.Errorf(ufmt.Sprintf("Size() must be 6. got: %v", size)) + } + + if (*Node)(nil).Size() != 0 { + t.Errorf(ufmt.Sprintf("Size() must be 0. got: %v", (*Node)(nil).Size())) + } +} + +func TestNode_Index(t *testing.T) { + root, err := Unmarshal([]byte(`[1, 2, 3, 4, 5, 6]`)) + if err != nil { + t.Error("error occurred while unmarshal") + } + + arr := root.MustArray() + for i, node := range arr { + if i != node.Index() { + t.Errorf(ufmt.Sprintf("Index() must be nil. got: %v", i)) + } + } +} + +func TestNode_Index_Fail(t *testing.T) { tests := []struct { - name string - value interface{} - expectType ValueType + name string + node *Node + want int }{ - {"string", "test value", String}, - {"integer", 123, Number}, - {"float", 1.23, Float}, - {"boolean", true, Boolean}, - {"null", nil, Null}, - {"array", []interface{}{1, 2, 3}, Array}, - {"object", map[string]interface{}{"key": "value"}, Object}, + {"nil node", (*Node)(nil), -1}, + {"null node", NullNode(""), -1}, + {"object node", ObjectNode("", nil), -1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.Index(); got != tt.want { + t.Errorf("Index() = %v, want %v", got, tt.want) + } + }) + + } +} + +func TestNode_GetIndex(t *testing.T) { + root, err := Unmarshal([]byte(`[1, 2, 3, 4, 5, 6]`)) + if err != nil { + t.Errorf("error occurred while unmarshal") + } + + expected := []int{1, 2, 3, 4, 5, 6} + for i, v := range expected { + val, err := root.GetIndex(i) + if err != nil { + t.Errorf("error occurred while getting index %d, %s", i, err) + } + + if val.MustNumeric() != float64(v) { + t.Errorf("value is not matched. expected: %d, got: %v", v, val.MustNumeric()) + } + } +} + +func TestNode_GetKey(t *testing.T) { + root, err := Unmarshal([]byte(`{"foo": true, "bar": null}`)) + if err != nil { + t.Error("error occurred while unmarshal") + } + + value, err := root.GetKey("foo") + if err != nil { + t.Errorf("error occurred while getting key, %s", err) + } + + if value.MustBool() != true { + t.Errorf("value is not matched. expected: true, got: %v", value.MustBool()) + } + + value, err = root.GetKey("bar") + if err != nil { + t.Errorf("error occurred while getting key, %s", err) + } + + _, err = root.GetKey("baz") + if err == nil { + t.Errorf("key baz is not exist. must be failed") + } + + if value.MustNull() != nil { + t.Errorf("value is not matched. expected: nil, got: %v", value.MustNull()) + } +} + +func TestNode_GetKey_Fail(t *testing.T) { + tests := []simpleNode{ + {"nil node", (*Node)(nil)}, + {"null node", NullNode("")}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - nv := &NodeValue{} - nv.Store(tt.value) + if _, err := tt.node.GetKey(""); err == nil { + t.Errorf("%s should be an error", tt.name) + } + }) + } +} + +func TestNode_EachKey(t *testing.T) { + tests := []struct { + name string + json string + expected []string + }{ + { + name: "simple foo/bar", + json: `{"foo": true, "bar": null}`, + expected: []string{"foo", "bar"}, + }, + { + name: "empty object", + json: `{}`, + expected: []string{}, + }, + { + name: "nested object", + json: `{ + "outer": { + "inner": { + "key": "value" + }, + "array": [1, 2, 3] + }, + "another": "item" + }`, + expected: []string{"outer", "inner", "key", "array", "another"}, + }, + { + name: "complex object", + json: `{ + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated": false, + "IDs": [116, 943, 234, 38793] + } + }`, + expected: []string{"Image", "Width", "Height", "Title", "Thumbnail", "Url", "Animated", "IDs"}, + }, + } - if nv.value != tt.value { - t.Errorf("Store: %s want %v got %v", tt.name, tt.value, nv.value) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root, err := Unmarshal([]byte(tt.json)) + if err != nil { + t.Errorf("error occurred while unmarshal") } - typ := typeOf(tt.value) - if typ != tt.expectType { - t.Errorf("Store: %s want type %v got %v", tt.name, tt.expectType.String(), typ.String()) + value := root.UniqueKeys() + if len(value) != len(tt.expected) { + t.Errorf("%s length must be %v. got: %v. retrieved keys: %s", tt.name, len(tt.expected), len(value), value) } - loadedValue := nv.Load() - if loadedValue != tt.value { - t.Errorf("Load: %s want %v got %v", tt.name, tt.value, loadedValue) + for _, key := range value { + if !contains(tt.expected, key) { + t.Errorf("EachKey() must be in %v. got: %v", tt.expected, key) + } } }) } } -func TestNodeSource(t *testing.T) { +// TODO: resolve stack overflow +func TestNode_IsEmpty(t *testing.T) { tests := []struct { name string node *Node - expected []byte + expected bool + }{ + {"nil node", (*Node)(nil), false}, // nil node is not empty. + // {"null node", NullNode(""), true}, + {"empty object", ObjectNode("", nil), true}, + {"empty array", ArrayNode("", nil), true}, + {"non-empty object", ObjectNode("", map[string]*Node{"foo": BoolNode("foo", true)}), false}, + {"non-empty array", ArrayNode("", []*Node{BoolNode("0", true)}), false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.Empty(); got != tt.expected { + t.Errorf("%s = %v, want %v", tt.name, got, tt.expected) + } + }) + } +} + +func TestNode_Index_EmptyList(t *testing.T) { + root, err := Unmarshal([]byte(`[]`)) + if err != nil { + t.Errorf("error occurred while unmarshal") + } + + array := root.MustArray() + for i, node := range array { + if i != node.Index() { + t.Errorf(ufmt.Sprintf("Index() must be nil. got: %v", i)) + } + } +} + +func TestNode_GetObject(t *testing.T) { + root, err := Unmarshal([]byte(`{"foo": true,"bar": null}`)) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + return + } + + value, err := root.GetObject() + if err != nil { + t.Errorf("Error on root.GetObject(): %s", err.Error()) + } + + if _, ok := value["foo"]; !ok { + t.Errorf("root.GetObject() is corrupted: foo") + } + + if _, ok := value["bar"]; !ok { + t.Errorf("root.GetObject() is corrupted: bar") + } +} + +func TestNode_GetObject_Fail(t *testing.T) { + tests := []simpleNode{ + {"nil node", (*Node)(nil)}, + {"get object from null node", NullNode("")}, + {"not object node", NumberNode("", 123)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := tt.node.GetObject(); err == nil { + t.Errorf("%s should be an error", tt.name) + } + }) + } +} + +func TestNode_ObjectEach(t *testing.T) { + tests := []struct { + name string + json string + expected map[string]int }{ { - name: "nil node", - node: nil, - expected: nil, + name: "empty object", + json: `{}`, + expected: make(map[string]int), }, { - name: "ready unmodified node with data", - node: &Node{ - data: []byte("test data"), - borders: [2]uint{0, 9}, - modified: false, - }, - expected: []byte("test data"), + name: "single key-value pair", + json: `{"key": 42}`, + expected: map[string]int{"key": 42}, + }, + { + name: "multiple key-value pairs", + json: `{"one": 1, "two": 2, "three": 3}`, + expected: map[string]int{"one": 1, "two": 2, "three": 3}, }, { - name: "ready modified node with data", + name: "multiple key-value pairs with some non-numeric values", + json: `{"one": 1, "two": "2", "three": 3, "four": "4"}`, + expected: map[string]int{"one": 1, "three": 3}, + }, + { + name: "non-object node", + json: `["not", "an", "object"]`, + expected: make(map[string]int), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root, err := Unmarshal([]byte(tc.json)) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + result := make(map[string]int) + root.ObjectEach(func(key string, value *Node) { + // extract integer values from the object + if val, err := strconv.Atoi(value.String()); err == nil { + result[key] = val + } + }) + + if len(result) != len(tc.expected) { + t.Errorf("%s: expected %d key-value pairs, got %d", tc.name, len(tc.expected), len(result)) + return + } + + for key, val := range tc.expected { + if result[key] != val { + t.Errorf("%s: expected value for key %s to be %d, got %d", tc.name, key, val, result[key]) + } + } + }) + } +} + +func TestNode_ExampleMust(t *testing.T) { + data := []byte(`{ + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated" : false, + "IDs": [116, 943, 234, 38793] + } + }`) + + root := Must(Unmarshal(data)) + if root.Size() != 1 { + t.Errorf("root.Size() must be 1. got: %v", root.Size()) + } + + ufmt.Sprintf("Object has %d inheritors inside", root.Size()) + // Output: + // Object has 1 inheritors inside +} + +// Calculate AVG price from different types of objects, JSON from: https://goessner.net/articles/JsonPath/index.html#e3 +func TestExampleUnmarshal(t *testing.T) { + data := []byte(`{ "store": { + "book": [ + { "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, + { "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { "color": "red", + "price": 19.95 + }, + "tools": null + } +}`) + + root, err := Unmarshal(data) + if err != nil { + t.Errorf("error occurred when unmarshal") + } + + store := root.MustKey("store").MustObject() + + var prices float64 + size := 0 + for _, objects := range store { + if objects.IsArray() && objects.Size() > 0 { + size += objects.Size() + for _, object := range objects.MustArray() { + prices += object.MustKey("price").MustNumeric() + } + } else if objects.IsObject() && objects.HasKey("price") { + size++ + prices += objects.MustKey("price").MustNumeric() + } + } + + result := int(prices / float64(size)) + ufmt.Sprintf("AVG price: %d", result) +} + +func TestNode_ExampleMust_panic(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("The code did not panic") + } + }() + data := []byte(`{]`) + root := Must(Unmarshal(data)) + ufmt.Sprintf("Object has %d inheritors inside", root.Size()) +} + +func TestNode_Path(t *testing.T) { + data := []byte(`{ + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated" : false, + "IDs": [116, 943, 234, 38793] + } + }`) + + root, err := Unmarshal(data) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + return + } + + if root.Path() != "$" { + t.Errorf("Wrong root.Path()") + } + + element := root.MustKey("Image").MustKey("Thumbnail").MustKey("Url") + if element.Path() != "$['Image']['Thumbnail']['Url']" { + t.Errorf("Wrong path found: %s", element.Path()) + } + + if (*Node)(nil).Path() != "" { + t.Errorf("Wrong (nil).Path()") + } +} + +func TestNode_Path2(t *testing.T) { + tests := []struct { + name string + node *Node + want string + }{ + { + name: "Node with key", node: &Node{ - data: []byte("test data"), - borders: [2]uint{0, 9}, - modified: true, + prev: &Node{}, + key: func() *string { s := "key"; return &s }(), }, - expected: nil, + want: "$['key']", }, { - name: "not ready node", + name: "Node with index", node: &Node{ - data: nil, - borders: [2]uint{0, 0}, - modified: false, + prev: &Node{}, + index: func() *int { i := 1; return &i }(), }, - expected: nil, + want: "$[1]", }, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - actual := tc.node.Source() - if string(actual) != string(tc.expected) { - t.Errorf("expected %v, got %v", tc.expected, actual) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.Path(); got != tt.want { + t.Errorf("Path() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNode_Root(t *testing.T) { + root := &Node{} + child := &Node{prev: root} + grandChild := &Node{prev: child} + + tests := []struct { + name string + node *Node + want *Node + }{ + { + name: "Root node", + node: root, + want: root, + }, + { + name: "Child node", + node: child, + want: root, + }, + { + name: "Grandchild node", + node: grandChild, + want: root, + }, + { + name: "Node is nil", + node: nil, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.root(); got != tt.want { + t.Errorf("root() = %v, want %v", got, tt.want) } }) } } + +func contains(slice []string, item string) bool { + for _, a := range slice { + if a == item { + return true + } + } + + return false +} + +// ignore the sequence of keys by ordering them. +// need to avoid import encoding/json and reflect package. +// because gno does not support them for now. +// TODO: use encoding/json to compare the result after if possible in gno. +func isSameObject(a, b string) bool { + aPairs := strings.Split(strings.Trim(a, "{}"), ",") + bPairs := strings.Split(strings.Trim(b, "{}"), ",") + + aMap := make(map[string]string) + bMap := make(map[string]string) + for _, pair := range aPairs { + kv := strings.Split(pair, ":") + key := strings.Trim(kv[0], `"`) + value := strings.Trim(kv[1], `"`) + aMap[key] = value + } + for _, pair := range bPairs { + kv := strings.Split(pair, ":") + key := strings.Trim(kv[0], `"`) + value := strings.Trim(kv[1], `"`) + bMap[key] = value + } + + aKeys := make([]string, 0, len(aMap)) + bKeys := make([]string, 0, len(bMap)) + for k := range aMap { + aKeys = append(aKeys, k) + } + + for k := range bMap { + bKeys = append(bKeys, k) + } + + sort.Strings(aKeys) + sort.Strings(bKeys) + + if len(aKeys) != len(bKeys) { + return false + } + + for i := range aKeys { + if aKeys[i] != bKeys[i] { + return false + } + + if aMap[aKeys[i]] != bMap[bKeys[i]] { + return false + } + } + + return true +} + +func compareNodes(n1, n2 *Node) bool { + if n1 == nil || n2 == nil { + return n1 == n2 + } + + if n1.key != n2.key { + return false + } + + if !bytes.Equal(n1.data, n2.data) { + return false + } + + if n1.index != n2.index { + return false + } + + if n1.borders != n2.borders { + return false + } + + if n1.modified != n2.modified { + return false + } + + if n1.nodeType != n2.nodeType { + return false + } + + if !compareNodes(n1.prev, n2.prev) { + return false + } + + if len(n1.next) != len(n2.next) { + return false + } + + for k, v := range n1.next { + if !compareNodes(v, n2.next[k]) { + return false + } + } + + return true +} diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index f4683541a1a..5daf72916e4 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -3,7 +3,6 @@ package json import ( "bytes" "errors" - "math" "strconv" ) @@ -13,113 +12,15 @@ const ( maxUint64 = 1<<64 - 1 ) -func tokenEnd(data []byte) int { - for i, tok := range data { - switch tok { - case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken, CommaToken, CurlyCloseToken, SquareCloseToken: - return i - } - } - - return len(data) -} - -func nextToken(data []byte) (int, error) { - for i, c := range data { - switch c { - case WhiteSpaceToken, NewLineToken, CarriageReturnToken, TabToken: - continue - default: - return i, nil - } - } - - return -1, errors.New("JSON Error: no token found") -} - -func stringEnd(data []byte) (lastCharIdx int, escaped bool, err error) { - escaped = false - for lastCharIdx, c := range data { - switch c { - case DoublyQuoteToken: - if !escaped { - return lastCharIdx + 1, false, nil - } - - prevCharIdx := lastCharIdx - 1 - for prevCharIdx >= 0 && data[prevCharIdx] == BackSlashToken { - prevCharIdx-- - } - - if (lastCharIdx-prevCharIdx)%2 == 0 { - return lastCharIdx + 1, true, nil - } - - return lastCharIdx + 1, false, nil - case BackSlashToken: - escaped = true - } - } - - return -1, escaped, errors.New("Invalid string input found while parsing string value") -} - -// Find end of the data structure, array or object. -// For array openSym and closeSym will be '[' and ']', for object '{' and '}' -func blockEnd(data []byte, openSym byte, closeSym byte) (int, error) { - level := 0 - - for i := 0; i < len(data); i++ { - switch data[i] { - case DoublyQuoteToken: - if se, _, err := stringEnd(data[i+1:]); err != nil { - return -1, err - } else { - i += se - } - case openSym: - level += 1 - case closeSym: - level -= 1 - if level == 0 { - return i + 1, nil - } - } - } - - return -1, errors.New("JSON Error: malformed data structure found while parsing container value") -} - -func keyMatched( - key []byte, - keyEscaped bool, - stackbuf [unescapeStackBufSize]byte, - keys []string, level int, -) (keyUnesc []byte, err error) { - if !keyEscaped { - keyUnesc = key - } - - if ku, err := unescape(key, stackbuf[:]); err != nil { - return nil, err - } else { - keyUnesc = ku - } - - if level > len(keys) { - return nil, errors.New("key level exceeds the length of the keys") - } - - return keyUnesc, err -} +const unescapeStackBufSize = 64 // PaseStringLiteral parses a string from the given byte slice. func ParseStringLiteral(data []byte) (string, error) { var buf [unescapeStackBufSize]byte - bf, err := unescape(data, buf[:]) + bf, err := Unescape(data, buf[:]) if err != nil { - return "", errors.New("Invalid string input found while parsing string value") + return "", errors.New("invalid string input found while parsing string value") } return string(bf), nil @@ -141,7 +42,6 @@ func ParseBoolLiteral(data []byte) (bool, error) { // // It utilizes double-precision (64-bit) floating-point format as defined // by the IEEE 754 standard, providing a decimal precision of approximately 15 digits. -// TODO: support for 32-bit floating-point format func ParseFloatLiteral(bytes []byte) (value float64, err error) { if len(bytes) == 0 { return -1, errors.New("JSON Error: empty byte slice found while parsing float value") @@ -237,7 +137,7 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { ) for _, c := range bytes { - if c == DotToken { + if c == dot { if decimalFound { return 0, 0, errors.New("JSON Error: multiple decimal points found while parsing float value") } @@ -265,131 +165,19 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { return man, exp10, nil } -// getTypeFromByteSlice is a function that takes a byte slice and an offset as input parameters. -// -// It returns a byte slice representing the data type, a ValueType indicating the type of data, -// an integer representing the end offset, and an error if any. -func getTypeFromByteSlice(data []byte, offset int) ([]byte, ValueType, int, error) { - if len(data) == 0 { - return nil, Unknown, offset, errors.New("JSON Error: empty byte slice found while parsing value") +func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { + if bytes[0] == minus { + return true, bytes[1:] } - dataType, endOffset, err := ParseValue(data, offset) - if err != nil { - return nil, dataType, offset, err - } - - return data[offset:endOffset], dataType, endOffset, nil -} - -// typeOf check the type of the given value and returns the corresponding ValueType. -func typeOf(v interface{}) ValueType { - switch v.(type) { - case string: - return String - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: - return Number - case float32, float64: - return Float - case bool: - return Boolean - case nil: - return Null - case map[string]interface{}: - return Object - case []interface{}: - return Array - default: - return Unknown - } -} - -// ParseValue parses a JSON value from the given data starting at the specified offset. -// It returns the parsed value, the new offset after parsing, and an error if any. -func ParseValue(data []byte, offset int) (ValueType, int, error) { - switch data[offset] { - case DoublyQuoteToken: - return parseString(data, offset) - case SquareOpenToken, CurlyOpenToken: - return parseContainer(data, offset) - } - - return parsePrimitive(data, offset) -} - -// parseString parses a JSON string and returns its type and the end position. -func parseString(data []byte, offset int) (ValueType, int, error) { - if idx, _, err := stringEnd(data[offset+1:]); err == nil { - return String, offset + idx + 1, nil - } - - return String, offset, errors.New("JSON Error: Invalid string input found while parsing string value") -} - -// parseContainer parses a JSON array or object and returns its type and the end position. -func parseContainer(data []byte, offset int) (ValueType, int, error) { - var ( - containerType ValueType - closing byte - ) - - if data[offset] == SquareOpenToken { - containerType = Array - closing = SquareCloseToken - } else { - containerType = Object - closing = CurlyCloseToken - } - - endOffset, err := blockEnd(data[offset:], data[offset], closing) - if endOffset == -1 { - return containerType, offset, err - } - - return containerType, offset + endOffset, nil + return false, bytes } -func parsePrimitive(data []byte, offset int) (ValueType, int, error) { - end := tokenEnd(data[offset:]) - if end == -1 { - return Unknown, offset, errors.New("JSON Error: malformed value found while parsing primitive value") - } - - value := data[offset : offset+end] - - switch data[offset] { - case 't', 'f': - if _b, err := ParseBoolLiteral(value); err != nil { - return Unknown, offset, err - } - - return Boolean, offset + end, nil - - case 'n': - if bytes.Equal(value, nullLiteral) { - return Null, offset + end, nil - } - - return Unknown, offset, errors.New("JSON Error: invalid null value found while parsing null value") - - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': - if !bytes.ContainsAny(value, ".eE") { - if _i, err := ParseIntLiteral(value); err != nil { - return Number, offset + end, err - } - } - - if _f, err := ParseFloatLiteral(value); err != nil { - return Number, offset + end, err - } - - return Number, offset + end, nil - } - - return Unknown, offset, errors.New("JSON Error: unknown value type found while parsing primitive value") +func notDigit(c byte) bool { + return (c & 0xF0) != 0x30 } -// TODO: may integrate this code with parsePrimitive in parser.go -func isNumeric(c byte) bool { - return bytes.Contains([]byte("0123456789+-.eE"), []byte{c}) +// lower converts a byte to lower case if it is an uppercase letter. +func lower(c byte) byte { + return c | 0x20 } diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index ff1406aac96..648ff92041f 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -1,7 +1,7 @@ package json import ( - "bytes" + "strconv" "testing" ) @@ -77,6 +77,8 @@ func TestParseFloatLiteral(t *testing.T) { {"-123", -123}, {"123.456", 123.456}, {"-123.456", -123.456}, + {"12345678.1234567890", 12345678.1234567890}, + {"-12345678.09123456789", -12345678.09123456789}, {"0.123", 0.123}, {"-0.123", -0.123}, {"", -1}, @@ -134,6 +136,25 @@ func TestParseFloatWithScientificNotation(t *testing.T) { } } +func TestParseFloat_May_Interoperability_Problem(t *testing.T) { + tests := []struct { + input string + shouldErr bool + }{ + {"3.141592653589793238462643383279", true}, + {"1E400", false}, // TODO: should error + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + _, err := ParseFloatLiteral([]byte(tt.input)) + if tt.shouldErr && err == nil { + t.Errorf("ParseFloatLiteral(%s): expected error, but not error", tt.input) + } + }) + } +} + func TestParseIntLiteral(t *testing.T) { tests := []struct { input string @@ -169,36 +190,123 @@ func TestParseIntLiteral(t *testing.T) { } } -func TestGetType(t *testing.T) { - tests := []struct { - data []byte - offset int - expected []byte - vt ValueType - err error - }{ - {[]byte(`{"name": "John", "age": 30}`), 0, []byte(`{"name": "John", "age": 30}`), Object, nil}, - {[]byte(`"Hello, World!"`), 0, []byte(`"Hello, World!"`), String, nil}, - {[]byte(`12345`), 0, []byte(`12345`), Number, nil}, - {[]byte(`true`), 0, []byte(`true`), Boolean, nil}, - {[]byte(`null`), 0, []byte(`null`), Null, nil}, - {[]byte(`[1, 2, 3]`), 0, []byte(`[1, 2, 3]`), Array, nil}, - {[]byte(`{}`), 0, []byte(`{}`), Object, nil}, - } +// TODO: remove comment after benchmark added. +// func benchmarkParseStringLiteral(b *testing.B, input []byte) { +// for i := 0; i < b.N; i++ { +// _, err := ParseStringLiteral(input) +// if err != nil { +// b.Fatal(err) +// } +// } +// } - for i, tt := range tests { - gotData, gotVT, _, gotErr := getTypeFromByteSlice(tt.data, tt.offset) +// func BenchmarkParseStringLiteral_Simple(b *testing.B) { +// benchmarkParseStringLiteral(b, []byte(`"Hello, World!"`)) +// } - if !bytes.Equal(gotData, tt.expected) { - t.Errorf("%d. expected data=%s, but got data=%s", i, tt.expected, gotData) - } +// func BenchmarkParseStringLiteral_Long(b *testing.B) { +// benchmarkParseStringLiteral(b, []byte(`"Lorem ipsum dolor sit amet, consectetur adipiscing elit."`)) +// } - if gotVT != tt.vt { - t.Errorf("%d. expected value type=%s, but got value type=%s", i, tt.vt, gotVT) - } +// func BenchmarkParseStringLiteral_Escaped(b *testing.B) { +// benchmarkParseStringLiteral(b, []byte(`"Hello, \"World!\""`)) +// } - if gotErr != tt.err { - t.Errorf("%d. expected error=%v, but got error=%v", i, tt.err, gotErr) - } - } -} +// func BenchmarkParseStringLiteral_Unicode(b *testing.B) { +// benchmarkParseStringLiteral(b, []byte(`"안녕, 世界!"`)) +// } + +// func benchmarkParseFloatLiteral(b *testing.B, input []byte) { +// for i := 0; i < b.N; i++ { +// ParseFloatLiteral(input) +// } +// } + +// func benchmarkStrconvParseFloat(b *testing.B, input string) { +// for i := 0; i < b.N; i++ { +// _, err := strconv.ParseFloat(input, 64) +// if err != nil { +// b.Fatal(err) +// } +// } +// } + +// func BenchmarkParseFloatLiteral_Simple(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("123.456")) +// } + +// func BenchmarkStrconvParseFloat_Simple(b *testing.B) { +// benchmarkStrconvParseFloat(b, "123.456") +// } + +// func BenchmarkParseFloatLiteral_Long(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("123456.78901")) +// } + +// func BenchmarkStrconvParseFloat_Long(b *testing.B) { +// benchmarkStrconvParseFloat(b, "123456.78901") +// } + +// func BenchmarkParseFloatLiteral_Negative(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("-123.456")) +// } + +// func BenchmarkStrconvParseFloat_Negative(b *testing.B) { +// benchmarkStrconvParseFloat(b, "-123.456") +// } + +// func BenchmarkParseFloatLiteral_Negative_Long(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("-123456.78901")) +// } + +// func BenchmarkStrconvParseFloat_Negative_Long(b *testing.B) { +// benchmarkStrconvParseFloat(b, "-123456.78901") +// } + +// func BenchmarkParseFloatLiteral_Decimal(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("123456")) +// } + +// func BenchmarkStrconvParseFloat_Decimal(b *testing.B) { +// benchmarkStrconvParseFloat(b, "123456") +// } + +// func BenchmarkParseFloatLiteral_Float_No_Decimal(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("123.")) +// } + +// func BenchmarkStrconvParseFloat_Float_No_Decimal(b *testing.B) { +// benchmarkStrconvParseFloat(b, "123.") +// } + +// func BenchmarkParseFloatLiteral_Complex(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("-123456.798012345")) +// } + +// func BenchmarkStrconvParseFloat_Complex(b *testing.B) { +// benchmarkStrconvParseFloat(b, "-123456.798012345") +// } + +// func BenchmarkParseFloatLiteral_Science_Notation(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("1e6")) +// } + +// func BenchmarkStrconvParseFloat_Science_Notation(b *testing.B) { +// benchmarkStrconvParseFloat(b, "1e6") +// } + +// func BenchmarkParseFloatLiteral_Science_Notation_Long(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("1.23e10")) +// } + +// func BenchmarkStrconvParseFloat_Science_Notation_Long(b *testing.B) { +// benchmarkStrconvParseFloat(b, "1.23e10") +// } + +// func BenchmarkParseFloatLiteral_Science_Notation_Negative(b *testing.B) { +// benchmarkParseFloatLiteral(b, []byte("-1.23e10")) +// } + +// func BenchmarkStrconvParseFloat_Science_Notation_Negative(b *testing.B) { +// benchmarkStrconvParseFloat(b, "-1.23e10") +// } diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index d1fbe3fbc9d..11859d677f2 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -4,9 +4,11 @@ import ( "errors" ) +// TODO: not fully implemented yet. + // ParsePath takes a JSONPath string and returns a slice of strings representing the path segments. func ParsePath(path string) ([]string, error) { - buf := NewBuffer([]byte(path)) + buf := newBuffer([]byte(path)) result := make([]string, 0) for { @@ -16,14 +18,14 @@ func ParsePath(path string) ([]string, error) { } switch { - case b == DollarToken || b == AtToken: + case b == dollarSign || b == atSign: result = append(result, string(b)) buf.step() - case b == DotToken: + case b == dot: buf.step() - if next, _ := buf.current(); next == DotToken { + if next, _ := buf.current(); next == dot { buf.step() result = append(result, "..") @@ -32,12 +34,12 @@ func ParsePath(path string) ([]string, error) { extractNextSegment(buf, &result) } - case b == SquareOpenToken: + case b == bracketOpen: start := buf.index buf.step() for { - if buf.index >= buf.length || buf.data[buf.index] == SquareCloseToken { + if buf.index >= buf.length || buf.data[buf.index] == bracketClose { break } diff --git a/examples/gno.land/p/demo/json/path_test.gno b/examples/gno.land/p/demo/json/path_test.gno index 7caff6f3b18..dd242849f03 100644 --- a/examples/gno.land/p/demo/json/path_test.gno +++ b/examples/gno.land/p/demo/json/path_test.gno @@ -48,3 +48,17 @@ func TestParseJSONPath(t *testing.T) { }) } } + +func isEqualSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for i, v := range a { + if v != b[i] { + return false + } + } + + return true +} diff --git a/examples/gno.land/p/demo/json/state_machine.gno b/examples/gno.land/p/demo/json/state_machine.gno deleted file mode 100644 index 356f2c3ba3a..00000000000 --- a/examples/gno.land/p/demo/json/state_machine.gno +++ /dev/null @@ -1,755 +0,0 @@ -package json - -import ( - "bytes" - "errors" - "regexp" - "strconv" - - "gno.land/p/demo/ufmt" -) - -const unescapeStackBufSize = 64 - -var ( - tabRegex = regexp.MustCompile(`(?m)^\t+`) - lastLineSpaceRegex = regexp.MustCompile(`(?m)^\\s+\\}`) -) - -// extractValueTypeFromToken recoginizes the type of JSON value by peeking at the first byte of the value. -func extractValueTypeFromToken(b byte, value []byte) (dataType ValueType, offset int, err error) { - if len(b) == 0 { - return Unknown, 0, errors.New("cannot determine type of empty value") - } - - switch b { - case 't', 'f': - if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { - dataType = Boolean - } - - return Unknown, offset, errors.New("invalid boolean value. expected: true or false") - - case 'u', 'n': - if bytes.Equal(value, nullLiteral) { - dataType = Null - } - - return Unknown, offset, errors.New("invalid null value. expected: null") - - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': - dataType = Number - - default: - return Unknown, offset, errors.New("unsupported value type or invalid JSON type") - } -} - -// findValueIndex is a function that takes a byte slice of JSON data and a series of keys. -// It traverses the JSON data to find the position where the value of the final key in the series begins. -// The position is returned as an integer representing the index in the byte slice. -func findValueIndex(data []byte, keys ...string) (int, error) { - if len(keys) == 0 { - return 0, nil - } - - var ( - keyLevel int - level int - lastMatched bool = true - stackbuf [unescapeStackBufSize]byte - ) - - for i := 0; i < len(data); i++ { - switch data[i] { - case DoublyQuoteToken: - if level < 1 { - return -1, errors.New("invalid JSON format") - } - - i += 1 - keyBegin := i - - strEnd, keyEscaped, err := stringEnd(data[i:]) - if err != nil { - return -1, err - } - - i += strEnd - keyEnd := i - 1 - - valueOffset, err := nextToken(data[i:]) - if err != nil { - return -1, err - } - - i += valueOffset - if data[i] != ColonToken { - i -= 1 - continue - } - - key := data[keyBegin:keyEnd] - keyUnesc, err := keyMatched(key, keyEscaped, stackbuf, keys, level) - if err != nil { - return -1, err - } - - lastMatched = bytes.Equal(keyUnesc, []byte(keys[level-1])) - if lastMatched && keyLevel == level-1 && keyLevel != len(keys) { - keyLevel += 1 - } - - if keyLevel == len(keys) { - return i + 1, nil - } - case CurlyOpenToken: - // in case parent key is matched then only we will increase the level otherwise can directly - // can move to the end of this block - if !lastMatched { - end, err := blockEnd(data[i:], CurlyOpenToken, CurlyCloseToken) - if end == -1 { - return -1, err - } - - i += end - 1 - } else { - level += 1 - } - case CurlyCloseToken: - level, keyLevel = decreaseLevel(level, keyLevel) - case SquareOpenToken: - // If we want to get array element by index - if keyLevel == level && keys[level][0] == SquareOpenToken { - keyLen := len(keys[level]) - if keyLen < 3 || keys[level][0] != SquareOpenToken || keys[level][keyLen-1] != SquareCloseToken { - return -1, errors.New("invalid array index") - } - - aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1]) - if err != nil { - return -1, errors.New("invalid array index") - } - - var ( - curIdx int - valueFound []byte - valueOffset int - ) - - curI := i - - ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { - if curIdx == aIdx { - valueFound = value - valueOffset = offset - - if dataType == String { - valueOffset = valueOffset - 2 - valueFound = data[curI+valueOffset : curI+valueOffset+len(value)+2] - } - } - - curIdx += 1 - }) - - if valueFound == nil { - return -1, errors.New("array index not found") - } - - subIndex, err := findValueIndex(valueFound, keys[level+1:]...) - if err != nil { - return -1, err - } - - return i + valueOffset + subIndex, nil - } else { - // Do not search for keys inside arrays - if arraySkip, err := blockEnd(data[i:], SquareOpenToken, SquareCloseToken); arraySkip == -1 { - return -1, err - } else { - i += arraySkip - 1 - } - } - case ColonToken: - return -1, errors.New("invalid JSON format") - } - } - - return -1, errors.New("key path not found in the JSON structure") -} - -func decreaseLevel(level, keyLevel int) (int, int) { - level -= 1 - - if level-1 == keyLevel { - keyLevel -= 1 - } - - return level, keyLevel -} - -// findKeyStart finds the start of a specific key in a given byte array. -func findKeyStart(data []byte, key string) (int, error) { - i, err := nextToken(data) - if err != nil { - return -1, err - } - - ln := len(data) - if ln > 0 && (data[i] == CurlyOpenToken || data[i] == SquareOpenToken) { - i += 1 - } - - var stackbuf [unescapeStackBufSize]byte - if ku, err := unescape([]byte(key), stackbuf[:]); err == nil { - key = string(ku) - } - - for _; i < ln; i++ { - switch data[i] { - case DoublyQuoteToken: - i++ - keyBegin := i - - strEnd, keyEscaped, _ := stringEnd(data[i:]) - if strEnd == -1 { - break - } - - i += strEnd - keyEnd := i - 1 - - valueOffset, err := nextToken(data[i:]) - if err != nil { - break - } - - i += valueOffset - - // if string is a key, and key level match - k := data[keyBegin:keyEnd] - - // for unescape: if there are no escape sequences, this is cheap; if there are, it is a - // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize - if keyEscaped { - if ku, err := unescape(k, stackbuf[:]); err != nil { - break - } else { - k = ku - } - } - - if data[i] == ColonToken && len(key) == len(k) && string(k) == key { - return keyBegin - 1, nil - } - - case SquareOpenToken, CurlyOpenToken: - if end, _ := blockEnd(data[i:], data[i], SquareCloseToken); end != -1 { - i = i + end - } - } - } - - return -1, errors.New("key path not found in the JSON structure") -} - -func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { - if len(keys) > 0 { - offset, err = findValueIndex(data, keys...) - if err != nil { - return nil, NotExist, -1, -1, err - } - } - - // Go to closest value - nO, err := nextToken(data[offset:]) - if err != nil { - return nil, NotExist, offset, -1, errors.New("key path not found in the JSON structure") - } - - offset += nO - if value, dataType, endOffset, err = getTypeFromByteSlice(data, offset); err != nil { - return value, dataType, offset, endOffset, err - } - - // strip quotes from string values - if dataType == String { - value = value[1 : len(value)-1] - } - - // remove unnecessary whitespace characters to make more readable - if dataType == Object { - value = adjustIndentation(value) - } - - return value[:len(value):len(value)], dataType, offset, endOffset, nil -} - -// adjustIndentation is a kind of hack to remove the extra tabs and spaces from the Object values. -// TODO: storing depth information while parsing step, may allows us to avoid regex. -func adjustIndentation(value []byte) []byte { - value = tabRegex.ReplaceAll(value, []byte("\t")) - value = lastLineSpaceRegex.ReplaceAll(value, []byte("}")) - - return value -} - -// Get is a function that retrieves a value from the given data based on the provided keys. -// It returns the value, data type, offset (position), and any error encountered during the retrieval process. -// -// For example: -// -// ``` gno -// data := []byte(`{"name":{"first":"Janet","last":"Prichard"},"age":47}`) -// value, dataType, offset, err := json.Get(data, "name", "last") -// ``` -// -// It returns the value `Prichard` with data type `String`, the offset `37`, and `nil` for the error. -func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { - a, b, _, d, e := internalGet(data, keys...) - - return a, b, d, e -} - -func ArrayEach( - data []byte, - cb func(value []byte, dataType ValueType, offset int, err error), - keys ...string, -) (offset int, err error) { - if len(data) == 0 { - return -1, errors.New("empty JSON data provided") - } - - if next, err := nextToken(data); err != nil { - return -1, err - } else { - offset += next - } - - if len(keys) > 0 { - if offset, err = findValueIndex(data, keys...); err != nil { - return -1, errors.New(ufmt.Sprintf("key path not found. keys: %v", keys)) - } - - // go to closest value - if n0, err := nextToken(data[offset:]); err != nil { - return -1, errors.New( - ufmt.Sprintf( - "invalid token found. offset: %d. got: %s", - offset, - data[offset:offset+1], - )) - } else { - offset += n0 - } - - if data[offset] != SquareOpenToken { - return -1, errors.New("invalid array. must start with `[`") - } else { - offset += 1 - } - } - - if n0, err := nextToken(data[offset:]); err != nil { - return -1, errors.New("next value not found") - } else { - offset += n0 - } - - if data[offset] == SquareCloseToken { - return -1, nil - } - - for true { - v, t, o, e := Get(data[offset:]) - - if e != nil { - return -1, e - } - - if o == 0 { - break - } - - if t != NotExist { - cb(v, t, offset+o-len(v), e) - } - - offset += o - if skipToToken, err := nextToken(data[offset:]); err != nil { - return offset, errors.New( - ufmt.Sprintf( - "json.ArrayEach: invalid token at offset %d. got: %s", - offset, - data[offset:offset+1], - )) - } else { - offset += skipToToken - } - - currToken := data[offset] - if currToken == SquareCloseToken { - break - } - - if currToken != CommaToken { - return offset, errors.New( - ufmt.Sprintf( - "json.ArrayEach: cannot find comma at offset %d. got: %s", - offset, - data[offset:offset+1], - )) - } else { - offset += 1 - } - } - - return offset, nil -} - -func ObjectEach( - data []byte, - cb func(key, value []byte, dataType ValueType, offset int) error, - keys ...string, -) (err error) { - offset := 0 - - // descent to desired key - if len(keys) > 0 { - if off, err := findValueIndex(data, keys...); err != nil { - errors.New("key path not found in the JSON structure") - } else { - offset += off - } - } - - // validate and skip past opening token - if off, err := nextToken(data[offset:]); err != nil { - return errors.New("Object is not properly formed") - } else if offset += off; data[offset] != CurlyOpenToken { - return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) - } else { - offset += 1 - } - - // skip to the 1st token insize the object - if off, err := nextToken(data[offset:]); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) - } else if offset += off; data[offset] == CurlyCloseToken { - // find the end of the object. should return here - return nil - } - - for offset < len(data) { - // find the next key - var key []byte - - // check the next token kind - switch data[offset] { - case DoublyQuoteToken: - offset += 1 // consider as a string type and skip the first quote - case CurlyCloseToken: - return nil // end of the object - default: - return errors.New("Object is not properly formed") - } - - // find the end of the key - var escaped bool - if off, ok, err := stringEnd(data[offset:]); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: invalid string at offset %d", offset)) - } else { - key, escaped = data[offset:offset+off-1], ok - offset += off - } - - // unescape the key if necessary - if escaped { - var stackbuf [unescapeStackBufSize]byte - - if ku, err := unescape(key, stackbuf[:]); err != nil { - return errors.New("json.ObjectEach: string is not valid UTF-8") - } else { - key = ku - } - } - - // skip colon - if off, err := nextToken(data[offset:]); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: invalid token at offset %d", offset)) - } else if offset += off; data[offset] != ColonToken { - return errors.New(ufmt.Sprintf("json.ObjectEach: cannot find colon at expected offset %d", offset)) - } else { - offset += 1 - } - - // find nearby value, call callback - if value, typ, off, err := Get(data[offset:]); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: %s", err)) - } else if err := cb(key, value, typ, offset+off); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: %s", err)) - } else { - offset += off - } - - // skip the next comma to the following token - if off, err := nextToken(data[offset:]); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) - } else { - offset += off - - switch data[offset] { - case CurlyCloseToken: - return nil // end of the object. should stop here. - case CommaToken: - offset += 1 // skip the comma and continue - default: - return errors.New( - ufmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset), - ) - } - } - - // skip the next token if comma is found - if off, err := nextToken(data[offset:]); err != nil { - return errors.New(ufmt.Sprintf("json.ObjectEach: can't find next token at offset %d", offset)) - } else { - offset += off // skip the next token - } - - // reset the key - key = nil - } - - return errors.New(ufmt.Sprintf("json.ObjectEach: object is not properly terminated at offset %d", offset)) -} - -func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) (int, error) { - var ( - x struct{} - level int - pathsMatched int - i int - ) - - pathFlags := initPathFlags(paths) - pathsBuf, maxPath := initPathBuf(paths) - - ln := len(data) - for i < ln { - switch data[i] { - case DoublyQuoteToken: - i++ - keyBegin := i - - strEnd, keyEscaped, err := stringEnd(data[i:]) - if err != nil { - return -1, errors.New("invalid string") - } - - i += strEnd - keyEnd := i - 1 - - valueOffset, err := nextToken(data[i:]) - if err != nil { - return -1, errors.New("invalid token found") - } - - i += valueOffset - - // if string is a key, and key level match - if data[i] == ColonToken { - match := -1 - if maxPath > 0 && maxPath >= level { - keyUnesc, err := processKey(data[keyBegin:keyEnd], keyEscaped) - if err != nil { - return -1, err - } - - pathsBuf[level-1] = string(keyUnesc) - for pi, path := range paths { - if !isPathValid(path, level, pathsBuf) || pathFlags[pi] { - continue - } - - match = pi - - pathsMatched += 1 - pathFlags[pi] = true - - val, dataTyp, _, err := Get(data[i+1:]) - cb(pi, val, dataTyp, err) - - if pathsMatched == len(paths) { - break - } - } - - if pathsMatched == len(paths) { - return i, nil - } - } - - if match == -1 { - tokenOffset, err := nextToken(data[i+1:]) - i += tokenOffset - - if data[i] == CurlyOpenToken { - blockSkip, err := blockEnd(data[i:], CurlyOpenToken, CurlyCloseToken) - if blockSkip == -1 { - return -1, err - } - - i += blockSkip + 1 - } - } - - if i < ln { - switch data[i] { - case CurlyOpenToken, CurlyCloseToken, SquareOpenToken, DoublyQuoteToken: - i -= 1 - } - } - } else { - i -= 1 - } - case CurlyOpenToken: - level += 1 - case CurlyCloseToken: - level -= 1 - case SquareOpenToken: - var ok bool - - arrIdxFlags := make(map[int]struct{}) - pIdxFlags := initPathIdxFlags(paths) - - if level < 0 { - err := errors.New("invalid JSON format") - cb(-1, nil, Unknown, err) - - return -1, err - } - - for p, path := range paths { - if !isPathValid(path, level, pathsBuf) || pathFlags[p] { - continue - } - - currPos := path[level] - if len(currPos) >= 2 { - arrIdx, err := strconv.Atoi(currPos[1 : len(currPos)-1]) - if err != nil { - return -1, err - } - - pIdxFlags[p] = true - arrIdxFlags[arrIdx] = x - } - } - - if len(arrIdxFlags) > 0 { - level += 1 - - var currIdx int - arrOff, err := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { - if _, ok := arrIdxFlags[currIdx]; ok { - for i, path := range paths { - if pIdxFlags[i] { - currLevelPath := path[level-1] - arrIdx, _ := strconv.Atoi(currLevelPath[1 : len(currLevelPath)-1]) - - if currIdx != arrIdx { - continue - } - - off, _ := findValueIndex(value, path[level:]...) - - if off != -1 { - val, dataTyp, _, err := Get(value[off:]) - cb(i, val, dataTyp, err) - } - - pathsMatched += 1 - pathFlags[i] = true - } - } - } - - currIdx += 1 - }) - - if pathsMatched == len(paths) { - return i, nil - } - - i += arrOff - 1 - } else { - // do not search for keys inside arrays - if arraySkip, err := blockEnd(data[i:], SquareOpenToken, SquareCloseToken); arraySkip == -1 { - return -1, err - } else { - i += arraySkip - } - } - - case SquareCloseToken: - level -= 1 - } - - i += 1 - } - - return -1, errors.New("key path not found in the JSON structure") -} - -func initPathFlags(paths [][]string) []bool { - return make([]bool, len(paths)) -} - -func initPathBuf(paths [][]string) ([]string, int) { - var maxPath int - - for _, p := range paths { - if len(p) > maxPath { - maxPath = len(p) - } - } - - return make([]string, maxPath), maxPath -} - -func initPathIdxFlags(paths [][]string) []bool { - return make([]bool, len(paths)) -} - -func sameTree(p1, p2 []string) bool { - minLen := len(p1) - p2Len := len(p2) - - if p2Len < minLen { - minLen = p2Len - } - - for pi_1, p_1 := range p1[:minLen] { - if p2[pi_1] != p_1 { - return false - } - } - - return true -} - -func processKey(key []byte, keyEscaped bool) ([]byte, error) { - if !keyEscaped { - return key, nil - } - - var stackBuf [unescapeStackBufSize]byte - - return unescape(key, stackBuf[:]) -} - -func isPathValid(path []string, level int, pathsBuf []string) bool { - return len(path) == level && sameTree(path, pathsBuf[:level]) -} diff --git a/examples/gno.land/p/demo/json/state_machine_test.gno b/examples/gno.land/p/demo/json/state_machine_test.gno deleted file mode 100644 index 1e5967c0023..00000000000 --- a/examples/gno.land/p/demo/json/state_machine_test.gno +++ /dev/null @@ -1,764 +0,0 @@ -package json - -import ( - "bytes" - "errors" - "fmt" - "testing" - - "gno.land/p/demo/ufmt" -) - -func TestSearchKeys(t *testing.T) { - tests := []struct { - name string - data []byte - keys []string - expected int - }{ - { - name: "Test 1", - data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), - keys: []string{"key1"}, - expected: 8, - }, - { - name: "Test 2", - data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), - keys: []string{"key2"}, - expected: 26, - }, - { - name: "Test 3", - data: []byte(`{"key1": "value1", "key2": "value2", "key3": "value3"}`), - keys: []string{"key4"}, - expected: -1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got, _ := findValueIndex(tt.data, tt.keys...); got != tt.expected { - t.Errorf("findValueIndex() = %v, want %v", got, tt.expected) - } - }) - } -} - -func TestFindKeyStart(t *testing.T) { - tests := []struct { - name string - data []byte - key string - expected int - }{ - { - name: "Test 1", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - key: "key", - expected: 3, - }, - { - name: "Test 2", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - key: "array", - expected: 20, - }, - { - name: "Test 3", - data: []byte("{\n\t\"key\": \"value\",\n\t\"array\": [1, 2, 3]\n}"), - key: "value", - expected: -1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, _ := findKeyStart(tt.data, tt.key) - if got != tt.expected { - t.Errorf("findKeyStart() = %v, want %v", got, tt.expected) - } - }) - } -} - -type GetTest struct { - desc string - json string - path []string - - isErr bool - isFound bool - - data interface{} -} - -var getTests = []GetTest{ - // Trivial tests - { - desc: "read string", - json: `""`, - isFound: true, - data: ``, - }, - { - desc: "read number", - json: `0`, - isFound: true, - data: `0`, - }, - { - desc: "read object", - json: `{}`, - isFound: true, - data: `{}`, - }, - { - desc: "read array", - json: `[]`, - isFound: true, - data: `[]`, - }, - { - desc: "read boolean", - json: `true`, - isFound: true, - data: `true`, - }, - { - desc: "read boolean 2", - json: `false`, - isFound: true, - data: `false`, - }, - { - desc: "read null", - json: `null`, - isFound: true, - data: `null`, - }, - - // Found key tests - { - desc: "handling multiple nested keys with same name", - json: `{"a":[{"b":1},{"b":2},3],"c":{"d":[1,2]}} }`, - path: []string{"c", "d"}, - isFound: true, - data: `[1,2]`, - }, - { - desc: "read basic key", - json: `{"a":"b"}`, - path: []string{"a"}, - isFound: true, - data: `b`, - }, - { - desc: "read basic key with space", - json: `{"a": "b"}`, - path: []string{"a"}, - isFound: true, - data: `b`, - }, - { - desc: "read composite key", - json: `{"a": { "b":{"c":"d" }}}`, - path: []string{"a", "b", "c"}, - isFound: true, - data: `d`, - }, - { - desc: `read numberic value as string`, - json: `{"a": "b", "c": 1}`, - path: []string{"c"}, - isFound: true, - data: `1`, - }, - { - desc: `handle multiple nested keys with same name`, - json: `{"a":[{"b":1},{"b":2},3],"c":{"c":[1,2]}} }`, - path: []string{"c", "c"}, - isFound: true, - data: `[1,2]`, - }, - { - desc: `read object`, - json: `{"a": { "b":{"c":"d" }}}`, - path: []string{"a", "b"}, - isFound: true, - data: `{"c":"d" }`, - }, - { - desc: `empty path`, - json: `{"c":"d" }`, - path: []string{}, - isFound: true, - data: `{"c":"d" }`, - }, - { - desc: `formatted JSON value`, - json: "{\n \"a\": \"b\"\n}", - path: []string{"a"}, - isFound: true, - data: `b`, - }, - { - desc: `formatted JSON value 2`, - json: "{\n \"a\":\n {\n\"b\":\n {\"c\":\"d\",\n\"e\": \"f\"}\n}\n}", - path: []string{"a", "b"}, - isFound: true, - data: "{\"c\":\"d\",\n\"e\": \"f\"}", - }, - { - desc: `whitespace`, - json: " \n\r\t{ \n\r\t\"whitespace\" \n\r\t: \n\r\t333 \n\r\t} \n\r\t", - path: []string{"whitespace"}, - isFound: true, - data: "333", - }, - { - desc: `unescaped backslash quote`, - json: `{"a": "\\"}`, - path: []string{"a"}, - isFound: true, - data: `\\`, - }, - { - desc: `unicode in JSON`, - json: `{"a": "15°C"}`, - path: []string{"a"}, - isFound: true, - data: `15°C`, - }, - { - desc: `no padding + nested`, - json: `{"a":{"a":"1"},"b":2}`, - path: []string{"b"}, - isFound: true, - data: `2`, - }, - { - desc: `no padding + nested + array`, - json: `{"a":{"b":[1,2]},"c":3}`, - path: []string{"c"}, - isFound: true, - data: `3`, - }, - { - desc: `empty key`, - json: `{"":{"":{"":true}}}`, - path: []string{"", "", ""}, - isFound: true, - data: `true`, - }, - - // Escaped key tests - { - desc: `key with simple escape`, - json: `{"a\\b":1}`, - path: []string{"a\\b"}, - isFound: true, - data: `1`, - }, - { - desc: `key and value with whitespace escapes`, - json: `{"key\b\f\n\r\tkey":"value\b\f\n\r\tvalue"}`, - path: []string{"key\b\f\n\r\tkey"}, - isFound: true, - data: `value\b\f\n\r\tvalue`, // value is not unescaped since this is Get(), but the key should work correctly - }, - { - desc: `key with Unicode escape`, - json: `{"a\u00B0b":1}`, - path: []string{"a\u00B0b"}, - isFound: true, - data: `1`, - }, - { - desc: `key with complex escape`, - json: `{"a\uD83D\uDE03b":1}`, - path: []string{"a\U0001F603b"}, - isFound: true, - data: `1`, - }, - - { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance - desc: `malformed with trailing whitespace`, - json: `{"a":1 `, - path: []string{"a"}, - isFound: true, - data: `1`, - }, - { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance - desc: `malformed with wrong closing bracket`, - json: `{"a":1]`, - path: []string{"a"}, - isFound: true, - data: `1`, - }, - { - desc: `missing closing brace, but can still find key`, - json: `{"a":"b"`, - path: []string{"a"}, - isFound: true, - data: `b`, - }, - { - desc: `missing value closing quote`, - json: `{"a":"b`, - path: []string{"a"}, - isErr: true, - }, - { - desc: `missing value closing curly brace`, - json: `{"a": { "b": "c"`, - path: []string{"a"}, - isErr: true, - }, - { - desc: `missing value closing square bracket`, - json: `{"a": [1, 2, 3 }`, - path: []string{"a"}, - isErr: true, - }, - { - desc: `missing value 1`, - json: `{"a":`, - path: []string{"a"}, - isErr: true, - }, - { - desc: `missing value 2`, - json: `{"a": `, - path: []string{"a"}, - isErr: true, - }, - { - desc: `missing value 3`, - json: `{"a":}`, - path: []string{"a"}, - isErr: true, - }, - { - desc: `malformed array (no closing brace)`, - json: `{"a":[, "b":123}`, - path: []string{"b"}, - isFound: false, - isErr: true, - }, - { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance (this is not ideal) - desc: "malformed 'colon chain', lookup first string", - json: `{"a":"b":"c"}`, - path: []string{"a"}, - isFound: true, - data: "b", - }, - { // This test returns a match instead of a parse error, as checking for the malformed JSON would reduce performance (this is not ideal) - desc: "malformed 'colon chain', lookup second string", - json: `{"a":"b":"c"}`, - path: []string{"b"}, - isFound: true, - data: "c", - }, -} - -func TestGet(t *testing.T) { - for _, test := range getTests { - t.Run(test.desc, func(t *testing.T) { - data := []byte(test.json) - value, _, _, err := Get(data, test.path...) - if test.isErr { - if err == nil { - t.Errorf("Expected error, got nil") - } - } else { - if err != nil { - t.Errorf("Got error: %v", err) - } - } - - if test.isFound { - if string(value) != test.data { - t.Errorf("Expected '%s', got '%s'", test.data, value) - } - } else { - if value != nil { - t.Errorf("Expected nil, got '%s'", value) - } - } - }) - } -} - -func TestArrayEach(t *testing.T) { - mock := []byte(`{"a": { "b":[{"x": "1"} ,{"x": 2},{ "x":3}, {"x":4} ]}}`) - count := 0 - - ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err error) { - count++ - - switch count { - case 1: - if string(value) != `{"x": "1"}` { - t.Errorf("Wrong first item: %s", string(value)) - } - case 2: - if string(value) != `{"x": 2}` { - t.Errorf("Wrong second item: %s", string(value)) - } - case 3: - if string(value) != `{ "x":3}` { - t.Errorf("Wrong third item: %s", string(value)) - } - case 4: - if string(value) != `{"x":4}` { - t.Errorf("Wrong forth item: %s", string(value)) - } - default: - t.Errorf("Should process only 4 items") - } - }, "a", "b") -} - -func TestArrayEach2(t *testing.T) { - test := []byte(`{"a": ["string", 42, true, false, null, [1, 2, 3]]}`) - ArrayEach(test, func(value []byte, dataType ValueType, offset int, err error) { - if string(value) != "string" && string(value) != "42" && string(value) != "true" && string(value) != "false" && string(value) != "null" && string(value) != `{"b": 1}` && string(value) != "[1, 2, 3]" { - t.Errorf("Wrong item: %s", string(value)) - } - }, "a") -} - -func TestObjectEach(t *testing.T) { - tests := []struct { - name string - input []byte - expected map[string]string - root string - isErr bool - }{ - { - name: "empty object", - input: []byte(`{}`), - expected: map[string]string{}, - root: "", - }, - { - name: "single key", - input: []byte(`{"a": 1}`), - expected: map[string]string{"a": "1"}, - root: "", - }, - { - name: "two keys", - input: []byte(`{"a": {"b": 1, "c": 2}}`), - expected: map[string]string{"b": "1", "c": "2"}, - root: "a", - }, - { - name: "two keys 2", - input: []byte(`{"a": {"b": 1, "c": 2}, "d": 4}`), - expected: map[string]string{"b": "1", "c": "2"}, - root: "a", - }, - { - name: "another set of keys", - input: []byte(`{"x": {"y": 3, "z": 4}}`), - expected: map[string]string{"y": "3", "z": "4"}, - root: "x", - }, - { - name: "multiple key-value object with many value types", - input: []byte(`{ - "key1": null, - "key2": true, - "key3": 1.23, - "key4": "string value", - "key5": [1,2,3], - "key6": {"a":"b"} - }`), - expected: map[string]string{ - "key1": "null", - "key2": "true", - "key3": "1.23", - "key4": "string value", - "key5": "[1,2,3]", - "key6": `{"a":"b"}`, - }, - }, - { - name: "so many white spaces", - input: []byte(`{"a" : 1}`), - expected: map[string]string{"a": "1"}, - }, - /* Error Cases */ - { - name: "unmatched brace", - input: []byte(`{`), - isErr: true, - }, - { - name: "unmatched brace 2", - input: []byte(`{"a": 1`), - root: "a", - isErr: true, - }, - { - name: "unmatched brace 3", - input: []byte(`}`), - isErr: true, - }, - { - name: "unmatched brace 3", - input: []byte(`{{}}{`), - isErr: true, - }, - { - name: "no object present", - input: []byte(`\t\n\r`), - isErr: true, - }, - { - name: "malformed key", - input: []byte(`{"foo: 1}`), - isErr: true, - }, - { - name: "malformed key 2", - input: []byte(`{"foo": 1, "bar: "11"}`), - root: "foo", - isErr: true, - }, - { - name: "malformed key in nested object", - input: []byte(`{"foo": {"bar: 1}}`), - root: "foo", - isErr: true, - }, - { - name: "bad value", - input: []byte(`{"foo": bar}`), - isErr: true, - }, - { - name: "no colon", - input: []byte(`{"foo" "bar"}`), - isErr: true, - }, - { - name: "no colon 2", - input: []byte(`{"foo""bar"}`), - isErr: true, - }, - { - name: "no colon 3", - input: []byte(`{"foo"; "bar"}`), - isErr: true, - }, - { - name: "invalid colon", - input: []byte(`{"foo":: "bar"}`), - isErr: true, - }, - { - name: "no trailing comma", - input: []byte(`{"foo": "bar" "baz": "qux"}`), - root: "foo", - isErr: true, - }, - { - name: "key with unicode escape", - input: []byte(`{"b": 10, "c": {"a\uD83D\uDE03b":1, "aa": false}}`), - root: "b", - isErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := ObjectEach(tc.input, func(key, value []byte, dataType ValueType, offset int) error { - expectedValue, ok := tc.expected[string(key)] - if !ok { - t.Errorf("Unexpected key: %s", string(key)) - } else if string(value) != expectedValue { - t.Errorf("Wrong value for key %s: got %s, want %s", string(key), string(value), expectedValue) - } - return nil - }, tc.root) - - if (err != nil) != tc.isErr { - t.Errorf("ObjectEach(%s) error = %v, wantErr %v", tc.name, err, tc.isErr) - } - }) - } -} - -var testJson = []byte(`{ - "name": "Name", - "order": "Order", - "sum": 100, - "len": 12, - "isPaid": true, - "nested": {"a":"test", "b":2, "nested3":{"a":"test3","b":4}, "c": "unknown"}, - "nested2": { - "a":"test2", - "b":3 - }, - "arr": [ - { - "a":"zxc", - "b": 1 - }, - { - "a":"123", - "b":2 - } - ], - "arrInt": [1,2,3,4], - "intPtr": 10, -}`) - -func TestEachKey(t *testing.T) { - paths := [][]string{ - {"name"}, - {"order"}, - {"nested", "a"}, - {"nested", "b"}, - {"nested2", "a"}, - {"nested", "nested3", "b"}, - {"arr", "[1]", "b"}, - {"arrInt", "[3]"}, - {"arrInt", "[5]"}, - {"nested"}, - {"arr", "["}, - {"a\n", "b\n"}, - {"nested", "b"}, - } - - keysFound := 0 - EachKey(testJson, func(idx int, value []byte, vt ValueType, err error) { - keysFound++ - - expectedValues := []string{"Name", "Order", "test", "2", "test2", "4", "2", "4", "", `{"a":"test", "b":2, "nested3":{"a":"test3","b":4}, "c": "unknown"}`, "", "99", "2"} - if idx < len(expectedValues) { - if string(value) != expectedValues[idx] { - t.Errorf("Invalid key #%v. Expected: %v, Got: %v", idx, expectedValues[idx], string(value)) - } - } else { - t.Errorf("Should find only %v keys, but got an extra key with index %v and value %v", len(expectedValues), idx, string(value)) - } - }, paths...) - - expectedKeysFound := 8 - if keysFound != expectedKeysFound { - t.Errorf("Mismatch in number of keys found. Expected: %v, Got: %v", expectedKeysFound, keysFound) - } -} - -func TestIsPathValid(t *testing.T) { - tests := []struct { - name string - path []string - level int - pathsBuf []string - want bool - }{ - { - name: "Exact match", - path: []string{"users", "id"}, - level: 2, - pathsBuf: []string{"users", "id"}, - want: true, - }, - { - name: "Path shorter than level", - path: []string{"users"}, - level: 2, - pathsBuf: []string{"users", "id"}, - want: false, - }, - { - name: "Path length matches level but paths differ", - path: []string{"users", "name"}, - level: 2, - pathsBuf: []string{"users", "id"}, - want: false, - }, - { - name: "Path longer than level but matches up to level", - path: []string{"users", "id", "name"}, - level: 2, - pathsBuf: []string{"users", "id"}, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isPathValid(tt.path, tt.level, tt.pathsBuf) - if got != tt.want { - t.Errorf("isPathValid(%s) = %v, want %v", tt.name, got, tt.want) - } - }) - } -} - -func TestSameTree(t *testing.T) { - tests := []struct { - name string - p1 []string - p2 []string - want bool - }{ - { - name: "Completely identical", - p1: []string{"a", "b", "c"}, - p2: []string{"a", "b", "c"}, - want: true, - }, - { - name: "Partially identical", - p1: []string{"a", "b"}, - p2: []string{"a", "b", "c"}, - want: true, - }, - { - name: "Not identical", - p1: []string{"a", "x"}, - p2: []string{"a", "b", "c"}, - want: false, - }, - { - name: "One path longer than the other", - p1: []string{"a", "b", "c", "d"}, - p2: []string{"a", "b", "c"}, - want: true, - }, - { - name: "One path (p1) is empty", - p1: []string{}, - p2: []string{"a", "b", "c"}, - want: true, - }, - { - name: "One path (p2) is empty", - p1: []string{"a", "b", "c"}, - p2: []string{}, - want: true, - }, - { - name: "Both paths are empty", - p1: []string{}, - p2: []string{}, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := sameTree(tt.p1, tt.p2); got != tt.want { - t.Errorf("sameTree(%v, %v) = %v, want %v", tt.p1, tt.p2, got, tt.want) - } - }) - } -} diff --git a/examples/gno.land/p/demo/json/struct.gno b/examples/gno.land/p/demo/json/struct.gno deleted file mode 100644 index eb00065eb7b..00000000000 --- a/examples/gno.land/p/demo/json/struct.gno +++ /dev/null @@ -1,369 +0,0 @@ -package json - -import ( - "bytes" - "errors" - "regexp" - "strconv" - "strings" - - "gno.land/p/demo/ufmt" -) - -var jsonKeyRegex = regexp.MustCompile(`"([^"\\]|\\.)*"\\s*:`) - -// Field stores each field's information. -type Field struct { - // name holds the field name. - name string - // typ holds the field type. - typ ValueType - // value holds the field value. - value interface{} - // arrayValues store the each value of the array. - arrayValues []interface{} - // nestedStruct represents the JSON object. - nestedStruct *Struct -} - -// Struct holds the whole structure of target struct. -type Struct struct { - // fields holds the fields of the struct. - fields []Field -} - -// NewStruct create new empty struct placeholder. -func NewStruct() *Struct { - return &Struct{} -} - -func (s *Struct) Fields() []Field { - return s.fields -} - -// addField adds a field to the struct. The order of the fields is preserved. -func (s *Struct) addField(name string, fieldType ValueType, value interface{}) *Struct { - field := Field{name: name, typ: fieldType, value: value} - s.fields = append(s.fields, field) - return s -} - -func (s *Struct) AddStringField(name string, value string) *Struct { - return s.addField(name, String, value) -} - -func (s *Struct) AddIntField(name string, value int) *Struct { - return s.addField(name, Number, value) -} - -func (s *Struct) AddFloatField(name string, value float64) *Struct { - return s.addField(name, Float, value) -} - -func (s *Struct) AddBoolField(name string, value bool) *Struct { - return s.addField(name, Boolean, value) -} - -func (s *Struct) AddNullField(name string) *Struct { - return s.addField(name, Null, nil) -} - -func (s *Struct) AddObjectField(name string, inner *Struct) *Struct { - f := Field{ - name: name, - typ: Object, - nestedStruct: inner, - } - s.fields = append(s.fields, f) - - return s -} - -func (s *Struct) AddArrayField(name string, arrayStruct *Struct) *Struct { - f := Field{ - name: name, - typ: Array, - arrayValues: arrayStruct.Fields(), - } - s.fields = append(s.fields, f) - - return s -} - -/* Encoder */ - -// Marshaler marshals the struct into a JSON byte slice. -func (s *Struct) Marshal() ([]byte, error) { - var buf bytes.Buffer - buf.WriteByte('{') - - for i, field := range s.fields { - if i > 0 { - buf.WriteString(", ") - } - - // `"name": ` - buf.WriteString(strconv.Quote(field.name)) - buf.WriteString(": ") - - if field.nestedStruct != nil { - // nested struct are considered as object. - // need to apply marshaler to each field. - if nestedJSON, err := field.nestedStruct.Marshal(); err != nil { - return nil, err - } else { - buf.Write(nestedJSON) - } - } else if len(field.arrayValues) > 0 { - buf.WriteByte('[') - for i, value := range field.arrayValues { - if i > 0 { - buf.WriteString(", ") - } - // apply marshaler to each value - if valueStr, err := marshalValue(value); err != nil { - return nil, err - } else { - buf.Write(valueStr) - } - } - buf.WriteByte(']') - } else { - if valueStr, err := marshalValue(field.value); err != nil { - return nil, err - } else { - buf.Write(valueStr) - } - } - } - - buf.WriteByte('}') - return buf.Bytes(), nil -} - -// CustomMarshaller interface that the custom type should implement. -type CustomMarshaller interface { - MarshalCustom() ([]byte, error) -} - -// marshalValue marshals the value by its type. -func marshalValue(value interface{}) ([]byte, error) { - if customVal, ok := value.(CustomMarshaller); ok { - if marshaled, err := customVal.MarshalCustom(); err != nil { - return nil, err - } else { - return marshaled, nil - } - } - - var buf bytes.Buffer - - switch v := value.(type) { - // Object - case map[string]interface{}: - buf.WriteByte('{') - isFirst := true - for key, value := range v { - if !isFirst { - buf.WriteString(", ") - } - - // `"": ` - buf.WriteString(strconv.Quote(key)) - buf.WriteString(": ") - - if marshaledValue, err := marshalValue(value); err != nil { - return nil, err - } else { - buf.Write(marshaledValue) - } - - isFirst = false - } - buf.WriteByte('}') - // Array - case []interface{}: - buf.WriteByte('[') - for i, value := range v { - if i > 0 { - buf.WriteString(", ") - } - if marshaledValue, err := marshalValue(value); err != nil { - return nil, err - } else { - buf.Write(marshaledValue) - } - } - buf.WriteByte(']') - case string: - // must handle escape characters - quoted := strconv.Quote(v) - buf.WriteString(quoted) - case int, int8, int16, int32, int64: - buf.WriteString(ufmt.Sprintf("%d", v)) - case uint, uint8, uint16, uint32, uint64: - buf.WriteString(ufmt.Sprintf("%d", v)) - case float32, float64: - buf.WriteString(ufmt.Sprintf("%g", v)) - case bool: - buf.WriteString(ufmt.Sprintf("%t", v)) - case nil: - buf.WriteString("null") - default: - return nil, errors.New(ufmt.Sprintf("json.Marshal: unsupported type %s", value)) - } - - return buf.Bytes(), nil -} - -/* Decoder */ - -// Unmarshal unmarshals the JSON data into the struct. -// The struct should be declared as a Struct instance. -func Unmarshal(data []byte, s *Struct) error { - keys, err := extractKeysFromJSON(data) - if err != nil { - return err - } - - for _, key := range keys { - // regex-parsed key contains double quotes, so need to remove them. - // this is unnecessary if we don't use regex when retrieving keys. - key = strings.ReplaceAll(key, `"`, "") - val, typ, _, err := Get(data, key) - if err != nil { - return err - } - - switch typ { - case String: - s.AddStringField(key, string(val)) - case Number, Float: - if got, err := ParseFloatLiteral(val); err != nil { - return err - } else { - s.AddFloatField(key, got) - } - case Boolean: - if got, err := ParseBoolLiteral(val); err != nil { - return err - } else { - s.AddBoolField(key, got) - } - case Null: - s.AddNullField(key) - // case Object: - // nestedStruct := NewStruct() - // if err := ObjectEach(val, func(innerKey, innerValue []byte, innerDataType ValueType, offset int) error { - // return Unmarshal(innerValue, nestedStruct) - // }); err != nil { - // return err - // } - // s.AddObjectField(key, nestedStruct) - // case Array: - // arrayStruct := NewStruct() - // _, err := ArrayEach(val, func(value []byte, dataType ValueType, offset int, err error) { - // if err != nil { - // return - // } - // // unmarshal here - // }) - - // if err != nil { - // return err - // } - // s.AddArrayField(key, arrayStruct) - default: - return errors.New("json.Unmarshal: unknown data type") - } - } - - return nil -} - -func extractKeysFromJSON(data []byte) ([]string, error) { - if !jsonKeyRegex.Match(data) { - return nil, errors.New("Invalid JSON format") - } - - var keys []string - matches := jsonKeyRegex.FindAllSubmatch(data, -1) - for _, match := range matches { - // extract the key, trimming the trailing colon and any whitespace - key := strings.TrimSuffix(string(match[0]), ":") - key = strings.TrimSpace(key) - - // reject empty keys - if key == "" { - return nil, errors.New("json: key cannot be an empty string") - } - - keys = append(keys, key) - } - - return keys, nil -} - -// String apply the marshaler to the struct and return the JSON string. -func (s *Struct) String() string { - if json, err := s.Marshal(); err != nil { - return err.Error() - } else { - return string(json) - } -} - -/* CRUD */ -// CRUD operations for the struct. -// by using this methods, we can modify the struct also the JSON data easily. - -// Search finds a field by name and returns it. -func (s *Struct) Search(name string) (*Field, error) { - for i, field := range s.fields { - if field.name == name { - return &s.fields[i], nil - } - } - - return nil, errors.New(ufmt.Sprintf("json.Search: Field name %s not found", name)) -} - -// Remove removes a field from the struct instance. -func (s *Struct) Remove(name string) error { - for i, field := range s.fields { - if field.name == name { - // arranges the slice to remove the field. - s.fields = append(s.fields[:i], s.fields[i+1:]...) - - return nil - } - } - - return errors.New(ufmt.Sprintf("json.Search: Field name %s not found", name)) -} - -// Update updates the value of the field. -// -// The updated value must be the same type as the original one. -// Otherwise, it will return an error. -func (s *Struct) Update(name string, newVal interface{}) error { - // search field by name - f, err := s.Search(name) - if err != nil { - return errors.New(ufmt.Sprintf("json.Update: Field name %s not found", name)) - } - - // type assertion - newValType := typeOf(newVal) - if f.typ != newValType { - return errors.New( - ufmt.Sprintf("json.Update: Field type mismatch. Expected %s, got %s", - f.typ.String(), - newValType.String(), - )) - } - - // update the value - f.value = newVal - return nil -} diff --git a/examples/gno.land/p/demo/json/struct_test.gno b/examples/gno.land/p/demo/json/struct_test.gno deleted file mode 100644 index 6e68c306591..00000000000 --- a/examples/gno.land/p/demo/json/struct_test.gno +++ /dev/null @@ -1,188 +0,0 @@ -package json - -import ( - "errors" - "testing" -) - -var dummyStruct = &Struct{ - fields: []Field{ - {name: "field1", value: "value1"}, - {name: "field2", value: "value2"}, - {name: "field3", value: "value3"}, - }, -} - -func TestStructSearch(t *testing.T) { - s := dummyStruct - tests := []struct { - name string - searchName string - expectedName string - expectedErr error - }{ - { - name: "Test 1", - searchName: "field1", - expectedName: "field1", - expectedErr: nil, - }, - { - name: "Test 2", - searchName: "field2", - expectedName: "field2", - expectedErr: nil, - }, - { - name: "Test 3", - searchName: "field4", - expectedName: "", - expectedErr: errors.New("Field name not found"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - field, err := s.Search(tt.searchName) - - if err != nil && tt.expectedErr == nil { - t.Errorf("Search() error = %v, expectedErr %v", err, tt.expectedErr) - return - } - - if field != nil && field.name != tt.expectedName { - t.Errorf("Search() got = %v, want %v", field.name, tt.expectedName) - } - }) - } -} - -func TestRemoveField(t *testing.T) { - tests := []struct { - name string - fields []Field - remove string - expectedFields []string - }{ - { - name: "remove existing field", - fields: []Field{ - {name: "a", typ: String, value: "apple"}, - {name: "b", typ: String, value: "banana"}, - }, - remove: "a", - expectedFields: []string{"b"}, - }, - { - name: "remove non-existing field", - fields: []Field{ - {name: "a", typ: String, value: "apple"}, - }, - remove: "b", - expectedFields: []string{"a"}, - }, - { - name: "remove from empty struct", - fields: []Field{}, - remove: "a", - expectedFields: []string{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := NewStruct() - for _, f := range tt.fields { - s.addField(f.name, f.typ, f.value) - } - - s.Remove(tt.remove) - - if !fieldsMatch(s.fields, tt.expectedFields) { - t.Errorf("Remove() got = %v, want %v", fieldNames(s.fields), tt.expectedFields) - } - }) - } -} - -func TestUpdateField(t *testing.T) { - tests := []struct { - name string - init []Field - fName string - newValue interface{} - expectedFields []Field - expectError bool - }{ - { - name: "update existing field", - init: []Field{ - {name: "a", typ: String, value: "apple"}, - {name: "b", typ: Number, value: 10}, - }, - fName: "a", - newValue: "avocado", - expectedFields: []Field{ - {name: "a", typ: String, value: "avocado"}, - {name: "b", typ: Number, value: 10}, - }, - expectError: false, - }, - { - name: "update non-existing field", - init: []Field{ - {name: "a", typ: String, value: "apple"}, - }, - fName: "b", - newValue: "banana", - expectedFields: []Field{ - {name: "a", typ: String, value: "apple"}, - }, - expectError: true, - }, - { - name: "type mismatch update", - init: []Field{ - {name: "a", typ: String, value: "apple"}, - }, - fName: "a", - newValue: 100, // int instead of string - expectedFields: []Field{ - {name: "a", typ: String, value: "apple"}, - }, - expectError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := NewStruct() - for _, f := range tt.init { - s.addField(f.name, f.typ, f.value) - } - - err := s.Update(tt.fName, tt.newValue) - if (err != nil) != tt.expectError { - t.Errorf("Update() error = %v, expectErr %v", err, tt.expectError) - return - } - - // check if fields are equal - for _, ef := range tt.expectedFields { - f, err := s.Search(ef.name) - if err != nil { - t.Errorf("Update() error = %v, expectErr %v", err, tt.expectError) - return - } - - if f.value != ef.value { - t.Errorf("Update() got = %v, want %v", f.value, ef.value) - } - - if f.typ != ef.typ { - t.Errorf("Update() got = %v, want %v", f.typ, ef.typ) - } - } - }) - } -} diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 63e2c5a01dd..4791850bf46 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -1,51 +1,45 @@ package json const ( - SquareOpenToken = '[' - SquareCloseToken = ']' - RoundOpenToken = '(' - RoundCloseToken = ')' - CurlyOpenToken = '{' - CurlyCloseToken = '}' - CommaToken = ',' - DotToken = '.' - ColonToken = ':' - BackTickToken = '`' - QuoteToken = '\'' - DoublyQuoteToken = '"' - EmptyStringToken = "" - WhiteSpaceToken = ' ' - PlusToken = '+' - MinusToken = '-' - AesteriskToken = '*' - BangToken = '!' - QuestionToken = '?' - NewLineToken = '\n' - TabToken = '\t' - CarriageReturnToken = '\r' - FormFeedToken = '\f' - BackSpaceToken = '\b' - SlashToken = '/' - BackSlashToken = '\\' - UnderScoreToken = '_' - DollarToken = '$' - AtToken = '@' - AndToken = '&' - OrToken = '|' + bracketOpen = '[' + bracketClose = ']' + parenOpen = '(' + parenClose = ')' + curlyOpen = '{' + curlyClose = '}' + comma = ',' + dot = '.' + colon = ':' + backTick = '`' + singleQuote = '\'' + doubleQuote = '"' + emptyString = "" + whiteSpace = ' ' + plus = '+' + minus = '-' + aesterisk = '*' + bang = '!' + question = '?' + newLine = '\n' + tab = '\t' + carriageReturn = '\r' + formFeed = '\f' + backSpace = '\b' + slash = '/' + backSlash = '\\' + underScore = '_' + dollarSign = '$' + atSign = '@' + andSign = '&' + orSign = '|' ) -const ( +var ( trueLiteral = []byte("true") falseLiteral = []byte("false") nullLiteral = []byte("null") ) -const ( - StructLiteral = "struct" - InterfaceLiteral = "interface{}" - MapLiteral = "map" -) - type ValueType int const ( @@ -76,9 +70,7 @@ func (v ValueType) String() string { return "boolean" case Null: return "null" - case Unknown: - return "unknown" default: - return string(v) + return "unknown" } } diff --git a/examples/gno.land/p/demo/json/utils.gno b/examples/gno.land/p/demo/json/utils.gno deleted file mode 100644 index 14d595d0719..00000000000 --- a/examples/gno.land/p/demo/json/utils.gno +++ /dev/null @@ -1,87 +0,0 @@ -package json - -func notDigit(c byte) bool { - return (c & 0xF0) != 0x30 -} - -// lower converts a byte to lower case if it is an uppercase letter. -// -// In ASCII, the lowercase letters have the 6th bit set to 1, which is not set in their uppercase counterparts. -// This function sets the 6th bit of the byte, effectively converting uppercase letters to lowercase. -// It has no effect on bytes that are not uppercase letters. -func lower(c byte) byte { - return c | 0x20 -} - -const hexLookupTable = [256]int{ - '0': 0x0, '1': 0x1, '2': 0x2, '3': 0x3, '4': 0x4, - '5': 0x5, '6': 0x6, '7': 0x7, '8': 0x8, '9': 0x9, - 'A': 0xA, 'B': 0xB, 'C': 0xC, 'D': 0xD, 'E': 0xE, 'F': 0xF, - 'a': 0xA, 'b': 0xB, 'c': 0xC, 'd': 0xD, 'e': 0xE, 'f': 0xF, - // Fill unspecified index-value pairs with key and value of -1 - 'G': -1, 'H': -1, 'I': -1, 'J': -1, - 'K': -1, 'L': -1, 'M': -1, 'N': -1, - 'O': -1, 'P': -1, 'Q': -1, 'R': -1, - 'S': -1, 'T': -1, 'U': -1, 'V': -1, - 'W': -1, 'X': -1, 'Y': -1, 'Z': -1, - 'g': -1, 'h': -1, 'i': -1, 'j': -1, - 'k': -1, 'l': -1, 'm': -1, 'n': -1, - 'o': -1, 'p': -1, 'q': -1, 'r': -1, - 's': -1, 't': -1, 'u': -1, 'v': -1, - 'w': -1, 'x': -1, 'y': -1, 'z': -1, -} - -func h2i(c byte) int { - return hexLookupTable[c] -} - -func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { - if bytes[0] == MinusToken { - return true, bytes[1:] - } - - return false, bytes -} - -func equalStr(b *[]byte, s string) bool { - return string(*b) == s -} - -/* Testing Helper Function */ - -func isEqualSlice(a, b []string) bool { - if len(a) != len(b) { - return false - } - - for i, v := range a { - if v != b[i] { - return false - } - } - - return true -} - -func fieldsMatch(fs []Field, names []string) bool { - if len(fs) != len(names) { - return false - } - - for i, f := range fs { - if f.name != names[i] { - return false - } - } - - return true -} - -func fieldNames(fs []Field) []string { - names := make([]string, len(fs)) - for i, f := range fs { - names[i] = f.name - } - - return names -} From 03a9d171d931a9268d9750611d59f2ef7400e10f Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Mar 2024 20:13:21 +0900 Subject: [PATCH 62/72] use ryu algorithm to format float and reorganize the structure --- .../json/{ => eisel_lemire}/eisel_lemire.gno | 4 +- .../gno.land/p/demo/json/eisel_lemire/gno.mod | 1 + examples/gno.land/p/demo/json/encode.gno | 6 +- examples/gno.land/p/demo/json/encode_test.gno | 3 +- examples/gno.land/p/demo/json/gno.mod | 5 +- examples/gno.land/p/demo/json/parser.gno | 4 +- .../gno.land/p/demo/json/ryu/floatconv.gno | 146 ++++ .../p/demo/json/ryu/floatconv_test.gno | 33 + examples/gno.land/p/demo/json/ryu/gno.mod | 1 + examples/gno.land/p/demo/json/ryu/ryu64.gno | 365 ++++++++++ examples/gno.land/p/demo/json/ryu/table.gno | 678 ++++++++++++++++++ 11 files changed, 1239 insertions(+), 7 deletions(-) rename examples/gno.land/p/demo/json/{ => eisel_lemire}/eisel_lemire.gno (99%) create mode 100644 examples/gno.land/p/demo/json/eisel_lemire/gno.mod create mode 100644 examples/gno.land/p/demo/json/ryu/floatconv.gno create mode 100644 examples/gno.land/p/demo/json/ryu/floatconv_test.gno create mode 100644 examples/gno.land/p/demo/json/ryu/gno.mod create mode 100644 examples/gno.land/p/demo/json/ryu/ryu64.gno create mode 100644 examples/gno.land/p/demo/json/ryu/table.gno diff --git a/examples/gno.land/p/demo/json/eisel_lemire.gno b/examples/gno.land/p/demo/json/eisel_lemire/eisel_lemire.gno similarity index 99% rename from examples/gno.land/p/demo/json/eisel_lemire.gno rename to examples/gno.land/p/demo/json/eisel_lemire/eisel_lemire.gno index ef95b76f9da..6a29f7f1350 100644 --- a/examples/gno.land/p/demo/json/eisel_lemire.gno +++ b/examples/gno.land/p/demo/json/eisel_lemire/eisel_lemire.gno @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package json +package eisel_lemire // This file implements the Eisel-Lemire ParseFloat algorithm, published in // 2020 and discussed extensively at @@ -46,7 +46,7 @@ const ( // the leading zeros. This is followed by the main algorithm logic that // converts the normalized mantissa and exponent into a 64-bit floating-point number. // The function returns this number along with a boolean indicating the success of the operation. -func eiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { +func EiselLemire64(man uint64, exp10 int, neg bool) (f float64, ok bool) { // The terse comments in this function body refer to sections of the // https://nigeltao.github.io/blog/2020/eisel-lemire.html blog post. diff --git a/examples/gno.land/p/demo/json/eisel_lemire/gno.mod b/examples/gno.land/p/demo/json/eisel_lemire/gno.mod new file mode 100644 index 00000000000..d6670de82e2 --- /dev/null +++ b/examples/gno.land/p/demo/json/eisel_lemire/gno.mod @@ -0,0 +1 @@ +module gno.land/p/demo/json/eisel_lemire diff --git a/examples/gno.land/p/demo/json/encode.gno b/examples/gno.land/p/demo/json/encode.gno index 00506813f8e..18e5bb75ad5 100644 --- a/examples/gno.land/p/demo/json/encode.gno +++ b/examples/gno.land/p/demo/json/encode.gno @@ -5,6 +5,7 @@ import ( "errors" "strconv" + "gno.land/p/demo/json/ryu" "gno.land/p/demo/ufmt" ) @@ -38,8 +39,9 @@ func Marshal(node *Node) ([]byte, error) { num := ufmt.Sprintf("%d", int64(nVal)) buf.WriteString(num) } else { - // TODO: fix float formatter - num := ufmt.Sprintf("%f", nVal) + // use ryu algorithm to convert float to string + // num := ufmt.Sprintf("%f", nVal) + num := ryu.FormatFloat64(nVal) buf.WriteString(num) } diff --git a/examples/gno.land/p/demo/json/encode_test.gno b/examples/gno.land/p/demo/json/encode_test.gno index 97081c4381b..33a1fae3d4e 100644 --- a/examples/gno.land/p/demo/json/encode_test.gno +++ b/examples/gno.land/p/demo/json/encode_test.gno @@ -39,8 +39,9 @@ func TestMarshal_Primitive(t *testing.T) { name: "42", node: NumberNode("", 42), }, + // TODO: fix output for not to use scientific notation { - name: "100.5", + name: "1.005e+02", node: NumberNode("", 100.5), }, { diff --git a/examples/gno.land/p/demo/json/gno.mod b/examples/gno.land/p/demo/json/gno.mod index ef794458c56..06fbfed60ad 100644 --- a/examples/gno.land/p/demo/json/gno.mod +++ b/examples/gno.land/p/demo/json/gno.mod @@ -1,3 +1,6 @@ module gno.land/p/demo/json -require gno.land/p/demo/ufmt v0.0.0-latest +require ( + gno.land/p/demo/json/eisel_lemire v0.0.0-latest + gno.land/p/demo/ufmt v0.0.0-latest +) diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 5daf72916e4..73b30e46f68 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -4,6 +4,8 @@ import ( "bytes" "errors" "strconv" + + el "gno.land/p/demo/json/eisel_lemire" ) const ( @@ -72,7 +74,7 @@ func ParseFloatLiteral(bytes []byte) (value float64, err error) { } // for fast float64 conversion - f, success := eiselLemire64(man, exp10, neg) + f, success := el.EiselLemire64(man, exp10, neg) if !success { return 0, nil } diff --git a/examples/gno.land/p/demo/json/ryu/floatconv.gno b/examples/gno.land/p/demo/json/ryu/floatconv.gno new file mode 100644 index 00000000000..ce0f7f2c574 --- /dev/null +++ b/examples/gno.land/p/demo/json/ryu/floatconv.gno @@ -0,0 +1,146 @@ +// Copyright 2018 Ulf Adams +// Modifications copyright 2019 Caleb Spare +// +// The contents of this file may be used under the terms of the Apache License, +// Version 2.0. +// +// (See accompanying file LICENSE or copy at +// http://www.apache.org/licenses/LICENSE-2.0) +// +// Unless required by applicable law or agreed to in writing, this software +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. +// +// The code in this file is part of a Go translation of the C code originally written by +// Ulf Adams, which can be found at https://github.com/ulfjack/ryu. The original source +// code is licensed under the Apache License 2.0. This code is a derivative work thereof, +// adapted and modified to meet the specifications of the Gno language project. +// +// original Go implementation can be found at https://github.com/cespare/ryu. +// +// Please note that the modifications are also under the Apache License 2.0 unless +// otherwise specified. + +// Package ryu implements the Ryu algorithm for quickly converting floating +// point numbers into strings. +package ryu + +import ( + "math" +) + +// TODO: move to strconv package? +// TODO: support %g and %f formats in ufmt package. + +const ( + mantBits32 = 23 + expBits32 = 8 + bias32 = 127 + + mantBits64 = 52 + expBits64 = 11 + bias64 = 1023 +) + +// FormatFloat64 converts a 64-bit floating point number f to a string. +// It behaves like strconv.FormatFloat(f, 'e', -1, 64). +func FormatFloat64(f float64) string { + b := make([]byte, 0, 24) + b = AppendFloat64(b, f) + return string(b) +} + +// AppendFloat64 appends the string form of the 64-bit floating point number f, +// as generated by FormatFloat64, to b and returns the extended buffer. +func AppendFloat64(b []byte, f float64) []byte { + // Step 1: Decode the floating-point number. + // Unify normalized and subnormal cases. + u := math.Float64bits(f) + neg := u>>(mantBits64+expBits64) != 0 + mant := u & (uint64(1)<> mantBits64) & (uint64(1)<= 0, "e >= 0") + assert(e <= 1650, "e <= 1650") + return (uint32(e) * 78913) >> 18 +} + +// log10Pow5 returns floor(log_10(5^e)). +func log10Pow5(e int32) uint32 { + // The first value this approximation fails for is 5^2621 + // which is just greater than 10^1832. + assert(e >= 0, "e >= 0") + assert(e <= 2620, "e <= 2620") + return (uint32(e) * 732923) >> 20 +} + +// pow5Bits returns ceil(log_2(5^e)), or else 1 if e==0. +func pow5Bits(e int32) int32 { + // This approximation works up to the point that the multiplication + // overflows at e = 3529. If the multiplication were done in 64 bits, + // it would fail at 5^4004 which is just greater than 2^9297. + assert(e >= 0, "e >= 0") + assert(e <= 3528, "e <= 3528") + return int32((uint32(e)*1217359)>>19 + 1) +} diff --git a/examples/gno.land/p/demo/json/ryu/floatconv_test.gno b/examples/gno.land/p/demo/json/ryu/floatconv_test.gno new file mode 100644 index 00000000000..5184094e61b --- /dev/null +++ b/examples/gno.land/p/demo/json/ryu/floatconv_test.gno @@ -0,0 +1,33 @@ +package ryu + +import ( + "math" + "testing" +) + +func TestFormatFloat64(t *testing.T) { + tests := []struct { + name string + value float64 + expected string + }{ + {"positive infinity", math.Inf(1), "+Inf"}, + {"negative infinity", math.Inf(-1), "-Inf"}, + {"NaN", math.NaN(), "NaN"}, + {"zero", 0.0, "0e+00"}, + {"negative zero", -0.0, "0e+00"}, + {"positive number", 3.14159, "3.14159e+00"}, + {"negative number", -2.71828, "-2.71828e+00"}, + {"very small number", 1.23e-20, "1.23e-20"}, + {"very large number", 1.23e+20, "1.23e+20"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := FormatFloat64(test.value) + if result != test.expected { + t.Errorf("FormatFloat64(%v) = %q, expected %q", test.value, result, test.expected) + } + }) + } +} diff --git a/examples/gno.land/p/demo/json/ryu/gno.mod b/examples/gno.land/p/demo/json/ryu/gno.mod new file mode 100644 index 00000000000..86a1988b052 --- /dev/null +++ b/examples/gno.land/p/demo/json/ryu/gno.mod @@ -0,0 +1 @@ +module gno.land/p/demo/json/ryu diff --git a/examples/gno.land/p/demo/json/ryu/ryu64.gno b/examples/gno.land/p/demo/json/ryu/ryu64.gno new file mode 100644 index 00000000000..7dca83dd4db --- /dev/null +++ b/examples/gno.land/p/demo/json/ryu/ryu64.gno @@ -0,0 +1,365 @@ +// Copyright 2018 Ulf Adams +// Modifications copyright 2019 Caleb Spare +// +// The contents of this file may be used under the terms of the Apache License, +// Version 2.0. +// +// (See accompanying file LICENSE or copy at +// http://www.apache.org/licenses/LICENSE-2.0) +// +// Unless required by applicable law or agreed to in writing, this software +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. +// +// The code in this file is part of a Go translation of the C code originally written by +// Ulf Adams, which can be found at https://github.com/ulfjack/ryu. The original source +// code is licensed under the Apache License 2.0. This code is a derivative work thereof, +// adapted and modified to meet the specifications of the Gno language project. +// +// Please note that the modifications are also under the Apache License 2.0 unless +// otherwise specified. + +package ryu + +import ( + "math/bits" +) + +type uint128 struct { + lo uint64 + hi uint64 +} + +// dec64 is a floating decimal type representing m * 10^e. +type dec64 struct { + m uint64 + e int32 +} + +func (d dec64) append(b []byte, neg bool) []byte { + // Step 5: Print the decimal representation. + if neg { + b = append(b, '-') + } + + out := d.m + outLen := decimalLen64(out) + bufLen := outLen + if bufLen > 1 { + bufLen++ // extra space for '.' + } + + // Print the decimal digits. + n := len(b) + if cap(b)-len(b) >= bufLen { + // Avoid function call in the common case. + b = b[:len(b)+bufLen] + } else { + b = append(b, make([]byte, bufLen)...) + } + + // Avoid expensive 64-bit divisions. + // We have at most 17 digits, and uint32 can store 9 digits. + // If the output doesn't fit into a uint32, cut off 8 digits + // so the rest will fit into a uint32. + var i int + if out>>32 > 0 { + var out32 uint32 + out, out32 = out/1e8, uint32(out%1e8) + for ; i < 8; i++ { + b[n+outLen-i] = '0' + byte(out32%10) + out32 /= 10 + } + } + out32 := uint32(out) + for ; i < outLen-1; i++ { + b[n+outLen-i] = '0' + byte(out32%10) + out32 /= 10 + } + b[n] = '0' + byte(out32%10) + + // Print the '.' if needed. + if outLen > 1 { + b[n+1] = '.' + } + + // Print the exponent. + b = append(b, 'e') + exp := d.e + int32(outLen) - 1 + if exp < 0 { + b = append(b, '-') + exp = -exp + } else { + // Unconditionally print a + here to match strconv's formatting. + b = append(b, '+') + } + // Always print at least two digits to match strconv's formatting. + d2 := exp % 10 + exp /= 10 + d1 := exp % 10 + d0 := exp / 10 + if d0 > 0 { + b = append(b, '0'+byte(d0)) + } + b = append(b, '0'+byte(d1), '0'+byte(d2)) + + return b +} + +func float64ToDecimalExactInt(mant, exp uint64) (d dec64, ok bool) { + e := exp - bias64 + if e > mantBits64 { + return d, false + } + shift := mantBits64 - e + mant |= 1 << mantBits64 // implicit 1 + d.m = mant >> shift + if d.m<= 0 { + // This expression is slightly faster than max(0, log10Pow2(e2) - 1). + q := log10Pow2(e2) - boolToUint32(e2 > 3) + e10 = int32(q) + k := pow5InvNumBits64 + pow5Bits(int32(q)) - 1 + i := -e2 + int32(q) + k + mul := pow5InvSplit64[q] + vr = mulShift64(4*m2, mul, i) + vp = mulShift64(4*m2+2, mul, i) + vm = mulShift64(4*m2-1-mmShift, mul, i) + if q <= 21 { + // This should use q <= 22, but I think 21 is also safe. + // Smaller values may still be safe, but it's more + // difficult to reason about them. Only one of mp, mv, + // and mm can be a multiple of 5, if any. + if mv%5 == 0 { + vrIsTrailingZeros = multipleOfPowerOfFive64(mv, q) + } else if acceptBounds { + // Same as min(e2 + (^mm & 1), pow5Factor64(mm)) >= q + // <=> e2 + (^mm & 1) >= q && pow5Factor64(mm) >= q + // <=> true && pow5Factor64(mm) >= q, since e2 >= q. + vmIsTrailingZeros = multipleOfPowerOfFive64(mv-1-mmShift, q) + } else if multipleOfPowerOfFive64(mv+2, q) { + vp-- + } + } + } else { + // This expression is slightly faster than max(0, log10Pow5(-e2) - 1). + q := log10Pow5(-e2) - boolToUint32(-e2 > 1) + e10 = int32(q) + e2 + i := -e2 - int32(q) + k := pow5Bits(i) - pow5NumBits64 + j := int32(q) - k + mul := pow5Split64[i] + vr = mulShift64(4*m2, mul, j) + vp = mulShift64(4*m2+2, mul, j) + vm = mulShift64(4*m2-1-mmShift, mul, j) + if q <= 1 { + // {vr,vp,vm} is trailing zeros if {mv,mp,mm} has at least q trailing 0 bits. + // mv = 4 * m2, so it always has at least two trailing 0 bits. + vrIsTrailingZeros = true + if acceptBounds { + // mm = mv - 1 - mmShift, so it has 1 trailing 0 bit iff mmShift == 1. + vmIsTrailingZeros = mmShift == 1 + } else { + // mp = mv + 2, so it always has at least one trailing 0 bit. + vp-- + } + } else if q < 63 { // TODO(ulfjack/cespare): Use a tighter bound here. + // We need to compute min(ntz(mv), pow5Factor64(mv) - e2) >= q - 1 + // <=> ntz(mv) >= q - 1 && pow5Factor64(mv) - e2 >= q - 1 + // <=> ntz(mv) >= q - 1 (e2 is negative and -e2 >= q) + // <=> (mv & ((1 << (q - 1)) - 1)) == 0 + // We also need to make sure that the left shift does not overflow. + vrIsTrailingZeros = multipleOfPowerOfTwo64(mv, q-1) + } + } + + // Step 4: Find the shortest decimal representation + // in the interval of valid representations. + var removed int32 + var lastRemovedDigit uint8 + var out uint64 + // On average, we remove ~2 digits. + if vmIsTrailingZeros || vrIsTrailingZeros { + // General case, which happens rarely (~0.7%). + for { + vpDiv10 := vp / 10 + vmDiv10 := vm / 10 + if vpDiv10 <= vmDiv10 { + break + } + vmMod10 := vm % 10 + vrDiv10 := vr / 10 + vrMod10 := vr % 10 + vmIsTrailingZeros = vmIsTrailingZeros && vmMod10 == 0 + vrIsTrailingZeros = vrIsTrailingZeros && lastRemovedDigit == 0 + lastRemovedDigit = uint8(vrMod10) + vr = vrDiv10 + vp = vpDiv10 + vm = vmDiv10 + removed++ + } + if vmIsTrailingZeros { + for { + vmDiv10 := vm / 10 + vmMod10 := vm % 10 + if vmMod10 != 0 { + break + } + vpDiv10 := vp / 10 + vrDiv10 := vr / 10 + vrMod10 := vr % 10 + vrIsTrailingZeros = vrIsTrailingZeros && lastRemovedDigit == 0 + lastRemovedDigit = uint8(vrMod10) + vr = vrDiv10 + vp = vpDiv10 + vm = vmDiv10 + removed++ + } + } + if vrIsTrailingZeros && lastRemovedDigit == 5 && vr%2 == 0 { + // Round even if the exact number is .....50..0. + lastRemovedDigit = 4 + } + out = vr + // We need to take vr + 1 if vr is outside bounds + // or we need to round up. + if (vr == vm && (!acceptBounds || !vmIsTrailingZeros)) || lastRemovedDigit >= 5 { + out++ + } + } else { + // Specialized for the common case (~99.3%). + // Percentages below are relative to this. + roundUp := false + for vp/100 > vm/100 { + // Optimization: remove two digits at a time (~86.2%). + roundUp = vr%100 >= 50 + vr /= 100 + vp /= 100 + vm /= 100 + removed += 2 + } + // Loop iterations below (approximately), without optimization above: + // 0: 0.03%, 1: 13.8%, 2: 70.6%, 3: 14.0%, 4: 1.40%, 5: 0.14%, 6+: 0.02% + // Loop iterations below (approximately), with optimization above: + // 0: 70.6%, 1: 27.8%, 2: 1.40%, 3: 0.14%, 4+: 0.02% + for vp/10 > vm/10 { + roundUp = vr%10 >= 5 + vr /= 10 + vp /= 10 + vm /= 10 + removed++ + } + // We need to take vr + 1 if vr is outside bounds + // or we need to round up. + out = vr + boolToUint64(vr == vm || roundUp) + } + + return dec64{m: out, e: e10 + removed} +} + +var powersOf10 = [...]uint64{ + 1e0, + 1e1, + 1e2, + 1e3, + 1e4, + 1e5, + 1e6, + 1e7, + 1e8, + 1e9, + 1e10, + 1e11, + 1e12, + 1e13, + 1e14, + 1e15, + 1e16, + 1e17, + // We only need to find the length of at most 17 digit numbers. +} + +func decimalLen64(u uint64) int { + // http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10 + log2 := 64 - bits.LeadingZeros64(u) - 1 + t := (log2 + 1) * 1233 >> 12 + return t - boolToInt(u < powersOf10[t]) + 1 +} + +func mulShift64(m uint64, mul uint128, shift int32) uint64 { + hihi, hilo := bits.Mul64(m, mul.hi) + lohi, _ := bits.Mul64(m, mul.lo) + sum := uint128{hi: hihi, lo: lohi + hilo} + if sum.lo < lohi { + sum.hi++ // overflow + } + return shiftRight128(sum, shift-64) +} + +func shiftRight128(v uint128, shift int32) uint64 { + // The shift value is always modulo 64. + // In the current implementation of the 64-bit version + // of Ryu, the shift value is always < 64. + // (It is in the range [2, 59].) + // Check this here in case a future change requires larger shift + // values. In this case this function needs to be adjusted. + assert(shift < 64, "shift < 64") + return (v.hi << uint64(64-shift)) | (v.lo >> uint(shift)) +} + +func pow5Factor64(v uint64) uint32 { + for n := uint32(0); ; n++ { + q, r := v/5, v%5 + if r != 0 { + return n + } + v = q + } +} + +func multipleOfPowerOfFive64(v uint64, p uint32) bool { + return pow5Factor64(v) >= p +} + +func multipleOfPowerOfTwo64(v uint64, p uint32) bool { + return uint32(bits.TrailingZeros64(v)) >= p +} diff --git a/examples/gno.land/p/demo/json/ryu/table.gno b/examples/gno.land/p/demo/json/ryu/table.gno new file mode 100644 index 00000000000..fe33ad90a57 --- /dev/null +++ b/examples/gno.land/p/demo/json/ryu/table.gno @@ -0,0 +1,678 @@ +// Code generated by running "go generate". DO NOT EDIT. + +// Copyright 2018 Ulf Adams +// Modifications copyright 2019 Caleb Spare +// +// The contents of this file may be used under the terms of the Apache License, +// Version 2.0. +// +// (See accompanying file LICENSE or copy at +// http://www.apache.org/licenses/LICENSE-2.0) +// +// Unless required by applicable law or agreed to in writing, this software +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. +// +// The code in this file is part of a Go translation of the C code written by +// Ulf Adams which may be found at https://github.com/ulfjack/ryu. That source +// code is licensed under Apache 2.0 and this code is derivative work thereof. + +package ryu + +const pow5NumBits32 = 61 + +var pow5Split32 = [...]uint64{ + 1152921504606846976, 1441151880758558720, 1801439850948198400, 2251799813685248000, + 1407374883553280000, 1759218604441600000, 2199023255552000000, 1374389534720000000, + 1717986918400000000, 2147483648000000000, 1342177280000000000, 1677721600000000000, + 2097152000000000000, 1310720000000000000, 1638400000000000000, 2048000000000000000, + 1280000000000000000, 1600000000000000000, 2000000000000000000, 1250000000000000000, + 1562500000000000000, 1953125000000000000, 1220703125000000000, 1525878906250000000, + 1907348632812500000, 1192092895507812500, 1490116119384765625, 1862645149230957031, + 1164153218269348144, 1455191522836685180, 1818989403545856475, 2273736754432320594, + 1421085471520200371, 1776356839400250464, 2220446049250313080, 1387778780781445675, + 1734723475976807094, 2168404344971008868, 1355252715606880542, 1694065894508600678, + 2117582368135750847, 1323488980084844279, 1654361225106055349, 2067951531382569187, + 1292469707114105741, 1615587133892632177, 2019483917365790221, +} + +const pow5InvNumBits32 = 59 + +var pow5InvSplit32 = [...]uint64{ + 576460752303423489, 461168601842738791, 368934881474191033, 295147905179352826, + 472236648286964522, 377789318629571618, 302231454903657294, 483570327845851670, + 386856262276681336, 309485009821345069, 495176015714152110, 396140812571321688, + 316912650057057351, 507060240091291761, 405648192073033409, 324518553658426727, + 519229685853482763, 415383748682786211, 332306998946228969, 531691198313966350, + 425352958651173080, 340282366920938464, 544451787073501542, 435561429658801234, + 348449143727040987, 557518629963265579, 446014903970612463, 356811923176489971, + 570899077082383953, 456719261665907162, 365375409332725730, +} + +const pow5NumBits64 = 121 + +var pow5Split64 = [...]uint128{ + {0, 72057594037927936}, + {0, 90071992547409920}, + {0, 112589990684262400}, + {0, 140737488355328000}, + {0, 87960930222080000}, + {0, 109951162777600000}, + {0, 137438953472000000}, + {0, 85899345920000000}, + {0, 107374182400000000}, + {0, 134217728000000000}, + {0, 83886080000000000}, + {0, 104857600000000000}, + {0, 131072000000000000}, + {0, 81920000000000000}, + {0, 102400000000000000}, + {0, 128000000000000000}, + {0, 80000000000000000}, + {0, 100000000000000000}, + {0, 125000000000000000}, + {0, 78125000000000000}, + {0, 97656250000000000}, + {0, 122070312500000000}, + {0, 76293945312500000}, + {0, 95367431640625000}, + {0, 119209289550781250}, + {4611686018427387904, 74505805969238281}, + {10376293541461622784, 93132257461547851}, + {8358680908399640576, 116415321826934814}, + {612489549322387456, 72759576141834259}, + {14600669991935148032, 90949470177292823}, + {13639151471491547136, 113686837721616029}, + {3213881284082270208, 142108547152020037}, + {4314518811765112832, 88817841970012523}, + {781462496279003136, 111022302462515654}, + {10200200157203529728, 138777878078144567}, + {13292654125893287936, 86736173798840354}, + {7392445620511834112, 108420217248550443}, + {4628871007212404736, 135525271560688054}, + {16728102434789916672, 84703294725430033}, + {7075069988205232128, 105879118406787542}, + {18067209522111315968, 132348898008484427}, + {8986162942105878528, 82718061255302767}, + {6621017659204960256, 103397576569128459}, + {3664586055578812416, 129246970711410574}, + {16125424340018921472, 80779356694631608}, + {1710036351314100224, 100974195868289511}, + {15972603494424788992, 126217744835361888}, + {9982877184015493120, 78886090522101180}, + {12478596480019366400, 98607613152626475}, + {10986559581596820096, 123259516440783094}, + {2254913720070624656, 77037197775489434}, + {12042014186943056628, 96296497219361792}, + {15052517733678820785, 120370621524202240}, + {9407823583549262990, 75231638452626400}, + {11759779479436578738, 94039548065783000}, + {14699724349295723422, 117549435082228750}, + {4575641699882439235, 73468396926392969}, + {10331238143280436948, 91835496157991211}, + {8302361660673158281, 114794370197489014}, + {1154580038986672043, 143492962746861268}, + {9944984561221445835, 89683101716788292}, + {12431230701526807293, 112103877145985365}, + {1703980321626345405, 140129846432481707}, + {17205888765512323542, 87581154020301066}, + {12283988920035628619, 109476442525376333}, + {1519928094762372062, 136845553156720417}, + {12479170105294952299, 85528470722950260}, + {15598962631618690374, 106910588403687825}, + {5663645234241199255, 133638235504609782}, + {17374836326682913246, 83523897190381113}, + {7883487353071477846, 104404871487976392}, + {9854359191339347308, 130506089359970490}, + {10770660513014479971, 81566305849981556}, + {13463325641268099964, 101957882312476945}, + {2994098996302961243, 127447352890596182}, + {15706369927971514489, 79654595556622613}, + {5797904354682229399, 99568244445778267}, + {2635694424925398845, 124460305557222834}, + {6258995034005762182, 77787690973264271}, + {3212057774079814824, 97234613716580339}, + {17850130272881932242, 121543267145725423}, + {18073860448192289507, 75964541966078389}, + {8757267504958198172, 94955677457597987}, + {6334898362770359811, 118694596821997484}, + {13182683513586250689, 74184123013748427}, + {11866668373555425458, 92730153767185534}, + {5609963430089506015, 115912692208981918}, + {17341285199088104971, 72445432630613698}, + {12453234462005355406, 90556790788267123}, + {10954857059079306353, 113195988485333904}, + {13693571323849132942, 141494985606667380}, + {17781854114260483896, 88434366004167112}, + {3780573569116053255, 110542957505208891}, + {114030942967678664, 138178696881511114}, + {4682955357782187069, 86361685550944446}, + {15077066234082509644, 107952106938680557}, + {5011274737320973344, 134940133673350697}, + {14661261756894078100, 84337583545844185}, + {4491519140835433913, 105421979432305232}, + {5614398926044292391, 131777474290381540}, + {12732371365632458552, 82360921431488462}, + {6692092170185797382, 102951151789360578}, + {17588487249587022536, 128688939736700722}, + {15604490549419276989, 80430587335437951}, + {14893927168346708332, 100538234169297439}, + {14005722942005997511, 125672792711621799}, + {15671105866394830300, 78545495444763624}, + {1142138259283986260, 98181869305954531}, + {15262730879387146537, 122727336632443163}, + {7233363790403272633, 76704585395276977}, + {13653390756431478696, 95880731744096221}, + {3231680390257184658, 119850914680120277}, + {4325643253124434363, 74906821675075173}, + {10018740084832930858, 93633527093843966}, + {3300053069186387764, 117041908867304958}, + {15897591223523656064, 73151193042065598}, + {10648616992549794273, 91438991302581998}, + {4087399203832467033, 114298739128227498}, + {14332621041645359599, 142873423910284372}, + {18181260187883125557, 89295889943927732}, + {4279831161144355331, 111619862429909666}, + {14573160988285219972, 139524828037387082}, + {13719911636105650386, 87203017523366926}, + {7926517508277287175, 109003771904208658}, + {684774848491833161, 136254714880260823}, + {7345513307948477581, 85159196800163014}, + {18405263671790372785, 106448996000203767}, + {18394893571310578077, 133061245000254709}, + {13802651491282805250, 83163278125159193}, + {3418256308821342851, 103954097656448992}, + {4272820386026678563, 129942622070561240}, + {2670512741266674102, 81214138794100775}, + {17173198981865506339, 101517673492625968}, + {3019754653622331308, 126897091865782461}, + {4193189667727651020, 79310682416114038}, + {14464859121514339583, 99138353020142547}, + {13469387883465536574, 123922941275178184}, + {8418367427165960359, 77451838296986365}, + {15134645302384838353, 96814797871232956}, + {471562554271496325, 121018497339041196}, + {9518098633274461011, 75636560836900747}, + {7285937273165688360, 94545701046125934}, + {18330793628311886258, 118182126307657417}, + {4539216990053847055, 73863828942285886}, + {14897393274422084627, 92329786177857357}, + {4786683537745442072, 115412232722321697}, + {14520892257159371055, 72132645451451060}, + {18151115321449213818, 90165806814313825}, + {8853836096529353561, 112707258517892282}, + {1843923083806916143, 140884073147365353}, + {12681666973447792349, 88052545717103345}, + {2017025661527576725, 110065682146379182}, + {11744654113764246714, 137582102682973977}, + {422879793461572340, 85988814176858736}, + {528599741826965425, 107486017721073420}, + {660749677283706782, 134357522151341775}, + {7330497575943398595, 83973451344588609}, + {13774807988356636147, 104966814180735761}, + {3383451930163631472, 131208517725919702}, + {15949715511634433382, 82005323578699813}, + {6102086334260878016, 102506654473374767}, + {3015921899398709616, 128133318091718459}, + {18025852251620051174, 80083323807324036}, + {4085571240815512351, 100104154759155046}, + {14330336087874166247, 125130193448943807}, + {15873989082562435760, 78206370905589879}, + {15230800334775656796, 97757963631987349}, + {5203442363187407284, 122197454539984187}, + {946308467778435600, 76373409087490117}, + {5794571603150432404, 95466761359362646}, + {16466586540792816313, 119333451699203307}, + {7985773578781816244, 74583407312002067}, + {5370530955049882401, 93229259140002584}, + {6713163693812353001, 116536573925003230}, + {18030785363914884337, 72835358703127018}, + {13315109668038829614, 91044198378908773}, + {2808829029766373305, 113805247973635967}, + {17346094342490130344, 142256559967044958}, + {6229622945628943561, 88910349979403099}, + {3175342663608791547, 111137937474253874}, + {13192550366365765242, 138922421842817342}, + {3633657960551215372, 86826513651760839}, + {18377130505971182927, 108533142064701048}, + {4524669058754427043, 135666427580876311}, + {9745447189362598758, 84791517238047694}, + {2958436949848472639, 105989396547559618}, + {12921418224165366607, 132486745684449522}, + {12687572408530742033, 82804216052780951}, + {11247779492236039638, 103505270065976189}, + {224666310012885835, 129381587582470237}, + {2446259452971747599, 80863492239043898}, + {12281196353069460307, 101079365298804872}, + {15351495441336825384, 126349206623506090}, + {14206370669262903769, 78968254139691306}, + {8534591299723853903, 98710317674614133}, + {15279925143082205283, 123387897093267666}, + {14161639232853766206, 77117435683292291}, + {13090363022639819853, 96396794604115364}, + {16362953778299774816, 120495993255144205}, + {12532689120651053212, 75309995784465128}, + {15665861400813816515, 94137494730581410}, + {10358954714162494836, 117671868413226763}, + {4168503687137865320, 73544917758266727}, + {598943590494943747, 91931147197833409}, + {5360365506546067587, 114913933997291761}, + {11312142901609972388, 143642417496614701}, + {9375932322719926695, 89776510935384188}, + {11719915403399908368, 112220638669230235}, + {10038208235822497557, 140275798336537794}, + {10885566165816448877, 87672373960336121}, + {18218643725697949000, 109590467450420151}, + {18161618638695048346, 136988084313025189}, + {13656854658398099168, 85617552695640743}, + {12459382304570236056, 107021940869550929}, + {1739169825430631358, 133777426086938662}, + {14922039196176308311, 83610891304336663}, + {14040862976792997485, 104513614130420829}, + {3716020665709083144, 130642017663026037}, + {4628355925281870917, 81651261039391273}, + {10397130925029726550, 102064076299239091}, + {8384727637859770284, 127580095374048864}, + {5240454773662356427, 79737559608780540}, + {6550568467077945534, 99671949510975675}, + {3576524565420044014, 124589936888719594}, + {6847013871814915412, 77868710555449746}, + {17782139376623420074, 97335888194312182}, + {13004302183924499284, 121669860242890228}, + {17351060901807587860, 76043662651806392}, + {3242082053549933210, 95054578314757991}, + {17887660622219580224, 118818222893447488}, + {11179787888887237640, 74261389308404680}, + {13974734861109047050, 92826736635505850}, + {8245046539531533005, 116033420794382313}, + {16682369133275677888, 72520887996488945}, + {7017903361312433648, 90651109995611182}, + {17995751238495317868, 113313887494513977}, + {8659630992836983623, 141642359368142472}, + {5412269370523114764, 88526474605089045}, + {11377022731581281359, 110658093256361306}, + {4997906377621825891, 138322616570451633}, + {14652906532082110942, 86451635356532270}, + {9092761128247862869, 108064544195665338}, + {2142579373455052779, 135080680244581673}, + {12868327154477877747, 84425425152863545}, + {2250350887815183471, 105531781441079432}, + {2812938609768979339, 131914726801349290}, + {6369772649532999991, 82446704250843306}, + {17185587848771025797, 103058380313554132}, + {3035240737254230630, 128822975391942666}, + {6508711479211282048, 80514359619964166}, + {17359261385868878368, 100642949524955207}, + {17087390713908710056, 125803686906194009}, + {3762090168551861929, 78627304316371256}, + {4702612710689827411, 98284130395464070}, + {15101637925217060072, 122855162994330087}, + {16356052730901744401, 76784476871456304}, + {1998321839917628885, 95980596089320381}, + {7109588318324424010, 119975745111650476}, + {13666864735807540814, 74984840694781547}, + {12471894901332038114, 93731050868476934}, + {6366496589810271835, 117163813585596168}, + {3979060368631419896, 73227383490997605}, + {9585511479216662775, 91534229363747006}, + {2758517312166052660, 114417786704683758}, + {12671518677062341634, 143022233380854697}, + {1002170145522881665, 89388895863034186}, + {10476084718758377889, 111736119828792732}, + {13095105898447972362, 139670149785990915}, + {5878598177316288774, 87293843616244322}, + {16571619758500136775, 109117304520305402}, + {11491152661270395161, 136396630650381753}, + {264441385652915120, 85247894156488596}, + {330551732066143900, 106559867695610745}, + {5024875683510067779, 133199834619513431}, + {10058076329834874218, 83249896637195894}, + {3349223375438816964, 104062370796494868}, + {4186529219298521205, 130077963495618585}, + {14145795808130045513, 81298727184761615}, + {13070558741735168987, 101623408980952019}, + {11726512408741573330, 127029261226190024}, + {7329070255463483331, 79393288266368765}, + {13773023837756742068, 99241610332960956}, + {17216279797195927585, 124052012916201195}, + {8454331864033760789, 77532508072625747}, + {5956228811614813082, 96915635090782184}, + {7445286014518516353, 121144543863477730}, + {9264989777501460624, 75715339914673581}, + {16192923240304213684, 94644174893341976}, + {1794409976670715490, 118305218616677471}, + {8039035263060279037, 73940761635423419}, + {5437108060397960892, 92425952044279274}, + {16019757112352226923, 115532440055349092}, + {788976158365366019, 72207775034593183}, + {14821278253238871236, 90259718793241478}, + {9303225779693813237, 112824648491551848}, + {11629032224617266546, 141030810614439810}, + {11879831158813179495, 88144256634024881}, + {1014730893234310657, 110180320792531102}, + {10491785653397664129, 137725400990663877}, + {8863209042587234033, 86078375619164923}, + {6467325284806654637, 107597969523956154}, + {17307528642863094104, 134497461904945192}, + {10817205401789433815, 84060913690590745}, + {18133192770664180173, 105076142113238431}, + {18054804944902837312, 131345177641548039}, + {18201782118205355176, 82090736025967524}, + {4305483574047142354, 102613420032459406}, + {14605226504413703751, 128266775040574257}, + {2210737537617482988, 80166734400358911}, + {16598479977304017447, 100208418000448638}, + {11524727934775246001, 125260522500560798}, + {2591268940807140847, 78287826562850499}, + {17074144231291089770, 97859783203563123}, + {16730994270686474309, 122324729004453904}, + {10456871419179046443, 76452955627783690}, + {3847717237119032246, 95566194534729613}, + {9421332564826178211, 119457743168412016}, + {5888332853016361382, 74661089480257510}, + {16583788103125227536, 93326361850321887}, + {16118049110479146516, 116657952312902359}, + {16991309721690548428, 72911220195563974}, + {12015765115258409727, 91139025244454968}, + {15019706394073012159, 113923781555568710}, + {9551260955736489391, 142404726944460888}, + {5969538097335305869, 89002954340288055}, + {2850236603241744433, 111253692925360069}, +} + +const pow5InvNumBits64 = 122 + +var pow5InvSplit64 = [...]uint128{ + {1, 288230376151711744}, + {3689348814741910324, 230584300921369395}, + {2951479051793528259, 184467440737095516}, + {17118578500402463900, 147573952589676412}, + {12632330341676300947, 236118324143482260}, + {10105864273341040758, 188894659314785808}, + {15463389048156653253, 151115727451828646}, + {17362724847566824558, 241785163922925834}, + {17579528692795369969, 193428131138340667}, + {6684925324752475329, 154742504910672534}, + {18074578149087781173, 247588007857076054}, + {18149011334012135262, 198070406285660843}, + {3451162622983977240, 158456325028528675}, + {5521860196774363583, 253530120045645880}, + {4417488157419490867, 202824096036516704}, + {7223339340677503017, 162259276829213363}, + {7867994130342094503, 259614842926741381}, + {2605046489531765280, 207691874341393105}, + {2084037191625412224, 166153499473114484}, + {10713157136084480204, 265845599156983174}, + {12259874523609494487, 212676479325586539}, + {13497248433629505913, 170141183460469231}, + {14216899864323388813, 272225893536750770}, + {11373519891458711051, 217780714829400616}, + {5409467098425058518, 174224571863520493}, + {4965798542738183305, 278759314981632789}, + {7661987648932456967, 223007451985306231}, + {2440241304404055250, 178405961588244985}, + {3904386087046488400, 285449538541191976}, + {17880904128604832013, 228359630832953580}, + {14304723302883865611, 182687704666362864}, + {15133127457049002812, 146150163733090291}, + {16834306301794583852, 233840261972944466}, + {9778096226693756759, 187072209578355573}, + {15201174610838826053, 149657767662684458}, + {2185786488890659746, 239452428260295134}, + {5437978005854438120, 191561942608236107}, + {15418428848909281466, 153249554086588885}, + {6222742084545298729, 245199286538542217}, + {16046240111861969953, 196159429230833773}, + {1768945645263844993, 156927543384667019}, + {10209010661905972635, 251084069415467230}, + {8167208529524778108, 200867255532373784}, + {10223115638361732810, 160693804425899027}, + {1599589762411131202, 257110087081438444}, + {4969020624670815285, 205688069665150755}, + {3975216499736652228, 164550455732120604}, + {13739044029062464211, 263280729171392966}, + {7301886408508061046, 210624583337114373}, + {13220206756290269483, 168499666669691498}, + {17462981995322520850, 269599466671506397}, + {6591687966774196033, 215679573337205118}, + {12652048002903177473, 172543658669764094}, + {9175230360419352987, 276069853871622551}, + {3650835473593572067, 220855883097298041}, + {17678063637842498946, 176684706477838432}, + {13527506561580357021, 282695530364541492}, + {3443307619780464970, 226156424291633194}, + {6443994910566282300, 180925139433306555}, + {5155195928453025840, 144740111546645244}, + {15627011115008661990, 231584178474632390}, + {12501608892006929592, 185267342779705912}, + {2622589484121723027, 148213874223764730}, + {4196143174594756843, 237142198758023568}, + {10735612169159626121, 189713759006418854}, + {12277838550069611220, 151771007205135083}, + {15955192865369467629, 242833611528216133}, + {1696107848069843133, 194266889222572907}, + {12424932722681605476, 155413511378058325}, + {1433148282581017146, 248661618204893321}, + {15903913885032455010, 198929294563914656}, + {9033782293284053685, 159143435651131725}, + {14454051669254485895, 254629497041810760}, + {11563241335403588716, 203703597633448608}, + {16629290697806691620, 162962878106758886}, + {781423413297334329, 260740604970814219}, + {4314487545379777786, 208592483976651375}, + {3451590036303822229, 166873987181321100}, + {5522544058086115566, 266998379490113760}, + {4418035246468892453, 213598703592091008}, + {10913125826658934609, 170878962873672806}, + {10082303693170474728, 273406340597876490}, + {8065842954536379782, 218725072478301192}, + {17520720807854834795, 174980057982640953}, + {5897060404116273733, 279968092772225526}, + {1028299508551108663, 223974474217780421}, + {15580034865808528224, 179179579374224336}, + {17549358155809824511, 286687326998758938}, + {2971440080422128639, 229349861599007151}, + {17134547323305344204, 183479889279205720}, + {13707637858644275364, 146783911423364576}, + {14553522944347019935, 234854258277383322}, + {4264120725993795302, 187883406621906658}, + {10789994210278856888, 150306725297525326}, + {9885293106962350374, 240490760476040522}, + {529536856086059653, 192392608380832418}, + {7802327114352668369, 153914086704665934}, + {1415676938738538420, 246262538727465495}, + {1132541550990830736, 197010030981972396}, + {15663428499760305882, 157608024785577916}, + {17682787970132668764, 252172839656924666}, + {10456881561364224688, 201738271725539733}, + {15744202878575200397, 161390617380431786}, + {17812026976236499989, 258224987808690858}, + {3181575136763469022, 206579990246952687}, + {13613306553636506187, 165263992197562149}, + {10713244041592678929, 264422387516099439}, + {12259944048016053467, 211537910012879551}, + {6118606423670932450, 169230328010303641}, + {2411072648389671274, 270768524816485826}, + {16686253377679378312, 216614819853188660}, + {13349002702143502650, 173291855882550928}, + {17669055508687693916, 277266969412081485}, + {14135244406950155133, 221813575529665188}, + {240149081334393137, 177450860423732151}, + {11452284974360759988, 283921376677971441}, + {5472479164746697667, 227137101342377153}, + {11756680961281178780, 181709681073901722}, + {2026647139541122378, 145367744859121378}, + {18000030682233437097, 232588391774594204}, + {18089373360528660001, 186070713419675363}, + {3403452244197197031, 148856570735740291}, + {16513570034941246220, 238170513177184465}, + {13210856027952996976, 190536410541747572}, + {3189987192878576934, 152429128433398058}, + {1414630693863812771, 243886605493436893}, + {8510402184574870864, 195109284394749514}, + {10497670562401807014, 156087427515799611}, + {9417575270359070576, 249739884025279378}, + {14912757845771077107, 199791907220223502}, + {4551508647133041040, 159833525776178802}, + {10971762650154775986, 255733641241886083}, + {16156107749607641435, 204586912993508866}, + {9235537384944202825, 163669530394807093}, + {11087511001168814197, 261871248631691349}, + {12559357615676961681, 209496998905353079}, + {13736834907283479668, 167597599124282463}, + {18289587036911657145, 268156158598851941}, + {10942320814787415393, 214524926879081553}, + {16132554281313752961, 171619941503265242}, + {11054691591134363444, 274591906405224388}, + {16222450902391311402, 219673525124179510}, + {12977960721913049122, 175738820099343608}, + {17075388340318968271, 281182112158949773}, + {2592264228029443648, 224945689727159819}, + {5763160197165465241, 179956551781727855}, + {9221056315464744386, 287930482850764568}, + {14755542681855616155, 230344386280611654}, + {15493782960226403247, 184275509024489323}, + {1326979923955391628, 147420407219591459}, + {9501865507812447252, 235872651551346334}, + {11290841220991868125, 188698121241077067}, + {1653975347309673853, 150958496992861654}, + {10025058185179298811, 241533595188578646}, + {4330697733401528726, 193226876150862917}, + {14532604630946953951, 154581500920690333}, + {1116074521063664381, 247330401473104534}, + {4582208431592841828, 197864321178483627}, + {14733813189500004432, 158291456942786901}, + {16195403473716186445, 253266331108459042}, + {5577625149489128510, 202613064886767234}, + {8151448934333213131, 162090451909413787}, + {16731667109675051333, 259344723055062059}, + {17074682502481951390, 207475778444049647}, + {6281048372501740465, 165980622755239718}, + {6360328581260874421, 265568996408383549}, + {8777611679750609860, 212455197126706839}, + {10711438158542398211, 169964157701365471}, + {9759603424184016492, 271942652322184754}, + {11497031554089123517, 217554121857747803}, + {16576322872755119460, 174043297486198242}, + {11764721337440549842, 278469275977917188}, + {16790474699436260520, 222775420782333750}, + {13432379759549008416, 178220336625867000}, + {3045063541568861850, 285152538601387201}, + {17193446092222730773, 228122030881109760}, + {13754756873778184618, 182497624704887808}, + {18382503128506368341, 145998099763910246}, + {3586563302416817083, 233596959622256395}, + {2869250641933453667, 186877567697805116}, + {17052795772514404226, 149502054158244092}, + {12527077977055405469, 239203286653190548}, + {17400360011128145022, 191362629322552438}, + {2852241564676785048, 153090103458041951}, + {15631632947708587046, 244944165532867121}, + {8815957543424959314, 195955332426293697}, + {18120812478965698421, 156764265941034957}, + {14235904707377476180, 250822825505655932}, + {4010026136418160298, 200658260404524746}, + {17965416168102169531, 160526608323619796}, + {2919224165770098987, 256842573317791675}, + {2335379332616079190, 205474058654233340}, + {1868303466092863352, 164379246923386672}, + {6678634360490491686, 263006795077418675}, + {5342907488392393349, 210405436061934940}, + {4274325990713914679, 168324348849547952}, + {10528270399884173809, 269318958159276723}, + {15801313949391159694, 215455166527421378}, + {1573004715287196786, 172364133221937103}, + {17274202803427156150, 275782613155099364}, + {17508711057483635243, 220626090524079491}, + {10317620031244997871, 176500872419263593}, + {12818843235250086271, 282401395870821749}, + {13944423402941979340, 225921116696657399}, + {14844887537095493795, 180736893357325919}, + {15565258844418305359, 144589514685860735}, + {6457670077359736959, 231343223497377177}, + {16234182506113520537, 185074578797901741}, + {9297997190148906106, 148059663038321393}, + {11187446689496339446, 236895460861314229}, + {12639306166338981880, 189516368689051383}, + {17490142562555006151, 151613094951241106}, + {2158786396894637579, 242580951921985771}, + {16484424376483351356, 194064761537588616}, + {9498190686444770762, 155251809230070893}, + {11507756283569722895, 248402894768113429}, + {12895553841597688639, 198722315814490743}, + {17695140702761971558, 158977852651592594}, + {17244178680193423523, 254364564242548151}, + {10105994129412828495, 203491651394038521}, + {4395446488788352473, 162793321115230817}, + {10722063196803274280, 260469313784369307}, + {1198952927958798777, 208375451027495446}, + {15716557601334680315, 166700360821996356}, + {17767794532651667857, 266720577315194170}, + {14214235626121334286, 213376461852155336}, + {7682039686155157106, 170701169481724269}, + {1223217053622520399, 273121871170758831}, + {15735968901865657612, 218497496936607064}, + {16278123936234436413, 174797997549285651}, + {219556594781725998, 279676796078857043}, + {7554342905309201445, 223741436863085634}, + {9732823138989271479, 178993149490468507}, + {815121763415193074, 286389039184749612}, + {11720143854957885429, 229111231347799689}, + {13065463898708218666, 183288985078239751}, + {6763022304224664610, 146631188062591801}, + {3442138057275642729, 234609900900146882}, + {13821756890046245153, 187687920720117505}, + {11057405512036996122, 150150336576094004}, + {6623802375033462826, 240240538521750407}, + {16367088344252501231, 192192430817400325}, + {13093670675402000985, 153753944653920260}, + {2503129006933649959, 246006311446272417}, + {13070549649772650937, 196805049157017933}, + {17835137349301941396, 157444039325614346}, + {2710778055689733971, 251910462920982955}, + {2168622444551787177, 201528370336786364}, + {5424246770383340065, 161222696269429091}, + {1300097203129523457, 257956314031086546}, + {15797473021471260058, 206365051224869236}, + {8948629602435097724, 165092040979895389}, + {3249760919670425388, 264147265567832623}, + {9978506365220160957, 211317812454266098}, + {15361502721659949412, 169054249963412878}, + {2442311466204457120, 270486799941460606}, + {16711244431931206989, 216389439953168484}, + {17058344360286875914, 173111551962534787}, + {12535955717491360170, 276978483140055660}, + {10028764573993088136, 221582786512044528}, + {15401709288678291155, 177266229209635622}, + {9885339602917624555, 283625966735416996}, + {4218922867592189321, 226900773388333597}, + {14443184738299482427, 181520618710666877}, + {4175850161155765295, 145216494968533502}, + {10370709072591134795, 232346391949653603}, + {15675264887556728482, 185877113559722882}, + {5161514280561562140, 148701690847778306}, + {879725219414678777, 237922705356445290}, + {703780175531743021, 190338164285156232}, + {11631070584651125387, 152270531428124985}, + {162968861732249003, 243632850284999977}, + {11198421533611530172, 194906280227999981}, + {5269388412147313814, 155925024182399985}, + {8431021459435702103, 249480038691839976}, + {3055468352806651359, 199584030953471981}, + {17201769941212962380, 159667224762777584}, + {16454785461715008838, 255467559620444135}, + {13163828369372007071, 204374047696355308}, + {17909760324981426303, 163499238157084246}, + {2830174816776909822, 261598781051334795}, + {2264139853421527858, 209279024841067836}, + {16568707141704863579, 167423219872854268}, + {4373838538276319787, 267877151796566830}, + {3499070830621055830, 214301721437253464}, + {6488605479238754987, 171441377149802771}, + {3003071137298187333, 274306203439684434}, + {6091805724580460189, 219444962751747547}, + {15941491023890099121, 175555970201398037}, + {10748990379256517301, 280889552322236860}, + {8599192303405213841, 224711641857789488}, + {14258051472207991719, 179769313486231590}, +} From cc4cd3e67373bbab0bdbcaba039ccf4662e664f5 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 12 Mar 2024 20:15:31 +0900 Subject: [PATCH 63/72] tidy --- examples/gno.land/p/demo/json/gno.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/gno.land/p/demo/json/gno.mod b/examples/gno.land/p/demo/json/gno.mod index 06fbfed60ad..8a380644acc 100644 --- a/examples/gno.land/p/demo/json/gno.mod +++ b/examples/gno.land/p/demo/json/gno.mod @@ -2,5 +2,6 @@ module gno.land/p/demo/json require ( gno.land/p/demo/json/eisel_lemire v0.0.0-latest + gno.land/p/demo/json/ryu v0.0.0-latest gno.land/p/demo/ufmt v0.0.0-latest ) From 22189815665bdd60e9e82023c2ef322951d0bba7 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Mar 2024 15:11:39 +0900 Subject: [PATCH 64/72] fix DeleteIndex --- examples/gno.land/p/demo/json/LICENSE | 6 +- examples/gno.land/p/demo/json/decode.gno | 16 +- examples/gno.land/p/demo/json/node.gno | 262 ++---------------- examples/gno.land/p/demo/json/node_test.gno | 288 ++++---------------- examples/gno.land/p/demo/json/ryu/License | 21 ++ examples/gno.land/p/demo/json/ryu/ryu64.gno | 21 -- 6 files changed, 107 insertions(+), 507 deletions(-) create mode 100644 examples/gno.land/p/demo/json/ryu/License diff --git a/examples/gno.land/p/demo/json/LICENSE b/examples/gno.land/p/demo/json/LICENSE index 7dad6acda0f..954b7ef406d 100644 --- a/examples/gno.land/p/demo/json/LICENSE +++ b/examples/gno.land/p/demo/json/LICENSE @@ -1,6 +1,6 @@ -MIT License +# MIT License -Copyright (c) 2016 Leonid Bugaev +Copyright (c) 2019 Pyzhov Stepan Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/examples/gno.land/p/demo/json/decode.gno b/examples/gno.land/p/demo/json/decode.gno index 79f287433f7..485e4ab36c3 100644 --- a/examples/gno.land/p/demo/json/decode.gno +++ b/examples/gno.land/p/demo/json/decode.gno @@ -1,3 +1,5 @@ +// ref: https://github.com/spyzhov/ajson/blob/master/decode.go + package json import ( @@ -11,10 +13,6 @@ import ( // This is permitted by https://tools.ietf.org/html/rfc7159#section-9 const maxNestingDepth = 10000 -// State machine transition logic and grammar references -// [1] https://github.com/spyzhov/ajson/blob/master/decode.go -// [2] https://cs.opensource.google/go/go/+/refs/tags/go1.22.1:src/encoding/json/scanner.go - // Unmarshal parses the JSON-encoded data and returns a Node. // The data must be a valid JSON-encoded value. // @@ -283,16 +281,6 @@ func checkNestingDepth(nesting int) (int, error) { return nesting + 1, nil } -func cptrs(cpy *string) *string { - if cpy == nil { - return nil - } - - val := *cpy - - return &val -} - func UnmarshalSafe(data []byte) (*Node, error) { var safe []byte safe = append(safe, data...) diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index 0d70131be76..a9c628b2570 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -113,8 +113,8 @@ func (n *Node) MustKey(key string) *Node { return val } -// UniqueKeys traverses the current JSON nodes and collects all the unique keys. -func (n *Node) UniqueKeys() []string { +// UniqueKeyLists traverses the current JSON nodes and collects all the unique keys. +func (n *Node) UniqueKeyLists() []string { var collectKeys func(*Node) []string collectKeys = func(node *Node) []string { if node == nil || !node.IsObject() { @@ -249,7 +249,7 @@ func (n *Node) Delete() error { return n.prev.remove(n) } -// Size returns the number of sub-nodes of the current Array node. +// Size returns the size (length) of the current array node. // // Usage: // @@ -329,6 +329,10 @@ func (n *Node) GetIndex(idx int) (*Node, error) { return nil, errors.New("node is not array") } + if idx > n.Size() { + return nil, errors.New("input index exceeds the array size") + } + if idx < 0 { idx += len(n.next) } @@ -348,7 +352,7 @@ func (n *Node) DeleteIndex(idx int) error { return err } - return node.remove(node) + return n.remove(node) } // NullNode creates a new null type node. @@ -507,6 +511,9 @@ func (n *Node) IsNumber() bool { return n.nodeType == Number } +// ready checks the current node is ready or not. +// +// the meaning of ready is the current node is parsed and has a valid value. func (n *Node) ready() bool { return n.borders[1] != 0 } @@ -606,83 +613,6 @@ func (n *Node) GetNumeric() (float64, error) { return v, nil } -// GetInts traverses the current JSON nodes and collects all integer values. -// -// The Number type stores both int and float types together as float64, -// but the GetInts function only retrieves values of the int type -// because it fetches values only if there is no fractional part when compared with float64 values. -// -// Usage: -// -// root := Must(Unmarshal([]byte(`{"key": 10.5, "key2": 10, "key3": "foo"}`))) -// ints := root.GetInts() -// if len(ints) != 1 { -// t.Errorf("GetInts returns wrong result: %v", ints) -// } -func (n *Node) GetInts() []int { - var collectInts func(*Node) []int - collectInts = func(node *Node) []int { - if node == nil { - return nil - } - - result := []int{} - if node.IsNumber() { - numVal, err := node.GetNumeric() - if err == nil && numVal == float64(int(numVal)) { // doesn't have a decimal part - result = append(result, int(numVal)) - } - } - - for _, childNode := range node.next { - childInts := collectInts(childNode) - result = append(result, childInts...) - } - - return result - } - - return collectInts(n) -} - -// GetFloats traverses the current JSON nodes and collects all float values. -// -// The Number type combines int and float types into float64 for storage, -// but the GetFloats function only accurately retrieves float types because it checks whether the numbers have a fractional part. -// -// Usage: -// -// root := Must(Unmarshal([]byte(`{"key": 10.5, "key2": 10, "key3": "foo"}`))) -// floats := root.GetFloats() -// if len(floats) != 1 { -// t.Errorf("GetFloats returns wrong result: %v", floats) -// } -func (n *Node) GetFloats() []float64 { - var collectFloats func(*Node) []float64 - collectFloats = func(node *Node) []float64 { - if node == nil { - return nil - } - - result := []float64{} - if node.IsNumber() { - numVal, err := node.GetNumeric() - if err == nil && numVal != float64(int(numVal)) { // check if it's a float - result = append(result, numVal) - } - } - - for _, childNode := range node.next { - childFloats := collectFloats(childNode) - result = append(result, childFloats...) - } - - return result - } - - return collectFloats(n) -} - // MustNumeric returns the numeric (int/float) value if current node is number type. // // It panics if the current node is not number type. @@ -732,41 +662,6 @@ func (n *Node) GetString() (string, error) { return v, nil } -// GetStrings traverses the current JSON nodes and collects all string values. -// -// Usage: -// -// root := Must(Unmarshal([]byte(`{"key": "value", "key2": 10, "key3": "foo"}`)) -// strs := root.GetStrings() -// if len(strs) != 2 { -// t.Errorf("GetStrings returns wrong result: %v", strs) -// } -func (n *Node) GetStrings() []string { - var collectStrings func(*Node) []string - collectStrings = func(node *Node) []string { - if node == nil { - return nil - } - - result := []string{} - if node.IsString() { - strVal, err := node.GetString() - if err == nil { - result = append(result, strVal) - } - } - - for _, childNode := range node.next { - childStrings := collectStrings(childNode) - result = append(result, childStrings...) - } - - return result - } - - return collectStrings(n) -} - // MustString returns the string value if current node is string type. // // It panics if the current node is not string type. @@ -779,40 +674,6 @@ func (n *Node) MustString() string { return v } -// GetBools traverses the current JSON nodes and collects all boolean values. -// -// Usage: -// -// root := Must(Unmarshal([]byte(`{"key": true, "key2": 10, "key3": "foo"}`))) -// bools := root.GetBools() -// if len(bools) != 1 { -// t.Errorf("GetBools returns wrong result: %v", bools) -// } -func (n *Node) GetBools() []bool { - var collectBools func(*Node) []bool - collectBools = func(node *Node) []bool { - if node == nil { - return nil - } - - result := []bool{} - if node.IsBool() { - if boolVal, err := node.GetBool(); err == nil { - result = append(result, boolVal) - } - } - - for _, childNode := range node.next { - childBools := collectBools(childNode) - result = append(result, childBools...) - } - - return result - } - - return collectBools(n) -} - // GetBool returns the boolean value if current node is boolean type. // // Usage: @@ -1099,63 +960,6 @@ func (n *Node) mark() { } } -// clear clears the current node's value -func (n *Node) clear() { - n.data = nil - n.borders[1] = 0 - - for key := range n.next { - n.next[key].prev = nil - } - - n.next = nil -} - -// validate checks the current node value is matching the given type. -func (n *Node) validate(t ValueType, v interface{}) error { - if n == nil { - return errors.New("node is nil") - } - - switch t { - case Null: - if v != nil { - return errors.New("invalid null value") - } - - // TODO: support uint256, int256 type later. - case Number: - switch v.(type) { - case float64, int, uint: - return nil - default: - return errors.New("invalid number value") - } - - case String: - if _, ok := v.(string); !ok { - return errors.New("invalid string value") - } - - case Boolean: - if _, ok := v.(bool); !ok { - return errors.New("invalid boolean value") - } - - case Array: - if _, ok := v.([]*Node); !ok { - return errors.New("invalid array value") - } - - case Object: - if _, ok := v.(map[string]*Node); !ok { - return errors.New("invalid object value") - } - } - - return nil -} - // isContainer checks the current node type is array or object. func (n *Node) isContainer() bool { return n.IsArray() || n.IsObject() @@ -1245,45 +1049,25 @@ func (n *Node) isParentNode(nd *Node) bool { return false } -func (n *Node) setRef(prev *Node, key *string, idx *int) { - n.prev = prev - - if key == nil { - n.key = nil - } else { - tmp := *key - n.key = &tmp +// cptrs returns the pointer of the given string value. +func cptrs(cpy *string) *string { + if cpy == nil { + return nil } - n.index = idx -} - -// Clone creates a new node instance with the same value of the current node. -func (n *Node) Clone() *Node { - node := n.clone() - node.setRef(nil, nil, nil) + val := *cpy - return node + return &val } -func (n *Node) clone() *Node { - node := &Node{ - prev: n.prev, - next: make(map[string]*Node, len(n.next)), - key: n.key, - data: n.data, - value: n.value, - nodeType: n.nodeType, - index: n.index, - borders: n.borders, - modified: n.modified, - } - - for k, v := range n.next { - node.next[k] = v +// cptri returns the pointer of the given integer value. +func cptri(i *int) *int { + if i == nil { + return nil } - return node + val := *i + return &val } // Must panics if the given node is not fulfilled the expectation. diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index 81eb0c2f825..7931cce853a 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -262,26 +262,18 @@ func TestNode_AppendArray(t *testing.T) { t.Errorf("Marshal returns wrong value: %s", string(value)) } - // TODO: Number is not handled properly. - // number does not make problem if they positioned in the array or object. if err := root.AppendArray( - // NumberNode("", 1), - // StringNode("", "foo"), + NumberNode("", 1), + StringNode("", "foo"), Must(Unmarshal([]byte(`[0,1,null,true,"example"]`))), Must(Unmarshal([]byte(`{"foo": true, "bar": null, "baz": 123}`))), ); err != nil { t.Errorf("AppendArray returns error: %v", err) } - // if value, err := Marshal(root); err != nil { - // t.Errorf("Marshal returns error: %v", err) - // } else if string(value) != `[{"foo":"bar"},null,1,"foo",[0,1,null,true,"example"],{"foo": true, "bar": null, "baz": 123}]` { - // t.Errorf("Marshal returns wrong value: %s", string(value)) - // } - if value, err := Marshal(root); err != nil { t.Errorf("Marshal returns error: %v", err) - } else if string(value) != `[{"foo":"bar"},null,[0,1,null,true,"example"],{"foo": true, "bar": null, "baz": 123}]` { + } else if string(value) != `[{"foo":"bar"},null,1,"foo",[0,1,null,true,"example"],{"foo": true, "bar": null, "baz": 123}]` { t.Errorf("Marshal returns wrong value: %s", string(value)) } } @@ -320,70 +312,6 @@ func TestNode_GetBool_Fail(t *testing.T) { } } -// countBooleans test function that counts the number of true and false values in a slice of booleans. -// because map doesn't maintain order, we can't use it to test the order of the values. -func countBooleans(bools []bool) (trueCount, falseCount int) { - for _, b := range bools { - if b { - trueCount++ - } else { - falseCount++ - } - } - return -} - -// TODO: resolve stack overflow -func TestNode_GetAllBools(t *testing.T) { - tests := []struct { - name string - json string - expectedTrues int - expectedFalses int - }{ - // { - // name: "no booleans", - // json: `{"foo": "bar", "number": 123}`, - // expectedTrues: 0, - // expectedFalses: 0, - // }, - // { - // name: "single boolean true", - // json: `{"isTrue": true}`, - // expectedTrues: 1, - // expectedFalses: 0, - // }, - // { - // name: "single boolean false", - // json: `{"isFalse": false}`, - // expectedTrues: 0, - // expectedFalses: 1, - // }, - // { - // name: "multiple booleans", - // json: `{"first": true, "info": {"second": false, "third": true}, "array": [false, true, false]}`, - // expectedTrues: 3, - // expectedFalses: 3, - // }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - root, err := Unmarshal([]byte(tc.json)) - if err != nil { - t.Errorf("Failed to unmarshal JSON: %v", err) - return - } - - bools := root.GetBools() - trueCount, falseCount := countBooleans(bools) - if trueCount != tc.expectedTrues || falseCount != tc.expectedFalses { - t.Errorf("%s: expected %d true and %d false booleans, got %d true and %d false", tc.name, tc.expectedTrues, tc.expectedFalses, trueCount, falseCount) - } - }) - } -} - func TestNode_IsBool(t *testing.T) { tests := []simpleNode{ {"true", BoolNode("", true)}, @@ -535,108 +463,6 @@ func TestNode_GetNumeric_Fail(t *testing.T) { } } -// TODO: resolve stack overflow -func TestNode_GetAllIntsFromNode(t *testing.T) { - tests := []struct { - name string - json string - expected []int - }{ - // { - // name: "no numbers", - // json: `{"foo": "bar", "baz": "qux"}`, - // expected: []int{}, - // }, - // { - // name: "mixed numbers", - // json: `{"int": 42, "float": 3.14, "nested": {"int2": 24, "array": [1, 2.34, 3]}}`, - // expected: []int{42, 24, 1, 3}, - // }, - // { - // name: "all floats", - // json: `{"float1": 0.1, "float2": 4.5}`, - // expected: []int{}, - // }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - root, err := Unmarshal([]byte(tc.json)) - if err != nil { - t.Fatalf("Unmarshal failed: %v", err) - } - - got := root.GetInts() - for _, exp := range tc.expected { - if !containsInts(got, exp) { - t.Errorf("Expected integer %d is missing", exp) - } - } - }) - } -} - -// containsInts checks if a slice of integers contains a specific value. -func containsInts(slice []int, val int) bool { - for _, item := range slice { - if item == val { - return true - } - } - return false -} - -// TODO: resolve stack overflow -func TestNode_GetAllFloatsFromNode(t *testing.T) { - tests := []struct { - name string - json string - expected []float64 - }{ - // { - // name: "no numbers", - // json: `{"foo": "bar", "baz": "qux"}`, - // expected: []float64{}, - // }, - // { - // name: "mixed numbers", - // json: `{"int": 42, "float": 3.14, "nested": {"float2": 2.34, "array": [1, 2.34, 3]}}`, - // expected: []float64{3.14, 2.34, 2.34}, - // }, - // { - // name: "all ints", - // json: `{"int1": 1, "int2": 2}`, - // expected: []float64{}, - // }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - root, err := Unmarshal([]byte(tc.json)) - if err != nil { - t.Fatalf("Unmarshal failed: %v", err) - } - - got := root.GetFloats() - for _, exp := range tc.expected { - if !containsFloats(got, exp) { - t.Errorf("Expected float %f is missing", exp) - } - } - }) - } -} - -// containsFloats checks if a slice of floats contains a specific value. -func containsFloats(slice []float64, val float64) bool { - for _, item := range slice { - if item == val { - return true - } - } - return false -} - func TestNode_GetString(t *testing.T) { root, err := Unmarshal([]byte(`"123foobar 3456"`)) if err != nil { @@ -669,53 +495,6 @@ func TestNode_GetString_Fail(t *testing.T) { } } -// TODO: resolve stack overflow -func TestNode_GetAllStringsFromNode(t *testing.T) { - tests := []struct { - name string - json string - expected []string - }{ - // { - // name: "no strings", - // json: `{"int": 42, "float": 3.14}`, - // expected: []string{}, - // }, - // { - // name: "multiple strings", - // json: `{"str": "hello", "nested": {"str2": "world"}, "array": ["foo", "bar"]}`, - // expected: []string{"hello", "world", "foo", "bar"}, - // }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - root, err := Unmarshal([]byte(tc.json)) - if err != nil { - t.Fatalf("Unmarshal failed: %v", err) - } - - got := root.GetStrings() - if len(got) != len(tc.expected) { - t.Errorf("Expected %d strings, got %d", len(tc.expected), len(got)) - return - } - for _, exp := range tc.expected { - found := false - for _, str := range got { - if str == exp { - found = true - break - } - } - if !found { - t.Errorf("Expected string '%s' not found", exp) - } - } - }) - } -} - func TestNode_MustString(t *testing.T) { tests := []struct { name string @@ -992,12 +771,14 @@ func TestNode_Index_Fail(t *testing.T) { } func TestNode_GetIndex(t *testing.T) { - root, err := Unmarshal([]byte(`[1, 2, 3, 4, 5, 6]`)) - if err != nil { - t.Errorf("error occurred while unmarshal") + root := Must(Unmarshal([]byte(`[1, 2, 3, 4, 5, 6]`))) + expected := []int{1, 2, 3, 4, 5, 6} + + if len(expected) != root.Size() { + t.Errorf("length is not matched. expected: %d, got: %d", len(expected), root.Size()) } - expected := []int{1, 2, 3, 4, 5, 6} + // TODO: if length exceeds, stack overflow occurs. need to fix for i, v := range expected { val, err := root.GetIndex(i) if err != nil { @@ -1010,6 +791,53 @@ func TestNode_GetIndex(t *testing.T) { } } +func TestNode_GetIndex_InputIndex_Exceed_Original_Node_Index(t *testing.T) { + root, err := Unmarshal([]byte(`[1, 2, 3, 4, 5, 6]`)) + if err != nil { + t.Errorf("error occurred while unmarshal") + } + + _, err = root.GetIndex(10) + if err == nil { + t.Errorf("GetIndex should return error") + } +} + +func TestNode_DeleteIndex(t *testing.T) { + tests := []struct { + name string + expected string + index int + ok bool + }{ + {`null`, ``, 0, false}, + {`1`, ``, 0, false}, + {`{}`, ``, 0, false}, + {`{"foo":"bar"}`, ``, 0, false}, + {`true`, ``, 0, false}, + {`[]`, ``, 0, false}, + {`[]`, ``, -1, false}, + {`[1]`, `[]`, 0, true}, + {`[{}]`, `[]`, 0, true}, + {`[{}, [], 42]`, `[{}, []]`, -1, true}, + {`[{}, [], 42]`, `[[], 42]`, 0, true}, + {`[{}, [], 42]`, `[{}, 42]`, 1, true}, + {`[{}, [], 42]`, `[{}, []]`, 2, true}, + {`[{}, [], 42]`, ``, 10, false}, + {`[{}, [], 42]`, ``, -10, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := Must(Unmarshal([]byte(tt.name))) + err := root.DeleteIndex(tt.index) + if err != nil && tt.ok { + t.Errorf("DeleteIndex returns error: %v", err) + } + }) + } +} + func TestNode_GetKey(t *testing.T) { root, err := Unmarshal([]byte(`{"foo": true, "bar": null}`)) if err != nil { @@ -1055,7 +883,7 @@ func TestNode_GetKey_Fail(t *testing.T) { } } -func TestNode_EachKey(t *testing.T) { +func TestNode_GetUniqueKeyList(t *testing.T) { tests := []struct { name string json string @@ -1111,7 +939,7 @@ func TestNode_EachKey(t *testing.T) { t.Errorf("error occurred while unmarshal") } - value := root.UniqueKeys() + value := root.UniqueKeyLists() if len(value) != len(tt.expected) { t.Errorf("%s length must be %v. got: %v. retrieved keys: %s", tt.name, len(tt.expected), len(value), value) } diff --git a/examples/gno.land/p/demo/json/ryu/License b/examples/gno.land/p/demo/json/ryu/License new file mode 100644 index 00000000000..55beeadce54 --- /dev/null +++ b/examples/gno.land/p/demo/json/ryu/License @@ -0,0 +1,21 @@ +# Apache License + +Copyright 2018 Ulf Adams +Modifications copyright 2019 Caleb Spare + +The contents of this file may be used under the terms of the Apache License, +Version 2.0. + + (See accompanying file LICENSE or copy at + ) + +Unless required by applicable law or agreed to in writing, this software +is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. + +The code in this file is part of a Go translation of the C code originally written by +Ulf Adams, which can be found at . The original source +code is licensed under the Apache License 2.0. This code is a derivative work thereof, +adapted and modified to meet the specifications of the Gno language project. + +Please note that the modifications are also under the Apache License 2.0 unless otherwise specified. diff --git a/examples/gno.land/p/demo/json/ryu/ryu64.gno b/examples/gno.land/p/demo/json/ryu/ryu64.gno index 7dca83dd4db..249e3d0f526 100644 --- a/examples/gno.land/p/demo/json/ryu/ryu64.gno +++ b/examples/gno.land/p/demo/json/ryu/ryu64.gno @@ -1,24 +1,3 @@ -// Copyright 2018 Ulf Adams -// Modifications copyright 2019 Caleb Spare -// -// The contents of this file may be used under the terms of the Apache License, -// Version 2.0. -// -// (See accompanying file LICENSE or copy at -// http://www.apache.org/licenses/LICENSE-2.0) -// -// Unless required by applicable law or agreed to in writing, this software -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. -// -// The code in this file is part of a Go translation of the C code originally written by -// Ulf Adams, which can be found at https://github.com/ulfjack/ryu. The original source -// code is licensed under the Apache License 2.0. This code is a derivative work thereof, -// adapted and modified to meet the specifications of the Gno language project. -// -// Please note that the modifications are also under the Apache License 2.0 unless -// otherwise specified. - package ryu import ( From faf989020de6255e41d455aeb7cd274b512310ea Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Mar 2024 16:13:15 +0900 Subject: [PATCH 65/72] Improve determining int and float type --- examples/gno.land/p/demo/json/encode.gno | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/gno.land/p/demo/json/encode.gno b/examples/gno.land/p/demo/json/encode.gno index 18e5bb75ad5..17001f86023 100644 --- a/examples/gno.land/p/demo/json/encode.gno +++ b/examples/gno.land/p/demo/json/encode.gno @@ -3,6 +3,7 @@ package json import ( "bytes" "errors" + "math" "strconv" "gno.land/p/demo/json/ryu" @@ -35,12 +36,12 @@ func Marshal(node *Node) ([]byte, error) { // ufmt does not support %g. by doing so, we need to check if the number is an integer // after then, apply the correct format for each float and integer numbers. - if nVal == float64(int64(nVal)) { - num := ufmt.Sprintf("%d", int64(nVal)) + if math.Mod(nVal, 1.0) == 0 { + // must convert float to integer. otherwise it will be overflowed. + num := ufmt.Sprintf("%d", int(nVal)) buf.WriteString(num) } else { // use ryu algorithm to convert float to string - // num := ufmt.Sprintf("%f", nVal) num := ryu.FormatFloat64(nVal) buf.WriteString(num) } From 33ab79ae2a6d8bf6e8fd5f8bb61a8d89ae2aa5b7 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Mar 2024 16:56:30 +0900 Subject: [PATCH 66/72] resolve conflict --- gnovm/pkg/gnolang/precompile.go | 398 -------------------------------- 1 file changed, 398 deletions(-) delete mode 100644 gnovm/pkg/gnolang/precompile.go diff --git a/gnovm/pkg/gnolang/precompile.go b/gnovm/pkg/gnolang/precompile.go deleted file mode 100644 index b93d2e492de..00000000000 --- a/gnovm/pkg/gnolang/precompile.go +++ /dev/null @@ -1,398 +0,0 @@ -package gnolang - -import ( - "bytes" - "fmt" - "go/ast" - "go/format" - "go/parser" - goscanner "go/scanner" - "go/token" - "os" - "os/exec" - "path/filepath" - "regexp" - "sort" - "strconv" - "strings" - - "go.uber.org/multierr" - "golang.org/x/tools/go/ast/astutil" - - "github.com/gnolang/gno/tm2/pkg/std" -) - -const ( - GnoRealmPkgsPrefixBefore = "gno.land/r/" - GnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/" - GnoPackagePrefixBefore = "gno.land/p/demo/" - GnoPackagePrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/demo/" - GnoStdPkgBefore = "std" - GnoStdPkgAfter = "github.com/gnolang/gno/gnovm/stdlibs/stdshim" -) - -var stdlibWhitelist = []string{ - // go - "bufio", - "bytes", - "compress/gzip", - "context", - "crypto/md5", - "crypto/sha1", - "crypto/chacha20", - "crypto/cipher", - "crypto/sha256", - "encoding/base64", - "encoding/binary", - "encoding/hex", - "encoding/json", - "encoding/xml", - "errors", - "hash", - "hash/adler32", - "internal/bytealg", - "internal/os", - "flag", - "fmt", - "io", - "io/util", - "math", - "math/big", - "math/bits", - "math/rand", - "net/url", - "path", - "regexp", - "sort", - "strconv", - "strings", - "text/template", - "time", - "unicode", - "unicode/utf8", - "unicode/utf16", - - // gno - "std", -} - -var importPrefixWhitelist = []string{ - "github.com/gnolang/gno/_test", -} - -const ImportPrefix = "github.com/gnolang/gno" - -type precompileResult struct { - Imports []*ast.ImportSpec - Translated string -} - -// TODO: func PrecompileFile: supports caching. -// TODO: func PrecompilePkg: supports directories. - -func guessRootDir(fileOrPkg string, goBinary string) (string, error) { - abs, err := filepath.Abs(fileOrPkg) - if err != nil { - return "", err - } - args := []string{"list", "-m", "-mod=mod", "-f", "{{.Dir}}", ImportPrefix} - cmd := exec.Command(goBinary, args...) - cmd.Dir = abs - out, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("can't guess --root-dir") - } - rootDir := strings.TrimSpace(string(out)) - return rootDir, nil -} - -// GetPrecompileFilenameAndTags returns the filename and tags for precompiled files. -func GetPrecompileFilenameAndTags(gnoFilePath string) (targetFilename, tags string) { - nameNoExtension := strings.TrimSuffix(filepath.Base(gnoFilePath), ".gno") - switch { - case strings.HasSuffix(gnoFilePath, "_filetest.gno"): - tags = "gno && filetest" - targetFilename = "." + nameNoExtension + ".gno.gen.go" - case strings.HasSuffix(gnoFilePath, "_test.gno"): - tags = "gno && test" - targetFilename = "." + nameNoExtension + ".gno.gen_test.go" - default: - tags = "gno" - targetFilename = nameNoExtension + ".gno.gen.go" - } - return -} - -func PrecompileAndCheckMempkg(mempkg *std.MemPackage) error { - gofmt := "gofmt" - - tmpDir, err := os.MkdirTemp("", mempkg.Name) - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) //nolint: errcheck - - var errs error - for _, mfile := range mempkg.Files { - if !strings.HasSuffix(mfile.Name, ".gno") { - continue // skip spurious file. - } - res, err := Precompile(mfile.Body, "gno,tmp", mfile.Name) - if err != nil { - errs = multierr.Append(errs, err) - continue - } - tmpFile := filepath.Join(tmpDir, mfile.Name) - err = os.WriteFile(tmpFile, []byte(res.Translated), 0o644) - if err != nil { - errs = multierr.Append(errs, err) - continue - } - err = PrecompileVerifyFile(tmpFile, gofmt) - if err != nil { - errs = multierr.Append(errs, err) - continue - } - } - - if errs != nil { - return fmt.Errorf("precompile package: %w", errs) - } - return nil -} - -func Precompile(source string, tags string, filename string) (*precompileResult, error) { - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, filename, source, parser.ParseComments) - if err != nil { - return nil, fmt.Errorf("parse: %w", err) - } - - isTestFile := strings.HasSuffix(filename, "_test.gno") || strings.HasSuffix(filename, "_filetest.gno") - shouldCheckWhitelist := !isTestFile - - transformed, err := precompileAST(fset, f, shouldCheckWhitelist) - if err != nil { - return nil, fmt.Errorf("precompileAST: %w", err) - } - - header := "// Code generated by github.com/gnolang/gno. DO NOT EDIT.\n\n" - if tags != "" { - header += "//go:build " + tags + "\n\n" - } - var out bytes.Buffer - out.WriteString(header) - err = format.Node(&out, fset, transformed) - if err != nil { - return nil, fmt.Errorf("format.Node: %w", err) - } - - res := &precompileResult{ - Imports: f.Imports, - Translated: out.String(), - } - return res, nil -} - -// PrecompileVerifyFile tries to run `go fmt` against a precompiled .go file. -// -// This is fast and won't look the imports. -func PrecompileVerifyFile(path string, gofmtBinary string) error { - // TODO: use cmd/parser instead of exec? - - args := strings.Split(gofmtBinary, " ") - args = append(args, []string{"-l", "-e", path}...) - cmd := exec.Command(args[0], args[1:]...) - out, err := cmd.CombinedOutput() - if err != nil { - fmt.Fprintln(os.Stderr, string(out)) - return fmt.Errorf("%s: %w", gofmtBinary, err) - } - return nil -} - -// PrecompileBuildPackage tries to run `go build` against the precompiled .go files. -// -// This method is the most efficient to detect errors but requires that -// all the import are valid and available. -func PrecompileBuildPackage(fileOrPkg, goBinary string) error { - // TODO: use cmd/compile instead of exec? - // TODO: find the nearest go.mod file, chdir in the same folder, rim prefix? - // TODO: temporarily create an in-memory go.mod or disable go modules for gno? - // TODO: ignore .go files that were not generated from gno? - // TODO: automatically precompile if not yet done. - - files := []string{} - - info, err := os.Stat(fileOrPkg) - if err != nil { - return fmt.Errorf("invalid file or package path %s: %w", fileOrPkg, err) - } - if !info.IsDir() { - file := fileOrPkg - files = append(files, file) - } else { - pkgDir := fileOrPkg - goGlob := filepath.Join(pkgDir, "*.go") - goMatches, err := filepath.Glob(goGlob) - if err != nil { - return fmt.Errorf("glob %s: %w", goGlob, err) - } - for _, goMatch := range goMatches { - switch { - case strings.HasPrefix(goMatch, "."): // skip - case strings.HasSuffix(goMatch, "_filetest.go"): // skip - case strings.HasSuffix(goMatch, "_filetest.gno.gen.go"): // skip - case strings.HasSuffix(goMatch, "_test.go"): // skip - case strings.HasSuffix(goMatch, "_test.gno.gen.go"): // skip - default: - files = append(files, goMatch) - } - } - } - - sort.Strings(files) - args := append([]string{"build", "-v", "-tags=gno"}, files...) - cmd := exec.Command(goBinary, args...) - rootDir, err := guessRootDir(fileOrPkg, goBinary) - if err == nil { - cmd.Dir = rootDir - } - out, err := cmd.CombinedOutput() - if _, ok := err.(*exec.ExitError); ok { - // exit error - return parseGoBuildErrors(string(out)) - } - return err -} - -var errorRe = regexp.MustCompile(`(?m)^(\S+):(\d+):(\d+): (.+)$`) - -// parseGoBuildErrors returns a scanner.ErrorList filled with all errors found -// in out, which is supposed to be the output of the `go build` command. -// Each errors are translated into their correlated gno files, by: -// - changing the filename from *.gno.gen.go to *.gno -// - shifting line number according to the added header in generated go files -// (see [Precompile] for that header). -func parseGoBuildErrors(out string) error { - var errList goscanner.ErrorList - matches := errorRe.FindAllStringSubmatch(out, -1) - for _, match := range matches { - filename := match[1] - line, err := strconv.Atoi(match[2]) - if err != nil { - return fmt.Errorf("parse line go build error %s: %w", match, err) - } - - column, err := strconv.Atoi(match[3]) - if err != nil { - return fmt.Errorf("parse column go build error %s: %w", match, err) - } - msg := match[4] - errList.Add(token.Position{ - // Remove .gen.go extension, we want to target the gno file - Filename: strings.TrimSuffix(filename, ".gen.go"), - // Shift the 4 lines header added in *.gen.go files. - // NOTE(tb): the 4 lines shift below assumes there's always a //go:build - // directive. But the tags are optional in the Precompile() function - // so that leaves some doubts... We might want something more reliable than - // constants to shift lines. - Line: line - 4, - Column: column, - }, msg) - } - return errList.Err() -} - -func precompileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.Node, error) { - var errs goscanner.ErrorList - - imports := astutil.Imports(fset, f) - - // import whitelist - if checkWhitelist { - for _, paragraph := range imports { - for _, importSpec := range paragraph { - importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`) - - if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) { - continue - } - - if strings.HasPrefix(importPath, GnoPackagePrefixBefore) { - continue - } - - valid := false - for _, whitelisted := range stdlibWhitelist { - if importPath == whitelisted { - valid = true - break - } - } - if valid { - continue - } - - for _, whitelisted := range importPrefixWhitelist { - if strings.HasPrefix(importPath, whitelisted) { - valid = true - break - } - } - if valid { - continue - } - - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("import %q is not in the whitelist", importPath)) - } - } - } - - // rewrite imports - for _, paragraph := range imports { - for _, importSpec := range paragraph { - importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`) - - // std package - if importPath == GnoStdPkgBefore { - if !astutil.RewriteImport(fset, f, GnoStdPkgBefore, GnoStdPkgAfter) { - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("failed to replace the %q package with %q", GnoStdPkgBefore, GnoStdPkgAfter)) - } - } - - // p/pkg packages - if strings.HasPrefix(importPath, GnoPackagePrefixBefore) { - target := GnoPackagePrefixAfter + strings.TrimPrefix(importPath, GnoPackagePrefixBefore) - - if !astutil.RewriteImport(fset, f, importPath, target) { - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("failed to replace the %q package with %q", importPath, target)) - } - } - - // r/realm packages - if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) { - target := GnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, GnoRealmPkgsPrefixBefore) - - if !astutil.RewriteImport(fset, f, importPath, target) { - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("failed to replace the %q package with %q", importPath, target)) - } - } - } - } - - // custom handler - node := astutil.Apply(f, - // pre - func(c *astutil.Cursor) bool { - // do things here - return true - }, - // post - func(c *astutil.Cursor) bool { - // and here - return true - }, - ) - return node, errs.Err() -} From a54cfab13b66072df73a9994ba0a03158018055a Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 13 Mar 2024 17:03:54 +0900 Subject: [PATCH 67/72] remove utf16 package from json --- docs/reference/go-gno-compatibility.md | 2 +- gnovm/stdlibs/unicode/utf16/export_test.gno | 11 - gnovm/stdlibs/unicode/utf16/utf16.gno | 125 ----------- gnovm/stdlibs/unicode/utf16/utf16_test.gno | 231 -------------------- 4 files changed, 1 insertion(+), 368 deletions(-) delete mode 100644 gnovm/stdlibs/unicode/utf16/export_test.gno delete mode 100644 gnovm/stdlibs/unicode/utf16/utf16.gno delete mode 100644 gnovm/stdlibs/unicode/utf16/utf16_test.gno diff --git a/docs/reference/go-gno-compatibility.md b/docs/reference/go-gno-compatibility.md index 9c993691cd9..f36eb67730a 100644 --- a/docs/reference/go-gno-compatibility.md +++ b/docs/reference/go-gno-compatibility.md @@ -258,7 +258,7 @@ Legend: | time | `full`[^7] | | time/tzdata | `tbd` | | unicode | `full` | -| unicode/utf16 | `full` | +| unicode/utf16 | `todo` | | unicode/utf8 | `full` | | unsafe | `nondet` | diff --git a/gnovm/stdlibs/unicode/utf16/export_test.gno b/gnovm/stdlibs/unicode/utf16/export_test.gno deleted file mode 100644 index e0c57f52aef..00000000000 --- a/gnovm/stdlibs/unicode/utf16/export_test.gno +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package utf16 - -// Extra names for constants so we can validate them during testing. -const ( - MaxRune = maxRune - ReplacementChar = replacementChar -) diff --git a/gnovm/stdlibs/unicode/utf16/utf16.gno b/gnovm/stdlibs/unicode/utf16/utf16.gno deleted file mode 100644 index 38d8be60602..00000000000 --- a/gnovm/stdlibs/unicode/utf16/utf16.gno +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package utf16 implements encoding and decoding of UTF-16 sequences. -package utf16 - -// The conditions replacementChar==unicode.ReplacementChar and -// maxRune==unicode.MaxRune are verified in the tests. -// Defining them locally avoids this package depending on package unicode. - -const ( - replacementChar = '\uFFFD' // Unicode replacement character - maxRune = '\U0010FFFF' // Maximum valid Unicode code point. -) - -const ( - // 0xd800-0xdc00 encodes the high 10 bits of a pair. - // 0xdc00-0xe000 encodes the low 10 bits of a pair. - // the value is those 20 bits plus 0x10000. - surr1 = 0xd800 - surr2 = 0xdc00 - surr3 = 0xe000 - - surrSelf = 0x10000 -) - -// IsSurrogate reports whether the specified Unicode code point -// can appear in a surrogate pair. -func IsSurrogate(r rune) bool { - return surr1 <= r && r < surr3 -} - -// DecodeRune returns the UTF-16 decoding of a surrogate pair. -// If the pair is not a valid UTF-16 surrogate pair, DecodeRune returns -// the Unicode replacement code point U+FFFD. -func DecodeRune(r1, r2 rune) rune { - if surr1 <= r1 && r1 < surr2 && surr2 <= r2 && r2 < surr3 { - return (r1-surr1)<<10 | (r2 - surr2) + surrSelf - } - return replacementChar -} - -// EncodeRune returns the UTF-16 surrogate pair r1, r2 for the given rune. -// If the rune is not a valid Unicode code point or does not need encoding, -// EncodeRune returns U+FFFD, U+FFFD. -func EncodeRune(r rune) (r1, r2 rune) { - if r < surrSelf || r > maxRune { - return replacementChar, replacementChar - } - r -= surrSelf - return surr1 + (r>>10)&0x3ff, surr2 + r&0x3ff -} - -// Encode returns the UTF-16 encoding of the Unicode code point sequence s. -func Encode(s []rune) []uint16 { - n := len(s) - for _, v := range s { - if v >= surrSelf { - n++ - } - } - - a := make([]uint16, n) - n = 0 - for _, v := range s { - switch { - case 0 <= v && v < surr1, surr3 <= v && v < surrSelf: - // normal rune - a[n] = uint16(v) - n++ - case surrSelf <= v && v <= maxRune: - // needs surrogate sequence - r1, r2 := EncodeRune(v) - a[n] = uint16(r1) - a[n+1] = uint16(r2) - n += 2 - default: - a[n] = uint16(replacementChar) - n++ - } - } - return a[:n] -} - -// AppendRune appends the UTF-16 encoding of the Unicode code point r -// to the end of p and returns the extended buffer. If the rune is not -// a valid Unicode code point, it appends the encoding of U+FFFD. -func AppendRune(a []uint16, r rune) []uint16 { - // This function is inlineable for fast handling of ASCII. - switch { - case 0 <= r && r < surr1, surr3 <= r && r < surrSelf: - // normal rune - return append(a, uint16(r)) - case surrSelf <= r && r <= maxRune: - // needs surrogate sequence - r1, r2 := EncodeRune(r) - return append(a, uint16(r1), uint16(r2)) - } - return append(a, replacementChar) -} - -// Decode returns the Unicode code point sequence represented -// by the UTF-16 encoding s. -func Decode(s []uint16) []rune { - a := make([]rune, len(s)) - n := 0 - for i := 0; i < len(s); i++ { - switch r := s[i]; { - case r < surr1, surr3 <= r: - // normal rune - a[n] = rune(r) - case surr1 <= r && r < surr2 && i+1 < len(s) && - surr2 <= s[i+1] && s[i+1] < surr3: - // valid surrogate sequence - a[n] = DecodeRune(rune(r), rune(s[i+1])) - i++ - default: - // invalid surrogate sequence - a[n] = replacementChar - } - n++ - } - return a[:n] -} diff --git a/gnovm/stdlibs/unicode/utf16/utf16_test.gno b/gnovm/stdlibs/unicode/utf16/utf16_test.gno deleted file mode 100644 index 75b3af4ebbe..00000000000 --- a/gnovm/stdlibs/unicode/utf16/utf16_test.gno +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package utf16 - -import ( - "testing" - "unicode" - "unicode/utf16" -) - -// Validate the constants redefined from unicode. -func TestConstants(t *testing.T) { - if MaxRune != unicode.MaxRune { - t.Errorf("utf16.maxRune is wrong: %x should be %x", MaxRune, unicode.MaxRune) - } - if ReplacementChar != unicode.ReplacementChar { - t.Errorf("utf16.replacementChar is wrong: %x should be %x", ReplacementChar, unicode.ReplacementChar) - } -} - -type encodeTest struct { - in []rune - out []uint16 -} - -var encodeTests = []encodeTest{ - {[]rune{1, 2, 3, 4}, []uint16{1, 2, 3, 4}}, - { - []rune{0xffff, 0x10000, 0x10001, 0x12345, 0x10ffff}, - []uint16{0xffff, 0xd800, 0xdc00, 0xd800, 0xdc01, 0xd808, 0xdf45, 0xdbff, 0xdfff}, - }, - { - []rune{'a', 'b', 0xd7ff, 0xd800, 0xdfff, 0xe000, 0x110000, -1}, - []uint16{'a', 'b', 0xd7ff, 0xfffd, 0xfffd, 0xe000, 0xfffd, 0xfffd}, - }, -} - -func slicesEqual(a, b []uint16) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} - -func TestEncode(t *testing.T) { - for _, tt := range encodeTests { - out := Encode(tt.in) - if !slicesEqual(out, tt.out) { - t.Errorf("Encode(%x) = %x; want %x", tt.in, out, tt.out) - } - } -} - -func TestEncodeRune(t *testing.T) { - for i, tt := range encodeTests { - j := 0 - for _, r := range tt.in { - r1, r2 := EncodeRune(r) - if r < 0x10000 || r > unicode.MaxRune { - if j >= len(tt.out) { - t.Errorf("#%d: ran out of tt.out", i) - break - } - if r1 != unicode.ReplacementChar || r2 != unicode.ReplacementChar { - t.Errorf("EncodeRune(%#x) = %#x, %#x; want 0xfffd, 0xfffd", r, r1, r2) - } - j++ - } else { - if j+1 >= len(tt.out) { - t.Errorf("#%d: ran out of tt.out", i) - break - } - if r1 != rune(tt.out[j]) || r2 != rune(tt.out[j+1]) { - t.Errorf("EncodeRune(%#x) = %#x, %#x; want %#x, %#x", r, r1, r2, tt.out[j], tt.out[j+1]) - } - j += 2 - dec := DecodeRune(r1, r2) - if dec != r { - t.Errorf("DecodeRune(%#x, %#x) = %#x; want %#x", r1, r2, dec, r) - } - } - } - if j != len(tt.out) { - t.Errorf("#%d: EncodeRune didn't generate enough output", i) - } - } -} - -type decodeTest struct { - in []uint16 - out []rune -} - -var decodeTests = []decodeTest{ - {[]uint16{1, 2, 3, 4}, []rune{1, 2, 3, 4}}, - { - []uint16{0xffff, 0xd800, 0xdc00, 0xd800, 0xdc01, 0xd808, 0xdf45, 0xdbff, 0xdfff}, - []rune{0xffff, 0x10000, 0x10001, 0x12345, 0x10ffff}, - }, - {[]uint16{0xd800, 'a'}, []rune{0xfffd, 'a'}}, - {[]uint16{0xdfff}, []rune{0xfffd}}, -} - -func TestDecode(t *testing.T) { - for _, tt := range decodeTests { - out := Decode(tt.in) - if !runesEqual(out, tt.out) { - t.Errorf("Decode(%x) = %x; want %x", tt.in, out, tt.out) - } - } -} - -func runesEqual(a, b []rune) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} - -var decodeRuneTests = []struct { - r1, r2 rune - want rune -}{ - {0xd800, 0xdc00, 0x10000}, - {0xd800, 0xdc01, 0x10001}, - {0xd808, 0xdf45, 0x12345}, - {0xdbff, 0xdfff, 0x10ffff}, - {0xd800, 'a', 0xfffd}, // illegal, replacement rune substituted -} - -func TestDecodeRune(t *testing.T) { - for i, tt := range decodeRuneTests { - got := DecodeRune(tt.r1, tt.r2) - if got != tt.want { - t.Errorf("%d: DecodeRune(%q, %q) = %v; want %v", i, tt.r1, tt.r2, got, tt.want) - } - } -} - -var surrogateTests = []struct { - r rune - want bool -}{ - // from https://en.wikipedia.org/wiki/UTF-16 - {'\u007A', false}, // LATIN SMALL LETTER Z - {'\u6C34', false}, // CJK UNIFIED IDEOGRAPH-6C34 (water) - {'\uFEFF', false}, // Byte Order Mark - {'\U00010000', false}, // LINEAR B SYLLABLE B008 A (first non-BMP code point) - {'\U0001D11E', false}, // MUSICAL SYMBOL G CLEF - {'\U0010FFFD', false}, // PRIVATE USE CHARACTER-10FFFD (last Unicode code point) - - {rune(0xd7ff), false}, // surr1-1 - {rune(0xd800), true}, // surr1 - {rune(0xdc00), true}, // surr2 - {rune(0xe000), false}, // surr3 - {rune(0xdfff), true}, // surr3-1 -} - -func TestIsSurrogate(t *testing.T) { - for i, tt := range surrogateTests { - got := IsSurrogate(tt.r) - if got != tt.want { - t.Errorf("%d: IsSurrogate(%q) = %v; want %v", i, tt.r, got, tt.want) - } - } -} - -func BenchmarkDecodeValidASCII(b *testing.B) { - // "hello world" - data := []uint16{104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100} - for i := 0; i < b.N; i++ { - Decode(data) - } -} - -func BenchmarkDecodeValidJapaneseChars(b *testing.B) { - // "日本語日本語日本語" - data := []uint16{26085, 26412, 35486, 26085, 26412, 35486, 26085, 26412, 35486} - for i := 0; i < b.N; i++ { - Decode(data) - } -} - -func BenchmarkDecodeRune(b *testing.B) { - rs := make([]rune, 10) - // U+1D4D0 to U+1D4D4: MATHEMATICAL BOLD SCRIPT CAPITAL LETTERS - for i, u := range []rune{'𝓐', '𝓑', '𝓒', '𝓓', '𝓔'} { - rs[2*i], rs[2*i+1] = EncodeRune(u) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 5; j++ { - DecodeRune(rs[2*j], rs[2*j+1]) - } - } -} - -func BenchmarkEncodeValidASCII(b *testing.B) { - data := []rune{'h', 'e', 'l', 'l', 'o'} - for i := 0; i < b.N; i++ { - Encode(data) - } -} - -func BenchmarkEncodeValidJapaneseChars(b *testing.B) { - data := []rune{'日', '本', '語'} - for i := 0; i < b.N; i++ { - Encode(data) - } -} - -func BenchmarkEncodeRune(b *testing.B) { - for i := 0; i < b.N; i++ { - for _, u := range []rune{'𝓐', '𝓑', '𝓒', '𝓓', '𝓔'} { - EncodeRune(u) - } - } -} From 88e93cce4a8a29e37afc50d27ef91e75fb1be1a4 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 28 Mar 2024 18:33:34 +0900 Subject: [PATCH 68/72] refactor --- examples/gno.land/p/demo/json/buffer.gno | 6 +- examples/gno.land/p/demo/json/decode.gno | 2 +- examples/gno.land/p/demo/json/encode.gno | 2 +- examples/gno.land/p/demo/json/escape.gno | 57 +++++---- examples/gno.land/p/demo/json/node.gno | 9 +- examples/gno.land/p/demo/json/parser.gno | 6 +- examples/gno.land/p/demo/json/parser_test.gno | 121 ------------------ 7 files changed, 46 insertions(+), 157 deletions(-) diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index 10faffeb35c..23fb53fb0ea 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -128,10 +128,10 @@ func (b *buffer) skipAny(endTokens map[byte]bool) error { tokens = append(tokens, string(token)) } - return errors.New(ufmt.Sprintf( + return ufmt.Errorf( "EOF reached before encountering one of the expected tokens: %s", strings.Join(tokens, ", "), - )) + ) } // skipAndReturnIndex moves the buffer index forward by one and returns the new index. @@ -478,7 +478,7 @@ func numberKind2f64(value interface{}) (result float64, err error) { case uint64: result = float64(typed) default: - err = errors.New(ufmt.Sprintf("invalid number type: %T", value)) + err = ufmt.Errorf("invalid number type: %T", value) } return diff --git a/examples/gno.land/p/demo/json/decode.gno b/examples/gno.land/p/demo/json/decode.gno index 485e4ab36c3..4e909f5805e 100644 --- a/examples/gno.land/p/demo/json/decode.gno +++ b/examples/gno.land/p/demo/json/decode.gno @@ -244,7 +244,7 @@ func getString(b *buffer) (*string, error) { } func unexpectedTokenError(data []byte, index int) error { - return errors.New(ufmt.Sprintf("unexpected token at index %d. data %b", index, data)) + return ufmt.Errorf("unexpected token at index %d. data %b", index, data) } func createNode(current *Node, buf *buffer, nodeType ValueType, key **string) (*Node, error) { diff --git a/examples/gno.land/p/demo/json/encode.gno b/examples/gno.land/p/demo/json/encode.gno index 17001f86023..6c0e66c7e6f 100644 --- a/examples/gno.land/p/demo/json/encode.gno +++ b/examples/gno.land/p/demo/json/encode.gno @@ -74,7 +74,7 @@ func Marshal(node *Node) ([]byte, error) { elem, ok := node.next[strconv.Itoa(i)] if !ok { - return nil, errors.New(ufmt.Sprintf("array element %d is not found", i)) + return nil, ufmt.Errorf("array element %d is not found", i) } oVal, err = Marshal(elem) diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index 9f4186fffed..a3421bc470f 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -157,13 +157,15 @@ var escapeByteSet = [256]byte{ 't': tab, } -// Unquote takes a byte slice and unquotes it by removing the surrounding quotes and unescaping the contents. -func Unquote(s []byte, border byte) (t string, ok bool) { - s, ok = unquoteBytes(s, border) - t = string(s) - return +// Unquote takes a byte slice and unquotes it by removing +// the surrounding quotes and unescaping the contents. +func Unquote(s []byte, border byte) (string, bool) { + s, ok := unquoteBytes(s, border) + return string(s), ok } +// unquoteBytes takes a byte slice and unquotes it by removing +// TODO: consider to move this function to the strconv package. func unquoteBytes(s []byte, border byte) ([]byte, bool) { if len(s) < 2 || s[0] != border || s[len(s)-1] != border { return nil, false @@ -269,25 +271,30 @@ func unquoteBytes(s []byte, border byte) ([]byte, bool) { // // If the escape sequence is invalid, or if 'in' does not completely enclose the escape sequence, // function returns (-1, -1) to indicate an error. -func processEscapedUTF8(in, out []byte) (inLen int, outLen int, err error) { - if len(in) < 2 || in[0] != backSlash { - return -1, -1, errors.New("invalid escape sequence") - } +func processEscapedUTF8(in, out []byte) (int, int, error) { + if len(in) < 2 || in[0] != backSlash { + return -1, -1, errors.New("invalid escape sequence") + } - escapeSeqLen := 2 - escapeChar := in[1] - if escapeChar == 'u' { - if r, size := decodeUnicodeEscape(in); size != -1 { - outLen = utf8.EncodeRune(out, r) - return size, outLen, nil - } - } else { - val := escapeByteSet[escapeChar] - if val != 0 { - out[0] = val - return escapeSeqLen, 1, nil - } - } + escapeSeqLen := 2 + escapeChar := in[1] - return -1, -1, errors.New("invalid escape sequence") -} + if escapeChar != 'u' { + val := escapeByteSet[escapeChar] + if val == 0 { + return -1, -1, errors.New("invalid escape sequence") + } + + out[0] = val + return escapeSeqLen, 1, nil + } + + r, size := decodeUnicodeEscape(in) + if size == -1 { + return -1, -1, errors.New("invalid escape sequence") + } + + outLen := utf8.EncodeRune(out, r) + + return size, outLen, nil +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index a9c628b2570..658b1fdfcc6 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -92,12 +92,12 @@ func (n *Node) GetKey(key string) (*Node, error) { } if n.Type() != Object { - return nil, errors.New(ufmt.Sprintf("target node is not object type. got: %s", n.Type().String())) + return nil, ufmt.Errorf("target node is not object type. got: %s", n.Type().String()) } value, ok := n.next[key] if !ok { - return nil, errors.New(ufmt.Sprintf("key not found: %s", key)) + return nil, ufmt.Errorf("key not found: %s", key) } return value, nil @@ -968,7 +968,10 @@ func (n *Node) isContainer() bool { // remove removes the value from the current container type node. func (n *Node) remove(v *Node) error { if !n.isContainer() { - return errors.New(ufmt.Sprintf("can't remove value from non-array or non-object node. got=%s", n.Type().String())) + return ufmt.Errorf( + "can't remove value from non-array or non-object node. got=%s", + n.Type().String(), + ) } if v.prev != n { diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 73b30e46f68..9a2c3a8c817 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -44,7 +44,7 @@ func ParseBoolLiteral(data []byte) (bool, error) { // // It utilizes double-precision (64-bit) floating-point format as defined // by the IEEE 754 standard, providing a decimal precision of approximately 15 digits. -func ParseFloatLiteral(bytes []byte) (value float64, err error) { +func ParseFloatLiteral(bytes []byte) (float64, error) { if len(bytes) == 0 { return -1, errors.New("JSON Error: empty byte slice found while parsing float value") } @@ -82,7 +82,7 @@ func ParseFloatLiteral(bytes []byte) (value float64, err error) { return f, nil } -func ParseIntLiteral(bytes []byte) (v int64, err error) { +func ParseIntLiteral(bytes []byte) (int64, error) { if len(bytes) == 0 { return 0, errors.New("JSON Error: empty byte slice found while parsing integer value") } @@ -167,7 +167,7 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { return man, exp10, nil } -func trimNegativeSign(bytes []byte) (neg bool, trimmed []byte) { +func trimNegativeSign(bytes []byte) (bool, []byte) { if bytes[0] == minus { return true, bytes[1:] } diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index 648ff92041f..44a2fee6404 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -189,124 +189,3 @@ func TestParseIntLiteral(t *testing.T) { }) } } - -// TODO: remove comment after benchmark added. -// func benchmarkParseStringLiteral(b *testing.B, input []byte) { -// for i := 0; i < b.N; i++ { -// _, err := ParseStringLiteral(input) -// if err != nil { -// b.Fatal(err) -// } -// } -// } - -// func BenchmarkParseStringLiteral_Simple(b *testing.B) { -// benchmarkParseStringLiteral(b, []byte(`"Hello, World!"`)) -// } - -// func BenchmarkParseStringLiteral_Long(b *testing.B) { -// benchmarkParseStringLiteral(b, []byte(`"Lorem ipsum dolor sit amet, consectetur adipiscing elit."`)) -// } - -// func BenchmarkParseStringLiteral_Escaped(b *testing.B) { -// benchmarkParseStringLiteral(b, []byte(`"Hello, \"World!\""`)) -// } - -// func BenchmarkParseStringLiteral_Unicode(b *testing.B) { -// benchmarkParseStringLiteral(b, []byte(`"안녕, 世界!"`)) -// } - -// func benchmarkParseFloatLiteral(b *testing.B, input []byte) { -// for i := 0; i < b.N; i++ { -// ParseFloatLiteral(input) -// } -// } - -// func benchmarkStrconvParseFloat(b *testing.B, input string) { -// for i := 0; i < b.N; i++ { -// _, err := strconv.ParseFloat(input, 64) -// if err != nil { -// b.Fatal(err) -// } -// } -// } - -// func BenchmarkParseFloatLiteral_Simple(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("123.456")) -// } - -// func BenchmarkStrconvParseFloat_Simple(b *testing.B) { -// benchmarkStrconvParseFloat(b, "123.456") -// } - -// func BenchmarkParseFloatLiteral_Long(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("123456.78901")) -// } - -// func BenchmarkStrconvParseFloat_Long(b *testing.B) { -// benchmarkStrconvParseFloat(b, "123456.78901") -// } - -// func BenchmarkParseFloatLiteral_Negative(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("-123.456")) -// } - -// func BenchmarkStrconvParseFloat_Negative(b *testing.B) { -// benchmarkStrconvParseFloat(b, "-123.456") -// } - -// func BenchmarkParseFloatLiteral_Negative_Long(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("-123456.78901")) -// } - -// func BenchmarkStrconvParseFloat_Negative_Long(b *testing.B) { -// benchmarkStrconvParseFloat(b, "-123456.78901") -// } - -// func BenchmarkParseFloatLiteral_Decimal(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("123456")) -// } - -// func BenchmarkStrconvParseFloat_Decimal(b *testing.B) { -// benchmarkStrconvParseFloat(b, "123456") -// } - -// func BenchmarkParseFloatLiteral_Float_No_Decimal(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("123.")) -// } - -// func BenchmarkStrconvParseFloat_Float_No_Decimal(b *testing.B) { -// benchmarkStrconvParseFloat(b, "123.") -// } - -// func BenchmarkParseFloatLiteral_Complex(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("-123456.798012345")) -// } - -// func BenchmarkStrconvParseFloat_Complex(b *testing.B) { -// benchmarkStrconvParseFloat(b, "-123456.798012345") -// } - -// func BenchmarkParseFloatLiteral_Science_Notation(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("1e6")) -// } - -// func BenchmarkStrconvParseFloat_Science_Notation(b *testing.B) { -// benchmarkStrconvParseFloat(b, "1e6") -// } - -// func BenchmarkParseFloatLiteral_Science_Notation_Long(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("1.23e10")) -// } - -// func BenchmarkStrconvParseFloat_Science_Notation_Long(b *testing.B) { -// benchmarkStrconvParseFloat(b, "1.23e10") -// } - -// func BenchmarkParseFloatLiteral_Science_Notation_Negative(b *testing.B) { -// benchmarkParseFloatLiteral(b, []byte("-1.23e10")) -// } - -// func BenchmarkStrconvParseFloat_Science_Notation_Negative(b *testing.B) { -// benchmarkStrconvParseFloat(b, "-1.23e10") -// } From 8bffa4846e52ab7272a2c3e0d81fb6366d5ba9ec Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 28 Mar 2024 20:20:50 +0900 Subject: [PATCH 69/72] some lint --- examples/gno.land/p/demo/json/decode.gno | 11 +-- examples/gno.land/p/demo/json/encode.gno | 2 +- examples/gno.land/p/demo/json/escape.gno | 38 +++++----- examples/gno.land/p/demo/json/node.gno | 2 +- examples/gno.land/p/demo/json/node_test.gno | 9 +-- .../gno.land/p/demo/json/ryu/floatconv.gno | 71 +++++++++---------- .../p/demo/json/ryu/floatconv_test.gno | 50 ++++++------- 7 files changed, 90 insertions(+), 93 deletions(-) diff --git a/examples/gno.land/p/demo/json/decode.gno b/examples/gno.land/p/demo/json/decode.gno index 4e909f5805e..634626bccb4 100644 --- a/examples/gno.land/p/demo/json/decode.gno +++ b/examples/gno.land/p/demo/json/decode.gno @@ -17,11 +17,12 @@ const maxNestingDepth = 10000 // The data must be a valid JSON-encoded value. // // Usage: -// node, err := json.Unmarshal([]byte(`{"key": "value"}`)) -// if err != nil { -// ufmt.Println(err) -// } -// println(node) // {"key": "value"} +// +// node, err := json.Unmarshal([]byte(`{"key": "value"}`)) +// if err != nil { +// ufmt.Println(err) +// } +// println(node) // {"key": "value"} func Unmarshal(data []byte) (*Node, error) { buf := newBuffer(data) diff --git a/examples/gno.land/p/demo/json/encode.gno b/examples/gno.land/p/demo/json/encode.gno index 6c0e66c7e6f..b2da1d9c1e5 100644 --- a/examples/gno.land/p/demo/json/encode.gno +++ b/examples/gno.land/p/demo/json/encode.gno @@ -33,7 +33,7 @@ func Marshal(node *Node) ([]byte, error) { if err != nil { return nil, err } - + // ufmt does not support %g. by doing so, we need to check if the number is an integer // after then, apply the correct format for each float and integer numbers. if math.Mod(nVal, 1.0) == 0 { diff --git a/examples/gno.land/p/demo/json/escape.gno b/examples/gno.land/p/demo/json/escape.gno index a3421bc470f..5a834068127 100644 --- a/examples/gno.land/p/demo/json/escape.gno +++ b/examples/gno.land/p/demo/json/escape.gno @@ -272,29 +272,29 @@ func unquoteBytes(s []byte, border byte) ([]byte, bool) { // If the escape sequence is invalid, or if 'in' does not completely enclose the escape sequence, // function returns (-1, -1) to indicate an error. func processEscapedUTF8(in, out []byte) (int, int, error) { - if len(in) < 2 || in[0] != backSlash { - return -1, -1, errors.New("invalid escape sequence") - } + if len(in) < 2 || in[0] != backSlash { + return -1, -1, errors.New("invalid escape sequence") + } - escapeSeqLen := 2 - escapeChar := in[1] + escapeSeqLen := 2 + escapeChar := in[1] - if escapeChar != 'u' { - val := escapeByteSet[escapeChar] - if val == 0 { - return -1, -1, errors.New("invalid escape sequence") - } + if escapeChar != 'u' { + val := escapeByteSet[escapeChar] + if val == 0 { + return -1, -1, errors.New("invalid escape sequence") + } out[0] = val - return escapeSeqLen, 1, nil - } + return escapeSeqLen, 1, nil + } - r, size := decodeUnicodeEscape(in) - if size == -1 { - return -1, -1, errors.New("invalid escape sequence") - } + r, size := decodeUnicodeEscape(in) + if size == -1 { + return -1, -1, errors.New("invalid escape sequence") + } - outLen := utf8.EncodeRune(out, r) + outLen := utf8.EncodeRune(out, r) - return size, outLen, nil -} \ No newline at end of file + return size, outLen, nil +} diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index 658b1fdfcc6..e69e38eaa06 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -442,7 +442,7 @@ func ArrayNode(key string, value []*Node) *Node { curr.value = value for i, v := range value { - var idx = i + idx := i curr.next[strconv.Itoa(i)] = v v.prev = curr diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index 7931cce853a..af4c835bf3f 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -214,7 +214,6 @@ func TestNode_AppendObject(t *testing.T) { } val, err := Marshal(root) - if err != nil { t.Errorf("Marshal returns error: %v", err) } @@ -526,7 +525,6 @@ func TestNode_MustString(t *testing.T) { func TestUnmarshal_Array(t *testing.T) { root, err := Unmarshal([]byte(" [1,[\"1\",[1,[1,2,3]]]]\r\n")) - if err != nil { t.Errorf("Error on Unmarshal: %s", err.Error()) } @@ -766,7 +764,6 @@ func TestNode_Index_Fail(t *testing.T) { t.Errorf("Index() = %v, want %v", got, tt.want) } }) - } } @@ -805,10 +802,10 @@ func TestNode_GetIndex_InputIndex_Exceed_Original_Node_Index(t *testing.T) { func TestNode_DeleteIndex(t *testing.T) { tests := []struct { - name string + name string expected string - index int - ok bool + index int + ok bool }{ {`null`, ``, 0, false}, {`1`, ``, 0, false}, diff --git a/examples/gno.land/p/demo/json/ryu/floatconv.gno b/examples/gno.land/p/demo/json/ryu/floatconv.gno index ce0f7f2c574..ea28c439104 100644 --- a/examples/gno.land/p/demo/json/ryu/floatconv.gno +++ b/examples/gno.land/p/demo/json/ryu/floatconv.gno @@ -18,7 +18,7 @@ // // original Go implementation can be found at https://github.com/cespare/ryu. // -// Please note that the modifications are also under the Apache License 2.0 unless +// Please note that the modifications are also under the Apache License 2.0 unless // otherwise specified. // Package ryu implements the Ryu algorithm for quickly converting floating @@ -45,48 +45,48 @@ const ( // FormatFloat64 converts a 64-bit floating point number f to a string. // It behaves like strconv.FormatFloat(f, 'e', -1, 64). func FormatFloat64(f float64) string { - b := make([]byte, 0, 24) - b = AppendFloat64(b, f) - return string(b) + b := make([]byte, 0, 24) + b = AppendFloat64(b, f) + return string(b) } // AppendFloat64 appends the string form of the 64-bit floating point number f, // as generated by FormatFloat64, to b and returns the extended buffer. func AppendFloat64(b []byte, f float64) []byte { - // Step 1: Decode the floating-point number. - // Unify normalized and subnormal cases. - u := math.Float64bits(f) - neg := u>>(mantBits64+expBits64) != 0 - mant := u & (uint64(1)<> mantBits64) & (uint64(1)<>(mantBits64+expBits64) != 0 + mant := u & (uint64(1)<> mantBits64) & (uint64(1)< Date: Fri, 29 Mar 2024 12:25:46 +0900 Subject: [PATCH 70/72] save --- examples/gno.land/p/demo/json/decode_test.gno | 432 +++++++++--------- examples/gno.land/p/demo/json/encode.gno | 12 +- examples/gno.land/p/demo/json/node.gno | 3 - examples/gno.land/p/demo/json/node_test.gno | 11 - examples/gno.land/p/demo/json/path.gno | 2 - .../gno.land/p/demo/json/ryu/floatconv.gno | 8 +- 6 files changed, 221 insertions(+), 247 deletions(-) diff --git a/examples/gno.land/p/demo/json/decode_test.gno b/examples/gno.land/p/demo/json/decode_test.gno index c731164feb1..5a04846224e 100644 --- a/examples/gno.land/p/demo/json/decode_test.gno +++ b/examples/gno.land/p/demo/json/decode_test.gno @@ -271,217 +271,6 @@ func TestUnmarshal_ArraySimpleCorrupted(t *testing.T) { } } -var glossary = `{ - "glossary": { - "title": "example glossary", - "GlossDiv": { - "title": "S", - "GlossList": { - "GlossEntry": { - "ID": "SGML", - "SortAs": "SGML", - "GlossTerm": "Standard Generalized Markup Language", - "Acronym": "SGML", - "Abbrev": "ISO 8879:1986", - "GlossDef": { - "para": "A meta-markup language, used to create markup languages such as DocBook.", - "GlossSeeAlso": ["GML", "XML"] - }, - "GlossSee": "markup" - } - } - } - } -}` - -var file = `{"menu": { - "id": "file", - "value": "File", - "popup": { - "menuitem": [ - {"value": "New", "onclick": "CreateNewDoc()"}, - {"value": "Open", "onclick": "OpenDoc()"}, - {"value": "Close", "onclick": "CloseDoc()"} - ] - } -}}` - -var widget = `{"widget": { - "debug": "on", - "window": { - "title": "Sample Konfabulator Widget", - "name": "main_window", - "width": 500, - "height": 500 - }, - "image": { - "src": "Images/Sun.png", - "name": "sun1", - "hOffset": 250, - "vOffset": 250, - "alignment": "center" - }, - "text": { - "data": "Click Here", - "size": 36, - "style": "bold", - "name": "text1", - "hOffset": 250, - "vOffset": 100, - "alignment": "center", - "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" - } -}} ` - -var webApp = `{"web-app": { - "servlet": [ - { - "servlet-name": "cofaxCDS", - "servlet-class": "org.cofax.cds.CDSServlet", - "init-param": { - "configGlossary:installationAt": "Philadelphia, PA", - "configGlossary:adminEmail": "ksm@pobox.com", - "configGlossary:poweredBy": "Cofax", - "configGlossary:poweredByIcon": "/images/cofax.gif", - "configGlossary:staticPath": "/content/static", - "templateProcessorClass": "org.cofax.WysiwygTemplate", - "templateLoaderClass": "org.cofax.FilesTemplateLoader", - "templatePath": "templates", - "templateOverridePath": "", - "defaultListTemplate": "listTemplate.htm", - "defaultFileTemplate": "articleTemplate.htm", - "useJSP": false, - "jspListTemplate": "listTemplate.jsp", - "jspFileTemplate": "articleTemplate.jsp", - "cachePackageTagsTrack": 200, - "cachePackageTagsStore": 200, - "cachePackageTagsRefresh": 60, - "cacheTemplatesTrack": 100, - "cacheTemplatesStore": 50, - "cacheTemplatesRefresh": 15, - "cachePagesTrack": 200, - "cachePagesStore": 100, - "cachePagesRefresh": 10, - "cachePagesDirtyRead": 10, - "searchEngineListTemplate": "forSearchEnginesList.htm", - "searchEngineFileTemplate": "forSearchEngines.htm", - "searchEngineRobotsDb": "WEB-INF/robots.db", - "useDataStore": true, - "dataStoreClass": "org.cofax.SqlDataStore", - "redirectionClass": "org.cofax.SqlRedirection", - "dataStoreName": "cofax", - "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", - "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", - "dataStoreUser": "sa", - "dataStorePassword": "dataStoreTestQuery", - "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", - "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", - "dataStoreInitConns": 10, - "dataStoreMaxConns": 100, - "dataStoreConnUsageLimit": 100, - "dataStoreLogLevel": "debug", - "maxUrlLength": 500}}, - { - "servlet-name": "cofaxEmail", - "servlet-class": "org.cofax.cds.EmailServlet", - "init-param": { - "mailHost": "mail1", - "mailHostOverride": "mail2"}}, - { - "servlet-name": "cofaxAdmin", - "servlet-class": "org.cofax.cds.AdminServlet"}, - - { - "servlet-name": "fileServlet", - "servlet-class": "org.cofax.cds.FileServlet"}, - { - "servlet-name": "cofaxTools", - "servlet-class": "org.cofax.cms.CofaxToolsServlet", - "init-param": { - "templatePath": "toolstemplates/", - "log": 1, - "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", - "logMaxSize": "", - "dataLog": 1, - "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", - "dataLogMaxSize": "", - "removePageCache": "/content/admin/remove?cache=pages&id=", - "removeTemplateCache": "/content/admin/remove?cache=templates&id=", - "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", - "lookInContext": 1, - "adminGroupID": 4, - "betaServer": true}}], - "servlet-mapping": { - "cofaxCDS": "/", - "cofaxEmail": "/cofaxutil/aemail/*", - "cofaxAdmin": "/admin/*", - "fileServlet": "/static/*", - "cofaxTools": "/tools/*"}, - - "taglib": { - "taglib-uri": "cofax.tld", - "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}` - -var svgViewer = `{"menu": { - "header": "SVG Viewer", - "items": [ - {"id": "Open"}, - {"id": "OpenNew", "label": "Open New"}, - null, - {"id": "ZoomIn", "label": "Zoom In"}, - {"id": "ZoomOut", "label": "Zoom Out"}, - {"id": "OriginalView", "label": "Original View"}, - null, - {"id": "Quality"}, - {"id": "Pause"}, - {"id": "Mute"}, - null, - {"id": "Find", "label": "Find..."}, - {"id": "FindAgain", "label": "Find Again"}, - {"id": "Copy"}, - {"id": "CopyAgain", "label": "Copy Again"}, - {"id": "CopySVG", "label": "Copy SVG"}, - {"id": "ViewSVG", "label": "View SVG"}, - {"id": "ViewSource", "label": "View Source"}, - {"id": "SaveAs", "label": "Save As"}, - null, - {"id": "Help"}, - {"id": "About", "label": "About Adobe CVG Viewer..."} - ] -}}` - -var bookStore = `{ "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - } -}` - // Examples from https://json.org/example.html func TestUnmarshal(t *testing.T) { tests := []struct { @@ -489,24 +278,193 @@ func TestUnmarshal(t *testing.T) { value string }{ { - name: "glossary", - value: glossary, + name: "glossary", + value: `{ + "glossary": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } + }`, }, { - name: "menu", - value: file, + name: "menu", + value: `{"menu": { + "id": "file", + "value": "File", + "popup": { + "menuitem": [ + {"value": "New", "onclick": "CreateNewDoc()"}, + {"value": "Open", "onclick": "OpenDoc()"}, + {"value": "Close", "onclick": "CloseDoc()"} + ] + } + }}`, }, { - name: "widget", - value: widget, + name: "widget", + value: `{"widget": { + "debug": "on", + "window": { + "title": "Sample Konfabulator Widget", + "name": "main_window", + "width": 500, + "height": 500 + }, + "image": { + "src": "Images/Sun.png", + "name": "sun1", + "hOffset": 250, + "vOffset": 250, + "alignment": "center" + }, + "text": { + "data": "Click Here", + "size": 36, + "style": "bold", + "name": "text1", + "hOffset": 250, + "vOffset": 100, + "alignment": "center", + "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;" + } + }} `, }, { - name: "web-app", - value: webApp, + name: "web-app", + value: `{"web-app": { + "servlet": [ + { + "servlet-name": "cofaxCDS", + "servlet-class": "org.cofax.cds.CDSServlet", + "init-param": { + "configGlossary:installationAt": "Philadelphia, PA", + "configGlossary:adminEmail": "ksm@pobox.com", + "configGlossary:poweredBy": "Cofax", + "configGlossary:poweredByIcon": "/images/cofax.gif", + "configGlossary:staticPath": "/content/static", + "templateProcessorClass": "org.cofax.WysiwygTemplate", + "templateLoaderClass": "org.cofax.FilesTemplateLoader", + "templatePath": "templates", + "templateOverridePath": "", + "defaultListTemplate": "listTemplate.htm", + "defaultFileTemplate": "articleTemplate.htm", + "useJSP": false, + "jspListTemplate": "listTemplate.jsp", + "jspFileTemplate": "articleTemplate.jsp", + "cachePackageTagsTrack": 200, + "cachePackageTagsStore": 200, + "cachePackageTagsRefresh": 60, + "cacheTemplatesTrack": 100, + "cacheTemplatesStore": 50, + "cacheTemplatesRefresh": 15, + "cachePagesTrack": 200, + "cachePagesStore": 100, + "cachePagesRefresh": 10, + "cachePagesDirtyRead": 10, + "searchEngineListTemplate": "forSearchEnginesList.htm", + "searchEngineFileTemplate": "forSearchEngines.htm", + "searchEngineRobotsDb": "WEB-INF/robots.db", + "useDataStore": true, + "dataStoreClass": "org.cofax.SqlDataStore", + "redirectionClass": "org.cofax.SqlRedirection", + "dataStoreName": "cofax", + "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", + "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", + "dataStoreUser": "sa", + "dataStorePassword": "dataStoreTestQuery", + "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", + "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", + "dataStoreInitConns": 10, + "dataStoreMaxConns": 100, + "dataStoreConnUsageLimit": 100, + "dataStoreLogLevel": "debug", + "maxUrlLength": 500}}, + { + "servlet-name": "cofaxEmail", + "servlet-class": "org.cofax.cds.EmailServlet", + "init-param": { + "mailHost": "mail1", + "mailHostOverride": "mail2"}}, + { + "servlet-name": "cofaxAdmin", + "servlet-class": "org.cofax.cds.AdminServlet"}, + + { + "servlet-name": "fileServlet", + "servlet-class": "org.cofax.cds.FileServlet"}, + { + "servlet-name": "cofaxTools", + "servlet-class": "org.cofax.cms.CofaxToolsServlet", + "init-param": { + "templatePath": "toolstemplates/", + "log": 1, + "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", + "logMaxSize": "", + "dataLog": 1, + "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", + "dataLogMaxSize": "", + "removePageCache": "/content/admin/remove?cache=pages&id=", + "removeTemplateCache": "/content/admin/remove?cache=templates&id=", + "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", + "lookInContext": 1, + "adminGroupID": 4, + "betaServer": true}}], + "servlet-mapping": { + "cofaxCDS": "/", + "cofaxEmail": "/cofaxutil/aemail/*", + "cofaxAdmin": "/admin/*", + "fileServlet": "/static/*", + "cofaxTools": "/tools/*"}, + + "taglib": { + "taglib-uri": "cofax.tld", + "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}`, }, { - name: "SVG Viewer", - value: svgViewer, + name: "SVG Viewer", + value: `{"menu": { + "header": "SVG Viewer", + "items": [ + {"id": "Open"}, + {"id": "OpenNew", "label": "Open New"}, + null, + {"id": "ZoomIn", "label": "Zoom In"}, + {"id": "ZoomOut", "label": "Zoom Out"}, + {"id": "OriginalView", "label": "Original View"}, + null, + {"id": "Quality"}, + {"id": "Pause"}, + {"id": "Mute"}, + null, + {"id": "Find", "label": "Find..."}, + {"id": "FindAgain", "label": "Find Again"}, + {"id": "Copy"}, + {"id": "CopyAgain", "label": "Copy Again"}, + {"id": "CopySVG", "label": "Copy SVG"}, + {"id": "ViewSVG", "label": "View SVG"}, + {"id": "ViewSource", "label": "View Source"}, + {"id": "SaveAs", "label": "Save As"}, + null, + {"id": "Help"}, + {"id": "About", "label": "About Adobe CVG Viewer..."} + ] + }}`, }, } for _, test := range tests { @@ -520,7 +478,37 @@ func TestUnmarshal(t *testing.T) { } func TestUnmarshalSafe(t *testing.T) { - json := []byte(bookStore) + json := []byte(`{ "store": { + "book": [ + { "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, + { "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + } + }`) safe, err := UnmarshalSafe(json) if err != nil { t.Errorf("Error on Unmarshal: %s", err.Error()) diff --git a/examples/gno.land/p/demo/json/encode.gno b/examples/gno.land/p/demo/json/encode.gno index b2da1d9c1e5..f6e3cdda69d 100644 --- a/examples/gno.land/p/demo/json/encode.gno +++ b/examples/gno.land/p/demo/json/encode.gno @@ -12,8 +12,8 @@ import ( // Marshal returns the JSON encoding of a Node. func Marshal(node *Node) ([]byte, error) { - var buf bytes.Buffer var ( + buf bytes.Buffer sVal string bVal bool nVal float64 @@ -23,7 +23,13 @@ func Marshal(node *Node) ([]byte, error) { if node == nil { return nil, errors.New("node is nil") - } else if node.modified { + } + + if !node.modified && !node.ready() { + return nil, errors.New("node is not ready") + } + + if node.modified { switch node.nodeType { case Null: buf.Write(nullLiteral) @@ -114,8 +120,6 @@ func Marshal(node *Node) ([]byte, error) { } } else if node.ready() { buf.Write(node.source()) - } else { - return nil, errors.New("node is not modified") } return buf.Bytes(), nil diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index e69e38eaa06..1e71a101e62 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -140,9 +140,6 @@ func (n *Node) UniqueKeyLists() []string { return collectKeys(n) } -// TODO: implement the EachKey method. this takes a callback function and executes it for each key in the object node. -// func (n *Node) EachKey(callback func(key string, value *Node)) { ... } - // Empty returns true if the current node is empty. func (n *Node) Empty() bool { if n == nil { diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index af4c835bf3f..dbc82369f68 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -37,17 +37,6 @@ func TestNode_CreateNewNode(t *testing.T) { expectErr bool expectPanic bool }{ - // { - // name: "object with empty key", - // args: _args{ - // prev: ObjectNode("", make(map[string]*Node)), - // buf: newBuffer(make([]byte, 10)), - // typ: Boolean, - // key: &nilKey, - // }, - // expectCurr: nil, - // expectPanic: true, - // }, { name: "child for non container type", args: _args{ diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 11859d677f2..31f7e04633f 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -4,8 +4,6 @@ import ( "errors" ) -// TODO: not fully implemented yet. - // ParsePath takes a JSONPath string and returns a slice of strings representing the path segments. func ParsePath(path string) ([]string, error) { buf := newBuffer([]byte(path)) diff --git a/examples/gno.land/p/demo/json/ryu/floatconv.gno b/examples/gno.land/p/demo/json/ryu/floatconv.gno index ea28c439104..617141d2734 100644 --- a/examples/gno.land/p/demo/json/ryu/floatconv.gno +++ b/examples/gno.land/p/demo/json/ryu/floatconv.gno @@ -29,9 +29,6 @@ import ( "math" ) -// TODO: move to strconv package? -// TODO: support %g and %f formats in ufmt package. - const ( mantBits32 = 23 expBits32 = 8 @@ -79,10 +76,11 @@ func appendSpecial(b []byte, neg, expZero, mantZero bool) []byte { if !expZero { if neg { return append(b, "-Inf"...) - } else { - return append(b, "+Inf"...) } + + return append(b, "+Inf"...) } + if neg { b = append(b, '-') } From bd14f5089abe8f210b86937e21e353e370b3bf4d Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 29 Mar 2024 16:37:08 +0900 Subject: [PATCH 71/72] refactor and update README --- examples/gno.land/p/demo/json/README.md | 175 ++++++++++++++++- examples/gno.land/p/demo/json/decode.gno | 233 +++++++++++++++-------- examples/gno.land/p/demo/json/encode.gno | 6 +- 3 files changed, 329 insertions(+), 85 deletions(-) diff --git a/examples/gno.land/p/demo/json/README.md b/examples/gno.land/p/demo/json/README.md index ef9fce85617..b5f0c7b3db5 100644 --- a/examples/gno.land/p/demo/json/README.md +++ b/examples/gno.land/p/demo/json/README.md @@ -1,13 +1,174 @@ -## Source +# JSON Parser -### JSON Parser +The JSON parser is a package that provides functionality for parsing and processing JSON strings. This package accepts JSON strings as byte slices. -The code for this project/package can be found at [here](https://github.com/buger/jsonparser). This code is provided for actual project development purposes and has been slightly modified from its original version for specific use within this project. +Currently, gno does not [support the `reflect` package](https://docs.gno.land/concepts/effective-gno#reflection-is-never-clear), so it cannot retrieve type information at runtime. Therefore, it is designed to infer and handle type information when parsing JSON strings using a state machine approach. -For more detailed information on the modifications and their purposes, please refer to the documentation within the project repository. +After passing through the state machine, JSON strings are represented as the `Node` type. The `Node` type represents nodes for JSON data, including various types such as `ObjectNode`, `ArrayNode`, `StringNode`, `NumberNode`, `BoolNode`, and `NullNode`. -#### License +This package provides methods for manipulating, searching, and extracting the Node type. -This project is licensed under the MIT License. This permits the use, copying, modification, merging, publishing, distribution, sublicensing, and/or selling of copies of the software, and allows those to whom the software is furnished to do so as well, under the same terms. +## State Machine -The full text of the license can be found in the 'LICENSE' file located in the package's root directory. This license specifies the terms under which you agree to use this project. For more detailed information on the license and its terms, please refer to the original license documentation in the [project's repository](https://github.com/buger/jsonparser) or the 'LICENSE.txt' file. +To parse JSON strings, a [finite state machine](https://en.wikipedia.org/wiki/Finite-state_machine) approach is used. The state machine transitions to the next state based on the current state and the input character while parsing the JSON string. Through this method, type information can be inferred and processed without reflect, and the amount of parser code can be significantly reduced. + +The image below shows the state transitions of the state machine according to the states and input characters. + +```mermaid +stateDiagram-v2 + [*] --> __: Start + __ --> ST: String + __ --> MI: Number + __ --> ZE: Zero + __ --> IN: Integer + __ --> T1: Boolean (true) + __ --> F1: Boolean (false) + __ --> N1: Null + __ --> ec: Empty Object End + __ --> cc: Object End + __ --> bc: Array End + __ --> co: Object Begin + __ --> bo: Array Begin + __ --> cm: Comma + __ --> cl: Colon + __ --> OK: Success/End + ST --> OK: String Complete + MI --> OK: Number Complete + ZE --> OK: Zero Complete + IN --> OK: Integer Complete + T1 --> OK: True Complete + F1 --> OK: False Complete + N1 --> OK: Null Complete + ec --> OK: Empty Object Complete + cc --> OK: Object Complete + bc --> OK: Array Complete + co --> OB: Inside Object + bo --> AR: Inside Array + cm --> KE: Expecting New Key + cm --> VA: Expecting New Value + cl --> VA: Expecting Value + OB --> ST: String in Object (Key) + OB --> ec: Empty Object + OB --> cc: End Object + AR --> ST: String in Array + AR --> bc: End Array + KE --> ST: String as Key + VA --> ST: String as Value + VA --> MI: Number as Value + VA --> T1: True as Value + VA --> F1: False as Value + VA --> N1: Null as Value + OK --> [*]: End +``` + +## Examples + +This package provides parsing functionality along with encoding and decoding functionality. The following examples demonstrate how to use this package. + +### Decoding + +Decoding (or Unmarshaling) is the functionality that converts an input byte slice JSON string into a `Node` type. + +The converted `Node` type allows you to modify the JSON data or search and extract data that meets specific conditions. + +```go +package main + +import ( + "fmt" + "gno.land/p/demo/json" + "gno.land/p/demo/ufmt" +) + +func main() { + node, err := json.Unmarshal([]byte(`{"foo": "var"}`)) + if err != nil { + ufmt.Errorf("error: %v", err) + } + + ufmt.Sprintf("node: %v", node) +} +``` + +### Encoding + +Encoding (or Marshaling) is the functionality that converts JSON data represented as a Node type into a byte slice JSON string. + +> ⚠️ Caution: Converting a large `Node` type into a JSON string may _impact performance_. or might be cause _unexpected behavior_. + +```go +package main + +import ( + "fmt" + "gno.land/p/demo/json" + "gno.land/p/demo/ufmt" +) + +func main() { + node := ObjectNode("", map[string]*Node{ + "foo": StringNode("foo", "bar"), + "baz": NumberNode("baz", 100500), + "qux": NullNode("qux"), + }) + + b, err := json.Marshal(node) + if err != nil { + ufmt.Errorf("error: %v", err) + } + + ufmt.Sprintf("json: %s", string(b)) +} +``` + +### Searching + +Node 타입으로 변환된 JSON 데이터를 이용하여 특정 조건을 만족하는 데이터를 검색, 추출할 수 있습니다. 예를 들어, 특정 타입을 가진 데이터를 찾거나, 특정 키를 가진 데이터를 찾을 수 있습니다. 이 기능을 사용하려면 GetXXX 형태의 메서드를 사용합니다. MustXXX 메서드 역시 GetXXX 메서드와 동일한 기능을 제공하지만, 만약 조건을 만족하는 데이터가 없을 경우 panic을 발생시킵니다. + +아래는 특정 키를 가진 데이터를 찾는 예제입니다. 더 많은 예제는 [node.gno](node.gno) 파일을 참고하세요. + +Once the JSON data converted into a `Node` type, you can **search** and **extract** data that satisfy specific conditions. For example, you can find data with a specific type or data with a specific key. + +To use this functionality, you can use methods in the `GetXXX` prefixed methods. The `MustXXX` methods also provide the same functionality as the former methods, but they will **panic** if data doesn't satisfies the condition. + +Here is an example of finding data with a specific key. For more examples, please refer to the [node.gno](node.gno) file. + +```go +package main + +import ( + "fmt" + "gno.land/p/demo/json" + "gno.land/p/demo/ufmt" +) + +func main() { + root, err := Unmarshal([]byte(`{"foo": true, "bar": null}`)) + if err != nil { + ufmt.Errorf("error: %v", err) + } + + value, err := root.GetKey("foo") + if err != nil { + ufmt.Errorf("error occurred while getting key, %s", err) + } + + if value.MustBool() != true { + ufmt.Errorf("value is not true") + } + + value, err = root.GetKey("bar") + if err != nil { + t.Errorf("error occurred while getting key, %s", err) + } + + _, err = root.GetKey("baz") + if err == nil { + t.Errorf("key baz is not exist. must be failed") + } +} +``` + +## Contributing + +Please submit any issues or pull requests for this package through the GitHub repository at [gnolang/gno](). diff --git a/examples/gno.land/p/demo/json/decode.gno b/examples/gno.land/p/demo/json/decode.gno index 634626bccb4..8a6e2b99846 100644 --- a/examples/gno.land/p/demo/json/decode.gno +++ b/examples/gno.land/p/demo/json/decode.gno @@ -61,16 +61,13 @@ func Unmarshal(data []byte) (*Node, error) { buf.state = CO } else { - if nesting, err = checkNestingDepth(nesting); err != nil { - return nil, err - } - - current, err = createNode(current, buf, String, useKey()) + current, nesting, err = createNestedNode(current, buf, String, nesting, useKey()) if err != nil { return nil, err } - if err = buf.string(doubleQuote, false); err != nil { + err = buf.string(doubleQuote, false) + if err != nil { return nil, err } @@ -79,52 +76,27 @@ func Unmarshal(data []byte) (*Node, error) { } case MI, ZE, IN: // number - if current, err = createNode(current, buf, Number, useKey()); err != nil { - return nil, err - } - - if err = buf.numeric(false); err != nil { + current, err = processNumericNode(current, buf, useKey()) + if err != nil { return nil, err } - current.borders[1] = buf.index - if current.prev != nil { - current = current.prev - } - - buf.index -= 1 - buf.state = OK - case T1, F1: // boolean - if current, err = createNode(current, buf, Boolean, useKey()); err != nil { - return nil, err - } - - var literal []byte + literal := falseLiteral if buf.state == T1 { literal = trueLiteral - } else { - literal = falseLiteral } - if err = buf.word(literal); err != nil { + current, nesting, err = processLiteralNode(current, buf, Boolean, literal, useKey(), nesting) + if err != nil { return nil, err } - current, nesting = updateNode(current, buf, nesting, false) - buf.state = OK - case N1: // null - if current, err = createNode(current, buf, Null, useKey()); err != nil { - return nil, err - } - - if err = buf.word(nullLiteral); err != nil { + current, nesting, err = processLiteralNode(current, buf, Null, nullLiteral, useKey(), nesting) + if err != nil { return nil, err } - - current, nesting = updateNode(current, buf, nesting, false) - buf.state = OK } } else { // region action @@ -134,54 +106,43 @@ func Unmarshal(data []byte) (*Node, error) { return nil, unexpectedTokenError(buf.data, buf.index) } - if !isValidContainerType(current, Object) { - return nil, unexpectedTokenError(buf.data, buf.index) - } - - current, nesting = updateNode(current, buf, nesting, true) - buf.state = OK - - case bc: // ] - if !isValidContainerType(current, Array) { - return nil, unexpectedTokenError(buf.data, buf.index) - } - - current, nesting = updateNode(current, buf, nesting, true) - buf.state = OK - - case co: // { - if nesting, err = checkNestingDepth(nesting); err != nil { + current, nesting, err = updateNodeAndSetBufferState(current, buf, nesting, Object) + if err != nil { return nil, err } - if current, err = createNode(current, buf, Object, useKey()); err != nil { + case bc: // ] + current, nesting, err = updateNodeAndSetBufferState(current, buf, nesting, Array) + if err != nil { return nil, err } - buf.state = OB - - case bo: // [ - if nesting, err = checkNestingDepth(nesting); err != nil { - return nil, err + case co, bo: // { [ + valTyp, bState := Object, OB + if state == bo { + valTyp, bState = Array, AR } - if current, err = createNode(current, buf, Array, useKey()); err != nil { + current, nesting, err = createNestedNode(current, buf, valTyp, nesting, useKey()) + if err != nil { return nil, err } - buf.state = AR + buf.state = bState case cm: // , if current == nil { return nil, unexpectedTokenError(buf.data, buf.index) } + if !current.isContainer() { + return nil, unexpectedTokenError(buf.data, buf.index) + } + if current.IsObject() { buf.state = KE // key expected - } else if current.IsArray() { - buf.state = VA // value expected } else { - return nil, unexpectedTokenError(buf.data, buf.index) + buf.state = VA // value expected } case cl: // : @@ -218,6 +179,60 @@ func Unmarshal(data []byte) (*Node, error) { return root, err } +// UnmarshalSafe parses the JSON-encoded data and returns a Node. +func UnmarshalSafe(data []byte) (*Node, error) { + var safe []byte + safe = append(safe, data...) + return Unmarshal(safe) +} + +// processNumericNode creates a new node, processes a numeric value, +// sets the node's borders, and moves to the previous node. +func processNumericNode(current *Node, buf *buffer, key **string) (*Node, error) { + var err error + current, err = createNode(current, buf, Number, key) + if err != nil { + return nil, err + } + + if err = buf.numeric(false); err != nil { + return nil, err + } + + current.borders[1] = buf.index + if current.prev != nil { + current = current.prev + } + + buf.index -= 1 + buf.state = OK + + return current, nil +} + +// processLiteralNode creates a new node, processes a literal value, +// sets the node's borders, and moves to the previous node. +func processLiteralNode( + current *Node, + buf *buffer, + literalType ValueType, + literalValue []byte, + useKey **string, + nesting int, +) (*Node, int, error) { + var err error + current, nesting, err = createLiteralNode(current, buf, literalType, literalValue, useKey, nesting) + if err != nil { + return nil, nesting, err + } + return current, nesting, nil +} + +// isValidContainerType checks if the current node is a valid container (object or array). +// The container must satisfy the following conditions: +// 1. The current node must not be nil. +// 2. The current node must be an object or array. +// 3. The current node must not be ready. func isValidContainerType(current *Node, nodeType ValueType) bool { switch nodeType { case Object: @@ -244,11 +259,13 @@ func getString(b *buffer) (*string, error) { return &value, nil } -func unexpectedTokenError(data []byte, index int) error { - return ufmt.Errorf("unexpected token at index %d. data %b", index, data) -} - -func createNode(current *Node, buf *buffer, nodeType ValueType, key **string) (*Node, error) { +// createNode creates a new node and sets the key if it is not nil. +func createNode( + current *Node, + buf *buffer, + nodeType ValueType, + key **string, +) (*Node, error) { var err error current, err = NewNode(current, buf, nodeType, key) if err != nil { @@ -258,7 +275,55 @@ func createNode(current *Node, buf *buffer, nodeType ValueType, key **string) (* return current, nil } -func updateNode(current *Node, buf *buffer, nesting int, decreaseLevel bool) (*Node, int) { +// createNestedNode creates a new nested node (array or object) and sets the key if it is not nil. +func createNestedNode( + current *Node, + buf *buffer, + nodeType ValueType, + nesting int, + key **string, +) (*Node, int, error) { + var err error + if nesting, err = checkNestingDepth(nesting); err != nil { + return nil, nesting, err + } + + if current, err = createNode(current, buf, nodeType, key); err != nil { + return nil, nesting, err + } + + return current, nesting, nil +} + +// createLiteralNode creates a new literal node and sets the key if it is not nil. +// The literal is a byte slice that represents a boolean or null value. +func createLiteralNode( + current *Node, + buf *buffer, + literalType ValueType, + literal []byte, + useKey **string, + nesting int, +) (*Node, int, error) { + var err error + if current, err = createNode(current, buf, literalType, useKey); err != nil { + return nil, 0, err + } + + if err = buf.word(literal); err != nil { + return nil, 0, err + } + + current, nesting = updateNode(current, buf, nesting, false) + buf.state = OK + + return current, nesting, nil +} + +// updateNode updates the current node and returns the previous node. +func updateNode( + current *Node, buf *buffer, nesting int, decreaseLevel bool, +) (*Node, int) { current.borders[1] = buf.index + 1 prev := current.prev @@ -274,6 +339,24 @@ func updateNode(current *Node, buf *buffer, nesting int, decreaseLevel bool) (*N return current, nesting } +// updateNodeAndSetBufferState updates the current node and sets the buffer state to OK. +func updateNodeAndSetBufferState( + current *Node, + buf *buffer, + nesting int, + typ ValueType, +) (*Node, int, error) { + if !isValidContainerType(current, typ) { + return nil, nesting, unexpectedTokenError(buf.data, buf.index) + } + + current, nesting = updateNode(current, buf, nesting, true) + buf.state = OK + + return current, nesting, nil +} + +// checkNestingDepth checks if the nesting depth is within the maximum allowed depth. func checkNestingDepth(nesting int) (int, error) { if nesting >= maxNestingDepth { return nesting, errors.New("maximum nesting depth exceeded") @@ -282,8 +365,6 @@ func checkNestingDepth(nesting int) (int, error) { return nesting + 1, nil } -func UnmarshalSafe(data []byte) (*Node, error) { - var safe []byte - safe = append(safe, data...) - return Unmarshal(safe) +func unexpectedTokenError(data []byte, index int) error { + return ufmt.Errorf("unexpected token at index %d. data %b", index, data) } diff --git a/examples/gno.land/p/demo/json/encode.gno b/examples/gno.land/p/demo/json/encode.gno index f6e3cdda69d..be90d7aa73d 100644 --- a/examples/gno.land/p/demo/json/encode.gno +++ b/examples/gno.land/p/demo/json/encode.gno @@ -29,6 +29,10 @@ func Marshal(node *Node) ([]byte, error) { return nil, errors.New("node is not ready") } + if !node.modified && node.ready() { + buf.Write(node.source()) + } + if node.modified { switch node.nodeType { case Null: @@ -118,8 +122,6 @@ func Marshal(node *Node) ([]byte, error) { buf.WriteByte(curlyClose) } - } else if node.ready() { - buf.Write(node.source()) } return buf.Bytes(), nil From 209a7549baa0169d40bae76fdb0622e47fba1f32 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Fri, 29 Mar 2024 16:53:47 +0900 Subject: [PATCH 72/72] typo --- examples/gno.land/p/demo/json/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/gno.land/p/demo/json/README.md b/examples/gno.land/p/demo/json/README.md index b5f0c7b3db5..86bc9928194 100644 --- a/examples/gno.land/p/demo/json/README.md +++ b/examples/gno.land/p/demo/json/README.md @@ -123,10 +123,6 @@ func main() { ### Searching -Node 타입으로 변환된 JSON 데이터를 이용하여 특정 조건을 만족하는 데이터를 검색, 추출할 수 있습니다. 예를 들어, 특정 타입을 가진 데이터를 찾거나, 특정 키를 가진 데이터를 찾을 수 있습니다. 이 기능을 사용하려면 GetXXX 형태의 메서드를 사용합니다. MustXXX 메서드 역시 GetXXX 메서드와 동일한 기능을 제공하지만, 만약 조건을 만족하는 데이터가 없을 경우 panic을 발생시킵니다. - -아래는 특정 키를 가진 데이터를 찾는 예제입니다. 더 많은 예제는 [node.gno](node.gno) 파일을 참고하세요. - Once the JSON data converted into a `Node` type, you can **search** and **extract** data that satisfy specific conditions. For example, you can find data with a specific type or data with a specific key. To use this functionality, you can use methods in the `GetXXX` prefixed methods. The `MustXXX` methods also provide the same functionality as the former methods, but they will **panic** if data doesn't satisfies the condition.