From 24ff72fd69e58c21226c579a42c75b9c045a7dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Dunglas?= Date: Wed, 26 Aug 2020 17:35:58 +0200 Subject: [PATCH] net/http: add a package to parse and serialize Structured Field Values Structured Field Values fields value for HTTP is an upcoming RFC defining data types for headers and trailers. This new package implements the specification. New methods are also added to the Header type to manipulate structured values. --- .gitmodules | 3 + src/go/build/deps_test.go | 11 + src/net/http/header.go | 55 +++++ src/net/http/header_test.go | 61 +++++ src/net/http/sfv/bareitem.go | 106 +++++++++ src/net/http/sfv/bareitem_test.go | 65 +++++ src/net/http/sfv/binary.go | 63 +++++ src/net/http/sfv/binary_test.go | 50 ++++ src/net/http/sfv/boolean.go | 53 +++++ src/net/http/sfv/boolean_test.go | 54 +++++ src/net/http/sfv/decimal.go | 67 ++++++ src/net/http/sfv/decimal_test.go | 46 ++++ src/net/http/sfv/decode.go | 67 ++++++ src/net/http/sfv/decode_test.go | 24 ++ src/net/http/sfv/dictionary.go | 171 +++++++++++++ src/net/http/sfv/dictionary_test.go | 125 ++++++++++ src/net/http/sfv/encode.go | 46 ++++ src/net/http/sfv/encode_test.go | 31 +++ src/net/http/sfv/example_test.go | 35 +++ src/net/http/sfv/httpwg_test.go | 303 ++++++++++++++++++++++++ src/net/http/sfv/innerlist.go | 95 ++++++++ src/net/http/sfv/innerlist_test.go | 34 +++ src/net/http/sfv/integer.go | 124 ++++++++++ src/net/http/sfv/integer_test.go | 81 +++++++ src/net/http/sfv/item.go | 77 ++++++ src/net/http/sfv/item_test.go | 96 ++++++++ src/net/http/sfv/key.go | 85 +++++++ src/net/http/sfv/key_test.go | 69 ++++++ src/net/http/sfv/list.go | 99 ++++++++ src/net/http/sfv/list_test.go | 102 ++++++++ src/net/http/sfv/member.go | 13 + src/net/http/sfv/params.go | 141 +++++++++++ src/net/http/sfv/params_test.go | 114 +++++++++ src/net/http/sfv/string.go | 93 ++++++++ src/net/http/sfv/string_test.go | 79 ++++++ src/net/http/sfv/structured-field-tests | 1 + src/net/http/sfv/token.go | 75 ++++++ src/net/http/sfv/token_test.go | 91 +++++++ src/net/http/sfv/utils.go | 20 ++ 39 files changed, 2925 insertions(+) create mode 100644 .gitmodules create mode 100644 src/net/http/sfv/bareitem.go create mode 100644 src/net/http/sfv/bareitem_test.go create mode 100644 src/net/http/sfv/binary.go create mode 100644 src/net/http/sfv/binary_test.go create mode 100644 src/net/http/sfv/boolean.go create mode 100644 src/net/http/sfv/boolean_test.go create mode 100644 src/net/http/sfv/decimal.go create mode 100644 src/net/http/sfv/decimal_test.go create mode 100644 src/net/http/sfv/decode.go create mode 100644 src/net/http/sfv/decode_test.go create mode 100644 src/net/http/sfv/dictionary.go create mode 100644 src/net/http/sfv/dictionary_test.go create mode 100644 src/net/http/sfv/encode.go create mode 100644 src/net/http/sfv/encode_test.go create mode 100644 src/net/http/sfv/example_test.go create mode 100644 src/net/http/sfv/httpwg_test.go create mode 100644 src/net/http/sfv/innerlist.go create mode 100644 src/net/http/sfv/innerlist_test.go create mode 100644 src/net/http/sfv/integer.go create mode 100644 src/net/http/sfv/integer_test.go create mode 100644 src/net/http/sfv/item.go create mode 100644 src/net/http/sfv/item_test.go create mode 100644 src/net/http/sfv/key.go create mode 100644 src/net/http/sfv/key_test.go create mode 100644 src/net/http/sfv/list.go create mode 100644 src/net/http/sfv/list_test.go create mode 100644 src/net/http/sfv/member.go create mode 100644 src/net/http/sfv/params.go create mode 100644 src/net/http/sfv/params_test.go create mode 100644 src/net/http/sfv/string.go create mode 100644 src/net/http/sfv/string_test.go create mode 160000 src/net/http/sfv/structured-field-tests create mode 100644 src/net/http/sfv/token.go create mode 100644 src/net/http/sfv/token_test.go create mode 100644 src/net/http/sfv/utils.go diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000000..f8934e245dbb65 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/net/http/sfv/structured-field-tests"] + path = src/net/http/sfv/structured-field-tests + url = https://github.com/httpwg/structured-field-tests diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index fa8ecf10f42815..e8eaac38836f9f 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -414,6 +414,16 @@ var depsRules = ` < golang.org/x/net/idna < golang.org/x/net/http/httpguts, golang.org/x/net/http/httpproxy; + encoding/base64, + errors, + fmt, + io, + math, + reflect, + strconv, + strings + < net/http/sfv; + NET, crypto/tls < net/http/httptrace; @@ -423,6 +433,7 @@ var depsRules = ` golang.org/x/net/http2/hpack, net/http/internal, net/http/httptrace, + net/http/sfv, mime/multipart, log < net/http; diff --git a/src/net/http/header.go b/src/net/http/header.go index b9b53911f38441..6a6c9ac924e96c 100644 --- a/src/net/http/header.go +++ b/src/net/http/header.go @@ -7,6 +7,7 @@ package http import ( "io" "net/http/httptrace" + "net/http/sfv" "net/textproto" "sort" "strings" @@ -37,6 +38,24 @@ func (h Header) Set(key, value string) { textproto.MIMEHeader(h).Set(key, value) } +// SetStructured sets the header entries associated with key to the +// given structured value. +// It encodes the structured value as text before setting it. +// It replaces any existing values associated with key. +// The key is case insensitive; it is canonicalized by +// textproto.CanonicalMIMEHeaderKey. +// To use non-canonical keys, assign to the map directly. +func (h Header) SetStructured(key string, value sfv.StructuredFieldValue) error { + v, err := sfv.Marshal(value) + if err != nil { + return err + } + + h.Set(key, v) + + return nil +} + // Get gets the first value associated with the given key. If // there are no values associated with the key, Get returns "". // It is case insensitive; textproto.CanonicalMIMEHeaderKey is @@ -46,6 +65,42 @@ func (h Header) Get(key string) string { return textproto.MIMEHeader(h).Get(key) } +// GetItem returns the item +// (according to the Structured Field Values specification) +// associated with the headers having the given key. +// If the key doesn't exist or if the value format +// is not a valid item, an error is returned. +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical keys, +// access the map directly. +func (h Header) GetItem(key string) (sfv.Item, error) { + return sfv.UnmarshalItem(h.Values(key)) +} + +// GetList returns the list +// (according to the Structured Field Values specification) +// associated with the headers having the given key. +// If the key doesn't exist or if the value format +// is not a valid list, an error is returned. +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical keys, +// access the map directly. +func (h Header) GetList(key string) (sfv.List, error) { + return sfv.UnmarshalList(h.Values(key)) +} + +// GetDictionary returns the dictionary +// (according to the Structured Field Values specification) +// associated with the given key. +// If the key doesn't exist or if the value format +// is not a valid dictionary, an error is returned. +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// used to canonicalize the provided key. To use non-canonical keys, +// access the map directly. +func (h Header) GetDictionary(key string) (*sfv.Dictionary, error) { + return sfv.UnmarshalDictionary(h.Values(key)) +} + // Values returns all values associated with the given key. // It is case insensitive; textproto.CanonicalMIMEHeaderKey is // used to canonicalize the provided key. To use non-canonical diff --git a/src/net/http/header_test.go b/src/net/http/header_test.go index 47893629194b6a..ed6438366336b8 100644 --- a/src/net/http/header_test.go +++ b/src/net/http/header_test.go @@ -7,6 +7,7 @@ package http import ( "bytes" "internal/race" + "net/http/sfv" "reflect" "runtime" "testing" @@ -251,3 +252,63 @@ func TestCloneOrMakeHeader(t *testing.T) { }) } } + +func TestSetStructured(t *testing.T) { + bar := sfv.NewItem(sfv.Token("bar")) + bar.Params.Add("baz", 42) + + l := sfv.List{ + bar, + sfv.NewItem(false), + } + + d := sfv.NewDictionary() + d.Add("a", bar) + d.Add("b", sfv.NewItem(false)) + + tests := []struct { + key string + value sfv.StructuredFieldValue + err bool + serialized string + }{ + {"Item", sfv.NewItem("bar"), false, `"bar"`}, + {"List", l, false, `bar;baz=42, ?0`}, + {"Dict", d, false, `a=bar;baz=42, b=?0`}, + {"Dict", sfv.NewItem(999999999999999999), true, ""}, + } + + h := Header{} + for _, tt := range tests { + err := h.SetStructured(tt.key, tt.value) != nil + if err != tt.err { + t.Errorf("Got: %#v\nWant: %#v", err, tt.err) + } + + if err { + continue + } + + s := h.Get(tt.key) + if s != tt.serialized { + t.Errorf("Got: %#v\nWant: %#v", s, tt.serialized) + } + + var r sfv.StructuredFieldValue + switch tt.value.(type) { + case sfv.Item: + r, _ = h.GetItem(tt.key) + case sfv.List: + r, _ = h.GetList(tt.key) + case *sfv.Dictionary: + r, _ = h.GetDictionary(tt.key) + default: + panic("type not found") + } + + s, _ = sfv.Marshal(r) + if s != tt.serialized { + t.Errorf("Got: %#v\nWant: %#v", s, tt.serialized) + } + } +} diff --git a/src/net/http/sfv/bareitem.go b/src/net/http/sfv/bareitem.go new file mode 100644 index 00000000000000..b4566447b7e09a --- /dev/null +++ b/src/net/http/sfv/bareitem.go @@ -0,0 +1,106 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "fmt" + "reflect" + "strings" +) + +// ErrInvalidBareItem is returned when a bare item is invalid. +var ErrInvalidBareItem = errors.New( + "invalid bare item type (allowed types are bool, string, int64, float64, []byte and Token)", +) + +// assertBareItem asserts that v is a valid bare item +// according to https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#item. +// +// v can be either: +// +// * an integer (Section 3.3.1.) +// * a decimal (Section 3.3.2.) +// * a string (Section 3.3.3.) +// * a token (Section 3.3.4.) +// * a byte sequence (Section 3.3.5.) +// * a boolean (Section 3.3.6.) +func assertBareItem(v interface{}) { + switch v.(type) { + case bool, + string, + int, + int8, + int16, + int32, + int64, + uint, + uint8, + uint16, + uint32, + uint64, + float32, + float64, + []byte, + Token: + return + default: + panic(fmt.Errorf("%w: got %s", ErrInvalidBareItem, reflect.TypeOf(v))) + } +} + +// marshalBareItem serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-bare-item. +func marshalBareItem(b *strings.Builder, v interface{}) error { + switch v := v.(type) { + case bool: + return marshalBoolean(b, v) + case string: + return marshalString(b, v) + case int64: + return marshalInteger(b, v) + case int, int8, int16, int32: + return marshalInteger(b, reflect.ValueOf(v).Int()) + case uint, uint8, uint16, uint32, uint64: + // Casting an uint64 to an int64 is possible because the maximum allowed value is 999,999,999,999,999 + return marshalInteger(b, int64(reflect.ValueOf(v).Uint())) + case float32, float64: + return marshalDecimal(b, v.(float64)) + case []byte: + return marshalBinary(b, v) + case Token: + return v.marshalSFV(b) + default: + panic(ErrInvalidBareItem) + } +} + +// parseBareItem parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-bare-item. +func parseBareItem(s *scanner) (interface{}, error) { + if s.eof() { + return nil, &UnmarshalError{s.off, ErrUnexpectedEndOfString} + } + + c := s.data[s.off] + switch c { + case '"': + return parseString(s) + case '?': + return parseBoolean(s) + case '*': + return parseToken(s) + case ':': + return parseBinary(s) + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return parseNumber(s) + default: + if isAlpha(c) { + return parseToken(s) + } + + return nil, &UnmarshalError{s.off, ErrUnrecognizedCharacter} + } +} diff --git a/src/net/http/sfv/bareitem_test.go b/src/net/http/sfv/bareitem_test.go new file mode 100644 index 00000000000000..ffce294f4284aa --- /dev/null +++ b/src/net/http/sfv/bareitem_test.go @@ -0,0 +1,65 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "reflect" + "strings" + "testing" + "time" +) + +func TestParseBareItem(t *testing.T) { + data := []struct { + in string + out interface{} + err bool + }{ + {"?1", true, false}, + {"?0", false, false}, + {"22", int64(22), false}, + {"-2.2", -2.2, false}, + {`"foo"`, "foo", false}, + {"abc", Token("abc"), false}, + {"*abc", Token("*abc"), false}, + {":YWJj:", []byte("abc"), false}, + {"", nil, true}, + {"~", nil, true}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + i, err := parseBareItem(s) + if d.err && err == nil { + t.Errorf("parseBareItem(%s): error expected", d.in) + } + + if !d.err && !reflect.DeepEqual(d.out, i) { + t.Errorf("parseBareItem(%s) = %v, %v; %v, expected", d.in, i, err, d.out) + } + } +} + +func TestMarshalBareItem(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("The code did not panic") + } + }() + + var b strings.Builder + _ = marshalBareItem(&b, time.Second) +} + +func TestAssertBareItem(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("The code did not panic") + } + }() + + assertBareItem(time.Second) +} diff --git a/src/net/http/sfv/binary.go b/src/net/http/sfv/binary.go new file mode 100644 index 00000000000000..883cb2c9212f0a --- /dev/null +++ b/src/net/http/sfv/binary.go @@ -0,0 +1,63 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "encoding/base64" + "errors" + "strings" +) + +// ErrInvalidBinaryFormat is returned when the binary format is invalid. +var ErrInvalidBinaryFormat = errors.New("invalid binary format") + +// marshalBinary serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-binary. +func marshalBinary(b *strings.Builder, bs []byte) error { + if err := b.WriteByte(':'); err != nil { + return err + } + + buf := make([]byte, base64.StdEncoding.EncodedLen(len(bs))) + base64.StdEncoding.Encode(buf, bs) + + if _, err := b.Write(buf); err != nil { + return err + } + + return b.WriteByte(':') +} + +// parseBinary parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-binary. +func parseBinary(s *scanner) ([]byte, error) { + if s.eof() || s.data[s.off] != ':' { + return nil, &UnmarshalError{s.off, ErrInvalidBinaryFormat} + } + s.off++ + + start := s.off + + for !s.eof() { + c := s.data[s.off] + if c == ':' { + // base64decode + decoded, err := base64.StdEncoding.DecodeString(s.data[start:s.off]) + if err != nil { + return nil, &UnmarshalError{s.off, err} + } + s.off++ + + return decoded, nil + } + + if !isAlpha(c) && !isDigit(c) && c != '+' && c != '/' && c != '=' { + return nil, &UnmarshalError{s.off, ErrInvalidBinaryFormat} + } + s.off++ + } + + return nil, &UnmarshalError{s.off, ErrInvalidBinaryFormat} +} diff --git a/src/net/http/sfv/binary_test.go b/src/net/http/sfv/binary_test.go new file mode 100644 index 00000000000000..233947277393cc --- /dev/null +++ b/src/net/http/sfv/binary_test.go @@ -0,0 +1,50 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "bytes" + "strings" + "testing" +) + +func TestBinary(t *testing.T) { + var bd strings.Builder + _ = marshalBinary(&bd, []byte{4, 2}) + + if bd.String() != ":BAI=:" { + t.Error("marshalBinary(): invalid") + } +} + +func TestParseBinary(t *testing.T) { + data := []struct { + in string + out []byte + err bool + }{ + {":YWJj:", []byte("abc"), false}, + {":YW55IGNhcm5hbCBwbGVhc3VyZQ==:", []byte("any carnal pleasure"), false}, + {":YW55IGNhcm5hbCBwbGVhc3Vy:", []byte("any carnal pleasur"), false}, + {"", []byte{}, false}, + {":", []byte{}, false}, + {":YW55IGNhcm5hbCBwbGVhc3Vy", []byte{}, false}, + {":YW55IGNhcm5hbCBwbGVhc3Vy~", []byte{}, false}, + {":YW55IGNhcm5hbCBwbGVhc3VyZQ=:", []byte{}, false}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + i, err := parseBinary(s) + if d.err && err == nil { + t.Errorf("parseBinary(%s): error expected", d.in) + } + + if !d.err && !bytes.Equal(d.out, i) { + t.Errorf("parseBinary(%s) = %v, %v; %v, expected", d.in, i, err, d.out) + } + } +} diff --git a/src/net/http/sfv/boolean.go b/src/net/http/sfv/boolean.go new file mode 100644 index 00000000000000..b8611aa7b6e768 --- /dev/null +++ b/src/net/http/sfv/boolean.go @@ -0,0 +1,53 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "io" +) + +// ErrInvalidBooleanFormat is returned when a boolean format is invalid. +var ErrInvalidBooleanFormat = errors.New("invalid boolean format") + +// marshalBoolean serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-boolean. +func marshalBoolean(bd io.StringWriter, b bool) error { + if b { + _, err := bd.WriteString("?1") + + return err + } + + _, err := bd.WriteString("?0") + + return err +} + +// parseBoolean parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-boolean. +func parseBoolean(s *scanner) (bool, error) { + if s.eof() || s.data[s.off] != '?' { + return false, &UnmarshalError{s.off, ErrInvalidBooleanFormat} + } + s.off++ + + if s.eof() { + return false, &UnmarshalError{s.off, ErrInvalidBooleanFormat} + } + + switch s.data[s.off] { + case '0': + s.off++ + + return false, nil + case '1': + s.off++ + + return true, nil + } + + return false, &UnmarshalError{s.off, ErrInvalidBooleanFormat} +} diff --git a/src/net/http/sfv/boolean_test.go b/src/net/http/sfv/boolean_test.go new file mode 100644 index 00000000000000..1783fe63b8b27f --- /dev/null +++ b/src/net/http/sfv/boolean_test.go @@ -0,0 +1,54 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" + "testing" +) + +func TestBooleanMarshalSFV(t *testing.T) { + var b strings.Builder + + _ = marshalBoolean(&b, true) + + if b.String() != "?1" { + t.Error("Invalid marshaling") + } + + b.Reset() + _ = marshalBoolean(&b, false) + + if b.String() != "?0" { + t.Error("Invalid marshaling") + } +} + +func TestParseBoolean(t *testing.T) { + data := []struct { + in string + out bool + err bool + }{ + {"?1", true, false}, + {"?0", false, false}, + {"?2", false, true}, + {"", false, true}, + {"?", false, true}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + i, err := parseBoolean(s) + if d.err && err == nil { + t.Errorf("parseBoolean(%s): error expected", d.in) + } + + if !d.err && d.out != i { + t.Errorf("parseBoolean(%s) = %v, %v; %v, expected", d.in, i, err, d.out) + } + } +} diff --git a/src/net/http/sfv/decimal.go b/src/net/http/sfv/decimal.go new file mode 100644 index 00000000000000..7ca7950cae7d8c --- /dev/null +++ b/src/net/http/sfv/decimal.go @@ -0,0 +1,67 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "io" + "math" + "strconv" +) + +const maxDecDigit = 3 + +// ErrInvalidDecimal is returned when a decimal is invalid. +var ErrInvalidDecimal = errors.New("the integer portion is larger than 12 digits: invalid decimal") + +// marshalDecimal serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-decimal. +// +// TODO(dunglas): add support for decimal float type when one will be available +// (https://github.com/golang/go/issues/19787) +func marshalDecimal(b io.StringWriter, d float64) error { + const TH = 0.001 + + rounded := math.RoundToEven(d/TH) * TH + int, frac := math.Modf(math.RoundToEven(d/TH) * TH) + + if int < -999999999999 || int > 999999999999 { + return ErrInvalidDecimal + } + + if _, err := b.WriteString(strconv.FormatFloat(rounded, 'f', -1, 64)); err != nil { + return err + } + + if frac == 0 { + _, err := b.WriteString(".0") + + return err + } + + return nil +} + +func parseDecimal(s *scanner, decSepOff int, str string, neg bool) (float64, error) { + if decSepOff == s.off-1 { + return 0, &UnmarshalError{s.off, ErrInvalidDecimalFormat} + } + + if len(s.data[decSepOff+1:s.off]) > maxDecDigit { + return 0, &UnmarshalError{s.off, ErrNumberOutOfRange} + } + + i, err := strconv.ParseFloat(str, 64) + if err != nil { + // Should never happen + return 0, &UnmarshalError{s.off, err} + } + + if neg { + i = -i + } + + return i, nil +} diff --git a/src/net/http/sfv/decimal_test.go b/src/net/http/sfv/decimal_test.go new file mode 100644 index 00000000000000..2700a9d4f6381c --- /dev/null +++ b/src/net/http/sfv/decimal_test.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" + "testing" +) + +func TestDecimalMarshalSFV(t *testing.T) { + data := []struct { + in float64 + expected string + valid bool + }{ + {10.0, "10.0", true}, + {-10.123, "-10.123", true}, + {10.1236, "10.124", true}, + {-10.0, "-10.0", true}, + {0, "0.0", true}, + {-999999999999.0, "-999999999999.0", true}, + {999999999999.0, "999999999999.0", true}, + {9999999999999, "", false}, + {-9999999999999.0, "", false}, + {9999999999999.0, "", false}, + } + + var b strings.Builder + + for _, d := range data { + b.Reset() + + err := marshalDecimal(&b, d.in) + if d.valid && err != nil { + t.Errorf("error not expected for %v, got %v", d.in, err) + } else if !d.valid && err == nil { + t.Errorf("error expected for %v, got %v", d.in, err) + } + + if b.String() != d.expected { + t.Errorf("got %v; want %v", b.String(), d.expected) + } + } +} diff --git a/src/net/http/sfv/decode.go b/src/net/http/sfv/decode.go new file mode 100644 index 00000000000000..4ff65235037b82 --- /dev/null +++ b/src/net/http/sfv/decode.go @@ -0,0 +1,67 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "fmt" +) + +// ErrUnexpectedEndOfString is returned when the end of string is unexpected. +var ErrUnexpectedEndOfString = errors.New("unexpected end of string") + +// ErrUnrecognizedCharacter is returned when an unrecognized character in encountered. +var ErrUnrecognizedCharacter = errors.New("unrecognized character") + +// UnmarshalError contains the underlying parsing error and the position at which it occurred. +type UnmarshalError struct { + off int + err error +} + +func (e *UnmarshalError) Error() string { + if e.err != nil { + return fmt.Sprintf("%s: character %d", e.err, e.off) + } + + return fmt.Sprintf("unmarshal error: character %d", e.off) +} + +func (e *UnmarshalError) Unwrap() error { + return e.err +} + +type scanner struct { + data string + off int +} + +// scanWhileSp consumes spaces. +func (s *scanner) scanWhileSp() { + for !s.eof() { + if s.data[s.off] != ' ' { + return + } + + s.off++ + } +} + +// scanWhileOWS consumes optional white space (OWS) characters. +func (s *scanner) scanWhileOWS() { + for !s.eof() { + c := s.data[s.off] + if c != ' ' && c != '\t' { + return + } + + s.off++ + } +} + +// eof returns true if the parser consumed all available characters. +func (s *scanner) eof() bool { + return s.off == len(s.data) +} diff --git a/src/net/http/sfv/decode_test.go b/src/net/http/sfv/decode_test.go new file mode 100644 index 00000000000000..c5c05e5553ed57 --- /dev/null +++ b/src/net/http/sfv/decode_test.go @@ -0,0 +1,24 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import "testing" + +func TestDecodeError(t *testing.T) { + _, err := UnmarshalItem([]string{"invalid-é"}) + + if err.Error() != "unmarshal error: character 8" { + t.Error("invalid error") + } + + _, err = UnmarshalItem([]string{`"é"`}) + if err.Error() != "invalid string format: character 2" { + t.Error("invalid error") + } + + if err.(*UnmarshalError).Unwrap().Error() != "invalid string format" { + t.Error("invalid wrapped error") + } +} diff --git a/src/net/http/sfv/dictionary.go b/src/net/http/sfv/dictionary.go new file mode 100644 index 00000000000000..5618399f1329ed --- /dev/null +++ b/src/net/http/sfv/dictionary.go @@ -0,0 +1,171 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "strings" +) + +// Dictionary is an ordered map of name-value pairs. +// See https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#dictionary +// Values can be: +// * Item (Section 3.3.) +// * Inner List (Section 3.1.1.) +type Dictionary struct { + names []string + values map[string]Member +} + +// ErrInvalidDictionaryFormat is returned when a dictionary value is invalid. +var ErrInvalidDictionaryFormat = errors.New("invalid dictionary format") + +// NewDictionary creates a new ordered map. +func NewDictionary() *Dictionary { + d := Dictionary{} + d.names = []string{} + d.values = map[string]Member{} + + return &d +} + +// Get retrieves a member. +func (d *Dictionary) Get(k string) (Member, bool) { + v, ok := d.values[k] + + return v, ok +} + +// Add appends a new member to the ordered list. +func (d *Dictionary) Add(k string, v Member) { + if _, exists := d.values[k]; !exists { + d.names = append(d.names, k) + } + + d.values[k] = v +} + +// Del removes a member from the ordered list. +func (d *Dictionary) Del(key string) bool { + if _, ok := d.values[key]; !ok { + return false + } + + for i, k := range d.names { + if k == key { + d.names = append(d.names[:i], d.names[i+1:]...) + + break + } + } + + delete(d.values, key) + + return true +} + +// Names retrieves the list of member names in the appropriate order. +func (d *Dictionary) Names() []string { + return d.names +} + +func (d *Dictionary) marshalSFV(b *strings.Builder) error { + last := len(d.names) - 1 + + for m, k := range d.names { + if err := marshalKey(b, k); err != nil { + return err + } + + v := d.values[k] + + if item, ok := v.(Item); ok && item.Value == true { + if err := item.Params.marshalSFV(b); err != nil { + return err + } + } else { + if err := b.WriteByte('='); err != nil { + return err + } + if err := v.marshalSFV(b); err != nil { + return err + } + } + + if m != last { + if _, err := b.WriteString(", "); err != nil { + return err + } + } + } + + return nil +} + +// UnmarshalDictionary parses a dictionary as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-dictionary. +func UnmarshalDictionary(v []string) (*Dictionary, error) { + s := &scanner{ + data: strings.Join(v, ","), + } + + s.scanWhileSp() + + sfv, err := parseDictionary(s) + if err != nil { + return sfv, err + } + + return sfv, nil +} + +func parseDictionary(s *scanner) (*Dictionary, error) { + d := NewDictionary() + + for !s.eof() { + k, err := parseKey(s) + if err != nil { + return nil, err + } + + var m Member + + if !s.eof() && s.data[s.off] == '=' { + s.off++ + m, err = parseItemOrInnerList(s) + + if err != nil { + return nil, err + } + } else { + p, err := parseParams(s) + if err != nil { + return nil, err + } + m = Item{true, p} + } + + d.Add(k, m) + s.scanWhileOWS() + + if s.eof() { + return d, nil + } + + if s.data[s.off] != ',' { + return nil, &UnmarshalError{s.off, ErrInvalidDictionaryFormat} + } + s.off++ + + s.scanWhileOWS() + + if s.eof() { + // there is a trailing comma + return nil, &UnmarshalError{s.off, ErrInvalidDictionaryFormat} + } + } + + return d, nil +} diff --git a/src/net/http/sfv/dictionary_test.go b/src/net/http/sfv/dictionary_test.go new file mode 100644 index 00000000000000..3c7fd4ac016416 --- /dev/null +++ b/src/net/http/sfv/dictionary_test.go @@ -0,0 +1,125 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "reflect" + "strings" + "testing" +) + +func TestDictionnary(t *testing.T) { + dict := NewDictionary() + + add := []struct { + in string + expected Member + valid bool + }{ + {"f_o1o3-", NewItem(10.0), true}, + {"deleteme", NewItem(""), true}, + {"*f0.o*", NewItem(""), true}, + {"t", NewItem(true), true}, + {"f", NewItem(false), true}, + {"b", NewItem([]byte{0, 1}), true}, + {"0foo", NewItem(""), false}, + {"mAj", NewItem(""), false}, + {"_foo", NewItem(""), false}, + {"foo", NewItem(Token("é")), false}, + } + + var b strings.Builder + + for _, d := range add { + vDict := NewDictionary() + vDict.Add(d.in, d.expected) + + b.Reset() + + if valid := vDict.marshalSFV(&b) == nil; valid != d.valid { + t.Errorf("(%v, %v).isValid() = %v; %v expected", d.in, d.expected, valid, d.valid) + } + + if d.valid { + dict.Add(d.in, d.expected) + } + } + + i := NewItem(123.0) + dict.Add("f_o1o3-", i) + + newValue, _ := dict.Get("f_o1o3-") + if newValue != i { + t.Errorf(`Add("f_o1o3-") must overwrite the existing value`) + } + + if !dict.Del("deleteme") { + t.Errorf(`Del("deleteme") must return true`) + } + + if dict.Del("deleteme") { + t.Errorf(`the second call to Del("deleteme") must return false`) + } + + if v, ok := dict.Get("*f0.o*"); v.(Item).Value != "" || !ok { + t.Errorf(`Get("*f0.o*") = %v, %v; "", true expected`, v, ok) + } + + if v, ok := dict.Get("notexist"); v != nil || ok { + t.Errorf(`Get("notexist") = %v, %v; nil, false expected`, v, ok) + } + + k := dict.Names() + if len(k) != 5 { + t.Errorf(`Names() = %v; {"f_o1o3-", "*f0.o*"} expected`, k) + } + + m, _ := dict.Get("f_o1o3-") + i = m.(Item) + i.Params.Add("foo", 9.5) + + b.Reset() + _ = dict.marshalSFV(&b) + + if b.String() != `f_o1o3-=123.0;foo=9.5, *f0.o*="", t, f=?0, b=:AAE=:` { + t.Errorf(`Dictionnary.marshalSFV(): invalid serialization: %v`, b.String()) + } +} + +func TestUnmarshalDictionary(t *testing.T) { + d1 := NewDictionary() + d1.Add("a", NewItem(false)) + d1.Add("b", NewItem(true)) + + c := NewItem(true) + c.Params.Add("foo", Token("bar")) + d1.Add("c", c) + + data := []struct { + in []string + expected *Dictionary + valid bool + }{ + {[]string{"a=?0, b, c; foo=bar"}, d1, false}, + {[]string{"a=?0, b", "c; foo=bar"}, d1, false}, + {[]string{""}, NewDictionary(), false}, + {[]string{"é"}, nil, true}, + {[]string{`foo="é"`}, nil, true}, + {[]string{`foo;é`}, nil, true}, + {[]string{`f="foo" é`}, nil, true}, + {[]string{`f="foo",`}, nil, true}, + } + + for _, d := range data { + l, err := UnmarshalDictionary(d.in) + if d.valid && err == nil { + t.Errorf("UnmarshalDictionary(%s): error expected", d.in) + } + + if !d.valid && !reflect.DeepEqual(d.expected, l) { + t.Errorf("UnmarshalDictionary(%s) = %v, %v; %v, expected", d.in, l, err, d.expected) + } + } +} diff --git a/src/net/http/sfv/encode.go b/src/net/http/sfv/encode.go new file mode 100644 index 00000000000000..88eacd82c4d3f8 --- /dev/null +++ b/src/net/http/sfv/encode.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpsfv implements serializing and parsing +// of Structured Field Values for HTTP as defined in the draft-ietf-httpbis-header-structure Internet-Draft. +// +// Structured Field Values are either lists, dictionaries or items. Dedicated types are provided for all of them. +// Dedicated types are also used for tokens, parameters and inner lists. +// Other values are stored in native types: +// +// int64, for integers +// float64, for decimals +// string, for strings +// byte[], for byte sequences +// bool, for booleans +// +// The specification is available at https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html. +package sfv + +import ( + "strings" +) + +// marshaler is the interface implemented by types that can marshal themselves into valid SFV. +type marshaler interface { + marshalSFV(b *strings.Builder) error +} + +// StructuredFieldValue represents a List, a Dictionary or an Item. +type StructuredFieldValue interface { + marshaler +} + +// Marshal returns the HTTP Structured Value serialization of v +// as defined in https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#text-serialize. +// +// v must be a List, a Dictionary or an Item. +func Marshal(v StructuredFieldValue) (string, error) { + var b strings.Builder + if err := v.marshalSFV(&b); err != nil { + return "", err + } + + return b.String(), nil +} diff --git a/src/net/http/sfv/encode_test.go b/src/net/http/sfv/encode_test.go new file mode 100644 index 00000000000000..f51e1c38efc0c2 --- /dev/null +++ b/src/net/http/sfv/encode_test.go @@ -0,0 +1,31 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import "testing" + +func TestMarshal(t *testing.T) { + i := NewItem(22.1) + i.Params.Add("foo", true) + i.Params.Add("bar", Token("baz")) + + d := NewDictionary() + d.Add("i", i) + + tok := NewItem(Token("foo")) + tok.Params.Add("a", "b") + d.Add("tok", tok) + + res, _ := Marshal(d) + if res != `i=22.1;foo;bar=baz, tok=foo;a="b"` { + t.Errorf("marshal: bad result") + } +} + +func TestMarshalError(t *testing.T) { + if _, err := Marshal(NewItem(Token("à"))); err == nil { + t.Errorf("marshal: error expected") + } +} diff --git a/src/net/http/sfv/example_test.go b/src/net/http/sfv/example_test.go new file mode 100644 index 00000000000000..6881886e4751f1 --- /dev/null +++ b/src/net/http/sfv/example_test.go @@ -0,0 +1,35 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "fmt" + "log" +) + +func ExampleUnmarshalList() { + v, err := UnmarshalList([]string{`"/member/*/author", "/member/*/comments"`}) + if err != nil { + log.Fatalln("error: ", err) + } + + fmt.Println("authors selector: ", v[0].(Item).Value) + fmt.Println("comments selector: ", v[1].(Item).Value) + // Output: + // authors selector: /member/*/author + // comments selector: /member/*/comments +} + +func ExampleMarshal() { + p := List{NewItem("/member/*/author"), NewItem("/member/*/comments")} + + v, err := Marshal(p) + if err != nil { + log.Fatalln("error: ", err) + } + + fmt.Println(v) + // Output: "/member/*/author", "/member/*/comments" +} diff --git a/src/net/http/sfv/httpwg_test.go b/src/net/http/sfv/httpwg_test.go new file mode 100644 index 00000000000000..57db4884103d8b --- /dev/null +++ b/src/net/http/sfv/httpwg_test.go @@ -0,0 +1,303 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "encoding/base32" + "encoding/json" + "os" + "reflect" + "strings" + "testing" +) + +const ( + ITEM = "item" + LIST = "list" + DICTIONARY = "dictionary" +) + +// test represents a test from the official test suite for the specification. +// See https://github.com/httpwg/structured-field-tests. +type test struct { + Name string `json:"name"` + Raw []string `json:"raw"` + HeaderType string `json:"header_type"` + Expected interface{} `json:"expected"` + MustFail bool `json:"must_fail"` + CanFail bool `json:"can_fail"` + Canonical []string `json:"canonical"` +} + +func valToBareItem(e interface{}) interface{} { + bareItem, ok := e.(map[string]interface{}) + if !ok { + if number, ok := e.(json.Number); ok { + if strings.Contains(number.String(), ".") { + bi, _ := number.Float64() + + return bi + } + + bi, _ := number.Int64() + + return bi + } + + return e + } + + switch bareItem["__type"] { + case "binary": + bi, _ := base32.StdEncoding.DecodeString(bareItem["value"].(string)) + + return bi + case "token": + return Token(bareItem["value"].(string)) + default: + } + + panic("unknown type " + bareItem["__type"].(string)) +} + +func populateParams(p *Params, e interface{}) { + ex := e.([]interface{}) + for _, l := range ex { + v := l.([]interface{}) + p.Add(v[0].(string), valToBareItem(v[1])) + } +} + +func valToItem(e interface{}) Item { + if e == nil { + return Item{} + } + + ex := e.([]interface{}) + i := NewItem(valToBareItem(ex[0])) + populateParams(i.Params, ex[1]) + + return i +} + +func valToInnerList(e []interface{}) InnerList { + il := InnerList{} + il.Params = NewParams() + + for _, i := range e[0].([]interface{}) { + il.Items = append(il.Items, valToItem(i)) + } + + populateParams(il.Params, e[1]) + + return il +} + +func valToMember(e interface{}) Member { + il := e.([]interface{}) + if _, ok := il[0].([]interface{}); ok { + return valToInnerList(il) + } + + return valToItem(e) +} + +func valToList(e interface{}) List { + if e == nil { + return nil + } + + ex := e.([]interface{}) + if len(ex) == 0 { + return nil + } + + l := List{} + for _, m := range ex { + l = append(l, valToMember(m)) + } + + return l +} + +func valToDictionary(e interface{}) *Dictionary { + if e == nil { + return nil + } + + ex := e.([]interface{}) + d := NewDictionary() + + for _, v := range ex { + m := v.([]interface{}) + d.Add(m[0].(string), valToMember(m[1])) + } + + return d +} + +func TestOfficialTestSuiteParsing(t *testing.T) { + const dir = "structured-field-tests/" + f, _ := os.Open(dir) + files, _ := f.Readdir(-1) + + for _, fi := range files { + n := fi.Name() + if !strings.HasSuffix(n, ".json") { + continue + } + + file, _ := os.Open(dir + n) + dec := json.NewDecoder(file) + dec.UseNumber() + + var tests []test + _ = dec.Decode(&tests) + + for _, te := range tests { + var ( + expected, got StructuredFieldValue + err error + ) + + switch te.HeaderType { + case ITEM: + expected = valToItem(te.Expected) + got, err = UnmarshalItem(te.Raw) + case LIST: + expected = valToList(te.Expected) + got, err = UnmarshalList(te.Raw) + case DICTIONARY: + expected = valToDictionary(te.Expected) + got, err = UnmarshalDictionary(te.Raw) + default: + panic("unknown header type") + } + + if te.MustFail && err == nil { + t.Errorf("%s: %s: must fail", n, te.Name) + + continue + } + + if (!te.MustFail && !te.CanFail) && err != nil { + t.Errorf("%s: %s: must not fail, got error %s", n, te.Name, err) + + continue + } + + if err == nil && !reflect.DeepEqual(expected, got) { + t.Errorf("%s: %s: %#v expected, got %#v", n, te.Name, expected, got) + } + } + } +} + +func BenchmarkParsingOfficialExamples(b *testing.B) { + file, _ := os.Open("structured-field-tests/examples.json") + dec := json.NewDecoder(file) + + var tests []test + _ = dec.Decode(&tests) + + for n := 0; n < b.N; n++ { + for _, te := range tests { + switch te.HeaderType { + case ITEM: + _, _ = UnmarshalItem(te.Raw) + case LIST: + _, _ = UnmarshalList(te.Raw) + case DICTIONARY: + _, _ = UnmarshalDictionary(te.Raw) + } + } + } +} + +func BenchmarkSerializingOfficialExamples(b *testing.B) { + file, _ := os.Open("structured-field-tests/examples.json") + dec := json.NewDecoder(file) + dec.UseNumber() + + var tests []test + _ = dec.Decode(&tests) + + var sfv []StructuredFieldValue + + for _, te := range tests { + if te.CanFail || te.MustFail { + continue + } + + switch te.HeaderType { + case ITEM: + sfv = append(sfv, valToItem(te.Expected)) + case LIST: + sfv = append(sfv, valToList(te.Expected)) + case DICTIONARY: + sfv = append(sfv, valToDictionary(te.Expected)) + } + } + + for n := 0; n < b.N; n++ { + for _, v := range sfv { + _, _ = Marshal(v) + } + } +} + +func TestOfficialTestSuiteSerialization(t *testing.T) { + const dir = "structured-field-tests/serialisation-tests/" + + f, _ := os.Open(dir) + files, _ := f.Readdir(-1) + + for _, fi := range files { + n := fi.Name() + if !strings.HasSuffix(n, ".json") { + continue + } + + file, _ := os.Open(dir + n) + dec := json.NewDecoder(file) + dec.UseNumber() + + var tests []test + _ = dec.Decode(&tests) + + for _, te := range tests { + var sfv StructuredFieldValue + + switch te.HeaderType { + case ITEM: + sfv = valToItem(te.Expected) + case LIST: + sfv = valToList(te.Expected) + case DICTIONARY: + sfv = valToDictionary(te.Expected) + default: + panic("unknown header type") + } + + canonical, err := Marshal(sfv) + + if te.MustFail && err == nil { + t.Errorf("%s: %s: must fail", n, te.Name) + + continue + } + + if (!te.MustFail && !te.CanFail) && err != nil { + t.Errorf("%s: %s: must not fail, got error %s", n, te.Name, err) + + continue + } + + if err == nil && te.Canonical[0] != canonical { + t.Errorf("%s: %s: %#v expected, got %#v", n, te.Name, te.Canonical[0], canonical) + } + } + } +} diff --git a/src/net/http/sfv/innerlist.go b/src/net/http/sfv/innerlist.go new file mode 100644 index 00000000000000..24c329e3bbd576 --- /dev/null +++ b/src/net/http/sfv/innerlist.go @@ -0,0 +1,95 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "strings" +) + +// ErrInvalidInnerListFormat is returned when an inner list format is invalid. +var ErrInvalidInnerListFormat = errors.New("invalid inner list format") + +// InnerList represents an inner list as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#inner-list. +type InnerList struct { + Items []Item + Params *Params +} + +func (il InnerList) member() { +} + +// marshalSFV serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-innerlist. +func (il InnerList) marshalSFV(b *strings.Builder) error { + if err := b.WriteByte('('); err != nil { + return err + } + + l := len(il.Items) + for i := 0; i < l; i++ { + if err := il.Items[i].marshalSFV(b); err != nil { + return err + } + + if i != l-1 { + if err := b.WriteByte(' '); err != nil { + return err + } + } + } + + if err := b.WriteByte(')'); err != nil { + return err + } + + return il.Params.marshalSFV(b) +} + +// parseInnerList parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-item-or-list. +func parseInnerList(s *scanner) (InnerList, error) { + if s.eof() || s.data[s.off] != '(' { + return InnerList{}, &UnmarshalError{s.off, ErrInvalidInnerListFormat} + } + s.off++ + + il := InnerList{nil, nil} + + for !s.eof() { + s.scanWhileSp() + + if s.eof() { + return InnerList{}, &UnmarshalError{s.off, ErrInvalidInnerListFormat} + } + + if s.data[s.off] == ')' { + s.off++ + + p, err := parseParams(s) + if err != nil { + return InnerList{}, err + } + + il.Params = p + + return il, nil + } + + i, err := parseItem(s) + if err != nil { + return InnerList{}, err + } + + if s.eof() || (s.data[s.off] != ')' && s.data[s.off] != ' ') { + return InnerList{}, &UnmarshalError{s.off, ErrInvalidInnerListFormat} + } + + il.Items = append(il.Items, i) + } + + return InnerList{}, &UnmarshalError{s.off, ErrInvalidInnerListFormat} +} diff --git a/src/net/http/sfv/innerlist_test.go b/src/net/http/sfv/innerlist_test.go new file mode 100644 index 00000000000000..c69f63c65101e2 --- /dev/null +++ b/src/net/http/sfv/innerlist_test.go @@ -0,0 +1,34 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" + "testing" +) + +func TestInnerList(t *testing.T) { + foo := NewItem("foo") + foo.Params.Add("a", true) + foo.Params.Add("b", 1936) + + bar := NewItem(Token("bar")) + bar.Params.Add("y", []byte{1, 3, 1, 2}) + + params := NewParams() + params.Add("d", 18.71) + + i := InnerList{ + []Item{foo, bar}, + params, + } + + var b strings.Builder + _ = i.marshalSFV(&b) + + if b.String() != `("foo";a;b=1936 bar;y=:AQMBAg==:);d=18.71` { + t.Errorf("invalid marshalSFV(): %v", b.String()) + } +} diff --git a/src/net/http/sfv/integer.go b/src/net/http/sfv/integer.go new file mode 100644 index 00000000000000..a7a86198bbea70 --- /dev/null +++ b/src/net/http/sfv/integer.go @@ -0,0 +1,124 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "io" + "strconv" +) + +const maxDigit = 12 + +// ErrNotDigit is returned when a character should be a digit but isn't. +var ErrNotDigit = errors.New("character is not a digit") + +// ErrNumberOutOfRange is returned when the number is too large according to the specification. +var ErrNumberOutOfRange = errors.New("integer or decimal out of range") + +// ErrInvalidDecimalFormat is returned when the decimal format is invalid. +var ErrInvalidDecimalFormat = errors.New("invalid decimal format") + +const ( + typeInteger = iota + typeDecimal +) + +// marshalInteger serialized as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#integer. +func marshalInteger(b io.StringWriter, i int64) error { + if i < -999999999999999 || i > 999999999999999 { + return ErrNumberOutOfRange + } + + _, err := b.WriteString(strconv.FormatInt(i, 10)) + + return err +} + +// parseNumber parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-number. +func parseNumber(s *scanner) (interface{}, error) { + neg := isNeg(s) + if neg && s.eof() { + return 0, &UnmarshalError{s.off, ErrUnexpectedEndOfString} + } + + if !isDigit(s.data[s.off]) { + return 0, &UnmarshalError{s.off, ErrNotDigit} + } + + start := s.off + s.off++ + + var ( + decSepOff int + t int = typeInteger + ) + + for s.off < len(s.data) { + size := s.off - start + if (t == typeInteger && (size >= 15)) || size >= 16 { + return 0, &UnmarshalError{s.off, ErrNumberOutOfRange} + } + + c := s.data[s.off] + if isDigit(c) { + s.off++ + + continue + } + + if t == typeInteger && c == '.' { + if size > maxDigit { + return 0, &UnmarshalError{s.off, ErrNumberOutOfRange} + } + + t = typeDecimal + decSepOff = s.off + s.off++ + + continue + } + + break + } + + str := s.data[start:s.off] + + if t == typeInteger { + return parseInteger(str, neg, s.off) + } + + return parseDecimal(s, decSepOff, str, neg) +} + +func isNeg(s *scanner) bool { + if s.data[s.off] == '-' { + s.off++ + + return true + } + + return false +} + +func parseInteger(str string, neg bool, off int) (int64, error) { + i, err := strconv.ParseInt(str, 10, 64) + if err != nil { + // Should never happen + return 0, &UnmarshalError{off, err} + } + + if neg { + i = -i + } + + if i < -999999999999999 || i > 999999999999999 { + return 0, &UnmarshalError{off, ErrNumberOutOfRange} + } + + return i, err +} diff --git a/src/net/http/sfv/integer_test.go b/src/net/http/sfv/integer_test.go new file mode 100644 index 00000000000000..14b612f810b696 --- /dev/null +++ b/src/net/http/sfv/integer_test.go @@ -0,0 +1,81 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" + "testing" +) + +func TestIntegerMarshalSFV(t *testing.T) { + data := []struct { + in int64 + expected string + valid bool + }{ + {10, "10", true}, + {-10, "-10", true}, + {0, "0", true}, + {-999999999999999, "-999999999999999", true}, + {999999999999999, "999999999999999", true}, + {-9999999999999999, "", false}, + {9999999999999999, "", false}, + } + + var b strings.Builder + + for _, d := range data { + b.Reset() + + err := marshalInteger(&b, d.in) + if d.valid && err != nil { + t.Errorf("error not expected for %v, got %v", d.in, err) + } else if !d.valid && err == nil { + t.Errorf("error expected for %v, got %v", d.in, err) + } + + if b.String() != d.expected { + t.Errorf("got %v; want %v", b.String(), d.expected) + } + } +} + +func TestParseIntegerOrDecimal(t *testing.T) { + data := []struct { + in string + expected interface{} + valid bool + }{ + {"1871", int64(1871), false}, + {"-1871", int64(-1871), false}, + {"18.71", 18.71, false}, + {"-18.71", -18.71, false}, + {"1871next", int64(1871), false}, + {"-18.71next", -18.71, false}, + {"-18.710", -18.71, false}, + {"a", 0, true}, + {"10.", 0, true}, + {"10.1234", 0, true}, + {"-", 0, true}, + {"1234567890123456", 0, true}, + {"123456789012345.6", 0, true}, + {"1234567890123.", 0, true}, + {"-9999999999999991", 0, true}, + {"9999999999999991", 0, true}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + i, err := parseNumber(s) + if d.valid && err == nil { + t.Errorf("parseIntegerOrDecimal(%s): error expected", d.in) + } + + if !d.valid && d.expected != i { + t.Errorf("parseIntegerOrDecimal(%s) = %v, %v; %v, expected", d.in, i, err, d.expected) + } + } +} diff --git a/src/net/http/sfv/item.go b/src/net/http/sfv/item.go new file mode 100644 index 00000000000000..c747ebaa8b8e6d --- /dev/null +++ b/src/net/http/sfv/item.go @@ -0,0 +1,77 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" +) + +// Item is a bare value and associated parameters. +// See https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#item. +type Item struct { + Value interface{} + Params *Params +} + +// NewItem returns a new Item. +func NewItem(v interface{}) Item { + assertBareItem(v) + + return Item{v, NewParams()} +} + +func (i Item) member() { +} + +// marshalSFV serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-item. +func (i Item) marshalSFV(b *strings.Builder) error { + if i.Value == nil { + return ErrInvalidBareItem + } + + if err := marshalBareItem(b, i.Value); err != nil { + return err + } + + return i.Params.marshalSFV(b) +} + +// UnmarshalItem parses an item as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-item. +func UnmarshalItem(v []string) (Item, error) { + s := &scanner{ + data: strings.Join(v, ","), + } + + s.scanWhileSp() + + sfv, err := parseItem(s) + if err != nil { + return Item{}, err + } + + s.scanWhileSp() + + if !s.eof() { + return Item{}, &UnmarshalError{off: s.off} + } + + return sfv, nil +} + +func parseItem(s *scanner) (Item, error) { + bi, err := parseBareItem(s) + if err != nil { + return Item{}, err + } + + p, err := parseParams(s) + if err != nil { + return Item{}, err + } + + return Item{bi, p}, nil +} diff --git a/src/net/http/sfv/item_test.go b/src/net/http/sfv/item_test.go new file mode 100644 index 00000000000000..4875c662f79273 --- /dev/null +++ b/src/net/http/sfv/item_test.go @@ -0,0 +1,96 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "reflect" + "strings" + "testing" +) + +func TestMarshalItem(t *testing.T) { + data := []struct { + in Item + expected string + valid bool + }{ + {NewItem(0), "0", true}, + {NewItem(int8(-42)), "-42", true}, + {NewItem(int16(-42)), "-42", true}, + {NewItem(int32(-42)), "-42", true}, + {NewItem(int64(-42)), "-42", true}, + {NewItem(uint(42)), "42", true}, + {NewItem(uint8(42)), "42", true}, + {NewItem(uint16(42)), "42", true}, + {NewItem(uint32(42)), "42", true}, + {NewItem(uint64(42)), "42", true}, + {NewItem(1.1), "1.1", true}, + {NewItem(""), `""`, true}, + {NewItem(Token("foo")), "foo", true}, + {NewItem([]byte{0, 1}), ":AAE=:", true}, + {NewItem(false), "?0", true}, + {NewItem(int64(9999999999999999)), "", false}, + {NewItem(9999999999999999.22), "", false}, + {NewItem("Kévin"), "", false}, + {NewItem(Token("/foo")), "", false}, + {Item{}, "", false}, + } + + for _, d := range data { + r, err := Marshal(d.in) + if d.valid && err != nil { + t.Errorf("error not expected for %v, got %v", d.in, err) + } else if !d.valid && err == nil { + t.Errorf("error expected for %v, got %v", d.in, err) + } + + if r != d.expected { + t.Errorf("got %v; want %v", r, d.expected) + } + } +} + +func TestItemParamsMarshalSFV(t *testing.T) { + i := NewItem(Token("bar")) + i.Params.Add("foo", 0.0) + i.Params.Add("baz", true) + + var b strings.Builder + _ = i.marshalSFV(&b) + + if b.String() != "bar;foo=0.0;baz" { + t.Error("marshalSFV(): invalid") + } +} + +func TestUnmarshalItem(t *testing.T) { + i1 := NewItem(true) + i1.Params.Add("foo", true) + i1.Params.Add("*bar", Token("tok")) + + data := []struct { + in []string + expected Item + valid bool + }{ + {[]string{"?1;foo;*bar=tok"}, i1, false}, + {[]string{" ?1;foo;*bar=tok "}, i1, false}, + {[]string{`"foo`, `bar"`}, NewItem("foo,bar"), false}, + {[]string{"é", ""}, Item{}, true}, + {[]string{"tok;é"}, Item{}, true}, + {[]string{" ?1;foo;*bar=tok é"}, Item{}, true}, + } + + for _, d := range data { + i, err := UnmarshalItem(d.in) + if d.valid && err == nil { + t.Errorf("UnmarshalItem(%s): error expected", d.in) + } + + if !d.valid && !reflect.DeepEqual(d.expected, i) { + t.Errorf("UnmarshalItem(%s) = %v, %v; %v, expected", d.in, i, err, d.expected) + } + } +} diff --git a/src/net/http/sfv/key.go b/src/net/http/sfv/key.go new file mode 100644 index 00000000000000..567828a3bf5b90 --- /dev/null +++ b/src/net/http/sfv/key.go @@ -0,0 +1,85 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "fmt" + "io" +) + +// ErrInvalidKeyFormat is returned when the format of a parameter or dictionary key is invalid. +var ErrInvalidKeyFormat = errors.New("invalid key format") + +// isKeyChar checks if c is a valid key characters. +func isKeyChar(c byte) bool { + if isLowerCaseAlpha(c) || isDigit(c) { + return true + } + + switch c { + case '_', '-', '.', '*': + return true + } + + return false +} + +// checkKey checks if the given value is a valid parameter key according to +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#param. +func checkKey(k string) error { + if len(k) == 0 { + return fmt.Errorf("a key cannot be empty: %w", ErrInvalidKeyFormat) + } + + if !isLowerCaseAlpha(k[0]) && k[0] != '*' { + return fmt.Errorf("a key must start with a lower case alpha character or *: %w", ErrInvalidKeyFormat) + } + + for i := 1; i < len(k); i++ { + if !isKeyChar(k[i]) { + return fmt.Errorf("the character %c isn't allowed in a key: %w", k[i], ErrInvalidKeyFormat) + } + } + + return nil +} + +// marshalKey serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-key. +func marshalKey(b io.StringWriter, k string) error { + if err := checkKey(k); err != nil { + return err + } + + _, err := b.WriteString(k) + + return err +} + +// parseKey parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-key. +func parseKey(s *scanner) (string, error) { + if s.eof() { + return "", &UnmarshalError{s.off, ErrInvalidKeyFormat} + } + + c := s.data[s.off] + if !isLowerCaseAlpha(c) && c != '*' { + return "", &UnmarshalError{s.off, ErrInvalidKeyFormat} + } + + start := s.off + s.off++ + + for !s.eof() { + if !isKeyChar(s.data[s.off]) { + break + } + s.off++ + } + + return s.data[start:s.off], nil +} diff --git a/src/net/http/sfv/key_test.go b/src/net/http/sfv/key_test.go new file mode 100644 index 00000000000000..0de221588482de --- /dev/null +++ b/src/net/http/sfv/key_test.go @@ -0,0 +1,69 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" + "testing" +) + +func TestKeyMarshalSFV(t *testing.T) { + data := []struct { + in string + expected string + valid bool + }{ + {"f1oo", "f1oo", true}, + {"*foo0", "*foo0", true}, + {"", "", false}, + {"1foo", "", false}, + {"fOo", "", false}, + } + + var b strings.Builder + + for _, d := range data { + b.Reset() + + err := marshalKey(&b, d.in) + if d.valid && err != nil { + t.Errorf("error not expected for %v, got %v", d.in, err) + } else if !d.valid && err == nil { + t.Errorf("error expected for %v, got %v", d.in, err) + } + + if b.String() != d.expected { + t.Errorf("got %v; want %v", b.String(), d.expected) + } + } +} + +func TestParseKey(t *testing.T) { + data := []struct { + in string + expected string + err bool + }{ + {"t", "t", false}, + {"tok", "tok", false}, + {"*k-.*", "*k-.*", false}, + {"k=", "k", false}, + {"", "", true}, + {"é", "", true}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + i, err := parseKey(s) + if d.err && err == nil { + t.Errorf("parseKey(%s): error expected", d.in) + } + + if !d.err && d.expected != i { + t.Errorf("parseKey(%s) = %v, %v; %v, expected", d.in, i, err, d.expected) + } + } +} diff --git a/src/net/http/sfv/list.go b/src/net/http/sfv/list.go new file mode 100644 index 00000000000000..9dc3e0b642c2c9 --- /dev/null +++ b/src/net/http/sfv/list.go @@ -0,0 +1,99 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "strings" +) + +// ErrInvalidListFormat is returned when the format of a list is invalid. +var ErrInvalidListFormat = errors.New("invalid list format") + +// List contains items an inner lists. +// +// See https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#list +type List []Member + +// marshalSFV serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-list. +func (l List) marshalSFV(b *strings.Builder) error { + s := len(l) + for i := 0; i < s; i++ { + if err := l[i].marshalSFV(b); err != nil { + return err + } + + if i != s-1 { + if _, err := b.WriteString(", "); err != nil { + return err + } + } + } + + return nil +} + +// UnmarshalList parses a list as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-list. +func UnmarshalList(v []string) (List, error) { + s := &scanner{ + data: strings.Join(v, ","), + } + + s.scanWhileSp() + + sfv, err := parseList(s) + if err != nil { + return List{}, err + } + + return sfv, nil +} + +// parseList parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-list. +func parseList(s *scanner) (List, error) { + var l List + + for !s.eof() { + m, err := parseItemOrInnerList(s) + if err != nil { + return nil, err + } + + l = append(l, m) + + s.scanWhileOWS() + + if s.eof() { + return l, nil + } + + if s.data[s.off] != ',' { + return nil, &UnmarshalError{s.off, ErrInvalidListFormat} + } + s.off++ + + s.scanWhileOWS() + + if s.eof() { + // there is a trailing comma + return nil, &UnmarshalError{s.off, ErrInvalidListFormat} + } + } + + return l, nil +} + +// parseItemOrInnerList parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-item-or-list. +func parseItemOrInnerList(s *scanner) (Member, error) { + if s.data[s.off] == '(' { + return parseInnerList(s) + } + + return parseItem(s) +} diff --git a/src/net/http/sfv/list_test.go b/src/net/http/sfv/list_test.go new file mode 100644 index 00000000000000..c02dfd6c76443e --- /dev/null +++ b/src/net/http/sfv/list_test.go @@ -0,0 +1,102 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "reflect" + "testing" +) + +func TestList(t *testing.T) { + params := NewParams() + params.Add("foo", true) + params.Add("bar", Token("baz")) + + tokItem := NewItem(Token("tok")) + tokItem.Params.Add("tp1", 42.42) + tokItem.Params.Add("tp2", []byte{0, 1}) + + il := InnerList{ + []Item{NewItem("il"), tokItem}, + NewParams(), + } + il.Params.Add("ilp1", true) + il.Params.Add("ilp2", false) + + data := []struct { + in List + expected string + valid bool + }{ + {List{}, "", true}, + {List{NewItem(true)}, "?1", true}, + {List{Item{"hello", params}}, `"hello";foo;bar=baz`, true}, + {List{il, Item{"hi", params}}, `("il" tok;tp1=42.42;tp2=:AAE=:);ilp1;ilp2=?0, "hi";foo;bar=baz`, true}, + {List{NewItem(Token("é"))}, "", false}, + {List{Item{}}, "", false}, + } + + for _, d := range data { + r, err := Marshal(d.in) + if d.valid && err != nil { + t.Errorf("error not expected for %v, got %v", d.in, err) + } else if !d.valid && err == nil { + t.Errorf("error expected for %v, got %v", d.in, err) + } + + if r != d.expected { + t.Errorf("got %v; want %v", r, d.expected) + } + } +} + +func TestUnmarshalList(t *testing.T) { + l1 := List{Item{Token("foo"), NewParams()}, Item{Token("bar"), NewParams()}} + + il2 := Item{"foo", NewParams()} + l2 := List{il2} + il2.Params.Add("bar", true) + il2.Params.Add("baz", Token("tok")) + + il3 := InnerList{[]Item{{Token("foo"), NewParams()}, {Token("bar"), NewParams()}}, NewParams()} + il3.Params.Add("bat", true) + l3 := List{il3} + + data := []struct { + in []string + out List + err bool + }{ + {[]string{""}, nil, false}, + {[]string{"foo,bar"}, l1, false}, + {[]string{"foo, bar"}, l1, false}, + {[]string{"foo,\t bar"}, l1, false}, + {[]string{"foo", "bar"}, l1, false}, + {[]string{`"foo";bar;baz=tok`}, l2, false}, + {[]string{`(foo bar);bat`}, l3, false}, + {[]string{`()`}, List{InnerList{nil, NewParams()}}, false}, + {[]string{` "foo";bar;baz=tok, (foo bar);bat `}, List{il2, il3}, false}, + {[]string{`foo,bar,`}, nil, true}, + {[]string{`foo,baré`}, nil, true}, + {[]string{`é`}, nil, true}, + {[]string{`foo,"bar" é`}, nil, true}, + {[]string{`(foo `}, nil, true}, + {[]string{`(foo);é`}, nil, true}, + {[]string{`("é")`}, nil, true}, + {[]string{`(""`}, nil, true}, + {[]string{`(`}, nil, true}, + } + + for _, d := range data { + l, err := UnmarshalList(d.in) + if d.err && err == nil { + t.Errorf("UnmarshalList(%s): error expected", d.in) + } + + if !d.err && !reflect.DeepEqual(d.out, l) { + t.Errorf("UnmarshalList(%s) = %t, %v; %t, expected", d.in, l, err, d.out) + } + } +} diff --git a/src/net/http/sfv/member.go b/src/net/http/sfv/member.go new file mode 100644 index 00000000000000..89f6133f2e8ece --- /dev/null +++ b/src/net/http/sfv/member.go @@ -0,0 +1,13 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +// Member is a marker interface for members of dictionaries and lists. +// +// See https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#list. +type Member interface { + member() + marshaler +} diff --git a/src/net/http/sfv/params.go b/src/net/http/sfv/params.go new file mode 100644 index 00000000000000..e3a02f9d596a36 --- /dev/null +++ b/src/net/http/sfv/params.go @@ -0,0 +1,141 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "strings" +) + +// Params are an ordered map of key-value pairs that are associated with an item or an inner list. +// +// See https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#param. +type Params struct { + names []string + values map[string]interface{} +} + +// ErrInvalidParameterFormat is returned when the format of a parameter is invalid. +var ErrInvalidParameterFormat = errors.New("invalid parameter format") + +// ErrInvalidParameterValue is returned when a parameter key is invalid. +var ErrInvalidParameterValue = errors.New("invalid parameter value") + +// NewParams creates a new ordered map. +func NewParams() *Params { + p := Params{} + p.names = []string{} + p.values = map[string]interface{}{} + + return &p +} + +// Get retrieves a parameter. +func (p *Params) Get(k string) (interface{}, bool) { + v, ok := p.values[k] + + return v, ok +} + +// Add appends a new parameter to the ordered list. +// If the key already exists, overwrite its value. +func (p *Params) Add(k string, v interface{}) { + assertBareItem(v) + + if _, exists := p.values[k]; !exists { + p.names = append(p.names, k) + } + + p.values[k] = v +} + +// Del removes a parameter from the ordered list. +func (p *Params) Del(key string) bool { + if _, ok := p.values[key]; !ok { + return false + } + + for i, k := range p.names { + if k == key { + p.names = append(p.names[:i], p.names[i+1:]...) + + break + } + } + + delete(p.values, key) + + return true +} + +// Names retrieves the list of parameter names in the appropriate order. +func (p *Params) Names() []string { + return p.names +} + +// marshalSFV serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-params. +func (p *Params) marshalSFV(b *strings.Builder) error { + for _, k := range p.names { + if err := b.WriteByte(';'); err != nil { + return err + } + + if err := marshalKey(b, k); err != nil { + return err + } + + v := p.values[k] + if v == true { + continue + } + + if err := b.WriteByte('='); err != nil { + return err + } + + if err := marshalBareItem(b, v); err != nil { + return err + } + } + + return nil +} + +// parseParams parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-param. +func parseParams(s *scanner) (*Params, error) { + p := NewParams() + + for !s.eof() { + if s.data[s.off] != ';' { + break + } + s.off++ + s.scanWhileSp() + + k, err := parseKey(s) + if err != nil { + return nil, err + } + + var i interface{} + + if !s.eof() && s.data[s.off] == '=' { + s.off++ + + i, err = parseBareItem(s) + if err != nil { + return nil, err + } + } else { + i = true + } + + p.Add(k, i) + } + + return p, nil +} diff --git a/src/net/http/sfv/params_test.go b/src/net/http/sfv/params_test.go new file mode 100644 index 00000000000000..516b2f7bccff52 --- /dev/null +++ b/src/net/http/sfv/params_test.go @@ -0,0 +1,114 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "reflect" + "strings" + "testing" +) + +func TestParameters(t *testing.T) { + p := NewParams() + + add := []struct { + in string + expected interface{} + valid bool + }{ + {"f_o1o3-", 10.0, true}, + {"deleteme", "", true}, + {"*f0.o*", "", true}, + {"t", true, true}, + {"f", false, true}, + {"b", []byte{0, 1}, true}, + {"0foo", "", false}, + {"mAj", "", false}, + {"_foo", "", false}, + {"foo", "é", false}, + } + + var b strings.Builder + + for _, d := range add { + vParams := NewParams() + vParams.Add(d.in, d.expected) + + b.Reset() + + if valid := vParams.marshalSFV(&b) == nil; valid != d.valid { + t.Errorf("(%v, %v).isValid() = %v; %v expected", d.in, d.expected, valid, d.valid) + } + + if d.valid { + p.Add(d.in, d.expected) + } + } + + p.Add("f_o1o3-", 123.0) + + newValue, _ := p.Get("f_o1o3-") + if newValue != 123.0 { + t.Errorf(`Add("f_o1o3-") must overwrite the existing value`) + } + + if !p.Del("deleteme") { + t.Errorf(`Del("deleteme") must return true`) + } + + if p.Del("deleteme") { + t.Errorf(`the second call to Del("deleteme") must return false`) + } + + if v, ok := p.Get("*f0.o*"); v != "" || !ok { + t.Errorf(`Get("*f0.o*") = %v, %v; "", true expected`, v, ok) + } + + if v, ok := p.Get("notexist"); v != nil || ok { + t.Errorf(`Get("notexist") = %v, %v; nil, false expected`, v, ok) + } + + k := p.Names() + if len(k) != 5 { + t.Errorf(`Names() = %v; {"f_o1o3-", "*f0.o*"} expected`, k) + } + + b.Reset() + err := p.marshalSFV(&b) + + if b.String() != `;f_o1o3-=123.0;*f0.o*="";t;f=?0;b=:AAE=:` { + t.Errorf(`marshalSFV(): invalid serialization: %v (%v)`, b.String(), err) + } +} + +func TestParseParameters(t *testing.T) { + p0 := NewParams() + p0.Add("foo", true) + p0.Add("*bar", "baz") + + data := []struct { + in string + out *Params + err bool + }{ + {`;foo=?1;*bar="baz" foo`, p0, false}, + {`;foo;*bar="baz" foo`, p0, false}, + {`;é=?0`, p0, true}, + {`;foo=é`, p0, true}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + p, err := parseParams(s) + if d.err && err == nil { + t.Errorf("parseParameters(%s): error expected", d.in) + } + + if !d.err && !reflect.DeepEqual(p, d.out) { + t.Errorf("parseParameters(%s) = %v, %v; %v, expected", d.in, p, err, d.out) + } + } +} diff --git a/src/net/http/sfv/string.go b/src/net/http/sfv/string.go new file mode 100644 index 00000000000000..9c3e4a15b49fdf --- /dev/null +++ b/src/net/http/sfv/string.go @@ -0,0 +1,93 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "io" + "strings" + "unicode" +) + +// ErrInvalidStringFormat is returned when a string format is invalid. +var ErrInvalidStringFormat = errors.New("invalid string format") + +// marshalSFV serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-string. +func marshalString(b io.ByteWriter, s string) error { + if err := b.WriteByte('"'); err != nil { + return err + } + + for i := 0; i < len(s); i++ { + if s[i] <= '\u001F' || s[i] >= unicode.MaxASCII { + return ErrInvalidStringFormat + } + + switch s[i] { + case '"', '\\': + if err := b.WriteByte('\\'); err != nil { + return err + } + } + + if err := b.WriteByte(s[i]); err != nil { + return err + } + } + + if err := b.WriteByte('"'); err != nil { + return err + } + + return nil +} + +// parseString parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-string. +func parseString(s *scanner) (string, error) { + if s.eof() || s.data[s.off] != '"' { + return "", &UnmarshalError{s.off, ErrInvalidStringFormat} + } + s.off++ + + var b strings.Builder + + for !s.eof() { + c := s.data[s.off] + s.off++ + + switch c { + case '\\': + if s.eof() { + return "", &UnmarshalError{s.off, ErrInvalidStringFormat} + } + + n := s.data[s.off] + if n != '"' && n != '\\' { + return "", &UnmarshalError{s.off, ErrInvalidStringFormat} + } + s.off++ + + if err := b.WriteByte(n); err != nil { + return "", err + } + + continue + case '"': + return b.String(), nil + default: + if c <= '\u001F' || c >= unicode.MaxASCII { + return "", &UnmarshalError{s.off, ErrInvalidStringFormat} + } + + if err := b.WriteByte(c); err != nil { + return "", err + } + } + } + + return "", &UnmarshalError{s.off, ErrInvalidStringFormat} +} diff --git a/src/net/http/sfv/string_test.go b/src/net/http/sfv/string_test.go new file mode 100644 index 00000000000000..d5a5764c8636a2 --- /dev/null +++ b/src/net/http/sfv/string_test.go @@ -0,0 +1,79 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" + "testing" + "unicode" +) + +func TestStringMarshalSFV(t *testing.T) { + data := []struct { + in string + expected string + valid bool + }{ + {"foo", `"foo"`, true}, + {`f"oo`, `"f\"oo"`, true}, + {`f\oo`, `"f\\oo"`, true}, + {`f\"oo`, `"f\\\"oo"`, true}, + {"", `""`, true}, + {"H3lLo", `"H3lLo"`, true}, + {"hel\tlo", `"hel`, false}, + {"hel\x1flo", `"hel`, false}, + {"hel\x7flo", `"hel`, false}, + {"Kévin", `"K`, false}, + {"\t", `"`, false}, + } + + var b strings.Builder + + for _, d := range data { + b.Reset() + + err := marshalString(&b, d.in) + if d.valid && err != nil { + t.Errorf("error not expected for %v, got %v", d.in, err) + } else if !d.valid && err == nil { + t.Errorf("error expected for %v, got %v", d.in, err) + } + + if b.String() != d.expected { + t.Errorf("got %v; want %v", b.String(), d.expected) + } + } +} + +func TestParseString(t *testing.T) { + data := []struct { + in string + out string + err bool + }{ + {`"foo"`, "foo", false}, + {`"b\"a\\r"`, `b"a\r`, false}, + {"", "", true}, + {"a", "", true}, + {`"\`, "", true}, + {`"\o`, "", true}, + {string([]byte{'"', 0}), "", true}, + {string([]byte{'"', unicode.MaxASCII}), "", true}, + {`"foo`, "", true}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + i, err := parseString(s) + if d.err && err == nil { + t.Errorf("parse%s): error expected", d.in) + } + + if !d.err && d.out != i { + t.Errorf("parse%s) = %v, %v; %v, expected", d.in, i, err, d.out) + } + } +} diff --git a/src/net/http/sfv/structured-field-tests b/src/net/http/sfv/structured-field-tests new file mode 160000 index 00000000000000..4ac52e6de31a1a --- /dev/null +++ b/src/net/http/sfv/structured-field-tests @@ -0,0 +1 @@ +Subproject commit 4ac52e6de31a1a164cd1de0563edd701081a5b7b diff --git a/src/net/http/sfv/token.go b/src/net/http/sfv/token.go new file mode 100644 index 00000000000000..b495cbbeac4a79 --- /dev/null +++ b/src/net/http/sfv/token.go @@ -0,0 +1,75 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "errors" + "fmt" + "io" +) + +// isExtendedTchar checks if c is a valid token character as defined in the spec. +func isExtendedTchar(c byte) bool { + if isAlpha(c) || isDigit(c) { + return true + } + + switch c { + case '!', '#', '$', '%', '&', '\'', '*', '+', '-', '.', '^', '_', '`', '|', '~', ':', '/': + return true + } + + return false +} + +// ErrInvalidTokenFormat is returned when a token format is invalid. +var ErrInvalidTokenFormat = errors.New("invalid token format") + +// Token represents a token as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#token. +// A specific type is used to distinguish tokens from strings. +type Token string + +// marshalSFV serializes as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#ser-token. +func (t Token) marshalSFV(b io.StringWriter) error { + if len(t) == 0 { + return fmt.Errorf("a token cannot be empty: %w", ErrInvalidTokenFormat) + } + + if !isAlpha(t[0]) && t[0] != '*' { + return fmt.Errorf("a token must start with an alpha character or *: %w", ErrInvalidTokenFormat) + } + + for i := 1; i < len(t); i++ { + if !isExtendedTchar(t[i]) { + return fmt.Errorf("the character %c isn't allowed in a token: %w", t[i], ErrInvalidTokenFormat) + } + } + + _, err := b.WriteString(string(t)) + + return err +} + +// parseToken parses as defined in +// https://httpwg.org/http-extensions/draft-ietf-httpbis-header-structure.html#parse-token. +func parseToken(s *scanner) (Token, error) { + if s.eof() || (!isAlpha(s.data[s.off]) && s.data[s.off] != '*') { + return "", &UnmarshalError{s.off, ErrInvalidTokenFormat} + } + + start := s.off + s.off++ + + for !s.eof() { + if !isExtendedTchar(s.data[s.off]) { + break + } + s.off++ + } + + return Token(s.data[start:s.off]), nil +} diff --git a/src/net/http/sfv/token_test.go b/src/net/http/sfv/token_test.go new file mode 100644 index 00000000000000..e836d6b11330d7 --- /dev/null +++ b/src/net/http/sfv/token_test.go @@ -0,0 +1,91 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +import ( + "strings" + "testing" +) + +func TestTokenMarshalSFV(t *testing.T) { + data := []struct { + in string + valid bool + }{ + {"abc'!#$%*+-.^_|~:/`", true}, + {"H3lLo", true}, + {"a*foo", true}, + {"a!1", true}, + {"a#1", true}, + {"a$1", true}, + {"a%1", true}, + {"a&1", true}, + {"a'1", true}, + {"a*1", true}, + {"a+1", true}, + {"a-1", true}, + {"a.1", true}, + {"a^1", true}, + {"a_1", true}, + {"a`1", true}, + {"a|1", true}, + {"a~1", true}, + {"a:1", true}, + {"a/1", true}, + {`0foo`, false}, + {`!foo`, false}, + {"1abc", false}, + {"", false}, + {"hel\tlo", false}, + {"hel\x1flo", false}, + {"hel\x7flo", false}, + {"Kévin", false}, + } + + var b strings.Builder + + for _, d := range data { + b.Reset() + + err := Token(d.in).marshalSFV(&b) + if d.valid && err != nil { + t.Errorf("error not expected for %v, got %v", d.in, err) + } else if !d.valid && err == nil { + t.Errorf("error expected for %v, got %v", d.in, err) + } + + if d.valid && b.String() != d.in { + t.Errorf("got %v; want %v", b.String(), d.in) + } + } +} + +func TestParseToken(t *testing.T) { + data := []struct { + in string + out Token + err bool + }{ + {"t", Token("t"), false}, + {"tok", Token("tok"), false}, + {"*t!o&k", Token("*t!o&k"), false}, + {"t=", Token("t"), false}, + {"", Token(""), true}, + {"é", Token(""), true}, + } + + for _, d := range data { + s := &scanner{data: d.in} + + i, err := parseToken(s) + if d.err && err == nil { + t.Errorf("parseToken(%s): error expected", d.in) + } + + if !d.err && d.out != i { + t.Errorf("parseToken(%s) = %v, %v; %v, expected", d.in, i, err, d.out) + } + } +} diff --git a/src/net/http/sfv/utils.go b/src/net/http/sfv/utils.go new file mode 100644 index 00000000000000..f1b591cba5a232 --- /dev/null +++ b/src/net/http/sfv/utils.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sfv + +// isLowerCaseAlpha checks if c is a lower cased alpha character. +func isLowerCaseAlpha(c byte) bool { + return 'a' <= c && c <= 'z' +} + +// isAlpha checks if c is an alpha character. +func isAlpha(c byte) bool { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') +} + +// isDigit checks if c is a digit. +func isDigit(c byte) bool { + return '0' <= c && c <= '9' +}