Merge pull request #1099 from dolthub/fulghum/issue-3728

Adjust limits on varbinary/json data

enginetest/queries/insert_queries.go

@@ -1434,6 +1434,14 @@ var InsertErrorScripts = []ScriptTest{
 		Query:       "insert into bad values ('1234567890')",
 		ExpectedErr: sql.ErrLengthBeyondLimit,
 	},
+	{
+		Name: "try inserting varbinary larger than max limit",
+		SetUpScript: []string{
+			"create table bad (vb varbinary(65535))",
+		},
+		Query:       "insert into bad values (repeat('0', 65536))",
+		ExpectedErr: sql.ErrLengthBeyondLimit,
+	},
 }
 
 var InsertIgnoreScripts = []ScriptTest{

enginetest/queries/json_scripts.go

@@ -17,6 +17,30 @@ package queries
 import "github.com/dolthub/go-mysql-server/sql"
 
 var JsonScripts = []ScriptTest{
+	{
+		Name: "JSON under max length limit",
+		SetUpScript: []string{
+			"create table t (j JSON)",
+		},
+		Assertions: []ScriptTestAssertion{
+			{
+				Query:    `insert into t set j= concat('[', repeat('"word",', 10000000), '"word"]')`,
+				Expected: []sql.Row{{sql.OkResult{RowsAffected: 1}}},
+			},
+		},
+	},
+	{
+		Name: "JSON over max length limit",
+		SetUpScript: []string{
+			"create table t (j JSON)",
+		},
+		Assertions: []ScriptTestAssertion{
+			{
+				Query:       `insert into t set j= concat('[', repeat('"word",', 50000000), '"word"]')`,
+				ExpectedErr: sql.ErrLengthTooLarge,
+			},
+		},
+	},
 	{
 		Name: "JSON_ARRAYAGG on one column",
 		SetUpScript: []string{

sql/json.go

@@ -20,13 +20,12 @@ import (
 
 	"github.com/dolthub/vitess/go/sqltypes"
 	"github.com/dolthub/vitess/go/vt/proto/query"
-	"gopkg.in/src-d/go-errors.v1"
 )
 
 var (
-	ErrConvertingToJSON = errors.NewKind("value %v is not valid JSON")
-
 	jsonValueType = reflect.TypeOf((*JSONValue)(nil)).Elem()
+
+	MaxJsonFieldByteLength = int64(1024) * int64(1024) * int64(1024)
 )
 
 var JSON JsonType = jsonType{}
@@ -59,13 +58,24 @@ func (t jsonType) Convert(v interface{}) (doc interface{}, err error) {
 	case JSONValue:
 		return v, nil
 	case []byte:
+		if int64(len(v)) > MaxJsonFieldByteLength {
+			return nil, ErrLengthTooLarge.New(len(v), MaxJsonFieldByteLength)
+		}
 		err = json.Unmarshal(v, &doc)
 	case string:
+		charsetMaxLength := Collation_Default.CharacterSet().MaxLength()
+		length := int64(len(v)) * charsetMaxLength
+		if length > MaxJsonFieldByteLength {
+			return nil, ErrLengthTooLarge.New(length, MaxJsonFieldByteLength)
+		}
 		err = json.Unmarshal([]byte(v), &doc)
 	default:
 		// if |v| can be marshalled, it contains
 		// a valid JSON document representation
 		if b, berr := json.Marshal(v); berr == nil {
+			if int64(len(b)) > MaxJsonFieldByteLength {
+				return nil, ErrLengthTooLarge.New(len(b), MaxJsonFieldByteLength)
+			}
 			err = json.Unmarshal(b, &doc)
 		}
 	}

sql/stringtype.go

@@ -121,11 +121,17 @@ func CreateString(baseType query.Type, length int64, collation CollationID) (Str
 		if length > charBinaryMax {
 			return nil, ErrLengthTooLarge.New(length, charBinaryMax)
 		}
-	case sqltypes.VarChar, sqltypes.VarBinary:
-		// We limit on byte length, so acceptable character lengths are variable
+	case sqltypes.VarChar:
 		if byteLength > varcharVarbinaryMax {
 			return nil, ErrLengthTooLarge.New(length, varcharVarbinaryMax/charsetMaxLength)
 		}
+	case sqltypes.VarBinary:
+		// VarBinary fields transmitted over the wire could be for a VarBinary field,
+		// or a JSON field, so we validate against JSON's larger limit (1GB)
+		// instead of VarBinary's smaller limit (65k).
+		if byteLength > MaxJsonFieldByteLength {
+			return nil, ErrLengthTooLarge.New(length, MaxJsonFieldByteLength/charsetMaxLength)
+		}
 	case sqltypes.Text, sqltypes.Blob:
 		// We overall limit on character length, but determine tiny, medium, etc. based on byte length.
 		if length > longTextBlobMax {

sql/stringtype_test.go

@@ -205,7 +205,9 @@ func TestStringCreateString(t *testing.T) {
 		{sqltypes.Blob, longTextBlobMax + 1, Collation_binary, stringType{}, true},
 		{sqltypes.Char, charBinaryMax + 1, Collation_Default, stringType{}, true},
 		{sqltypes.Text, longTextBlobMax + 1, Collation_Default, stringType{}, true},
-		{sqltypes.VarBinary, varcharVarbinaryMax + 1, Collation_binary, stringType{}, true},
+
+		// JSON strings can also come in over the wire as VARBINARY types, and JSON allows a much larger length limit (1GB).
+		{sqltypes.VarBinary, MaxJsonFieldByteLength + 1, Collation_binary, stringType{}, true},
 		{sqltypes.VarChar, varcharVarbinaryMax, Collation_Default, stringType{}, true},
 
 		// Default collation is not valid for these types