Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SNOW-1669514] Honor the Valuer/Stringer methods to resolve #1209 #1211

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open

[SNOW-1669514] Honor the Valuer/Stringer methods to resolve #1209 #1211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
5a22458
try to recreate the issue with UUID to solve #1209
ChronosMasterOfAllTime Sep 19, 2024
cd958f0
[AB#1669514] fix UUID
ChronosMasterOfAllTime Sep 19, 2024
6d42602
[AB#1669514] remove commented code
ChronosMasterOfAllTime Sep 19, 2024
d4c0161
[AB#1669514] alternate approach; more generalized
ChronosMasterOfAllTime Sep 20, 2024
f84b52f
[AB#1669514] add stringer safe check; driver valuer check too
ChronosMasterOfAllTime Sep 20, 2024
c925d7f
[AB#1669514] Fix some comments
ChronosMasterOfAllTime Sep 20, 2024
97b609e
[AB#1669514] simplify stringer check
ChronosMasterOfAllTime Sep 20, 2024
8f86d58
[AB#1669514] simplify valuer
ChronosMasterOfAllTime Sep 20, 2024
1454aca
[AB#1669514] move the driver valuer short circuit above the other typ…
ChronosMasterOfAllTime Sep 20, 2024
1faf665
[AB#1669514] dont run stringer on everything arrays
ChronosMasterOfAllTime Sep 20, 2024
9dac29c
[AB#1669514] add array check too
ChronosMasterOfAllTime Sep 20, 2024
28bf47e
[AB#1669514] jsonFmtStr and ignore debug binary
ChronosMasterOfAllTime Sep 20, 2024
bb50506
[AB#1669514] make it simpler
ChronosMasterOfAllTime Sep 20, 2024
0d5b162
[AB#1669514] revert that change; wont handle complex types
ChronosMasterOfAllTime Sep 20, 2024
e0b541f
[AB#1669514] add test for SQL Null time
ChronosMasterOfAllTime Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
.idea/
.vscode/
parameters*.json
parameters*.bat
*.p8
Expand All @@ -11,6 +12,8 @@ wss-golang-agent.config
wss-unified-agent.jar
whitesource/
*.swp
cp.out
__debug_bin*

# exclude vendor
vendor
59 changes: 37 additions & 22 deletions converter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

const format = "2006-01-02 15:04:05.999999999"
const numberDefaultPrecision = 38
const jsonFormatStr = "json"
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sfc-gh-dszmolka can you please explain how the format string is being used by the execBindQuery struct, or what is the purpose of this field?


type timezoneType int

Expand Down Expand Up @@ -239,11 +240,21 @@ func valueToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
logger.Debugf("TYPE: %v, %v", reflect.TypeOf(v), reflect.ValueOf(v))
if v == nil {
if tsmode == objectType || tsmode == arrayType || tsmode == sliceType {
return bindingValue{nil, "json", nil}, nil
return bindingValue{nil, jsonFormatStr, nil}, nil
}
return bindingValue{nil, "", nil}, nil
}
v1 := reflect.Indirect(reflect.ValueOf(v))

if valuer, ok := v.(driver.Valuer); ok { // check for driver.Valuer satisfaction and honor that first
if value, err := valuer.Value(); err == nil && value != nil {
// if the output value is a valid string, return that
if strVal, ok := value.(string); ok {
return bindingValue{&strVal, "", nil}, nil
}
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this here as a suggestion; did we want to let a marshaler handle the driver's value if not a string by default?

EDIT: The below suggestion would break sql.NullTime. Tests pass if we dont use the below suggestion. This is unless we don't want the expectation to be Epoch Time for this case?

@sfc-gh-dszmolka ^^

Suggested change
}
} else if marshaled, err := json.Marshal(value); err == nil {
value := string(marshaled)
return bindingValue{&value, jsonFormatStr, nil}, nil
}

}
}

switch v1.Kind() {
case reflect.Bool:
s := strconv.FormatBool(v1.Bool())
Expand All @@ -257,7 +268,7 @@ func valueToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
case reflect.String:
s := v1.String()
if tsmode == objectType || tsmode == arrayType || tsmode == sliceType {
return bindingValue{&s, "json", nil}, nil
return bindingValue{&s, jsonFormatStr, nil}, nil
}
return bindingValue{&s, "", nil}, nil
case reflect.Slice, reflect.Array:
Expand All @@ -274,7 +285,7 @@ func valueToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
func arrayToString(v driver.Value, tsmode snowflakeType, params map[string]*string) (bindingValue, error) {
v1 := reflect.Indirect(reflect.ValueOf(v))
if v1.Kind() == reflect.Slice && v1.IsNil() {
return bindingValue{nil, "json", nil}, nil
return bindingValue{nil, jsonFormatStr, nil}, nil
}
if bd, ok := v.([][]byte); ok && tsmode == binaryType {
schema := bindingSchema{
Expand All @@ -289,14 +300,14 @@ func arrayToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
}
if len(bd) == 0 {
res := "[]"
return bindingValue{value: &res, format: "json", schema: &schema}, nil
return bindingValue{value: &res, format: jsonFormatStr, schema: &schema}, nil
}
s := ""
for _, b := range bd {
s += "\"" + hex.EncodeToString(b) + "\","
}
s = "[" + s[:len(s)-1] + "]"
return bindingValue{&s, "json", &schema}, nil
return bindingValue{&s, jsonFormatStr, &schema}, nil
} else if times, ok := v.([]time.Time); ok {
typ := driverTypeToSnowflake[tsmode]
sfFormat, err := dateTimeInputFormatByType(typ, params)
Expand All @@ -313,7 +324,7 @@ func arrayToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
}
res, err := json.Marshal(v)
if err != nil {
return bindingValue{nil, "json", &bindingSchema{
return bindingValue{nil, jsonFormatStr, &bindingSchema{
Typ: "array",
Nullable: true,
Fields: []fieldMetadata{
Expand All @@ -325,7 +336,7 @@ func arrayToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
}}, err
}
resString := string(res)
return bindingValue{&resString, "json", nil}, nil
return bindingValue{&resString, jsonFormatStr, nil}, nil
} else if isArrayOfStructs(v) {
stringEntries := make([]string, v1.Len())
sowcForSingleElement, err := buildSowcFromType(params, reflect.TypeOf(v).Elem())
Expand All @@ -337,7 +348,7 @@ func arrayToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
if sow, ok := potentialSow.Interface().(StructuredObjectWriter); ok {
bv, err := structValueToString(sow, tsmode, params)
if err != nil {
return bindingValue{nil, "json", nil}, err
return bindingValue{nil, jsonFormatStr, nil}, err
}
stringEntries[i] = *bv.value
}
Expand All @@ -354,14 +365,14 @@ func arrayToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
},
},
}
return bindingValue{&value, "json", arraySchema}, nil
return bindingValue{&value, jsonFormatStr, arraySchema}, nil
} else if reflect.ValueOf(v).Len() == 0 {
value := "[]"
return bindingValue{&value, "json", nil}, nil
return bindingValue{&value, jsonFormatStr, nil}, nil
} else if barr, ok := v.([]byte); ok {
if tsmode == binaryType {
res := hex.EncodeToString(barr)
return bindingValue{&res, "json", nil}, nil
return bindingValue{&res, jsonFormatStr, nil}, nil
}
schemaForBytes := bindingSchema{
Typ: "array",
Expand All @@ -375,23 +386,27 @@ func arrayToString(v driver.Value, tsmode snowflakeType, params map[string]*stri
}
if len(barr) == 0 {
res := "[]"
return bindingValue{&res, "json", &schemaForBytes}, nil
return bindingValue{&res, jsonFormatStr, &schemaForBytes}, nil
}
res := "["
for _, b := range barr {
res += fmt.Sprint(b) + ","
}
res = res[0:len(res)-1] + "]"
return bindingValue{&res, "json", &schemaForBytes}, nil
return bindingValue{&res, jsonFormatStr, &schemaForBytes}, nil
} else if stringer, ok := v1.Interface().(fmt.Stringer); ok && v1.Kind() == reflect.Array && v1.Type().Elem().Kind() == reflect.Uint8 && v1.Len() == 16 {
// special case for UUIDs (snowflake type and other implementers) check for stringer method and it's a len 16 byte array. Guarantees it's String() and returns string
value := stringer.String()
return bindingValue{&value, "", nil}, nil
} else if isSliceOfSlices(v) {
return bindingValue{}, errors.New("array of arrays is not supported")
}
res, err := json.Marshal(v)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After this, the bindingValue is set to "json". Should that be the case, or should it be string? Since it is already Marshaled, why set it to json again? That will cause it to be marshaled a second time. It feels like it should not marshal here if it also sets the binding value to json.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to be clear This is in reference to the code on Lines 406-409; the added code will not return the "json" string as it's already been transformed

if err != nil {
return bindingValue{nil, "json", nil}, err
return bindingValue{nil, jsonFormatStr, nil}, err
}
resString := string(res)
return bindingValue{&resString, "json", nil}, nil
return bindingValue{&resString, jsonFormatStr, nil}, nil
}

func mapToString(v driver.Value, tsmode snowflakeType, params map[string]*string) (bindingValue, error) {
Expand Down Expand Up @@ -549,7 +564,7 @@ func mapToString(v driver.Value, tsmode snowflakeType, params map[string]*string
Typ: "MAP",
Fields: []fieldMetadata{keyMetadata, *valueMetadata},
}
return bindingValue{&jsonString, "json", &schema}, nil
return bindingValue{&jsonString, jsonFormatStr, &schema}, nil
} else {
jsonBytes, err = json.Marshal(v)
if err != nil {
Expand All @@ -569,7 +584,7 @@ func mapToString(v driver.Value, tsmode snowflakeType, params map[string]*string
Typ: "MAP",
Fields: []fieldMetadata{keyMetadata, valueMetadata},
}
return bindingValue{&jsonString, "json", &schema}, nil
return bindingValue{&jsonString, jsonFormatStr, &schema}, nil
}

func toNullableInt64(val any) (int64, bool) {
Expand Down Expand Up @@ -771,7 +786,7 @@ func structValueToString(v driver.Value, tsmode snowflakeType, params map[string
case sql.NullString:
fmt := ""
if tsmode == objectType || tsmode == arrayType || tsmode == sliceType {
fmt = "json"
fmt = jsonFormatStr
}
if !typedVal.Valid {
return bindingValue{nil, fmt, nil}, nil
Expand All @@ -795,7 +810,7 @@ func structValueToString(v driver.Value, tsmode snowflakeType, params map[string
Nullable: true,
Fields: sowc.toFields(),
}
return bindingValue{&jsonString, "json", &schema}, nil
return bindingValue{&jsonString, jsonFormatStr, &schema}, nil
} else if typ, ok := v.(reflect.Type); ok && tsmode == nilArrayType {
metadata, err := goTypeToFieldMetadata(typ, tsmode, params)
if err != nil {
Expand All @@ -808,7 +823,7 @@ func structValueToString(v driver.Value, tsmode snowflakeType, params map[string
metadata,
},
}
return bindingValue{nil, "json", &schema}, nil
return bindingValue{nil, jsonFormatStr, &schema}, nil
} else if types, ok := v.(NilMapTypes); ok && tsmode == nilMapType {
keyMetadata, err := goTypeToFieldMetadata(types.Key, tsmode, params)
if err != nil {
Expand All @@ -823,7 +838,7 @@ func structValueToString(v driver.Value, tsmode snowflakeType, params map[string
Nullable: true,
Fields: []fieldMetadata{keyMetadata, valueMetadata},
}
return bindingValue{nil, "json", &schema}, nil
return bindingValue{nil, jsonFormatStr, &schema}, nil
} else if typ, ok := v.(reflect.Type); ok && tsmode == nilObjectType {
metadata, err := goTypeToFieldMetadata(typ, tsmode, params)
if err != nil {
Expand All @@ -834,7 +849,7 @@ func structValueToString(v driver.Value, tsmode snowflakeType, params map[string
Nullable: true,
Fields: metadata.Fields,
}
return bindingValue{nil, "json", &schema}, nil
return bindingValue{nil, jsonFormatStr, &schema}, nil
}
return bindingValue{}, fmt.Errorf("unknown binding for type %T and mode %v", v, tsmode)
}
Expand Down
40 changes: 37 additions & 3 deletions converter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,16 @@ func (o *testValueToStringStructuredObject) Write(sowc StructuredObjectWriterCon
return nil
}

type testSqlUuid UUID

func (uuid testSqlUuid) Value() (driver.Value, error) {
return uuid.String(), nil
}

func (u testSqlUuid) String() string {
return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:])
}

func TestValueToString(t *testing.T) {
v := cmplx.Sqrt(-5 + 12i) // should never happen as Go sql package must have already validated.
_, err := valueToString(v, nullType, nil)
Expand Down Expand Up @@ -269,22 +279,46 @@ func TestValueToString(t *testing.T) {
assertNilE(t, bv.schema)
assertEqualE(t, *bv.value, expectedString)

t.Run("SQL Time", func(t *testing.T) {
bv, err := valueToString(sql.NullTime{Time: localTime, Valid: true}, timestampLtzType, nil)
assertNilF(t, err)
assertEmptyStringE(t, bv.format)
assertNilE(t, bv.schema)
assertEqualE(t, *bv.value, expectedUnixTime)
})

t.Run("arrays", func(t *testing.T) {
bv, err := valueToString([2]int{1, 2}, objectType, nil)
assertNilF(t, err)
assertEqualE(t, bv.format, "json")
assertEqualE(t, bv.format, jsonFormatStr)
assertEqualE(t, *bv.value, "[1,2]")
})
t.Run("slices", func(t *testing.T) {
bv, err := valueToString([]int{1, 2}, objectType, nil)
assertNilF(t, err)
assertEqualE(t, bv.format, "json")
assertEqualE(t, bv.format, jsonFormatStr)
assertEqualE(t, *bv.value, "[1,2]")
})

t.Run("UUID - should return string", func(t *testing.T) {
u := NewUUID()
bv, err := valueToString(u, textType, nil)
assertNilF(t, err)
assertEmptyStringE(t, bv.format)
assertEqualE(t, *bv.value, u.String())
})

t.Run("database/sql/driver - Valuer interface", func(t *testing.T) {
u := testSqlUuid(NewUUID())
bv, err := valueToString(u, textType, nil)
assertNilF(t, err)
assertEmptyStringE(t, bv.format)
assertEqualE(t, *bv.value, u.String())
})

bv, err = valueToString(&testValueToStringStructuredObject{s: "some string", i: 123, date: time.Date(2024, time.May, 24, 0, 0, 0, 0, time.UTC)}, timestampLtzType, params)
assertNilF(t, err)
assertEqualE(t, bv.format, "json")
assertEqualE(t, bv.format, jsonFormatStr)
assertDeepEqualE(t, *bv.schema, bindingSchema{
Typ: "object",
Nullable: true,
Expand Down