diff --git a/test/common/response_check.go b/test/common/response_check.go index 2ceb8baf..64027bd2 100644 --- a/test/common/response_check.go +++ b/test/common/response_check.go @@ -147,11 +147,13 @@ func EqualColumn(t *testing.T, columnA entity.Column, columnB entity.Column) { require.ElementsMatch(t, columnA.(*entity.ColumnFloatVector).Data(), columnB.(*entity.ColumnFloatVector).Data()) case entity.FieldTypeBinaryVector: require.ElementsMatch(t, columnA.(*entity.ColumnBinaryVector).Data(), columnB.(*entity.ColumnBinaryVector).Data()) + case entity.FieldTypeArray: + log.Println("TODO support column element type") default: - log.Printf("The column type not in: [%v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v]", + log.Printf("The column type not in: [%v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v, %v]", entity.FieldTypeBool, entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString, - entity.FieldTypeVarChar, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector) + entity.FieldTypeVarChar, entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector) } } @@ -173,10 +175,10 @@ func CheckQueryResult(t *testing.T, actualColumns []entity.Column, expColumns [] func CheckOutputFields(t *testing.T, actualColumns []entity.Column, expFields []string) { actualFields := make([]string, 0) for _, actualColumn := range actualColumns { - log.Printf("column name: %s, column type: %s, column fieldData: %v", - actualColumn.Name(), actualColumn.Type(), actualColumn.FieldData()) actualFields = append(actualFields, actualColumn.Name()) } + log.Printf("actual fields: %v", actualFields) + log.Printf("expected fields: %v", expFields) require.ElementsMatchf(t, expFields, actualFields, fmt.Sprintf("Expected search output fields: %v, actual: %v", expFields, actualFields)) } diff --git a/test/common/utils.go b/test/common/utils.go index d06e45bb..3fef2537 100644 --- a/test/common/utils.go +++ b/test/common/utils.go @@ -1,6 +1,7 @@ package common import ( + "bytes" "encoding/json" "fmt" "log" @@ -18,12 +19,21 @@ const ( DefaultFloatFieldName = "float" DefaultVarcharFieldName = "varchar" DefaultJSONFieldName = "json" + DefaultArrayFieldName = "array" DefaultFloatVecFieldName = "floatVec" DefaultBinaryVecFieldName = "binaryVec" DefaultDynamicNumberField = "dynamicNumber" DefaultDynamicStringField = "dynamicString" DefaultDynamicBoolField = "dynamicBool" DefaultDynamicListField = "dynamicList" + DefaultBoolArrayField = "boolArray" + DefaultInt8ArrayField = "int8Array" + DefaultInt16ArrayField = "int16Array" + DefaultInt32ArrayField = "int32Array" + DefaultInt64ArrayField = "int64Array" + DefaultFloatArrayField = "floatArray" + DefaultDoubleArrayField = "doubleArray" + DefaultVarcharArrayField = "varcharArray" RowCount = "row_count" DefaultTimeout = 120 DefaultDim = int64(128) @@ -31,6 +41,8 @@ const ( DefaultNb = 3000 DefaultNq = 5 DefaultTopK = 10 + TestCapacity = 100 // default array field capacity + TestMaxLen = 100 // default varchar field max length ) // const default value from milvus @@ -45,10 +57,11 @@ const ( DefaultDb = "default" DefaultConsistencyLevel = entity.ClBounded MaxDim = 32768 - DefaultMaxLength = int64(65535) + MaxLength = int64(65535) MaxCollectionNameLen = 255 DefaultRgCapacity = 1000000 - RetentionDuration = 40 // common.retentionDuration + RetentionDuration = 40 // common.retentionDuration + MaxCapacity = 4096 // max array capacity ) var IndexStateValue = map[string]int32{ @@ -60,6 +73,28 @@ var IndexStateValue = map[string]int32{ "Retry": 5, } +var ArrayFieldType = []entity.FieldType{ + entity.FieldTypeBool, + entity.FieldTypeInt8, + entity.FieldTypeInt16, + entity.FieldTypeInt32, + entity.FieldTypeInt64, + entity.FieldTypeFloat, + entity.FieldTypeDouble, + //entity.FieldTypeVarChar, //t.Skip("Waiting for varchar bytes array fixed") +} + +var AllArrayFieldsName = []string{ + DefaultBoolArrayField, + DefaultInt8ArrayField, + DefaultInt16ArrayField, + DefaultInt32ArrayField, + DefaultInt64ArrayField, + DefaultFloatArrayField, + DefaultDoubleArrayField, + DefaultVarcharArrayField, +} + var r *rand.Rand func init() { @@ -98,6 +133,20 @@ func ColumnIndexFunc(data []entity.Column, fieldName string) int { return -1 } +func GenFloatVector(dim int64) []float32 { + vector := make([]float32, 0, dim) + for j := 0; j < int(dim); j++ { + vector = append(vector, rand.Float32()) + } + return vector +} + +func GenBinaryVector(dim int64) []byte { + vector := make([]byte, dim/8) + rand.Read(vector) + return vector +} + // --- common utils --- // --- gen fields --- @@ -127,7 +176,7 @@ func GenDefaultBinaryFields(autoID bool, dim int64) []*entity.Field { // GenDefaultVarcharFields gen default fields with varchar, floatVector field func GenDefaultVarcharFields(autoID bool) []*entity.Field { - varcharField := GenField(DefaultVarcharFieldName, entity.FieldTypeVarChar, WithIsPrimaryKey(true), WithAutoID(autoID), WithMaxLength(DefaultMaxLength)) + varcharField := GenField(DefaultVarcharFieldName, entity.FieldTypeVarChar, WithIsPrimaryKey(true), WithAutoID(autoID), WithMaxLength(MaxLength)) binaryVecField := GenField(DefaultBinaryVecFieldName, entity.FieldTypeBinaryVector, WithDim(DefaultDim)) fields := []*entity.Field{ varcharField, binaryVecField, @@ -135,21 +184,40 @@ func GenDefaultVarcharFields(autoID bool) []*entity.Field { return fields } +func GenAllArrayFields() []*entity.Field { + return GenAllArrayFieldsWithCapacity(TestCapacity) +} + +// GenAllArrayFieldsWithCapacity GenAllArrayFields gen all array fields +func GenAllArrayFieldsWithCapacity(capacity int64) []*entity.Field { + fields := []*entity.Field{ + GenField(DefaultBoolArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeBool), WithMaxCapacity(capacity)), + GenField(DefaultInt8ArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeInt8), WithMaxCapacity(capacity)), + GenField(DefaultInt16ArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeInt16), WithMaxCapacity(capacity)), + GenField(DefaultInt32ArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeInt32), WithMaxCapacity(capacity)), + GenField(DefaultInt64ArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeInt64), WithMaxCapacity(capacity)), + GenField(DefaultFloatArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeFloat), WithMaxCapacity(capacity)), + GenField(DefaultDoubleArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeDouble), WithMaxCapacity(capacity)), + GenField(DefaultVarcharArrayField, entity.FieldTypeArray, WithElementType(entity.FieldTypeVarChar), WithMaxLength(TestMaxLen), WithMaxCapacity(capacity)), + } + return fields +} + // GenAllFields gen fields with all scala field types func GenAllFields() []*entity.Field { allFields := []*entity.Field{ - GenField("int64", entity.FieldTypeInt64, WithIsPrimaryKey(true)), // int64 - GenField("bool", entity.FieldTypeBool), // bool - GenField("int8", entity.FieldTypeInt8), // int8 - GenField("int16", entity.FieldTypeInt16), // int16 - GenField("int32", entity.FieldTypeInt32), // int32 - GenField("float", entity.FieldTypeFloat), // float - GenField("double", entity.FieldTypeDouble), // double - GenField("varchar", entity.FieldTypeVarChar, WithMaxLength(DefaultMaxLength)), // varchar - GenField("json", entity.FieldTypeJSON), // json - GenField("floatVec", entity.FieldTypeFloatVector, WithDim(DefaultDim)), // float vector + GenField("int64", entity.FieldTypeInt64, WithIsPrimaryKey(true)), // int64 + GenField("bool", entity.FieldTypeBool), // bool + GenField("int8", entity.FieldTypeInt8), // int8 + GenField("int16", entity.FieldTypeInt16), // int16 + GenField("int32", entity.FieldTypeInt32), // int32 + GenField("float", entity.FieldTypeFloat), // float + GenField("double", entity.FieldTypeDouble), // double + GenField("varchar", entity.FieldTypeVarChar, WithMaxLength(MaxLength)), // varchar + GenField("json", entity.FieldTypeJSON), // json + GenField("floatVec", entity.FieldTypeFloatVector, WithDim(DefaultDim)), // float vector } - + allFields = append(allFields, GenAllArrayFields()...) return allFields } @@ -164,18 +232,6 @@ func GenDefaultColumnData(start int, nb int, dim int64) (entity.Column, entity.C GenColumnData(start, nb, entity.FieldTypeFloatVector, DefaultFloatVecFieldName, WithVectorDim(dim)) } -type GenColumnDataOption func(opt *genDataOpt) - -type genDataOpt struct { - dim int64 -} - -func WithVectorDim(dim int64) GenColumnDataOption { - return func(opt *genDataOpt) { - opt.dim = dim - } -} - // GenColumnData GenColumnDataOption func GenColumnData(start int, nb int, fieldType entity.FieldType, fieldName string, opts ...GenColumnDataOption) entity.Column { opt := &genDataOpt{} @@ -239,21 +295,20 @@ func GenColumnData(start int, nb int, fieldType entity.FieldType, fieldName stri } return entity.NewColumnVarChar(fieldName, varcharValues) + case entity.FieldTypeArray: + return GenArrayColumnData(start, nb, fieldName, opts...) + case entity.FieldTypeFloatVector: vecFloatValues := make([][]float32, 0, nb) for i := start; i < start+nb; i++ { - vec := make([]float32, 0, opt.dim) - for j := 0; j < int(opt.dim); j++ { - vec = append(vec, rand.Float32()) - } + vec := GenFloatVector(opt.dim) vecFloatValues = append(vecFloatValues, vec) } return entity.NewColumnFloatVector(fieldName, int(opt.dim), vecFloatValues) case entity.FieldTypeBinaryVector: binaryVectors := make([][]byte, 0, nb) for i := 0; i < nb; i++ { - vec := make([]byte, opt.dim/8) - rand.Read(vec) + vec := GenBinaryVector(opt.dim) binaryVectors = append(binaryVectors, vec) } return entity.NewColumnBinaryVector(fieldName, int(opt.dim), binaryVectors) @@ -262,6 +317,101 @@ func GenColumnData(start int, nb int, fieldType entity.FieldType, fieldName stri } } +func GenArrayColumnData(start int, nb int, fieldName string, opts ...GenColumnDataOption) entity.Column { + opt := &genDataOpt{} + for _, o := range opts { + o(opt) + } + eleType := opt.ElementType + capacity := int(opt.capacity) + switch eleType { + case entity.FieldTypeBool: + boolValues := make([][]bool, 0, nb) + for i := start; i < start+nb; i++ { + boolArray := make([]bool, 0, capacity) + for j := 0; j < capacity; j++ { + boolArray = append(boolArray, i%2 == 0) + } + boolValues = append(boolValues, boolArray) + } + return entity.NewColumnBoolArray(fieldName, boolValues) + case entity.FieldTypeInt8: + int8Values := make([][]int8, 0, nb) + for i := start; i < start+nb; i++ { + int8Array := make([]int8, 0, capacity) + for j := 0; j < capacity; j++ { + int8Array = append(int8Array, int8(i+j)) + } + int8Values = append(int8Values, int8Array) + } + return entity.NewColumnInt8Array(fieldName, int8Values) + case entity.FieldTypeInt16: + int16Values := make([][]int16, 0, nb) + for i := start; i < start+nb; i++ { + int16Array := make([]int16, 0, capacity) + for j := 0; j < capacity; j++ { + int16Array = append(int16Array, int16(i+j)) + } + int16Values = append(int16Values, int16Array) + } + return entity.NewColumnInt16Array(fieldName, int16Values) + case entity.FieldTypeInt32: + int32Values := make([][]int32, 0, nb) + for i := start; i < start+nb; i++ { + int32Array := make([]int32, 0, capacity) + for j := 0; j < capacity; j++ { + int32Array = append(int32Array, int32(i+j)) + } + int32Values = append(int32Values, int32Array) + } + return entity.NewColumnInt32Array(fieldName, int32Values) + case entity.FieldTypeInt64: + int64Values := make([][]int64, 0, nb) + for i := start; i < start+nb; i++ { + int64Array := make([]int64, 0, capacity) + for j := 0; j < capacity; j++ { + int64Array = append(int64Array, int64(i+j)) + } + int64Values = append(int64Values, int64Array) + } + return entity.NewColumnInt64Array(fieldName, int64Values) + case entity.FieldTypeFloat: + floatValues := make([][]float32, 0, nb) + for i := start; i < start+nb; i++ { + floatArray := make([]float32, 0, capacity) + for j := 0; j < capacity; j++ { + floatArray = append(floatArray, float32(i+j)) + } + floatValues = append(floatValues, floatArray) + } + return entity.NewColumnFloatArray(fieldName, floatValues) + case entity.FieldTypeDouble: + doubleValues := make([][]float64, 0, nb) + for i := start; i < start+nb; i++ { + doubleArray := make([]float64, 0, capacity) + for j := 0; j < capacity; j++ { + doubleArray = append(doubleArray, float64(i+j)) + } + doubleValues = append(doubleValues, doubleArray) + } + return entity.NewColumnDoubleArray(fieldName, doubleValues) + case entity.FieldTypeVarChar: + varcharValues := make([][][]byte, 0, nb) + for i := start; i < start+nb; i++ { + varcharArray := make([][]byte, 0, capacity) + for j := 0; j < capacity; j++ { + var buf bytes.Buffer + buf.WriteString(strconv.Itoa(i + j)) + varcharArray = append(varcharArray, buf.Bytes()) + } + varcharValues = append(varcharValues, varcharArray) + } + return entity.NewColumnVarCharArray(fieldName, varcharValues) + default: + return nil + } +} + // GenDefaultJSONData gen default column with data func GenDefaultJSONData(columnName string, start int, nb int) *entity.ColumnJSONBytes { type JSONStruct struct { @@ -309,7 +459,30 @@ func GenDefaultVarcharData(start int, nb int, dim int64) (entity.Column, entity. return varColumn, binaryColumn } -func GenAllFieldsData(start int, nb int, dim int64) []entity.Column { +func GenAllArrayData(start int, nb int, opts ...GenColumnDataOption) []entity.Column { + // how to pass different capacity for different column + opt := &genDataOpt{} + for _, o := range opts { + o(opt) + } + data := []entity.Column{ + GenArrayColumnData(start, nb, DefaultBoolArrayField, WithArrayElementType(entity.FieldTypeBool), WithArrayCapacity(opt.capacity)), + GenArrayColumnData(start, nb, DefaultInt8ArrayField, WithArrayElementType(entity.FieldTypeInt8), WithArrayCapacity(opt.capacity)), + GenArrayColumnData(start, nb, DefaultInt16ArrayField, WithArrayElementType(entity.FieldTypeInt16), WithArrayCapacity(opt.capacity)), + GenArrayColumnData(start, nb, DefaultInt32ArrayField, WithArrayElementType(entity.FieldTypeInt32), WithArrayCapacity(opt.capacity)), + GenArrayColumnData(start, nb, DefaultInt64ArrayField, WithArrayElementType(entity.FieldTypeInt64), WithArrayCapacity(opt.capacity)), + GenArrayColumnData(start, nb, DefaultFloatArrayField, WithArrayElementType(entity.FieldTypeFloat), WithArrayCapacity(opt.capacity)), + GenArrayColumnData(start, nb, DefaultDoubleArrayField, WithArrayElementType(entity.FieldTypeDouble), WithArrayCapacity(opt.capacity)), + GenArrayColumnData(start, nb, DefaultVarcharArrayField, WithArrayElementType(entity.FieldTypeVarChar), WithArrayCapacity(opt.capacity)), + } + return data +} + +func GenAllFieldsData(start int, nb int, dim int64, opts ...GenColumnDataOption) []entity.Column { + opt := &genDataOpt{} + for _, o := range opts { + o(opt) + } // prepare data data := []entity.Column{ GenColumnData(start, nb, entity.FieldTypeInt64, "int64"), @@ -323,6 +496,7 @@ func GenAllFieldsData(start int, nb int, dim int64) []entity.Column { GenDefaultJSONData("json", start, nb), GenColumnData(start, nb, entity.FieldTypeFloatVector, "floatVec", WithVectorDim(dim)), } + data = append(data, GenAllArrayData(start, nb, opts...)...) return data } @@ -337,27 +511,39 @@ type Dynamic struct { List []int64 `json:"dynamicList" milvus:"name:dynamicList"` } +type Array struct { + BoolArray []bool `json:"boolArray" milvus:"name:boolArray"` + Int8Array []int8 `json:"int8Array" milvus:"name:int8Array"` + Int16Array []int16 `json:"int16Array" milvus:"name:int16Array"` + Int32Array []int32 `json:"int32Array" milvus:"name:int32Array"` + Int64Array []int64 `json:"int64Array" milvus:"name:int64Array"` + FloatArray []float32 `json:"floatArray" milvus:"name:floatArray"` + DoubleArray []float64 `json:"doubleArray" milvus:"name:doubleArray"` + VarcharArray [][]byte `json:"varcharArray" milvus:"name:varcharArray"` +} + func GenDefaultRows(start int, nb int, dim int64, enableDynamicField bool) []interface{} { rows := make([]interface{}, 0, nb) - type DynamicRow struct { + // BaseRow generate insert rows + type BaseRow struct { Int64 int64 `json:"int64" milvus:"name:int64"` Float float32 `json:"float" milvus:"name:float"` FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` - Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` } - // BaseRow generate insert rows - type BaseRow struct { + type DynamicRow struct { Int64 int64 `json:"int64" milvus:"name:int64"` Float float32 `json:"float" milvus:"name:float"` FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` + Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` } for i := start; i < start+nb; i++ { - floatVec := make([]float32, 0, dim) - for j := 0; j < int(dim); j++ { - floatVec = append(floatVec, rand.Float32()) + baseRow := BaseRow{ + Int64: int64(i), + Float: float32(i), + FloatVec: GenFloatVector(dim), } if enableDynamicField { var dynamic Dynamic @@ -377,19 +563,15 @@ func GenDefaultRows(start int, nb int, dim int64, enableDynamicField bool) []int } dynamicRow := DynamicRow{ - Int64: int64(i), - Float: float32(i), - FloatVec: floatVec, + Int64: baseRow.Int64, + Float: baseRow.Float, + FloatVec: baseRow.FloatVec, Dynamic: dynamic, } rows = append(rows, dynamicRow) } else { - rows = append(rows, &BaseRow{ - Int64: int64(i), - Float: float32(i), - FloatVec: floatVec, - }) + rows = append(rows, &baseRow) } } return rows @@ -398,26 +580,26 @@ func GenDefaultRows(start int, nb int, dim int64, enableDynamicField bool) []int func GenDefaultBinaryRows(start int, nb int, dim int64, enableDynamicField bool) []interface{} { rows := make([]interface{}, 0, nb) - type DynamicRow struct { - Int64 int64 `json:"int64" milvus:"name:int64"` - Float float32 `json:"float" milvus:"name:float"` - BinaryVec [][]byte `json:"binaryVec" milvus:"name:binaryVec"` - Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` - } - // BaseRow generate insert rows type BaseRow struct { - Int64 int64 `json:"int64" milvus:"name:int64"` - Float float32 `json:"float" milvus:"name:float"` - BinaryVec [][]byte `json:"binaryVec" milvus:"name:binaryVec"` + Int64 int64 `json:"int64" milvus:"name:int64"` + Float float32 `json:"float" milvus:"name:float"` + BinaryVec []byte `json:"binaryVec" milvus:"name:binaryVec"` } - for i := start; i < start+nb; i++ { - binaryVec := make([][]byte, 0, nb) - vec := make([]byte, 0, dim/8) - rand.Read(vec) - binaryVec = append(binaryVec, vec) + type DynamicRow struct { + Int64 int64 `json:"int64" milvus:"name:int64"` + Float float32 `json:"float" milvus:"name:float"` + BinaryVec []byte `json:"binaryVec" milvus:"name:binaryVec"` + Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` + } + for i := start; i < start+nb; i++ { + baseRow := BaseRow{ + Int64: int64(i), + Float: float32(i), + BinaryVec: GenBinaryVector(dim), + } if enableDynamicField { dynamic := Dynamic{ Number: int32(i), @@ -427,19 +609,15 @@ func GenDefaultBinaryRows(start int, nb int, dim int64, enableDynamicField bool) } dynamicRow := DynamicRow{ - Int64: int64(i), - Float: float32(i), - BinaryVec: binaryVec, + Int64: baseRow.Int64, + Float: baseRow.Float, + BinaryVec: baseRow.BinaryVec, Dynamic: dynamic, } rows = append(rows, dynamicRow) } else { - rows = append(rows, &BaseRow{ - Int64: int64(i), - Float: float32(i), - BinaryVec: binaryVec, - }) + rows = append(rows, &baseRow) } } return rows @@ -448,23 +626,23 @@ func GenDefaultBinaryRows(start int, nb int, dim int64, enableDynamicField bool) func GenDefaultVarcharRows(start int, nb int, dim int64, enableDynamicField bool) []interface{} { rows := make([]interface{}, 0, nb) - type DynamicRow struct { - Varchar string `json:"varchar" milvus:"name:varchar"` - BinaryVec [][]byte `json:"binaryVec" milvus:"name:binaryVec"` - Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` - } - // BaseRow generate insert rows type BaseRow struct { - Varchar string `json:"varchar" milvus:"name:varchar"` - BinaryVec [][]byte `json:"binaryVec" milvus:"name:binaryVec"` + Varchar string `json:"varchar" milvus:"name:varchar"` + BinaryVec []byte `json:"binaryVec" milvus:"name:binaryVec"` + } + + type DynamicRow struct { + Varchar string `json:"varchar" milvus:"name:varchar"` + BinaryVec []byte `json:"binaryVec" milvus:"name:binaryVec"` + Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` } for i := start; i < start+nb; i++ { - binaryVec := make([][]byte, 0, nb) - vec := make([]byte, 0, dim/8) - rand.Read(vec) - binaryVec = append(binaryVec, vec) + baseRow := BaseRow{ + Varchar: strconv.Itoa(i), + BinaryVec: GenBinaryVector(dim), + } if enableDynamicField { dynamic := Dynamic{ @@ -475,17 +653,14 @@ func GenDefaultVarcharRows(start int, nb int, dim int64, enableDynamicField bool } dynamicRow := DynamicRow{ - Varchar: strconv.Itoa(i), - BinaryVec: binaryVec, + Varchar: baseRow.Varchar, + BinaryVec: baseRow.BinaryVec, Dynamic: dynamic, } rows = append(rows, dynamicRow) } else { - rows = append(rows, &BaseRow{ - Varchar: strconv.Itoa(i), - BinaryVec: binaryVec, - }) + rows = append(rows, &baseRow) } } return rows @@ -500,31 +675,26 @@ func GenDefaultJSONRows(start int, nb int, dim int64, enableDynamicField bool) [ List []int64 `json:"list" milvus:"name:list"` } - type BaseDynamicRow struct { + // BaseRow generate insert rows + type BaseRow struct { Int64 int64 `json:"int64" milvus:"name:int64"` Float float32 `json:"float" milvus:"name:float"` FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` JSON JSONStruct `json:"json" milvus:"name:json"` - Number int32 `json:"dynamicNumber" milvus:"name:dynamicNumber"` - String string `json:"dynamicString" milvus:"name:dynamicString"` - Bool bool `json:"dynamicBool" milvus:"name:dynamicBool"` - //List []int64 `json:"dynamicList" milvus:"name:dynamicList"` } - // BaseRow generate insert rows - type BaseRow struct { + type BaseDynamicRow struct { Int64 int64 `json:"int64" milvus:"name:int64"` Float float32 `json:"float" milvus:"name:float"` FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` JSON JSONStruct `json:"json" milvus:"name:json"` + Number int32 `json:"dynamicNumber" milvus:"name:dynamicNumber"` + String string `json:"dynamicString" milvus:"name:dynamicString"` + Bool bool `json:"dynamicBool" milvus:"name:dynamicBool"` + //List []int64 `json:"dynamicList" milvus:"name:dynamicList"` } for i := start; i < start+nb; i++ { - floatVec := make([]float32, 0, dim) - for j := 0; j < int(dim); j++ { - floatVec = append(floatVec, rand.Float32()) - } - // jsonStruct row and dynamic row var jsonStruct JSONStruct if i%2 == 0 { @@ -540,66 +710,224 @@ func GenDefaultJSONRows(start int, nb int, dim int64, enableDynamicField bool) [ List: []int64{int64(i), int64(i + 1)}, } } + // base row + baseRow := BaseRow{ + Int64: int64(i), + Float: float32(i), + FloatVec: GenFloatVector(dim), + JSON: jsonStruct, + } if enableDynamicField { baseDynamicRow := BaseDynamicRow{ - Int64: int64(i), - Float: float32(i), - FloatVec: floatVec, - JSON: jsonStruct, + Int64: baseRow.Int64, + Float: baseRow.Float, + FloatVec: baseRow.FloatVec, + JSON: baseRow.JSON, Number: int32(i), String: strconv.Itoa(i), Bool: i%2 == 0, //List: []int64{int64(i), int64(i + 1)}, } - rows = append(rows, baseDynamicRow) + rows = append(rows, &baseDynamicRow) } else { - rows = append(rows, &BaseRow{ - Int64: int64(i), - Float: float32(i), - FloatVec: floatVec, - JSON: jsonStruct, - }) + rows = append(rows, &baseRow) } } return rows } -func GenAllFieldsRows(start int, nb int, dim int64, enableDynamicField bool) []interface{} { +func GenAllArrayRow(index int, opts ...GenColumnDataOption) Array { + opt := &genDataOpt{} + for _, o := range opts { + o(opt) + } + var capacity int + if opt.capacity != 0 { + capacity = int(opt.capacity) + } else { + capacity = TestCapacity + } + + boolRow := make([]bool, 0, capacity) + int8Row := make([]int8, 0, capacity) + int16Row := make([]int16, 0, capacity) + int32Row := make([]int32, 0, capacity) + int64Row := make([]int64, 0, capacity) + floatRow := make([]float32, 0, capacity) + doubleRow := make([]float64, 0, capacity) + varcharRow := make([][]byte, 0, capacity) + for j := 0; j < capacity; j++ { + boolRow = append(boolRow, index%2 == 0) + int8Row = append(int8Row, int8(index+j)) + int16Row = append(int16Row, int16(index+j)) + int32Row = append(int32Row, int32(index+j)) + int64Row = append(int64Row, int64(index+j)) + floatRow = append(floatRow, float32(index+j)) + doubleRow = append(doubleRow, float64(index+j)) + var buf bytes.Buffer + buf.WriteString(strconv.Itoa(index + j)) + varcharRow = append(varcharRow, buf.Bytes()) + } + arrayRow := Array{ + BoolArray: boolRow, + Int8Array: int8Row, + Int16Array: int16Row, + Int32Array: int32Row, + Int64Array: int64Row, + FloatArray: floatRow, + DoubleArray: doubleRow, + VarcharArray: varcharRow, + } + return arrayRow +} + +func GenDefaultArrayRows(start int, nb int, dim int64, enableDynamicField bool, opts ...GenColumnDataOption) []interface{} { rows := make([]interface{}, 0, nb) + // BaseRow generate insert rows + type BaseRow struct { + Int64 int64 `json:"int64" milvus:"name:int64"` + Float float32 `json:"float" milvus:"name:float"` + FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` + BoolArray []bool `json:"boolArray" milvus:"name:boolArray"` + Int8Array []int8 `json:"int8Array" milvus:"name:int8Array"` + Int16Array []int16 `json:"int16Array" milvus:"name:int16Array"` + Int32Array []int32 `json:"int32Array" milvus:"name:int32Array"` + Int64Array []int64 `json:"int64Array" milvus:"name:int64Array"` + FloatArray []float32 `json:"floatArray" milvus:"name:floatArray"` + DoubleArray []float64 `json:"doubleArray" milvus:"name:doubleArray"` + VarcharArray [][]byte `json:"varcharArray" milvus:"name:varcharArray"` + } + type DynamicRow struct { - Int64 int64 `json:"int64" milvus:"name:int64"` - Bool bool `json:"bool" milvus:"name:bool"` - Int8 int8 `json:"int8" milvus:"name:int8"` - Int16 int16 `json:"int16" milvus:"name:int16"` - Int32 int32 `json:"int32" milvus:"name:int32"` - Float float32 `json:"float" milvus:"name:float"` - Double float64 `json:"double" milvus:"name:double"` - Varchar string `json:"varchar" milvus:"name:varchar"` - JSON Dynamic `json:"json" milvus:"name:json"` - FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` - Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` + Int64 int64 `json:"int64" milvus:"name:int64"` + Float float32 `json:"float" milvus:"name:float"` + FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` + BoolArray []bool `json:"boolArray" milvus:"name:boolArray"` + Int8Array []int8 `json:"int8Array" milvus:"name:int8Array"` + Int16Array []int16 `json:"int16Array" milvus:"name:int16Array"` + Int32Array []int32 `json:"int32Array" milvus:"name:int32Array"` + Int64Array []int64 `json:"int64Array" milvus:"name:int64Array"` + FloatArray []float32 `json:"floatArray" milvus:"name:floatArray"` + DoubleArray []float64 `json:"doubleArray" milvus:"name:doubleArray"` + VarcharArray [][]byte `json:"varcharArray" milvus:"name:varcharArray"` + Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` } + for i := start; i < start+nb; i++ { + // json and dynamic field + dynamicJSON := Dynamic{ + Number: int32(i), + String: strconv.Itoa(i), + Bool: i%2 == 0, + List: []int64{int64(i), int64(i + 1)}, + } + arrayRow := GenAllArrayRow(i, opts...) + baseRow := BaseRow{ + Int64: int64(i), + Float: float32(i), + FloatVec: GenFloatVector(dim), + BoolArray: arrayRow.BoolArray, + Int8Array: arrayRow.Int8Array, + Int16Array: arrayRow.Int16Array, + Int32Array: arrayRow.Int32Array, + Int64Array: arrayRow.Int64Array, + FloatArray: arrayRow.FloatArray, + DoubleArray: arrayRow.DoubleArray, + VarcharArray: arrayRow.VarcharArray, + } + if enableDynamicField { + dynamicRow := DynamicRow{ + Int64: baseRow.Int64, + Float: baseRow.Float, + FloatVec: baseRow.FloatVec, + BoolArray: arrayRow.BoolArray, + Int8Array: arrayRow.Int8Array, + Int16Array: arrayRow.Int16Array, + Int32Array: arrayRow.Int32Array, + Int64Array: arrayRow.Int64Array, + FloatArray: arrayRow.FloatArray, + DoubleArray: arrayRow.DoubleArray, + VarcharArray: arrayRow.VarcharArray, + Dynamic: dynamicJSON, + } + + rows = append(rows, dynamicRow) + } else { + rows = append(rows, &baseRow) + } + } + return rows +} + +func GenAllFieldsRows(start int, nb int, dim int64, enableDynamicField bool, opts ...GenColumnDataOption) []interface{} { + rows := make([]interface{}, 0, nb) + // BaseRow generate insert rows type BaseRow struct { - Int64 int64 `json:"int64" milvus:"name:int64"` - Bool bool `json:"bool" milvus:"name:bool"` - Int8 int8 `json:"int8" milvus:"name:int8"` - Int16 int16 `json:"int16" milvus:"name:int16"` - Int32 int32 `json:"int32" milvus:"name:int32"` - Float float32 `json:"float" milvus:"name:float"` - Double float64 `json:"double" milvus:"name:double"` - Varchar string `json:"varchar" milvus:"name:varchar"` - JSON Dynamic `json:"json" milvus:"name:json"` - FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` + Int64 int64 `json:"int64" milvus:"name:int64"` + Bool bool `json:"bool" milvus:"name:bool"` + Int8 int8 `json:"int8" milvus:"name:int8"` + Int16 int16 `json:"int16" milvus:"name:int16"` + Int32 int32 `json:"int32" milvus:"name:int32"` + Float float32 `json:"float" milvus:"name:float"` + Double float64 `json:"double" milvus:"name:double"` + Varchar string `json:"varchar" milvus:"name:varchar"` + JSON Dynamic `json:"json" milvus:"name:json"` + FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` + BoolArray []bool `json:"boolArray" milvus:"name:boolArray"` + Int8Array []int8 `json:"int8Array" milvus:"name:int8Array"` + Int16Array []int16 `json:"int16Array" milvus:"name:int16Array"` + Int32Array []int32 `json:"int32Array" milvus:"name:int32Array"` + Int64Array []int64 `json:"int64Array" milvus:"name:int64Array"` + FloatArray []float32 `json:"floatArray" milvus:"name:floatArray"` + DoubleArray []float64 `json:"doubleArray" milvus:"name:doubleArray"` + VarcharArray [][]byte `json:"varcharArray" milvus:"name:varcharArray"` + } + + type DynamicRow struct { + Int64 int64 `json:"int64" milvus:"name:int64"` + Bool bool `json:"bool" milvus:"name:bool"` + Int8 int8 `json:"int8" milvus:"name:int8"` + Int16 int16 `json:"int16" milvus:"name:int16"` + Int32 int32 `json:"int32" milvus:"name:int32"` + Float float32 `json:"float" milvus:"name:float"` + Double float64 `json:"double" milvus:"name:double"` + Varchar string `json:"varchar" milvus:"name:varchar"` + JSON Dynamic `json:"json" milvus:"name:json"` + FloatVec []float32 `json:"floatVec" milvus:"name:floatVec"` + BoolArray []bool `json:"boolArray" milvus:"name:boolArray"` + Int8Array []int8 `json:"int8Array" milvus:"name:int8Array"` + Int16Array []int16 `json:"int16Array" milvus:"name:int16Array"` + Int32Array []int32 `json:"int32Array" milvus:"name:int32Array"` + Int64Array []int64 `json:"int64Array" milvus:"name:int64Array"` + FloatArray []float32 `json:"floatArray" milvus:"name:floatArray"` + DoubleArray []float64 `json:"doubleArray" milvus:"name:doubleArray"` + VarcharArray [][]byte `json:"varcharArray" milvus:"name:varcharArray"` + Dynamic Dynamic `json:"dynamic" milvus:"name:dynamic"` } for i := start; i < start+nb; i++ { - floatVec := make([]float32, 0, dim) - for j := 0; j < int(dim); j++ { - floatVec = append(floatVec, rand.Float32()) + arrayRow := GenAllArrayRow(i, opts...) + baseRow := BaseRow{ + Int64: int64(i), + Bool: i%2 == 0, + Int8: int8(i), + Int16: int16(i), + Int32: int32(i), + Float: float32(i), + Double: float64(i), + Varchar: strconv.Itoa(i), + FloatVec: GenFloatVector(dim), + BoolArray: arrayRow.BoolArray, + Int8Array: arrayRow.Int8Array, + Int16Array: arrayRow.Int16Array, + Int32Array: arrayRow.Int32Array, + Int64Array: arrayRow.Int64Array, + FloatArray: arrayRow.FloatArray, + DoubleArray: arrayRow.DoubleArray, + VarcharArray: arrayRow.VarcharArray, } // json and dynamic field @@ -611,33 +939,28 @@ func GenAllFieldsRows(start int, nb int, dim int64, enableDynamicField bool) []i } if enableDynamicField { dynamicRow := DynamicRow{ - Int64: int64(i), - Bool: i%2 == 0, - Int8: int8(i), - Int16: int16(i), - Int32: int32(i), - Float: float32(i), - Double: float64(i), - Varchar: strconv.Itoa(i), - FloatVec: floatVec, - JSON: dynamicJSON, - Dynamic: dynamicJSON, + Int64: baseRow.Int64, + Bool: baseRow.Bool, + Int8: baseRow.Int8, + Int16: baseRow.Int16, + Int32: baseRow.Int32, + Float: baseRow.Float, + Double: baseRow.Double, + Varchar: baseRow.Varchar, + FloatVec: baseRow.FloatVec, + BoolArray: arrayRow.BoolArray, + Int8Array: arrayRow.Int8Array, + Int16Array: arrayRow.Int16Array, + Int32Array: arrayRow.Int32Array, + Int64Array: arrayRow.Int64Array, + FloatArray: arrayRow.FloatArray, + DoubleArray: arrayRow.DoubleArray, + VarcharArray: arrayRow.VarcharArray, + Dynamic: dynamicJSON, } - rows = append(rows, dynamicRow) } else { - rows = append(rows, &BaseRow{ - Int64: int64(i), - Bool: i%2 == 0, - Int8: int8(i), - Int16: int16(i), - Int32: int32(i), - Float: float32(i), - Double: float64(i), - Varchar: strconv.Itoa(i), - FloatVec: floatVec, - JSON: dynamicJSON, - }) + rows = append(rows, &baseRow) } } return rows @@ -741,23 +1064,19 @@ func GenSearchVectors(nq int, dim int64, dataType entity.FieldType) []entity.Vec switch dataType { case entity.FieldTypeFloatVector: for i := 0; i < nq; i++ { - vector := make([]float32, 0, dim) - for j := 0; j < int(dim); j++ { - vector = append(vector, rand.Float32()) - } + vector := GenFloatVector(dim) vectors = append(vectors, entity.FloatVector(vector)) } case entity.FieldTypeBinaryVector: for i := 0; i < nq; i++ { - vector := make([]byte, dim/8) - rand.Read(vector) + vector := GenBinaryVector(dim) vectors = append(vectors, entity.BinaryVector(vector)) } } return vectors } -// invalid expr +// InvalidExprStruct invalid expr type InvalidExprStruct struct { Expr string ErrNil bool @@ -779,6 +1098,8 @@ var InvalidExpressions = []InvalidExprStruct{ {Expr: fmt.Sprintf("json_contains_all (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_all operation element must be an array"}, {Expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_any operation element must be an array"}, {Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"}, + {Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"}, + {Expr: fmt.Sprintf("%s[-1] > %d", DefaultInt8ArrayField, TestCapacity), ErrNil: false, ErrMsg: "cannot parse expression"}, // array[-1] > } // --- search utils --- diff --git a/test/common/utils_client.go b/test/common/utils_client.go index 025f097a..da730684 100644 --- a/test/common/utils_client.go +++ b/test/common/utils_client.go @@ -45,6 +45,18 @@ func WithMaxLength(maxLen int64) CreateFieldOption { } } +func WithElementType(eleType entity.FieldType) CreateFieldOption { + return func(field *entity.Field) { + field.WithElementType(eleType) + } +} + +func WithMaxCapacity(maxCap int64) CreateFieldOption { + return func(field *entity.Field) { + field.WithMaxCapacity(maxCap) + } +} + func WithTypeParams(key string, value string) CreateFieldOption { return func(field *entity.Field) { field.WithTypeParams(key, value) @@ -84,7 +96,7 @@ func WithEnableDynamicField(enableDF bool) CreateSchemaOption { } } -// gen schema +// GenSchema gen schema func GenSchema(name string, autoID bool, fields []*entity.Field, opts ...CreateSchemaOption) *entity.Schema { schema := &entity.Schema{ CollectionName: name, @@ -98,3 +110,31 @@ func GenSchema(name string, autoID bool, fields []*entity.Field, opts ...CreateS } // -- create schema -- + +// GenColumnDataOption -- create column data -- +type GenColumnDataOption func(opt *genDataOpt) +type genDataOpt struct { + dim int64 + ElementType entity.FieldType + capacity int64 +} + +func WithVectorDim(dim int64) GenColumnDataOption { + return func(opt *genDataOpt) { + opt.dim = dim + } +} + +func WithArrayElementType(eleType entity.FieldType) GenColumnDataOption { + return func(opt *genDataOpt) { + opt.ElementType = eleType + } +} + +func WithArrayCapacity(capacity int64) GenColumnDataOption { + return func(opt *genDataOpt) { + opt.capacity = capacity + } +} + +// -- create column data -- diff --git a/test/testcases/collection_test.go b/test/testcases/collection_test.go index 13b6a9ef..da75823b 100644 --- a/test/testcases/collection_test.go +++ b/test/testcases/collection_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/milvus-io/milvus-sdk-go/v2/client" + "github.com/stretchr/testify/require" "github.com/milvus-io/milvus-sdk-go/v2/entity" @@ -446,6 +448,30 @@ func TestCreateCollectionDynamicSchema(t *testing.T) { // check describe collection collection, _ := mc.DescribeCollection(ctx, collName) common.CheckCollection(t, collection, collName, common.DefaultShards, schema, common.DefaultConsistencyLevel) + require.Truef(t, collection.Schema.EnableDynamicField, "Expected collection.Schema.EnableDynamicField is True") + + // check collName in ListCollections + collections, errListCollection := mc.ListCollections(ctx) + common.CheckErr(t, errListCollection, true) + common.CheckContainsCollection(t, collections, collName) +} + +// test create collection enable dynamic field by collection opt +func TestCreateCollectionDynamic(t *testing.T) { + t.Skip("Waiting for congqi to update schema.EnableDynamicField according to the CreateCollectionOption.EnableDynamicSchema") + ctx := createContext(t, time.Second*common.DefaultTimeout) + mc := createMilvusClient(ctx, t) + + collName := common.GenRandomString(6) + schema := common.GenSchema(collName, false, common.GenDefaultFields(false)) + + err := mc.CreateCollection(ctx, schema, common.DefaultShards, client.WithEnableDynamicSchema(true)) + common.CheckErr(t, err, true) + + // check describe collection + collection, _ := mc.DescribeCollection(ctx, collName) + common.CheckCollection(t, collection, collName, common.DefaultShards, schema, common.DefaultConsistencyLevel) + require.Truef(t, collection.Schema.EnableDynamicField, "Expected collection.Schema.EnableDynamicField is True") // check collName in ListCollections collections, errListCollection := mc.ListCollections(ctx) @@ -467,6 +493,88 @@ func TestCreateCollectionFieldMeta(t *testing.T) { common.CheckErr(t, err, false, fmt.Sprintf("Invalid field name: %s. The first character of a field name must be an underscore or letter", common.DefaultDynamicFieldName)) } +func TestCreateArrayFieldInvalidCapacity(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + mc := createMilvusClient(ctx, t) + + fields := common.GenDefaultFields(true) + // array field no Capacity + arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(entity.FieldTypeFloat)) + schema := common.GenSchema(common.GenRandomString(6), false, append(fields, arrayField), common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, false, "type param(max_capacity) should be specified for array field") + + // invalid Capacity + for _, invalidCapacity := range []int64{-1, 0, common.MaxCapacity + 1} { + arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(entity.FieldTypeFloat), common.WithMaxCapacity(invalidCapacity)) + schema := common.GenSchema(common.GenRandomString(6), false, append(fields, arrayField), common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, false, "the maximum capacity specified for a Array should be in (0, 4096]") + } +} + +// test create collection varchar array with invalid max length +func TestCreateVarcharArrayInvalidLength(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + mc := createMilvusClient(ctx, t) + + fields := common.GenDefaultFields(true) + varcharArrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(entity.FieldTypeVarChar), common.WithMaxCapacity(100)) + schema := common.GenSchema(common.GenRandomString(6), false, append(fields, varcharArrayField), common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, false, "type param(max_length) should be specified for varChar field") + + // invalid max length + for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} { + arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(entity.FieldTypeVarChar), + common.WithMaxCapacity(100), common.WithMaxLength(invalidLength)) + schema := common.GenSchema(common.GenRandomString(6), false, append(fields, arrayField), common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]: invalid parameter") + } +} + +// test create collection array field not supported type +func TestCreateArrayNotSupportedFieldType(t *testing.T) { + t.Parallel() + ctx := createContext(t, time.Second*common.DefaultTimeout) + mc := createMilvusClient(ctx, t) + + // not supported ElementType: Array, Json, FloatVector, BinaryVector + fields := common.GenDefaultFields(true) + for _, fieldType := range []entity.FieldType{entity.FieldTypeArray, entity.FieldTypeJSON, entity.FieldTypeBinaryVector, entity.FieldTypeFloatVector} { + arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(fieldType), common.WithMaxCapacity(100)) + + schema := common.GenSchema(common.GenRandomString(6), false, append(fields, arrayField), common.WithEnableDynamicField(true)) + + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, false, fmt.Sprintf("element type %s is not supported", fieldType.Name())) + } + + // NoneType ElementType + noneArrayFields := []*entity.Field{ + common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(entity.FieldTypeNone), common.WithMaxCapacity(100)), + common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithMaxCapacity(100)), + } + for _, noneArrayField := range noneArrayFields { + schema := common.GenSchema(common.GenRandomString(6), false, append(fields, noneArrayField), common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, false, "element data type None is not valid") + } +} + +func TestCreateCollectionAllFields(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + mc := createMilvusClient(ctx, t) + allFields := common.GenAllFields() + collName := common.GenRandomString(6) + schema := common.GenSchema(collName, false, allFields) + + // create collection + errCreateCollection := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, errCreateCollection, true) +} + // -- Get Collection Statistics -- func TestGetStaticsCollectionNotExisted(t *testing.T) { diff --git a/test/testcases/index_test.go b/test/testcases/index_test.go index a66c8b52..fbd73333 100644 --- a/test/testcases/index_test.go +++ b/test/testcases/index_test.go @@ -91,6 +91,41 @@ func TestCreateIndexJsonField(t *testing.T) { //common.CheckErr(t, err, false, "create index on json field is not supported") } +func TestCreateIndexArrayField(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVecArray, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: common.TestCapacity} + collName := createCollection(ctx, t, mc, cp) + + // prepare and insert data + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecArray, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} + _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(common.TestCapacity)) + + // flush and check row count + errFlush := mc.Flush(ctx, collName, false) + common.CheckErr(t, errFlush, true) + + // create scalar and vector index on array field + scalarIdx := entity.NewScalarIndex() + vectorIdx, _ := entity.NewIndexSCANN(entity.L2, 10, false) + collection, _ := mc.DescribeCollection(ctx, collName) + for _, field := range collection.Schema.Fields { + if field.DataType == entity.FieldTypeArray { + // create scalar index + err := mc.CreateIndex(ctx, collName, field.Name, scalarIdx, false, client.WithIndexName("scalar_index")) + common.CheckErr(t, err, false, "create index on Array field: invalid parameter") + // create vector index + err1 := mc.CreateIndex(ctx, collName, field.Name, vectorIdx, false, client.WithIndexName("vector_index")) + common.CheckErr(t, err1, false, "create index on Array field: invalid parameter") + } + } +} + // test create index with supported float vector index, Ip metric type func TestCreateIndexIp(t *testing.T) { t.Parallel() diff --git a/test/testcases/insert_test.go b/test/testcases/insert_test.go index 391f1483..3c94a1dd 100644 --- a/test/testcases/insert_test.go +++ b/test/testcases/insert_test.go @@ -160,7 +160,7 @@ func TestInsertAllFieldsData(t *testing.T) { // prepare fields, name, schema allFields := common.GenAllFields() collName := common.GenRandomString(6) - schema := common.GenSchema(collName, false, allFields) + schema := common.GenSchema(collName, false, allFields, common.WithEnableDynamicField(true)) // create collection errCreateCollection := mc.CreateCollection(ctx, schema, common.DefaultShards) @@ -168,6 +168,7 @@ func TestInsertAllFieldsData(t *testing.T) { // prepare and insert data data := common.GenAllFieldsData(0, common.DefaultNb, common.DefaultDim) + data = append(data, common.GenDynamicFieldData(0, common.DefaultNb)...) // insert data ids, errInsert := mc.Insert( ctx, @@ -407,3 +408,113 @@ func TestInsertDynamicFieldData(t *testing.T) { common.CheckErr(t, errInsert, false, "column size not match") } + +// test insert array column with empty data +func TestInsertEmptyArray(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVecArray, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: common.TestCapacity} + collName := createCollection(ctx, t, mc, cp) + + // prepare and insert data + var capacity int64 = 0 + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecArray, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} + _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(capacity)) + + // flush and check row count + errFlush := mc.Flush(ctx, collName, false) + common.CheckErr(t, errFlush, true) + stats, _ := mc.GetCollectionStatistics(ctx, collName) + require.Equal(t, strconv.Itoa(common.DefaultNb), stats[common.RowCount]) +} + +// test insert array type rows data +func TestInsertArrayRows(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // create collection + cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: false, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: common.TestCapacity} + collName := createCollection(ctx, t, mc, cp) + + // prepare and insert array rows data + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: AllFields, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: true} + _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(common.TestCapacity)) + + // flush and check row count + errFlush := mc.Flush(ctx, collName, false) + common.CheckErr(t, errFlush, true) + stats, _ := mc.GetCollectionStatistics(ctx, collName) + require.Equal(t, strconv.Itoa(common.DefaultNb), stats[common.RowCount]) +} + +// test insert array data type not match array field element type +func TestInsertArrayDataTypeNotMatch(t *testing.T) { + t.Parallel() + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + for _, arrayType := range common.ArrayFieldType { + + // fields: int64 + float + vector + array with TestCapacity + defaultFields := common.GenDefaultFields(false) + arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, + common.WithElementType(arrayType), common.WithMaxCapacity(100), common.WithMaxLength(100)) + fields := append(defaultFields, arrayField) + + // create collection + collName := common.GenRandomString(6) + schema := common.GenSchema(collName, false, fields, common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, true) + + // insert data type does not match schema array element type + var columnType entity.FieldType + if arrayType == entity.FieldTypeInt64 { + columnType = entity.FieldTypeBool + } else { + columnType = entity.FieldTypeInt64 + } + intColumn, floatColumn, vectorColumn := common.GenDefaultColumnData(0, common.DefaultNb, common.DefaultDim) + arrayColumn := common.GenArrayColumnData(0, common.DefaultNb, common.DefaultArrayFieldName, + common.WithArrayElementType(columnType)) + _, err = mc.Insert(ctx, collName, "", intColumn, floatColumn, vectorColumn, arrayColumn) + common.CheckErr(t, err, false, "insert data does not match") + } +} + +// test insert array column data that capacity exceeds max capacity +func TestInsertArrayDataCapacityExceed(t *testing.T) { + t.Parallel() + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + for _, arrayType := range common.ArrayFieldType { + // fields: int64 + float + vector + array with TestCapacity + defaultFields := common.GenDefaultFields(false) + arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, + common.WithElementType(arrayType), common.WithMaxCapacity(common.TestCapacity), common.WithMaxLength(100)) + fields := append(defaultFields, arrayField) + + // create collection + collName := common.GenRandomString(6) + schema := common.GenSchema(collName, false, fields, common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards) + common.CheckErr(t, err, true) + + // insert data capacity larger than TestCapacity + intColumn, floatColumn, vectorColumn := common.GenDefaultColumnData(0, common.DefaultNb, common.DefaultDim) + arrayColumn := common.GenArrayColumnData(0, common.DefaultNb, common.DefaultArrayFieldName, + common.WithArrayElementType(arrayType), common.WithArrayCapacity(common.TestCapacity+1)) + _, err = mc.Insert(ctx, collName, "", intColumn, floatColumn, vectorColumn, arrayColumn) + common.CheckErr(t, err, false, "array length exceeds max capacity") + } +} diff --git a/test/testcases/main_test.go b/test/testcases/main_test.go index fa79a8ee..7df17a10 100644 --- a/test/testcases/main_test.go +++ b/test/testcases/main_test.go @@ -4,7 +4,6 @@ import ( "context" "flag" "log" - "math/rand" "os" "strconv" "testing" @@ -243,11 +242,12 @@ func createVarcharCollectionWithDataIndex(ctx context.Context, t *testing.T, mc } const ( - Int64FloatVec CollectionFieldsType = "PkInt64FloatVec" // int64 + float + floatVec - Int64BinaryVec CollectionFieldsType = "Int64BinaryVec" // int64 + float + binaryVec - VarcharBinaryVec CollectionFieldsType = "PkVarcharBinaryVec" // varchar + binaryVec - Int64FloatVecJSON CollectionFieldsType = "PkInt64FloatVecJson" // int64 + float + floatVec + json - AllFields CollectionFieldsType = "AllFields" // all scalar fields + floatVec + Int64FloatVec CollectionFieldsType = "PkInt64FloatVec" // int64 + float + floatVec + Int64BinaryVec CollectionFieldsType = "Int64BinaryVec" // int64 + float + binaryVec + VarcharBinaryVec CollectionFieldsType = "PkVarcharBinaryVec" // varchar + binaryVec + Int64FloatVecJSON CollectionFieldsType = "PkInt64FloatVecJson" // int64 + float + floatVec + json + Int64FloatVecArray CollectionFieldsType = "Int64FloatVecArray" // int64 + float + floatVec + all array + AllFields CollectionFieldsType = "AllFields" // all scalar fields + floatVec ) func createCollection(ctx context.Context, t *testing.T, mc *base.MilvusClient, cp CollectionParams, opts ...client.CreateCollectionOption) string { @@ -267,6 +267,9 @@ func createCollection(ctx context.Context, t *testing.T, mc *base.MilvusClient, fields = common.GenDefaultFields(cp.AutoID) jsonField := common.GenField(common.DefaultJSONFieldName, entity.FieldTypeJSON) fields = append(fields, jsonField) + case Int64FloatVecArray: + fields = common.GenDefaultFields(cp.AutoID) + fields = append(fields, common.GenAllArrayFieldsWithCapacity(cp.MaxCapacity)...) case AllFields: fields = common.GenAllFields() } @@ -285,8 +288,7 @@ func createCollection(ctx context.Context, t *testing.T, mc *base.MilvusClient, return collName } -// insert nb data -func insertData(ctx context.Context, t *testing.T, mc *base.MilvusClient, dp DataParams) (entity.Column, error) { +func insertData(ctx context.Context, t *testing.T, mc *base.MilvusClient, dp DataParams, opts ...common.GenColumnDataOption) (entity.Column, error) { // todo autoid // prepare data var data []entity.Column @@ -328,11 +330,20 @@ func insertData(ctx context.Context, t *testing.T, mc *base.MilvusClient, dp Dat jsonColumn := common.GenDefaultJSONData(common.DefaultJSONFieldName, dp.start, dp.nb) data = append(data, intColumn, floatColumn, vecColumn, jsonColumn) } + case Int64FloatVecArray: + if dp.WithRows { + rows = common.GenDefaultArrayRows(dp.start, dp.nb, dp.dim, dp.EnableDynamicField) + } else { + data = common.GenAllArrayData(dp.start, dp.nb, opts...) + intColumn, floatColumn, vecColumn := common.GenDefaultColumnData(dp.start, dp.nb, dp.dim) + data = append(data, intColumn, floatColumn, vecColumn) + } + case AllFields: if dp.WithRows { - rows = common.GenAllFieldsRows(dp.start, dp.nb, dp.dim, dp.EnableDynamicField) + rows = common.GenAllFieldsRows(dp.start, dp.nb, dp.dim, dp.EnableDynamicField, opts...) } - data = common.GenAllFieldsData(dp.start, dp.nb, dp.dim) + data = common.GenAllFieldsData(dp.start, dp.nb, dp.dim, opts...) } if dp.EnableDynamicField && !dp.WithRows { @@ -349,7 +360,6 @@ func insertData(ctx context.Context, t *testing.T, mc *base.MilvusClient, dp Dat } common.CheckErr(t, err, true) require.Equalf(t, dp.nb, ids.Len(), "Expected insert id num: %d, actual: ", dp.nb, ids.Len()) - return ids, err } @@ -366,48 +376,8 @@ func createCollectionAllFields(ctx context.Context, t *testing.T, mc *base.Milvu errCreateCollection := mc.CreateCollection(ctx, schema, common.DefaultShards) common.CheckErr(t, errCreateCollection, true) - // prepare data - int64Values := make([]int64, 0, nb) - boolValues := make([]bool, 0, nb) - int8Values := make([]int8, 0, nb) - int16Values := make([]int16, 0, nb) - int32Values := make([]int32, 0, nb) - floatValues := make([]float32, 0, nb) - doubleValues := make([]float64, 0, nb) - varcharValues := make([]string, 0, nb) - floatVectors := make([][]float32, 0, nb) - for i := start; i < nb+start; i++ { - int64Values = append(int64Values, int64(i)) - boolValues = append(boolValues, i/2 == 0) - int8Values = append(int8Values, int8(i)) - int16Values = append(int16Values, int16(i)) - int32Values = append(int32Values, int32(i)) - floatValues = append(floatValues, float32(i)) - doubleValues = append(doubleValues, float64(i)) - varcharValues = append(varcharValues, strconv.Itoa(i)) - vec := make([]float32, 0, common.DefaultDim) - for j := 0; j < int(common.DefaultDim); j++ { - vec = append(vec, rand.Float32()) - } - floatVectors = append(floatVectors, vec) - } - - // insert data - ids, errInsert := mc.Insert( - ctx, - collName, - "", - entity.NewColumnInt64("int64", int64Values), - entity.NewColumnBool("bool", boolValues), - entity.NewColumnInt8("int8", int8Values), - entity.NewColumnInt16("int16", int16Values), - entity.NewColumnInt32("int32", int32Values), - entity.NewColumnFloat("float", floatValues), - entity.NewColumnDouble("double", doubleValues), - entity.NewColumnVarChar("varchar", varcharValues), - entity.NewColumnFloatVector("floatVec", int(common.DefaultDim), floatVectors), - common.GenDefaultJSONData("json", 0, nb), - ) + data := common.GenAllFieldsData(start, nb, common.DefaultDim, common.WithArrayCapacity(common.TestCapacity)) + ids, errInsert := mc.Insert(ctx, collName, "", data...) common.CheckErr(t, errInsert, true) require.Equal(t, nb, ids.Len()) return collName, ids diff --git a/test/testcases/option.go b/test/testcases/option.go index 659b5648..c41d4a4b 100644 --- a/test/testcases/option.go +++ b/test/testcases/option.go @@ -20,6 +20,7 @@ type CollectionParams struct { ShardsNum int32 Dim int64 MaxLength int64 + MaxCapacity int64 } type DataParams struct { diff --git a/test/testcases/partition_key_test.go b/test/testcases/partition_key_test.go index 2b88a480..42c9e864 100644 --- a/test/testcases/partition_key_test.go +++ b/test/testcases/partition_key_test.go @@ -25,7 +25,7 @@ func TestPartitionKeyDefaultInt64(t *testing.T) { // fields partitionKeyFieldName := "partitionKeyField" partitionKeyField := common.GenField(partitionKeyFieldName, entity.FieldTypeInt64, - common.WithIsPartitionKey(true), common.WithMaxLength(common.DefaultMaxLength)) + common.WithIsPartitionKey(true), common.WithMaxLength(common.MaxLength)) // schema schema := common.GenSchema(common.GenRandomString(6), false, common.GenDefaultFields(false)) @@ -105,7 +105,7 @@ func TestPartitionKeyDefaultVarchar(t *testing.T) { // fields partitionKeyFieldName := "partitionKeyField" partitionKeyField := common.GenField(partitionKeyFieldName, entity.FieldTypeVarChar, - common.WithIsPartitionKey(true), common.WithMaxLength(common.DefaultMaxLength)) + common.WithIsPartitionKey(true), common.WithMaxLength(common.MaxLength)) // schema schema := common.GenSchema(common.GenRandomString(6), false, common.GenDefaultFields(false)) diff --git a/test/testcases/query_test.go b/test/testcases/query_test.go index e7eaf0d6..ca279f99 100644 --- a/test/testcases/query_test.go +++ b/test/testcases/query_test.go @@ -223,7 +223,6 @@ func TestQueryEmptyIds(t *testing.T) { } // test query with non-primary field filter, and output scalar fields -// TODO "Issue: https://github.com/milvus-io/milvus-sdk-go/issues/366") func TestQueryNonPrimaryFields(t *testing.T) { t.Parallel() ctx := createContext(t, time.Second*common.DefaultTimeout) @@ -383,39 +382,43 @@ func TestQueryOutputBinaryAndVarchar(t *testing.T) { common.CheckOutputFields(t, queryResult, []string{common.DefaultBinaryVecFieldName, common.DefaultVarcharFieldName}) } -// test query output all scalar fields -func TestOutputAllScalarFields(t *testing.T) { +// test query output all fields +func TestOutputAllFields(t *testing.T) { ctx := createContext(t, time.Second*common.DefaultTimeout) // connect mc := createMilvusClient(ctx, t) - // create and insert - collName, _ := createCollectionAllFields(ctx, t, mc, common.DefaultNb, 0) - mc.Flush(ctx, collName, false) + for _, withRows := range []bool{true, false} { + // create collection + var capacity int64 = common.TestCapacity + cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: capacity} + collName := createCollection(ctx, t, mc, cp) - // create index - idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) - mc.CreateIndex(ctx, collName, "floatVec", idx, false) + // prepare and insert data + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: AllFields, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: withRows} + _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(capacity)) - // Load collection - errLoad := mc.LoadCollection(ctx, collName, false) - common.CheckErr(t, errLoad, true) + // flush and check row count + errFlush := mc.Flush(ctx, collName, false) + common.CheckErr(t, errFlush, true) - // query with non-primary field - queryIds := []entity.Column{ - entity.NewColumnBool("bool", []bool{true}), - entity.NewColumnInt8("int8", []int8{0}), - entity.NewColumnInt16("int16", []int16{0}), - entity.NewColumnInt32("int32", []int32{0}), - entity.NewColumnFloat("float", []float32{0}), - entity.NewColumnDouble("double", []float64{0}), - } + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) - queryResultAllScalar, errQuery := mc.QueryByPks(ctx, collName, []string{common.DefaultPartition}, - entity.NewColumnInt64(common.DefaultIntFieldName, []int64{0}), []string{"bool", "int8", "int16", "int32", "float", "double"}) - common.CheckErr(t, errQuery, true) - common.CheckQueryResult(t, queryResultAllScalar, queryIds) - common.CheckOutputFields(t, queryResultAllScalar, []string{"int64", "bool", "int8", "int16", "int32", "float", "double"}) + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query output all fields -> output all fields, includes vector and $meta field + allFieldsName := append(common.AllArrayFieldsName, "int64", "bool", "int8", "int16", "int32", "float", + "double", "varchar", "json", "floatVec", common.DefaultDynamicFieldName) + queryResultAll, errQuery := mc.Query(ctx, collName, []string{}, + fmt.Sprintf("%s == 0", common.DefaultIntFieldName), []string{"*"}) + common.CheckErr(t, errQuery, true) + common.CheckOutputFields(t, queryResultAll, allFieldsName) + } } // test query with an not existed field @@ -613,6 +616,135 @@ func TestQueryCountJsonDynamicExpr(t *testing.T) { } } +// test query with all kinds of array expr +func TestQueryArrayFieldExpr(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + for _, withRows := range []bool{true, false} { + // create collection + var capacity int64 = common.TestCapacity + cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: capacity} + collName := createCollection(ctx, t, mc, cp, client.WithConsistencyLevel(entity.ClStrong)) + + // prepare and insert data + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: AllFields, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: withRows} + _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(capacity)) + + // flush and check row count + errFlush := mc.Flush(ctx, collName, false) + common.CheckErr(t, errFlush, true) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + type exprCount struct { + expr string + count int64 + } + exprCounts := []exprCount{ + {expr: fmt.Sprintf("%s[0] == false", common.DefaultBoolArrayField), count: common.DefaultNb / 2}, // array[0] == + {expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt64ArrayField), count: common.DefaultNb - 1}, // array[0] > + {expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt8ArrayField), count: 1524}, // array[0] > int8 range: [-128, 127] + {expr: fmt.Sprintf("json_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), count: capacity}, // json_contains(array, 1) + {expr: fmt.Sprintf("array_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), count: capacity}, // array_contains(array, 1) + {expr: fmt.Sprintf("array_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // array_contains(array, 1) + {expr: fmt.Sprintf("json_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // json_contains(array, 1) + {expr: fmt.Sprintf("array_contains (%s, 1000000)", common.DefaultInt32ArrayField), count: 0}, // array_contains(array, 1) + {expr: fmt.Sprintf("json_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), count: 91}, // json_contains_all(array, [x]) + {expr: fmt.Sprintf("array_contains_all (%s, [1, 2])", common.DefaultInt64ArrayField), count: 2}, // array_contains_all(array, [x]) + {expr: fmt.Sprintf("array_contains_any (%s, [0, 100, 10000])", common.DefaultFloatArrayField), count: 101}, // array_contains_any(array, [x]) + {expr: fmt.Sprintf("json_contains_any (%s, [0, 100, 10])", common.DefaultFloatArrayField), count: 101}, // json_contains_any (array, [x]) + {expr: fmt.Sprintf("%s == [0, 1]", common.DefaultDoubleArrayField), count: 0}, // array == + {expr: fmt.Sprintf("array_length(%s) == 10", common.DefaultVarcharArrayField), count: 0}, // array_length + {expr: fmt.Sprintf("array_length(%s) == %d", common.DefaultDoubleArrayField, capacity), count: common.DefaultNb}, // array_length + } + + for _, _exprCount := range exprCounts { + log.Println(_exprCount.expr) + countRes, err := mc.Query(ctx, collName, + []string{}, _exprCount.expr, []string{common.QueryCountFieldName}) + common.CheckErr(t, err, true) + require.Equal(t, _exprCount.count, countRes.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0]) + } + } +} + +// test query different array rows has different element length +func TestQueryArrayDifferentLenBetweenRows(t *testing.T) { + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + + // fields + defaultFields := common.GenDefaultFields(false) + int64ArrayField := common.GenField(common.DefaultInt32ArrayField, entity.FieldTypeArray, + common.WithElementType(entity.FieldTypeInt32), common.WithMaxCapacity(common.TestCapacity)) + + // create collection with max + collName := common.GenRandomString(4) + schema := common.GenSchema(collName, false, append(defaultFields, int64ArrayField), common.WithEnableDynamicField(true)) + err := mc.CreateCollection(ctx, schema, common.DefaultShards, client.WithConsistencyLevel(entity.ClStrong)) + common.CheckErr(t, err, true) + + // insert data [0, 1500) with array length = capacity + intColumn, floatColumn, vecColumn := common.GenDefaultColumnData(0, common.DefaultNb/2, common.DefaultDim) + data := []entity.Column{ + intColumn, + floatColumn, + vecColumn, + common.GenArrayColumnData(0, common.DefaultNb/2, common.DefaultInt32ArrayField, + common.WithArrayElementType(entity.FieldTypeInt32), common.WithArrayCapacity(common.TestCapacity)), + } + ids, err := mc.Insert(ctx, collName, "", data...) + common.CheckErr(t, err, true) + + // insert data [1500, 3000) with array length < capacity + require.Equal(t, common.DefaultNb/2, ids.Len()) + intColumn1, floatColumn1, vecColumn1 := common.GenDefaultColumnData(common.DefaultNb/2, common.DefaultNb/2, common.DefaultDim) + data1 := []entity.Column{ + intColumn1, + floatColumn1, + vecColumn1, + common.GenArrayColumnData(common.DefaultNb/2, common.DefaultNb/2, common.DefaultInt32ArrayField, + common.WithArrayElementType(entity.FieldTypeInt32), common.WithArrayCapacity(common.TestCapacity/2)), + } + ids, err = mc.Insert(ctx, collName, "", data1...) + common.CheckErr(t, err, true) + require.Equal(t, common.DefaultNb/2, ids.Len()) + + // flush and check row count + errFlush := mc.Flush(ctx, collName, false) + common.CheckErr(t, errFlush, true) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + // query array idx invalid > capacity + countRes, err := mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s[%d] > 0", common.DefaultInt32ArrayField, common.TestCapacity), + []string{common.QueryCountFieldName}) + common.CheckErr(t, err, false, "index out of range") + + // query: some rows has element less than expr index + t.Skip("https://github.com/milvus-io/milvus/issues/28293") + countRes, err = mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s[%d] > 0", common.DefaultInt32ArrayField, common.TestCapacity/2+1), + []string{common.QueryCountFieldName}) + common.CheckErr(t, err, true) + log.Println(countRes.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0]) + +} + // test query with expr and verify output dynamic field data func TestQueryJsonDynamicExpr(t *testing.T) { t.Parallel() diff --git a/test/testcases/search_test.go b/test/testcases/search_test.go index fc7e83ca..b44ad826 100644 --- a/test/testcases/search_test.go +++ b/test/testcases/search_test.go @@ -405,13 +405,12 @@ func TestSearchOutputFields(t *testing.T) { // test search output all * fields when enable dynamic and insert dynamic column data func TestSearchOutputAllFields(t *testing.T) { - t.Skip("https://github.com/milvus-io/milvus-sdk-go/issues/596") t.Parallel() ctx := createContext(t, time.Second*common.DefaultTimeout) // connect mc := createMilvusClient(ctx, t) - for _, enableDynamic := range []bool{true, false} { + for _, enableDynamic := range []bool{false, true} { // create collection cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: enableDynamic, ShardsNum: common.DefaultShards, Dim: common.DefaultDim} @@ -433,41 +432,41 @@ func TestSearchOutputAllFields(t *testing.T) { // search vector output all scalar fields queryVec := common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector) - allScalarFields := []string{"int64", "bool", "int8", "int16", "int32", "float", "double", "varchar", "json"} - var allScalarFieldsDynamic []string + allFields := []string{"int64", "bool", "int8", "int16", "int32", "float", "double", "varchar", "json", "floatVec"} + allFields = append(allFields, common.AllArrayFieldsName...) + if enableDynamic { - allScalarFieldsDynamic = append(allScalarFields, common.DefaultDynamicFieldName) - } else { - allScalarFieldsDynamic = allScalarFields + allFields = append(allFields, common.DefaultDynamicFieldName) } sp, _ := entity.NewIndexHNSWSearchParam(74) searchRes, _ := mc.Search(ctx, collName, []string{}, "", - allScalarFieldsDynamic, + []string{"*"}, queryVec, common.DefaultFloatVecFieldName, entity.L2, common.DefaultTopK, sp, ) - common.CheckOutputFields(t, searchRes[0].Fields, allScalarFieldsDynamic) + common.CheckOutputFields(t, searchRes[0].Fields, allFields) // search with output * fields if enableDynamic { - //search output [*, a] fields + //search output [*, a] fields -> output all fields, no a field _, errNotExist := mc.Search(ctx, collName, []string{}, "", []string{"*", "a"}, queryVec, common.DefaultFloatVecFieldName, entity.L2, common.DefaultTopK, sp) - common.CheckErr(t, errNotExist, false, "not exist") + common.CheckErr(t, errNotExist, true) + common.CheckOutputFields(t, searchRes[0].Fields, allFields) - //search output [*, dynamicNumber] fields + //search output [*, dynamicNumber] fields -> -> output all fields, $meta replace by dynamicNumber searchRes, _ = mc.Search(ctx, collName, []string{}, "", []string{"*", common.DefaultDynamicNumberField}, queryVec, common.DefaultFloatVecFieldName, entity.L2, common.DefaultTopK, sp) - common.CheckOutputFields(t, searchRes[0].Fields, append(allScalarFields, common.DefaultFloatVecFieldName, common.DefaultDynamicNumberField)) + common.CheckOutputFields(t, searchRes[0].Fields, append(allFields, common.DefaultDynamicNumberField)) - // search output * fields - searchRes, _ = mc.Search(ctx, collName, []string{}, "", []string{"*"}, queryVec, common.DefaultFloatVecFieldName, - entity.L2, common.DefaultTopK, sp) - common.CheckOutputFields(t, searchRes[0].Fields, append(allScalarFields, common.DefaultFloatVecFieldName)) + ///search output [*, dynamicNumber] fields -> -> output all fields, $meta replace by dynamicNumber + searchRes, _ = mc.Search(ctx, collName, []string{}, "", []string{common.DefaultDynamicNumberField}, + queryVec, common.DefaultFloatVecFieldName, entity.L2, common.DefaultTopK, sp) + common.CheckOutputFields(t, searchRes[0].Fields, []string{common.DefaultDynamicNumberField}) } } } @@ -913,7 +912,7 @@ func TestSearchDynamicFieldExpr(t *testing.T) { // connect mc := createMilvusClient(ctx, t) - for _, withRows := range []bool{false, true} { + for _, withRows := range []bool{true, false} { // create collection cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true, ShardsNum: common.DefaultShards, Dim: common.DefaultDim} @@ -1001,6 +1000,75 @@ func TestSearchDynamicFieldExpr(t *testing.T) { } } +func TestSearchArrayFieldExpr(t *testing.T) { + t.Parallel() + + ctx := createContext(t, time.Second*common.DefaultTimeout) + // connect + mc := createMilvusClient(ctx, t) + var capacity int64 = common.TestCapacity + + for _, withRows := range []bool{true, false} { + // create collection + cp := CollectionParams{CollectionFieldsType: Int64FloatVecArray, AutoID: false, EnableDynamicField: true, + ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxCapacity: capacity} + collName := createCollection(ctx, t, mc, cp, client.WithConsistencyLevel(entity.ClStrong)) + + // prepare and insert data + dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecArray, + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: withRows} + _, _ = insertData(ctx, t, mc, dp, common.WithArrayCapacity(capacity)) + mc.Flush(ctx, collName, false) + + idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96) + _ = mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false) + + // Load collection + errLoad := mc.LoadCollection(ctx, collName, false) + common.CheckErr(t, errLoad, true) + + exprs := []string{ + fmt.Sprintf("%s[0] == false", common.DefaultBoolArrayField), // array[0] == + fmt.Sprintf("%s[0] > 0", common.DefaultInt64ArrayField), // array[0] > + fmt.Sprintf("json_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), // json_contains + fmt.Sprintf("array_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), // array_contains + fmt.Sprintf("json_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), // json_contains_all + fmt.Sprintf("array_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), // array_contains_all + fmt.Sprintf("array_contains_any (%s, [0, 100, 10000])", common.DefaultFloatArrayField), // array_contains_any + fmt.Sprintf("json_contains_any (%s, [0, 100, 10])", common.DefaultFloatArrayField), // json_contains_any + fmt.Sprintf("array_length(%s) == %d", common.DefaultDoubleArrayField, capacity), // array_length + } + + // search with jsonField expr key datatype and json data type mismatch + sp, _ := entity.NewIndexHNSWSearchParam(74) + for _, expr := range exprs { + log.Printf("search expr: %s", expr) + searchRes, errSearchEmpty := mc.Search( + ctx, collName, []string{}, + expr, common.AllArrayFieldsName, + common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector), + common.DefaultFloatVecFieldName, entity.L2, + common.DefaultTopK, sp, + ) + common.CheckErr(t, errSearchEmpty, true) + common.CheckOutputFields(t, searchRes[0].Fields, common.AllArrayFieldsName) + common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultTopK) + } + + // search hits empty + searchRes, errSearchEmpty := mc.Search( + ctx, collName, []string{}, + fmt.Sprintf("array_contains (%s, 1000000)", common.DefaultInt32ArrayField), + common.AllArrayFieldsName, + common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector), + common.DefaultFloatVecFieldName, entity.L2, common.DefaultTopK, + sp, + ) + common.CheckErr(t, errSearchEmpty, true) + require.Empty(t, searchRes) + } +} + // search with index scann search param ef < topK -> error func TestSearchInvalidScannReorderK(t *testing.T) { ctx := createContext(t, time.Second*common.DefaultTimeout) @@ -1061,13 +1129,13 @@ func TestSearchScannAllMetricsWithRawData(t *testing.T) { mc := createMilvusClient(ctx, t) // create collection - cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: false, + cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true, ShardsNum: common.DefaultShards, Dim: common.DefaultDim} collName := createCollection(ctx, t, mc, cp) // insert dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecJSON, - start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false} + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} _, _ = insertData(ctx, t, mc, dp) mc.Flush(ctx, collName, false) @@ -1100,7 +1168,7 @@ func TestSearchScannAllMetricsWithRawData(t *testing.T) { ) common.CheckErr(t, errSearch, true) common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, - common.DefaultJSONFieldName, common.DefaultFloatVecFieldName}) + common.DefaultJSONFieldName, common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName}) common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK) } } @@ -1113,13 +1181,13 @@ func TestRangeSearchScannL2(t *testing.T) { mc := createMilvusClient(ctx, t) // create collection - cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: false, + cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true, ShardsNum: common.DefaultShards, Dim: common.DefaultDim} collName := createCollection(ctx, t, mc, cp) // insert dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecJSON, - start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false} + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} _, _ = insertData(ctx, t, mc, dp) // create scann index @@ -1148,7 +1216,7 @@ func TestRangeSearchScannL2(t *testing.T) { common.CheckErr(t, errSearch, true) common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK) common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, - common.DefaultJSONFieldName, common.DefaultFloatVecFieldName}) + common.DefaultJSONFieldName, common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName}) for _, s := range resSearch[0].Scores { require.GreaterOrEqual(t, s, float32(15.0)) require.Less(t, s, float32(20.0)) @@ -1171,13 +1239,13 @@ func TestRangeSearchScannIPCosine(t *testing.T) { mc := createMilvusClient(ctx, t) // create collection - cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: false, + cp := CollectionParams{CollectionFieldsType: Int64FloatVecJSON, AutoID: false, EnableDynamicField: true, ShardsNum: common.DefaultShards, Dim: common.DefaultDim} collName := createCollection(ctx, t, mc, cp) // insert dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64FloatVecJSON, - start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false} + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} _, _ = insertData(ctx, t, mc, dp) mc.Flush(ctx, collName, false) @@ -1207,7 +1275,7 @@ func TestRangeSearchScannIPCosine(t *testing.T) { common.CheckErr(t, errSearch, true) common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK) common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, - common.DefaultJSONFieldName, common.DefaultFloatVecFieldName}) + common.DefaultJSONFieldName, common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName}) for _, s := range resSearch[0].Scores { require.GreaterOrEqual(t, s, float32(0)) require.Less(t, s, float32(100)) @@ -1232,13 +1300,13 @@ func TestRangeSearchScannBinary(t *testing.T) { mc := createMilvusClient(ctx, t) // create collection - cp := CollectionParams{CollectionFieldsType: Int64BinaryVec, AutoID: false, EnableDynamicField: false, + cp := CollectionParams{CollectionFieldsType: Int64BinaryVec, AutoID: false, EnableDynamicField: true, ShardsNum: common.DefaultShards, Dim: common.DefaultDim} collName := createCollection(ctx, t, mc, cp) // insert dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64BinaryVec, - start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false, WithRows: false} + start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false} _, _ = insertData(ctx, t, mc, dp) mc.Flush(ctx, collName, false) @@ -1267,7 +1335,8 @@ func TestRangeSearchScannBinary(t *testing.T) { // verify error nil, output all fields, range score common.CheckErr(t, errSearch, true) common.CheckSearchResult(t, resSearch, 1, common.DefaultTopK) - common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, common.DefaultBinaryVecFieldName}) + common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, + common.DefaultBinaryVecFieldName, common.DefaultDynamicFieldName}) for _, s := range resSearch[0].Scores { require.GreaterOrEqual(t, s, float32(0)) require.Less(t, s, float32(100)) diff --git a/test/testcases/upsert_test.go b/test/testcases/upsert_test.go index 83b30975..2d167f3b 100644 --- a/test/testcases/upsert_test.go +++ b/test/testcases/upsert_test.go @@ -4,7 +4,6 @@ package testcases import ( "fmt" - "log" "strconv" "testing" "time" @@ -159,23 +158,20 @@ func TestUpsertVarcharPk(t *testing.T) { _, binaryColumn1 := common.GenDefaultVarcharData(0, upsertNb, common.DefaultDim) ids, err := mc.Upsert(ctx, collName, "", varcharColumn1, binaryColumn1) common.CheckErr(t, err, true) - log.Println(ids.FieldData()) require.ElementsMatch(t, ids.(*entity.ColumnVarChar).Data(), varcharValues) // query old varchar pk (no space): ["1", ... "9"] resSet, err = mc.QueryByPks(ctx, collName, []string{}, pkColumn, []string{common.DefaultVarcharFieldName, common.DefaultBinaryVecFieldName}) - log.Println(resSet.GetColumn(common.DefaultVarcharFieldName).(*entity.ColumnVarChar).Data()) common.CheckErr(t, err, true) require.ElementsMatch(t, varcharColumn.(*entity.ColumnVarChar).Data()[:upsertNb], resSet.GetColumn(common.DefaultVarcharFieldName).(*entity.ColumnVarChar).Data()) require.ElementsMatch(t, binaryColumn.(*entity.ColumnBinaryVector).Data()[:upsertNb], resSet.GetColumn(common.DefaultBinaryVecFieldName).(*entity.ColumnBinaryVector).Data()) // query and verify the updated entities - pkColumn1 := entity.NewColumnVarChar(common.DefaultVarcharFieldName, varcharColumn1.Data()[:upsertNb]) + pkColumn1 := entity.NewColumnVarChar(common.DefaultVarcharFieldName, varcharColumn1.Data()) resSet, err = mc.QueryByPks(ctx, collName, []string{}, pkColumn1, []string{common.DefaultVarcharFieldName, common.DefaultBinaryVecFieldName}) - log.Println(resSet.GetColumn(common.DefaultVarcharFieldName).(*entity.ColumnVarChar).Data()) common.CheckErr(t, err, true) - require.ElementsMatch(t, varcharColumn1.Data()[:upsertNb], resSet.GetColumn(common.DefaultVarcharFieldName).(*entity.ColumnVarChar).Data()) - require.ElementsMatch(t, binaryColumn1.(*entity.ColumnBinaryVector).Data()[:upsertNb], resSet.GetColumn(common.DefaultBinaryVecFieldName).(*entity.ColumnBinaryVector).Data()) + require.ElementsMatch(t, varcharColumn1.Data(), resSet.GetColumn(common.DefaultVarcharFieldName).(*entity.ColumnVarChar).Data()) + require.ElementsMatch(t, binaryColumn1.(*entity.ColumnBinaryVector).Data(), resSet.GetColumn(common.DefaultBinaryVecFieldName).(*entity.ColumnBinaryVector).Data()) } // test upsert with partition @@ -309,7 +305,6 @@ func TestUpsertSamePksManyTimes(t *testing.T) { } func TestUpsertDynamicField(t *testing.T) { - t.Skip("https://github.com/milvus-io/milvus-sdk-go/issues/613") // enable dynamic field and insert dynamic column ctx := createContext(t, time.Second*common.DefaultTimeout) // connect @@ -338,14 +333,14 @@ func TestUpsertDynamicField(t *testing.T) { resSet, err = mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s < %d", common.DefaultDynamicNumberField, upsertNb), []string{common.DefaultDynamicNumberField}) require.Zero(t, resSet[0].Len()) - // 2. upsert not exist pk with dynamic column -> field dynamicNumber does not exist in collection ??? TODO issue + // 2. upsert not exist pk with dynamic column -> field dynamicNumber does not exist in collection intColumn2, floatColumn2, vecColumn2 := common.GenDefaultColumnData(common.DefaultNb, upsertNb, common.DefaultDim) dynamicData2 := common.GenDynamicFieldData(common.DefaultNb, upsertNb) _, err = mc.Upsert(ctx, collName, "", append(dynamicData2, intColumn2, floatColumn2, vecColumn2)...) common.CheckErr(t, err, true) // query and gets empty dynamic field - resSet, err = mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s > %d", common.DefaultDynamicNumberField, common.DefaultNb), []string{common.QueryCountFieldName}) + resSet, err = mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s >= %d", common.DefaultDynamicNumberField, common.DefaultNb), []string{common.QueryCountFieldName}) require.Equal(t, int64(upsertNb), resSet.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0]) } @@ -358,7 +353,7 @@ func TestUpsertPartitionKeyCollection(t *testing.T) { // create partition_key field partitionKeyFieldName := "partitionKeyField" partitionKeyField := common.GenField(partitionKeyFieldName, entity.FieldTypeInt64, - common.WithIsPartitionKey(true), common.WithMaxLength(common.DefaultMaxLength)) + common.WithIsPartitionKey(true), common.WithMaxLength(common.TestMaxLen)) // schema schema := common.GenSchema(common.GenRandomString(6), false, common.GenDefaultFields(false))