Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add test cases for groupby search #672

Merged
merged 1 commit into from
Mar 18, 2024
Merged

Add test cases for groupby search #672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 67 additions & 45 deletions test/common/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,28 +412,42 @@ func GenArrayColumnData(start int, nb int, fieldName string, opts ...GenColumnDa
}
}

type JSONStruct struct {
Number int32 `json:"number" milvus:"name:number"`
String string `json:"string" milvus:"name:string"`
Bool bool `json:"bool" milvus:"name:bool"`
List []int64 `json:"list" milvus:"name:list"`
}

// GenDefaultJSONData gen default column with data
func GenDefaultJSONData(columnName string, start int, nb int) *entity.ColumnJSONBytes {
type JSONStruct struct {
Number int32 `json:"number" milvus:"name:number"`
String string `json:"string" milvus:"name:string"`
Bool bool `json:"bool" milvus:"name:bool"`
List []int64 `json:"list" milvus:"name:list"`
}
jsonValues := make([][]byte, 0, nb)
var m JSONStruct
var m interface{}
for i := start; i < start+nb; i++ {
if i%2 == 0 {
m = JSONStruct{
String: strconv.Itoa(i),
Bool: i%2 == 0,
if i < (start+nb)/2 {
if i%2 == 0 {
m = JSONStruct{
String: strconv.Itoa(i),
Bool: i%2 == 0,
}
} else {
m = JSONStruct{
Number: int32(i),
String: strconv.Itoa(i),
Bool: i%2 == 0,
List: []int64{int64(i), int64(i + 1)},
}
}
} else {
m = JSONStruct{
Number: int32(i),
String: strconv.Itoa(i),
Bool: i%2 == 0,
List: []int64{int64(i), int64(i + 1)},
switch i % 4 {
case 0:
m = i
case 1:
m = float32(i)
case 2:
m = strconv.Itoa(i)
case 3:
m = []int64{int64(i), int64(i + 1)}
}
}
bs, err := json.Marshal(&m)
Expand Down Expand Up @@ -668,12 +682,6 @@ func GenDefaultVarcharRows(start int, nb int, dim int64, enableDynamicField bool

func GenDefaultJSONRows(start int, nb int, dim int64, enableDynamicField bool) []interface{} {
rows := make([]interface{}, 0, nb)
type JSONStruct struct {
Number int32 `json:"number" milvus:"name:number"`
String string `json:"string" milvus:"name:string"`
Bool bool `json:"bool" milvus:"name:bool"`
List []int64 `json:"list" milvus:"name:list"`
}

// BaseRow generate insert rows
type BaseRow struct {
Expand Down Expand Up @@ -1030,26 +1038,37 @@ func MergeColumnsToDynamic(nb int, columns []entity.Column) *entity.ColumnJSONBy
// --- gen row data ---

// --- index utils ---
var SupportFloatMetricType = []entity.MetricType{
entity.L2,
entity.IP,
entity.COSINE,
}

var SupportBinFlatMetricType = []entity.MetricType{
entity.JACCARD,
entity.HAMMING,
entity.SUBSTRUCTURE,
entity.SUPERSTRUCTURE,
}

var SupportBinIvfFlatMetricType = []entity.MetricType{
entity.JACCARD,
entity.HAMMING,
}

// GenAllFloatIndex gen all float vector index
func GenAllFloatIndex(metricType entity.MetricType) []entity.Index {
func GenAllFloatIndex() []entity.Index {
nlist := 128
idxFlat, _ := entity.NewIndexFlat(metricType)
idxIvfFlat, _ := entity.NewIndexIvfFlat(metricType, nlist)
idxIvfSq8, _ := entity.NewIndexIvfSQ8(metricType, nlist)
idxIvfPq, _ := entity.NewIndexIvfPQ(metricType, nlist, 16, 8)
idxHnsw, _ := entity.NewIndexHNSW(metricType, 8, 96)
idxScann, _ := entity.NewIndexSCANN(metricType, 16, false)
idxDiskAnn, _ := entity.NewIndexDISKANN(metricType)

allFloatIndex := []entity.Index{
idxFlat,
idxIvfFlat,
idxIvfSq8,
idxIvfPq,
idxHnsw,
idxScann,
idxDiskAnn,
var allFloatIndex []entity.Index
for _, metricType := range SupportFloatMetricType {
idxFlat, _ := entity.NewIndexFlat(metricType)
idxIvfFlat, _ := entity.NewIndexIvfFlat(metricType, nlist)
idxIvfSq8, _ := entity.NewIndexIvfSQ8(metricType, nlist)
idxIvfPq, _ := entity.NewIndexIvfPQ(metricType, nlist, 16, 8)
idxHnsw, _ := entity.NewIndexHNSW(metricType, 8, 96)
idxScann, _ := entity.NewIndexSCANN(metricType, 16, false)
idxDiskAnn, _ := entity.NewIndexDISKANN(metricType)
allFloatIndex = append(allFloatIndex, idxFlat, idxIvfFlat, idxIvfSq8, idxIvfPq, idxHnsw, idxScann, idxDiskAnn)
}
return allFloatIndex
}
Expand Down Expand Up @@ -1084,22 +1103,25 @@ type InvalidExprStruct struct {
}

var InvalidExpressions = []InvalidExprStruct{
{Expr: "id in [0]", ErrNil: true, ErrMsg: "fieldName(id) not found"}, // not exist field but no error
{Expr: "int64 in not [0]", ErrNil: false, ErrMsg: "cannot parse expression"}, // wrong term expr keyword
{Expr: "int64 < floatVec", ErrNil: false, ErrMsg: "not supported"}, // unsupported compare field
{Expr: "floatVec in [0]", ErrNil: false, ErrMsg: "cannot be casted to FloatVector"}, // value and field type mismatch
{Expr: fmt.Sprintf("%s == 1", DefaultJSONFieldName), ErrNil: false, ErrMsg: "can not comparisons jsonField directly"},
{Expr: fmt.Sprintf("%s == 1", DefaultDynamicFieldName), ErrNil: false, ErrMsg: "can not comparisons jsonField directly"},
{Expr: "id in [0]", ErrNil: true, ErrMsg: "fieldName(id) not found"}, // not exist field but no error
{Expr: "int64 in not [0]", ErrNil: false, ErrMsg: "cannot parse expression"}, // wrong term expr keyword
{Expr: "int64 < floatVec", ErrNil: false, ErrMsg: "not supported"}, // unsupported compare field
{Expr: "floatVec in [0]", ErrNil: false, ErrMsg: "cannot be casted to FloatVector"}, // value and field type mismatch
{Expr: fmt.Sprintf("%s == 1", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // hist empty
{Expr: fmt.Sprintf("%s like 'a%%' ", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // hist empty
{Expr: fmt.Sprintf("%s > 1", DefaultDynamicFieldName), ErrNil: true, ErrMsg: ""}, // hits empty
{Expr: fmt.Sprintf("%s[\"dynamicList\"] == [2, 3]", DefaultDynamicFieldName), ErrNil: true, ErrMsg: ""},
{Expr: fmt.Sprintf("%s['a'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // json field not exist
{Expr: fmt.Sprintf("%s['number'] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // field exist but type not match
{Expr: fmt.Sprintf("%s[0] == [2, 3]", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""}, // field exist but type not match
{Expr: fmt.Sprintf("json_contains (%s['number'], 2)", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""},
{Expr: fmt.Sprintf("json_contains (%s['list'], [2])", DefaultJSONFieldName), ErrNil: true, ErrMsg: ""},
{Expr: fmt.Sprintf("json_contains_all (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_all operation element must be an array"},
{Expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "contains_any operation element must be an array"},
{Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"},
{Expr: fmt.Sprintf("json_contains_aby (%s['list'], 2)", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression: json_contains_aby"},
{Expr: fmt.Sprintf("%s[-1] > %d", DefaultInt8ArrayField, TestCapacity), ErrNil: false, ErrMsg: "cannot parse expression"}, // array[-1] >
{Expr: fmt.Sprintf(fmt.Sprintf("%s[-1] > 1", DefaultJSONFieldName)), ErrNil: false, ErrMsg: "invalid expression"}, // json[-1] >
}

// --- search utils ---
41 changes: 38 additions & 3 deletions test/testcases/collection_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,19 +372,21 @@ func TestCreateCollectionDescription(t *testing.T) {
}

// test create collection with invalid dim
func TestCreateCollectionInvalidDim(t *testing.T) {
func TestCreateBinaryCollectionInvalidDim(t *testing.T) {
t.Parallel()
type invalidDimStruct struct {
dim string
errMsg string
}
invalidDims := []invalidDimStruct{
{dim: "10", errMsg: "should be multiple of 8"},
{dim: "0", errMsg: "should be in range 1 ~ 32768"},
{dim: "0", errMsg: "should be in range 2 ~ 32768"},
{dim: "1", errMsg: "should be in range 2 ~ 32768"},
{dim: "", errMsg: "invalid syntax"},
{dim: "中文", errMsg: "invalid syntax"},
{dim: "%$#", errMsg: "invalid syntax"},
{dim: fmt.Sprintf("%d", common.MaxDim+1), errMsg: "should be in range 1 ~ 32768"},
{dim: fmt.Sprintf("%d", common.MaxDim*9), errMsg: "binary vector dimension should be in range 2 ~ 262144"},
{dim: fmt.Sprintf("%d", common.MaxDim*8+1), errMsg: "binary vector dimension should be multiple of 8"},
}

// connect
Expand All @@ -405,6 +407,39 @@ func TestCreateCollectionInvalidDim(t *testing.T) {
}
}

func TestCreateFloatCollectionInvalidDim(t *testing.T) {
t.Parallel()
type invalidDimStruct struct {
dim string
errMsg string
}
invalidDims := []invalidDimStruct{
{dim: "0", errMsg: "should be in range 2 ~ 32768"},
{dim: "1", errMsg: "should be in range 2 ~ 32768"},
{dim: "", errMsg: "invalid syntax"},
{dim: "中文", errMsg: "invalid syntax"},
{dim: "%$#", errMsg: "invalid syntax"},
{dim: fmt.Sprintf("%d", common.MaxDim+1), errMsg: "float vector dimension should be in range 2 ~ 32768"},
}

// connect
ctx := createContext(t, time.Second*common.DefaultTimeout)
mc := createMilvusClient(ctx, t)

// create binary collection with autoID true
pkField := common.GenField("", entity.FieldTypeInt64, common.WithIsPrimaryKey(true), common.WithAutoID(true))
for _, invalidDim := range invalidDims {
collName := common.GenRandomString(6)
binaryFields := entity.NewField().
WithName(common.DefaultFloatVecFieldName).
WithDataType(entity.FieldTypeFloatVector).
WithTypeParams(entity.TypeParamDim, invalidDim.dim)
schema := common.GenSchema(collName, true, []*entity.Field{pkField, binaryFields})
errCreate := mc.CreateCollection(ctx, schema, common.DefaultShards)
common.CheckErr(t, errCreate, false, invalidDim.errMsg)
}
}

func TestCreateJsonCollection(t *testing.T) {
ctx := createContext(t, time.Second*common.DefaultTimeout)
mc := createMilvusClient(ctx, t)
Expand Down
Loading
Loading