[Bug]: [GOSDK] The bf16 vectors output by the query is slightly different from the inserted vectors

[ThreadDao]

Is there an existing issue for this?

• I have searched the existing issues

Environment

- Milvus version: master-20240613-512ea6be-amd64
- Deployment mode(standalone or cluster): cluster
- MQ type(rocksmq, pulsar or kafka):    
- SDK version(e.g. pymilvus v2.0.0rc2): go-sdk v2
- OS(Ubuntu or CentOS): 
- CPU/Memory: 
- GPU: 
- Others:

Current Behavior

The dim of bf16 field is 128. A bf16 vector type is slice []byte. The len of a inserted vector is 256 (128dim * 2, fp16 has 16bits 2 bytes). But the query output vector len is 128

• query output

 ([]uint8) (len=256) {
        	            	  00000000  45 3f 66 3e 63 3f 9b 3d  6b 3f 95 3d 8d 3d ed 3e  |E?f>c?.=k?.=.=.>|
        	            	  00000010  eb 3e bb 3d 50 3f 0b 3e  1e 3f 3d 3f c8 3e 42 3e  |.>.=P?.>.?=?.>B>|
        	            	  00000020  0a 3e e2 3e 5d 3e 83 3d  20 3f 1a 3f 1c 3e 75 3f  |.>.>]>.= ?.?.>u?|
        	            	  00000030  02 3f 8c 3d 26 3f 0d 3d  54 3e 6e 3d 0b 3f a3 3e  |.?.=&?.=T>n=.?.>|
        	            	  00000040  f7 3e 60 3f 78 3e 6e 3f  47 3f 52 3f 66 3f 3b 3f  |.>`?x>n?G?R?f?;?|
        	            	  00000050  5a 3f 70 3d ae 3e 30 3f  94 3d 3d 3f 52 3f 3e 3f  |Z?p=.>0?.==?R?>?|
        	            	  00000060  68 3f 4e 3e 9a 3e 43 3d  39 3f c4 3e 06 3c 64 3e  |h?N>.>C=9?.>.<d>|
        	            	  00000070  7b 3e 16 3e 59 3f 27 3f  a0 3d 3b 3f 66 3f 4e 3f  |{>.>Y?'?.=;?f?N?|
        	            	  00000080  6a 3f 2a 3f 88 3e 17 3f  47 3d 1f 3f 68 3c 59 3e  |j?*?.>.?G=.?h<Y>|
        	            	  00000090  18 3f 04 3f 44 3f 2d 3f  42 3f 24 3f 5a 3f d3 3d  |.?.?D?-?B?$?Z?.=|
        	            	  000000a0  24 3e 4e 3f 26 3f 3a 3f  58 3f 48 3f 43 3f 10 3f  |$>N?&?:?X?H?C?.?|
        	            	  000000b0  5f 3f 6d 3f 28 3f a9 3e  24 3f d4 3e 5b 3f 5d 3e  |_?m?(?.>$?.>[?]>|
        	            	  000000c0  d1 3e 0e 3f 5c 3f 48 3f  52 3e 00 3f d9 3e 35 3e  |.>.?\?H?R>.?.>5>|
        	            	  000000d0  5e 3f 35 3f ee 3e 86 3e  d1 3e 20 3f eb 3e a4 3e  |^?5?.>.>.> ?.>.>|
        	            	  000000e0  b8 3e 88 3e 0e 3e 03 3f  38 3f 2d 3f ec 3e 38 3e  |.>.>.>.?8?-?.>8>|
        	            	  000000f0  91 3e 98 3e 43 3f 3b 3f  87 3d 5a 3e e0 3e 19 3f  |.>.>C?;?.=Z>.>.?|
        	            	 },

• insert vector:

([][]uint8) (len=10) {
        	            	 ([]uint8) (len=128) {
        	            	  00000000  45 3f 66 3e 63 3f 9b 3d  6b 3f 95 3d 8d 3d ed 3e  |E?f>c?.=k?.=.=.>|
        	            	  00000010  eb 3e bb 3d 50 3f 0b 3e  1e 3f 3d 3f c8 3e 42 3e  |.>.=P?.>.?=?.>B>|
        	            	  00000020  0a 3e e2 3e 5d 3e 83 3d  20 3f 1a 3f 1c 3e 75 3f  |.>.>]>.= ?.?.>u?|
        	            	  00000030  02 3f 8c 3d 26 3f 0d 3d  54 3e 6e 3d 0b 3f a3 3e  |.?.=&?.=T>n=.?.>|
        	            	  00000040  f7 3e 60 3f 78 3e 6e 3f  47 3f 52 3f 66 3f 3b 3f  |.>`?x>n?G?R?f?;?|
        	            	  00000050  5a 3f 70 3d ae 3e 30 3f  94 3d 3d 3f 52 3f 3e 3f  |Z?p=.>0?.==?R?>?|
        	            	  00000060  68 3f 4e 3e 9a 3e 43 3d  39 3f c4 3e 06 3c 64 3e  |h?N>.>C=9?.>.<d>|
        	            	  00000070  7b 3e 16 3e 59 3f 27 3f  a0 3d 3b 3f 66 3f 4e 3f  |{>.>Y?'?.=;?f?N?|
        	            	 },

• case:

// test query output all fields and verify data
func TestOutputAllFieldsColumn(t *testing.T) {
	ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
	mc := createDefaultMilvusClient(ctx, t)

	// create collection
	for _, isDynamic := range [2]bool{true, false} {
		prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(isDynamic))
		prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
		prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))

		// insert
		columns := make([]column.Column, 0, len(schema.Fields)+1)
		dynamicColumns := make([]column.Column, 0, len(schema.Fields)+1)
		genDataOpt := hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity)
		for _, field := range schema.Fields {
			if field.IsDynamic {
				dynamicColumns = append(dynamicColumns, hp.GenDynamicColumnData(0, common.DefaultNb)...)
			} else {
				if field.DataType == entity.FieldTypeArray{
					genDataOpt.TWithElementType(field.ElementType)
				}
				columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *genDataOpt.TWithDim(common.DefaultDim)))
			}
		}
		ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...).WithColumns(dynamicColumns...))
		common.CheckErr(t, err, true)
		require.Equal(t, int64(common.DefaultNb), ids.InsertCount)
		prepare.FlushData(ctx, t, mc, schema.CollectionName)


		// query output all fields -> output all fields, includes vector and $meta field
		pos := 10
		allFieldsName := make([]string, 0, len(schema.Fields))
		for _, field := range schema.Fields {
			allFieldsName = append(allFieldsName, field.Name)
		}
		if isDynamic {
			allFieldsName = append(allFieldsName, common.DefaultDynamicFieldName)
		}
		queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).
			                                WithFilter(fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, pos)).WithOutputFields([]string{"*"}))
		common.CheckErr(t, errQuery, true)
		common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields)

		expColumns := make([]column.Column, 0, len(columns)+1)
		for _, _column := range columns {
			expColumns = append(expColumns, _column.Slice(0, pos))
			if _column.Type() == entity.FieldTypeBFloat16Vector {
				//log.Debug("_column", zap.ByteStrings("data", _column.(*column.ColumnBFloat16Vector).Data()))
				log.Debug("_column", zap.Int("len", len(_column.(*column.ColumnBFloat16Vector).Data()[0])))
			}
		}
		if isDynamic {
			expColumns = append(expColumns, hp.MergeColumnsToDynamic(pos, dynamicColumns))
		}
		common.CheckQueryResult(t, expColumns, queryResultAll.Fields)
	}
}

Expected Behavior

No response

Steps To Reproduce

No response

Milvus Log

No response

Anything else?

No response

[ThreadDao]

/assign @congqixia

[yanliang567]

/unassign

[congqixia]

/assign @ThreadDao
shall be fixed in #33684, could you please verify?

[ThreadDao]

@congqixia not fixed yet

[congqixia]

@congqixia not fixed yet

there is still some problem in bf16, tested with fp16 missing bf16 path

[adetunjii]

Hi @congqixia has this issue been resolved? It looks like you've fixed it, yet the issue remains open

[congqixia]

@adetunjii i think this issue is under verification by @ThreadDao

[ThreadDao]

@adetunjii @congqixia fixed.