From 8b9c0c357efb47d739686fd0033d19c7cd427807 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Mon, 9 Sep 2024 11:03:21 +0800 Subject: [PATCH 01/15] init 2.5 Signed-off-by: ryjiang --- .github/workflows/check.yml | 4 ++-- .gitmodules | 2 +- package.json | 2 +- proto | 2 +- test/grpc/MultipleVectors.spec.ts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8376c8f4..d90e01c6 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -4,12 +4,12 @@ on: pull_request: branches: - main - - 2.4 + - 2.5 types: [opened, synchronize] push: branches: - main - - 2.4 + - 2.5 jobs: publish: runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index f1271435..b5e19c85 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "proto"] path = proto url = https://github.com/milvus-io/milvus-proto.git - branch = 2.4 + branch = master diff --git a/package.json b/package.json index d4fc22c5..c388fc08 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@zilliz/milvus2-sdk-node", "author": "ued@zilliz.com", "version": "2.4.8", - "milvusVersion": "v2.4.10", + "milvusVersion": "master-20240908-208c8a23-amd64", "main": "dist/milvus", "files": [ "dist" diff --git a/proto b/proto index 32b0311b..d3a0eb38 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 32b0311b378567df2a3d93dde7b5cc64fd8b7349 +Subproject commit d3a0eb38fbf1c07a7e8174358b71cddca6effb04 diff --git a/test/grpc/MultipleVectors.spec.ts b/test/grpc/MultipleVectors.spec.ts index f8f24378..d31665c2 100644 --- a/test/grpc/MultipleVectors.spec.ts +++ b/test/grpc/MultipleVectors.spec.ts @@ -16,7 +16,7 @@ import { generateInsertData, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); const COLLECTION_NAME = GENERATE_NAME(); const dbParam = { From bedf3314ffe2f1c18f51d050feed3b9f709f42ca Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 11 Sep 2024 14:13:44 +0800 Subject: [PATCH 02/15] update milvus version Signed-off-by: ryjiang --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c388fc08..c074e13c 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@zilliz/milvus2-sdk-node", "author": "ued@zilliz.com", "version": "2.4.8", - "milvusVersion": "master-20240908-208c8a23-amd64", + "milvusVersion": "master-20240911-42eef490-amd64", "main": "dist/milvus", "files": [ "dist" From af3857f0ea5fbdbe0fcee56b9df3b003216fde81 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 11 Sep 2024 14:19:08 +0800 Subject: [PATCH 03/15] update proto Signed-off-by: ryjiang --- proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto b/proto index d3a0eb38..8f8ca678 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit d3a0eb38fbf1c07a7e8174358b71cddca6effb04 +Subproject commit 8f8ca67816cd2fee2b4c72f30c0ede66a7935087 From 845e8638a4fc21a1d4f3ed936541de8bf4deac51 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Thu, 12 Sep 2024 18:20:05 +0800 Subject: [PATCH 04/15] WIP: support default value & null Signed-off-by: ryjiang --- milvus/grpc/Collection.ts | 13 +++++++---- milvus/grpc/Data.ts | 38 ++++++++++++++++++++++---------- milvus/types/Collection.ts | 2 ++ milvus/types/Data.ts | 9 ++++++-- milvus/utils/Format.ts | 22 ++++++++++++++----- milvus/utils/Function.ts | 9 ++++++++ milvus/utils/Grpc.ts | 4 ++-- test/grpc/DynamicSchema.spec.ts | 39 ++++++++++++++++++++++++++++----- test/tools/collection.ts | 20 +++++++++++++---- test/tools/const.ts | 3 ++- test/tools/data.ts | 5 ++--- test/utils/Function.spec.ts | 28 +++++++++++++++++++++++ 12 files changed, 152 insertions(+), 40 deletions(-) diff --git a/milvus/grpc/Collection.ts b/milvus/grpc/Collection.ts index a1bf553d..d180a849 100644 --- a/milvus/grpc/Collection.ts +++ b/milvus/grpc/Collection.ts @@ -54,6 +54,7 @@ import { CreateCollectionWithFieldsReq, CreateCollectionWithSchemaReq, FieldSchema, + isVectorType, } from '../'; /** @@ -1041,9 +1042,12 @@ export class Collection extends Database { * }); * ``` */ - async getPkFieldName(data: DescribeCollectionReq): Promise { + async getPkFieldName( + data: DescribeCollectionReq, + desc?: DescribeCollectionResponse + ): Promise { // get collection info - const collectionInfo = await this.describeCollection(data); + const collectionInfo = desc ? desc : await this.describeCollection(data); // pk field let pkField = ''; @@ -1081,10 +1085,11 @@ export class Collection extends Database { * ``` */ async getPkFieldType( - data: DescribeCollectionReq + data: DescribeCollectionReq, + desc?: DescribeCollectionResponse ): Promise { // get collection info - const collectionInfo = await this.describeCollection(data); + const collectionInfo = desc ? desc : await this.describeCollection(data); // pk field type let pkFieldType: keyof typeof DataType = 'Int64'; diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index 298a5b62..bf1cfa92 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -48,7 +48,7 @@ import { buildDynamicRow, buildFieldDataMap, getDataKey, - Field, + _Field, buildFieldData, BinaryVector, RowData, @@ -65,6 +65,7 @@ import { sparseRowsToBytes, getSparseDim, f32ArrayToBinaryBytes, + getValidDataArray, } from '../'; import { Collection } from './Collection'; @@ -141,7 +142,7 @@ export class Data extends Collection { // Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`. // If primarykey is set `autoid = true`, you cannot insert the data. - const fieldMap = new Map( + const fieldMap = new Map( collectionInfo.schema.fields .filter(v => !v.is_primary_key || !v.autoID) .map(v => [ @@ -152,6 +153,8 @@ export class Data extends Collection { elementType: v.element_type, dim: Number(findKeyValue(v.type_params, 'dim')), data: [], // values container + nullable: v.nullable, + default_value: v.default_value, }, ]) ); @@ -164,6 +167,7 @@ export class Data extends Collection { type: 'JSON', elementType: 'None', data: [], // value container + nullable: false, }); } @@ -222,6 +226,10 @@ export class Data extends Collection { const dataKey = getDataKey(type); const elementType = DataTypeMap[field.elementType!]; const elementTypeKey = getDataKey(elementType); + const valid_data = getValidDataArray( + field.data, + data.fields_data?.length! + ); // build key value let keyValue; @@ -260,14 +268,16 @@ export class Data extends Collection { case DataType.Array: keyValue = { [dataKey]: { - data: field.data.map(d => { - return { - [elementTypeKey]: { - type: elementType, - data: d, - }, - }; - }), + data: field.data + .filter(v => v !== undefined) + .map(d => { + return { + [elementTypeKey]: { + type: elementType, + data: d, + }, + }; + }), element_type: elementType, }, }; @@ -275,17 +285,20 @@ export class Data extends Collection { default: keyValue = { [dataKey]: { - data: field.data, + data: field.data.filter(v => v !== undefined), }, }; break; } + const needValidData = field.nullable || field.default_value; + return { type, field_name: field.name, is_dynamic: field.name === DEFAULT_DYNAMIC_FIELD, [key]: keyValue, + valid_data: needValidData ? valid_data : [], }; }); @@ -924,9 +937,10 @@ export class Data extends Collection { // always get output_fields from fields_data const output_fields = promise.fields_data.map(f => f.field_name); + // build field data map const fieldsDataMap = buildFieldDataMap( promise.fields_data, - data.transformers + data.transformers, ); // For each output field, check if it has a fixed schema or not diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts index eeba9927..a5823ceb 100644 --- a/milvus/types/Collection.ts +++ b/milvus/types/Collection.ts @@ -32,6 +32,7 @@ export interface FieldSchema { is_partition_key?: boolean; is_dynamic?: boolean; is_clustering_key?: boolean; + nullable?: boolean; } export interface CollectionData { @@ -75,6 +76,7 @@ export interface FieldType { max_capacity?: TypeParam; max_length?: TypeParam; default_value?: number | string; + nullable?: boolean; } export interface ShowCollectionsReq extends GrpcTimeOut { diff --git a/milvus/types/Data.ts b/milvus/types/Data.ts index d2434515..9c9017a5 100644 --- a/milvus/types/Data.ts +++ b/milvus/types/Data.ts @@ -72,19 +72,22 @@ export type FieldData = | VarChar | JSON | Array - | VectorTypes; + | VectorTypes + | null; // Represents a row of data in Milvus. export interface RowData { [x: string]: FieldData; } -export interface Field { +export interface _Field { name: string; type: keyof typeof DataType; elementType?: keyof typeof DataType; data: FieldData[]; dim?: number; + nullable?: boolean; + default_value?: FieldData; } export interface FlushReq extends GrpcTimeOut { @@ -450,6 +453,8 @@ export interface QueryRes extends resStatusResponse { [x: string]: any; data: string; }; + is_dynamic: boolean; + valid_data: boolean[]; }[]; output_fields: string[]; collection_name: string; diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index 53da147a..1ed6cc3a 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -10,7 +10,7 @@ import { DescribeCollectionResponse, getDataKey, RowData, - Field, + _Field, JSON, FieldData, CreateCollectionWithFieldsReq, @@ -372,7 +372,7 @@ export const formatDescribedCol = ( */ export const buildDynamicRow = ( rowData: RowData, - fieldMap: Map, + fieldMap: Map, dynamicFieldName: string ) => { const originRow = cloneObj(rowData); @@ -488,17 +488,27 @@ export const buildFieldDataMap = ( }); } - // decode json switch (dataKey) { + // decode json case 'json_data': field_data.forEach((buffer: any, i: number) => { - // console.log(JSON.parse(buffer.toString())); - field_data[i] = JSON.parse(buffer.toString()); + field_data[i] = buffer.length + ? JSON.parse(buffer.toString()) + : null; }); break; default: break; } + + // set the field data with null if item.valid_data is not empty array, it the item in valid_data is false, set the field data with null + if (item.valid_data && item.valid_data.length) { + item.valid_data.forEach((v: any, i: number) => { + if (!v) { + field_data[i] = null; + } + }); + } } // Add the parsed data to the fieldsDataMap @@ -542,7 +552,7 @@ export const getAuthString = (data: { */ export const buildFieldData = ( rowData: RowData, - field: Field, + field: _Field, transformers?: InsertTransformers ): FieldData => { const { type, elementType, name } = field; diff --git a/milvus/utils/Function.ts b/milvus/utils/Function.ts index aeada05c..c3bf4756 100644 --- a/milvus/utils/Function.ts +++ b/milvus/utils/Function.ts @@ -7,6 +7,7 @@ import { DEFAULT_MIN_INT64, SearchResultData, SparseFloatVector, + FieldData, } from '../'; import { Pool } from 'generic-pool'; @@ -236,3 +237,11 @@ export const getSparseDim = (data: SparseFloatVector[]) => { } return dim; }; + +// get valid data +// create a length array with valid data, if the data is undefined or null, return false, otherwise return true +export const getValidDataArray = (data: FieldData[], length: number) => { + return Array.from({ length }).map((_, i) => { + return data[i] !== undefined && data[i] !== null; + }); +}; diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts index 28459b89..7a450c92 100644 --- a/milvus/utils/Grpc.ts +++ b/milvus/utils/Grpc.ts @@ -200,7 +200,7 @@ export const getRetryInterceptor = ({ logger.debug( `\x1b[32m[Response(${ Date.now() - startTime.getTime() - }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${msg}` + }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${string}` ); savedMessageNext(savedReceiveMessage); @@ -217,7 +217,7 @@ export const getRetryInterceptor = ({ const msg = string.length > 2048 ? string.slice(0, 2048) + '...' : string; logger.debug( - `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}` + `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${string}` ); savedSendMessage = message; next(message); diff --git a/test/grpc/DynamicSchema.spec.ts b/test/grpc/DynamicSchema.spec.ts index 64bebe98..94a9b22f 100644 --- a/test/grpc/DynamicSchema.spec.ts +++ b/test/grpc/DynamicSchema.spec.ts @@ -4,6 +4,7 @@ import { ErrorCode, ConsistencyLevelEnum, } from '../../milvus'; +import { DEFAULT_STRING_VALUE, DEFAULT_NUM_VALUE } from '../tools/const'; import { IP, genCollectionParams, @@ -12,7 +13,7 @@ import { dynamicFields, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); const COLLECTION = GENERATE_NAME(); const dbParam = { db_name: 'DynamicSchema', @@ -58,7 +59,7 @@ describe(`Dynamic schema API`, () => { it(`Insert data with dynamic field should success`, async () => { const data = generateInsertData( [...createCollectionParams.fields, ...dynamicFields], - 20 + 10 ); const insert = await milvusClient.insert({ @@ -100,21 +101,47 @@ describe(`Dynamic schema API`, () => { 'json', 'vector', 'id', + 'float', + 'bool', + 'default_value', + 'varChar', + 'json', 'dynamic_int64', 'dynamic_varChar', + 'int32_array', ], consistency_level: ConsistencyLevelEnum.Session, }); expect(query.status.error_code).toEqual(ErrorCode.SUCCESS); expect(query.data.length).toEqual(10); + // get test values + const varChars = query.data.map(v => v.varChar); + const defaultValues = query.data.map(v => v.default_value); + const jsons = query.data.map(v => v.json); + const arrays = query.data.map(v => v.int32_array); + const bools = query.data.map(v => v.bool); + const floats = query.data.map(v => v.float); + console.log('bool', floats, bools, varChars); + // some of floats should be equal to DEFAULT_NUM_VALUE + expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true); + // some of varChar should be equal to DEFAULT_STRING_VALUE + expect(varChars.some(v => v === DEFAULT_STRING_VALUE)).toEqual(true); + // some of default_value should be equal to DEFAULT_NUM_VALUE + expect(defaultValues.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual( + true + ); + // some of json should be null + expect(jsons.some(v => v === null)).toEqual(true); + // some of bools should be null + expect(bools.some(v => v === null)).toEqual(true); }); it(`search with dynamic field should success`, async () => { // search const search = await milvusClient.search({ collection_name: COLLECTION, - limit: 10, + limit: 5, data: [ [1, 2, 3, 4], [1, 2, 3, 4], @@ -126,17 +153,17 @@ describe(`Dynamic schema API`, () => { expect(search.status.error_code).toEqual(ErrorCode.SUCCESS); expect(search.results.length).toEqual(2); - expect(search.results[0].length).toEqual(10); + expect(search.results[0].length).toEqual(5); // search const search2 = await milvusClient.search({ collection_name: COLLECTION, - limit: 10, + limit: 5, data: [1, 2, 3, 4], expr: 'id > 0', output_fields: ['json', 'id', 'dynamic_int64', 'dynamic_varChar'], }); expect(search2.status.error_code).toEqual(ErrorCode.SUCCESS); - expect(search2.results.length).toEqual(10); + expect(search2.results.length).toEqual(5); }); }); diff --git a/test/tools/collection.ts b/test/tools/collection.ts index 7740c71b..cd40dcca 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -1,5 +1,11 @@ import { DataType, ConsistencyLevelEnum } from '../../milvus'; -import { VECTOR_FIELD_NAME, MAX_CAPACITY, MAX_LENGTH } from './const'; +import { + VECTOR_FIELD_NAME, + MAX_CAPACITY, + MAX_LENGTH, + DEFAULT_NUM_VALUE, + DEFAULT_STRING_VALUE, +} from './const'; import { GENERATE_VECTOR_NAME } from './'; export const dynamicFields = [ @@ -96,36 +102,42 @@ export const genCollectionParams = (data: { { name: 'float', description: 'Float field', + default_value: DEFAULT_NUM_VALUE, data_type: DataType.Float, }, { name: 'bool', description: 'bool field', + nullable: true, data_type: DataType.Bool, }, { name: 'default_value', - // default_value: DEFAULT_VALUE, + nullable: true, description: 'int64 field', - data_type: 'Int64', // test string data type + data_type: 'Int64', // }, { name: 'varChar', description: 'VarChar field', data_type: DataType.VarChar, + nullable: true, + default_value: DEFAULT_STRING_VALUE, max_length: MAX_LENGTH, is_partition_key: partitionKeyEnabled, }, { name: 'json', description: 'JSON field', + nullable: true, data_type: DataType.JSON, }, { name: 'int32_array', description: 'int array field', data_type: DataType.Array, - element_type: 'Int32', // test string element type + // nullable: true, + element_type: 'Int32', max_capacity: maxCapacity || MAX_CAPACITY, }, { diff --git a/test/tools/const.ts b/test/tools/const.ts index a63c2adb..5ddde9f5 100644 --- a/test/tools/const.ts +++ b/test/tools/const.ts @@ -3,7 +3,8 @@ export const INDEX_NAME = 'index_name'; export const DIMENSION = 4; export const INDEX_FILE_SIZE = 1024; export const PARTITION_TAG = 'random'; -export const DEFAULT_VALUE = '100'; +export const DEFAULT_NUM_VALUE = 100; +export const DEFAULT_STRING_VALUE = 'd'; export const MAX_LENGTH = 8; export const MAX_CAPACITY = 4; export const VECTOR_FIELD_NAME = 'vector'; diff --git a/test/tools/data.ts b/test/tools/data.ts index f1c4dfb2..1f8447d3 100644 --- a/test/tools/data.ts +++ b/test/tools/data.ts @@ -283,15 +283,14 @@ export const generateInsertData = ( for (const field of fields) { // Skip autoID and fields with default values - if (field.autoID || typeof field.default_value !== 'undefined') { + if (field.autoID) { continue; } // get data type const data_type = convertToDataType(field.data_type); - // Skip fields with default values - if (typeof field.default_value !== 'undefined') { + if ((field.nullable || field.default_value) && Math.random() < 0.5) { continue; } diff --git a/test/utils/Function.spec.ts b/test/utils/Function.spec.ts index d742657a..035fca5d 100644 --- a/test/utils/Function.spec.ts +++ b/test/utils/Function.spec.ts @@ -10,6 +10,7 @@ import { SparseFloatVector, getDataKey, DataType, + getValidDataArray, } from '../../milvus'; describe('Function API testing', () => { @@ -349,4 +350,31 @@ describe('Function API testing', () => { expect(getDataKey(DataType.JSON, true)).toEqual('jsonData'); expect(getDataKey(DataType.None, true)).toEqual('none'); }); + + it('should return the valid array', () => { + const a = [1, 2, 3]; + const length = 5; + const result = getValidDataArray(a, length); + expect(result).toEqual([true, true, true, false, false]); + + const b = [1, null, 3]; + const result2 = getValidDataArray(b, length); + expect(result2).toEqual([true, false, true, false, false]); + + const c: any = []; + const result3 = getValidDataArray(c, length); + expect(result3).toEqual([false, false, false, false, false]); + + const d: any = [1, 2, 3, 4, undefined]; + const result4 = getValidDataArray(d, length); + expect(result4).toEqual([true, true, true, true, false]); + + const e = [ + [1, 2], + [3, 4], + [5, 6], + ]; + const result5 = getValidDataArray(e, length); + expect(result5).toEqual([true, true, true, false, false]); + }); }); From 8121fbe23ec4a1f14b9c46ddbcb925c40662070e Mon Sep 17 00:00:00 2001 From: ryjiang Date: Fri, 13 Sep 2024 10:56:26 +0800 Subject: [PATCH 05/15] fix json format Signed-off-by: ryjiang --- milvus/utils/Format.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index 1ed6cc3a..af9f43bc 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -575,7 +575,9 @@ export const buildFieldData = ( ? f16Transformer(rowData[name] as Float16Vector) : rowData[name]; case DataType.JSON: - return Buffer.from(JSON.stringify(rowData[name] || {})); + return rowData[name] + ? Buffer.from(JSON.stringify(rowData[name] || {})) + : Buffer.alloc(0); case DataType.Array: const elementField = { ...field, type: elementType! }; return buildFieldData(rowData, elementField, transformers); From 41a23d02a1e6578ac5e3331d48aa3bf89a235401 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 18 Sep 2024 14:06:53 +0800 Subject: [PATCH 06/15] WIP Signed-off-by: ryjiang --- milvus/utils/Grpc.ts | 4 ++-- test/grpc/DynamicSchema.spec.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts index 28459b89..7a450c92 100644 --- a/milvus/utils/Grpc.ts +++ b/milvus/utils/Grpc.ts @@ -200,7 +200,7 @@ export const getRetryInterceptor = ({ logger.debug( `\x1b[32m[Response(${ Date.now() - startTime.getTime() - }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${msg}` + }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${string}` ); savedMessageNext(savedReceiveMessage); @@ -217,7 +217,7 @@ export const getRetryInterceptor = ({ const msg = string.length > 2048 ? string.slice(0, 2048) + '...' : string; logger.debug( - `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}` + `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${string}` ); savedSendMessage = message; next(message); diff --git a/test/grpc/DynamicSchema.spec.ts b/test/grpc/DynamicSchema.spec.ts index 94a9b22f..e0bbe473 100644 --- a/test/grpc/DynamicSchema.spec.ts +++ b/test/grpc/DynamicSchema.spec.ts @@ -26,7 +26,7 @@ const createCollectionParams = genCollectionParams({ dim: [4], vectorType: [DataType.FloatVector], autoID: false, - partitionKeyEnabled: true, + partitionKeyEnabled: false, numPartitions, enableDynamic: true, }); @@ -122,7 +122,7 @@ describe(`Dynamic schema API`, () => { const arrays = query.data.map(v => v.int32_array); const bools = query.data.map(v => v.bool); const floats = query.data.map(v => v.float); - console.log('bool', floats, bools, varChars); + console.log('varchar', varChars); // some of floats should be equal to DEFAULT_NUM_VALUE expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true); // some of varChar should be equal to DEFAULT_STRING_VALUE From db6bcce348f36ac4dcf767adaae82fdee49341ef Mon Sep 17 00:00:00 2001 From: ryjiang Date: Thu, 19 Sep 2024 16:25:49 +0800 Subject: [PATCH 07/15] WIP Signed-off-by: ryjiang --- milvus/grpc/Data.ts | 10 ++++++++-- test/tools/ip.ts | 2 +- test/utils/Format.spec.ts | 21 ++++++++++----------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index bf1cfa92..5a1d2f86 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -291,7 +291,13 @@ export class Data extends Collection { break; } - const needValidData = field.nullable || field.default_value; + const needValidData = + key !== 'vectors' && + (field.nullable === true || + (typeof field.default_value !== 'undefined' && + field.default_value !== null)); + + console.log('field', field.name, field.nullable, field.default_value, needValidData); return { type, @@ -940,7 +946,7 @@ export class Data extends Collection { // build field data map const fieldsDataMap = buildFieldDataMap( promise.fields_data, - data.transformers, + data.transformers ); // For each output field, check if it has a fixed schema or not diff --git a/test/tools/ip.ts b/test/tools/ip.ts index 2fec5d91..bd316310 100644 --- a/test/tools/ip.ts +++ b/test/tools/ip.ts @@ -1,3 +1,3 @@ // test IP -export const IP = '127.0.0.1:19530'; +export const IP = '10.102.7.208:19530'; export const ENDPOINT = `http://${IP}`; diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index e98d34d0..d48011de 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -25,7 +25,7 @@ import { getAuthString, buildFieldData, formatSearchResult, - Field, + _Field, formatSearchVector, buildSearchRequest, } from '../../milvus'; @@ -438,7 +438,7 @@ describe('utils/format', () => { name: 'key1', type: 'VarChar', data: [{ key1: 'value1' }], - } as Field, + } as _Field, ], ]); const dynamicField = 'dynamic'; @@ -458,7 +458,7 @@ describe('utils/format', () => { name: 'key1', type: 'VarChar', data: [{ key1: 'value1' }], - } as Field, + } as _Field, ], [ 'key2', @@ -466,7 +466,7 @@ describe('utils/format', () => { name: 'key2', type: 'VarChar', data: [{ key2: 'value2' }], - } as Field, + } as _Field, ], ]); const dynamicField = 'dynamic'; @@ -487,7 +487,7 @@ describe('utils/format', () => { name: 'key1', type: 'VarChar', data: [{ key1: 'value1' }], - } as Field, + } as _Field, ], ]); const dynamicField = 'dynamic'; @@ -519,16 +519,16 @@ describe('utils/format', () => { it('should return the value of the field for BinaryVector and FloatVector types', () => { const row = { name: 'John', vector: [1, 2, 3] }; const field = { type: 'BinaryVector', name: 'vector' }; - expect(buildFieldData(row, field as Field)).toEqual([1, 2, 3]); + expect(buildFieldData(row, field as _Field)).toEqual([1, 2, 3]); field.type = 'FloatVector'; - expect(buildFieldData(row, field as Field)).toEqual([1, 2, 3]); + expect(buildFieldData(row, field as _Field)).toEqual([1, 2, 3]); }); it('should return the JSON stringified value of the field for JSON type', () => { const row = { name: 'John', data: { age: 25, city: 'New York' } }; const field = { type: 'JSON', name: 'data' }; - expect(JSON.parse(buildFieldData(row, field as Field).toString())).toEqual({ + expect(JSON.parse(buildFieldData(row, field as _Field)!.toString())).toEqual({ age: 25, city: 'New York', }); @@ -537,13 +537,13 @@ describe('utils/format', () => { it('should recursively call buildFieldData for Array type', () => { const row = { name: 'John', array: [1, 2, 3] }; const field = { type: 'Array', elementType: 'Int', name: 'array' }; - expect(buildFieldData(row, field as Field)).toEqual([1, 2, 3]); + expect(buildFieldData(row, field as _Field)).toEqual([1, 2, 3]); }); it('should return the value of the field for other types', () => { const row = { name: 'John', age: 25 }; const field = { type: 'Int', name: 'age' }; - expect(buildFieldData(row, field as Field)).toEqual(25); + expect(buildFieldData(row, field as _Field)).toEqual(25); }); it('should format search results correctly', () => { @@ -866,7 +866,6 @@ describe('utils/format', () => { describeCollectionResponse, milvusProto ); - console.dir(searchRequest, { depth: null }); expect(searchRequest.isHybridSearch).toEqual(true); expect(searchRequest.request.collection_name).toEqual('test'); expect(searchRequest.request.output_fields).toEqual(['vector', 'vector1']); From 432a8485b6bddefe63484c97c3292d1026351ffd Mon Sep 17 00:00:00 2001 From: ryjiang Date: Fri, 20 Sep 2024 11:04:43 +0800 Subject: [PATCH 08/15] WIP Signed-off-by: ryjiang --- milvus/grpc/Data.ts | 2 -- test/grpc/DynamicSchema.spec.ts | 2 +- test/grpc/PartitionKey.spec.ts | 4 ++-- test/tools/collection.ts | 2 +- test/tools/ip.ts | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index 5a1d2f86..5523b48f 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -297,8 +297,6 @@ export class Data extends Collection { (typeof field.default_value !== 'undefined' && field.default_value !== null)); - console.log('field', field.name, field.nullable, field.default_value, needValidData); - return { type, field_name: field.name, diff --git a/test/grpc/DynamicSchema.spec.ts b/test/grpc/DynamicSchema.spec.ts index e0bbe473..fe155599 100644 --- a/test/grpc/DynamicSchema.spec.ts +++ b/test/grpc/DynamicSchema.spec.ts @@ -13,7 +13,7 @@ import { dynamicFields, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION = GENERATE_NAME(); const dbParam = { db_name: 'DynamicSchema', diff --git a/test/grpc/PartitionKey.spec.ts b/test/grpc/PartitionKey.spec.ts index aa4f53cd..d0d99874 100644 --- a/test/grpc/PartitionKey.spec.ts +++ b/test/grpc/PartitionKey.spec.ts @@ -12,7 +12,7 @@ import { generateInsertData, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP }); +const milvusClient = new MilvusClient({ address: IP , logLevel: 'debug' }); const COLLECTION_NAME = GENERATE_NAME(); const COLLECTION_NAME2 = GENERATE_NAME(); const COLLECTION_NAME3 = GENERATE_NAME(); @@ -132,7 +132,7 @@ describe(`Partition key API`, () => { it(`it should create collection successfully with partition_key_field set`, async () => { const createCollectionParams = genCollectionParams({ - collectionName: COLLECTION_NAME2, + collectionName: COLLECTION_NAME3, dim: [4], vectorType: [DataType.FloatVector], autoID: false, diff --git a/test/tools/collection.ts b/test/tools/collection.ts index cd40dcca..a45a2ac4 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -114,6 +114,7 @@ export const genCollectionParams = (data: { { name: 'default_value', nullable: true, + default_value: DEFAULT_NUM_VALUE, description: 'int64 field', data_type: 'Int64', // }, @@ -121,7 +122,6 @@ export const genCollectionParams = (data: { name: 'varChar', description: 'VarChar field', data_type: DataType.VarChar, - nullable: true, default_value: DEFAULT_STRING_VALUE, max_length: MAX_LENGTH, is_partition_key: partitionKeyEnabled, diff --git a/test/tools/ip.ts b/test/tools/ip.ts index bd316310..2fec5d91 100644 --- a/test/tools/ip.ts +++ b/test/tools/ip.ts @@ -1,3 +1,3 @@ // test IP -export const IP = '10.102.7.208:19530'; +export const IP = '127.0.0.1:19530'; export const ENDPOINT = `http://${IP}`; From 2e22bc9392c7f5966c6c5863d794fe9cdcd7e1bd Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 9 Oct 2024 15:25:30 +0800 Subject: [PATCH 09/15] add null test for array Signed-off-by: ryjiang --- test/grpc/DynamicSchema.spec.ts | 4 +++- test/tools/collection.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/grpc/DynamicSchema.spec.ts b/test/grpc/DynamicSchema.spec.ts index fe155599..8ef6c18e 100644 --- a/test/grpc/DynamicSchema.spec.ts +++ b/test/grpc/DynamicSchema.spec.ts @@ -122,7 +122,7 @@ describe(`Dynamic schema API`, () => { const arrays = query.data.map(v => v.int32_array); const bools = query.data.map(v => v.bool); const floats = query.data.map(v => v.float); - console.log('varchar', varChars); + console.dir(arrays, { depth: null }); // some of floats should be equal to DEFAULT_NUM_VALUE expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true); // some of varChar should be equal to DEFAULT_STRING_VALUE @@ -135,6 +135,8 @@ describe(`Dynamic schema API`, () => { expect(jsons.some(v => v === null)).toEqual(true); // some of bools should be null expect(bools.some(v => v === null)).toEqual(true); + // some of array should be null + expect(arrays.some(v => v === null)).toEqual(true); }); it(`search with dynamic field should success`, async () => { diff --git a/test/tools/collection.ts b/test/tools/collection.ts index a45a2ac4..ce80c323 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -136,7 +136,7 @@ export const genCollectionParams = (data: { name: 'int32_array', description: 'int array field', data_type: DataType.Array, - // nullable: true, + nullable: true, element_type: 'Int32', max_capacity: maxCapacity || MAX_CAPACITY, }, From c5502f2a06944e16681d2af25c9c3ebcba93921f Mon Sep 17 00:00:00 2001 From: ryjiang Date: Thu, 24 Oct 2024 14:37:26 +0800 Subject: [PATCH 10/15] recover ip Signed-off-by: ryjiang --- test/tools/ip.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tools/ip.ts b/test/tools/ip.ts index aff672cc..2fec5d91 100644 --- a/test/tools/ip.ts +++ b/test/tools/ip.ts @@ -1,3 +1,3 @@ // test IP -export const IP = '10.102.7.218:19530'; +export const IP = '127.0.0.1:19530'; export const ENDPOINT = `http://${IP}`; From 605fb8c266dfcb2f98d0c7a583027607ae5e4a16 Mon Sep 17 00:00:00 2001 From: shanghaikid Date: Mon, 28 Oct 2024 16:58:22 +0800 Subject: [PATCH 11/15] WIP Signed-off-by: shanghaikid --- milvus/utils/Grpc.ts | 4 +-- proto | 2 +- test/grpc/Data.spec.ts | 51 ++++++++++++++++++++++++++----- test/grpc/Functions.spec.ts | 2 +- test/grpc/MultipleVectors.spec.ts | 2 +- test/grpc/PartitionKey.spec.ts | 2 +- 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts index 7a450c92..28459b89 100644 --- a/milvus/utils/Grpc.ts +++ b/milvus/utils/Grpc.ts @@ -200,7 +200,7 @@ export const getRetryInterceptor = ({ logger.debug( `\x1b[32m[Response(${ Date.now() - startTime.getTime() - }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${string}` + }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${msg}` ); savedMessageNext(savedReceiveMessage); @@ -217,7 +217,7 @@ export const getRetryInterceptor = ({ const msg = string.length > 2048 ? string.slice(0, 2048) + '...' : string; logger.debug( - `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${string}` + `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}` ); savedSendMessage = message; next(message); diff --git a/proto b/proto index 85ccff4d..4d5c88b0 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 85ccff4d57fe9c510c88ee4eaf1ba33ef4ef1188 +Subproject commit 4d5c88b00cf7280b17542940d3b49041f1c92f66 diff --git a/test/grpc/Data.spec.ts b/test/grpc/Data.spec.ts index 2d67b40c..0f30cbca 100644 --- a/test/grpc/Data.spec.ts +++ b/test/grpc/Data.spec.ts @@ -31,6 +31,14 @@ const createCollectionParams = genCollectionParams({ dim: [4], vectorType: [DataType.FloatVector], autoID: false, + fields: [ + { + name: 'varChar2', + description: 'VarChar2 field', + data_type: DataType.VarChar, + max_length: 100, + }, + ], }); const createCollectionParamsVarcharID = genCollectionParams({ collectionName: VARCHAR_ID_COLLECTION_NAME, @@ -62,6 +70,7 @@ describe(`Data.API`, () => { collection_name: COLLECTION_NAME, data: generateInsertData(createCollectionParams.fields, 1024), }); + await milvusClient.insert({ collection_name: VARCHAR_ID_COLLECTION_NAME, data: generateInsertData(createCollectionParamsVarcharID.fields, 1024), @@ -101,13 +110,25 @@ describe(`Data.API`, () => { }); afterAll(async () => { - await milvusClient.dropCollection({ + const searchParams = { collection_name: COLLECTION_NAME, - }); - await milvusClient.dropCollection({ - collection_name: VARCHAR_ID_COLLECTION_NAME, - }); - await milvusClient.dropDatabase(dbParam); + // partition_names: [], + filter: 'json["number"] >= 0', + data: [1, 2, 3, 4], + limit: 4, + output_fields: ['id', 'json'], + }; + const res = await milvusClient.search(searchParams); + + console.log('xxx2', res); + + // await milvusClient.dropCollection({ + // collection_name: COLLECTION_NAME, + // }); + // await milvusClient.dropCollection({ + // collection_name: VARCHAR_ID_COLLECTION_NAME, + // }); + // await milvusClient.dropDatabase(dbParam); }); it(`it should insert successfully`, async () => { @@ -232,15 +253,27 @@ describe(`Data.API`, () => { it(`Exec simple search without params and output fields should success`, async () => { const limit = 4; - // collection search + const describe = await milvusClient.describeCollection({ + collection_name: COLLECTION_NAME, + }); + + // find varchar2 field + const varChar2Field = describe.schema.fields.find( + f => f.name === 'varChar2' + ) + + console.dir(varChar2Field, { depth: null }); + const searchWithData = await milvusClient.search({ collection_name: COLLECTION_NAME, filter: '', data: [1, 2, 3, 4], limit: limit, - group_by_field: 'varChar', + group_by_field: 'varChar2', }); + console.log('searchWithData', searchWithData); + expect(searchWithData.status.error_code).toEqual(ErrorCode.SUCCESS); const searchWithData2 = await milvusClient.search({ @@ -382,6 +415,8 @@ describe(`Data.API`, () => { }; const res = await milvusClient.search(searchParams); + console.log('xxx', res); + expect(res.status.error_code).toEqual(ErrorCode.SUCCESS); expect( res.results.forEach(r => { diff --git a/test/grpc/Functions.spec.ts b/test/grpc/Functions.spec.ts index f68c1301..d9ee95f2 100644 --- a/test/grpc/Functions.spec.ts +++ b/test/grpc/Functions.spec.ts @@ -14,7 +14,7 @@ import { dynamicFields, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION = GENERATE_NAME(); const dbParam = { db_name: 'Functions', diff --git a/test/grpc/MultipleVectors.spec.ts b/test/grpc/MultipleVectors.spec.ts index d31665c2..f8f24378 100644 --- a/test/grpc/MultipleVectors.spec.ts +++ b/test/grpc/MultipleVectors.spec.ts @@ -16,7 +16,7 @@ import { generateInsertData, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION_NAME = GENERATE_NAME(); const dbParam = { diff --git a/test/grpc/PartitionKey.spec.ts b/test/grpc/PartitionKey.spec.ts index d0d99874..c05d6156 100644 --- a/test/grpc/PartitionKey.spec.ts +++ b/test/grpc/PartitionKey.spec.ts @@ -12,7 +12,7 @@ import { generateInsertData, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP , logLevel: 'debug' }); +const milvusClient = new MilvusClient({ address: IP , logLevel: 'info' }); const COLLECTION_NAME = GENERATE_NAME(); const COLLECTION_NAME2 = GENERATE_NAME(); const COLLECTION_NAME3 = GENERATE_NAME(); From d9b8d71aa36b68c26fc0f6612c23b049956c85b7 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Tue, 5 Nov 2024 18:11:21 +0800 Subject: [PATCH 12/15] finish default_null Signed-off-by: ryjiang --- proto | 2 +- test/grpc/Data.spec.ts | 30 +++++++++++------------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/proto b/proto index 4d5c88b0..7e04a1bc 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 4d5c88b00cf7280b17542940d3b49041f1c92f66 +Subproject commit 7e04a1bcb0d6c20d33bf7353c4dde0fea93b34d6 diff --git a/test/grpc/Data.spec.ts b/test/grpc/Data.spec.ts index 0f30cbca..a0d18f82 100644 --- a/test/grpc/Data.spec.ts +++ b/test/grpc/Data.spec.ts @@ -19,7 +19,7 @@ import { timeoutTest } from '../tools'; const milvusClient = new MilvusClient({ address: IP, - logLevel: 'debug', + logLevel: 'info', }); const COLLECTION_NAME = GENERATE_NAME(); const VARCHAR_ID_COLLECTION_NAME = GENERATE_NAME(); @@ -120,15 +120,13 @@ describe(`Data.API`, () => { }; const res = await milvusClient.search(searchParams); - console.log('xxx2', res); - - // await milvusClient.dropCollection({ - // collection_name: COLLECTION_NAME, - // }); - // await milvusClient.dropCollection({ - // collection_name: VARCHAR_ID_COLLECTION_NAME, - // }); - // await milvusClient.dropDatabase(dbParam); + await milvusClient.dropCollection({ + collection_name: COLLECTION_NAME, + }); + await milvusClient.dropCollection({ + collection_name: VARCHAR_ID_COLLECTION_NAME, + }); + await milvusClient.dropDatabase(dbParam); }); it(`it should insert successfully`, async () => { @@ -258,11 +256,9 @@ describe(`Data.API`, () => { }); // find varchar2 field - const varChar2Field = describe.schema.fields.find( - f => f.name === 'varChar2' - ) - - console.dir(varChar2Field, { depth: null }); + describe.schema.fields.find(f => f.name === 'varChar2'); + + // console.dir(varChar2Field, { depth: null }); const searchWithData = await milvusClient.search({ collection_name: COLLECTION_NAME, @@ -272,8 +268,6 @@ describe(`Data.API`, () => { group_by_field: 'varChar2', }); - console.log('searchWithData', searchWithData); - expect(searchWithData.status.error_code).toEqual(ErrorCode.SUCCESS); const searchWithData2 = await milvusClient.search({ @@ -415,8 +409,6 @@ describe(`Data.API`, () => { }; const res = await milvusClient.search(searchParams); - console.log('xxx', res); - expect(res.status.error_code).toEqual(ErrorCode.SUCCESS); expect( res.results.forEach(r => { From f513acb4495be04530ac34d0efbf7aa8b8add37e Mon Sep 17 00:00:00 2001 From: ryjiang Date: Tue, 5 Nov 2024 18:12:11 +0800 Subject: [PATCH 13/15] update test version Signed-off-by: ryjiang --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cc7931a2..d85d0868 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@zilliz/milvus2-sdk-node", "author": "ued@zilliz.com", - "milvusVersion": "master-20241024-f78f6112-amd64", + "milvusVersion": "master-20241105-bd04cac4-amd64", "version": "2.4.9", "main": "dist/milvus", "files": [ From dd5422d9876ad17e5501689c60f1f36d70169084 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 6 Nov 2024 15:13:39 +0800 Subject: [PATCH 14/15] fix test Signed-off-by: ryjiang --- test/build/Collection.spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/build/Collection.spec.ts b/test/build/Collection.spec.ts index 0df9f9d0..9fc91428 100644 --- a/test/build/Collection.spec.ts +++ b/test/build/Collection.spec.ts @@ -91,7 +91,11 @@ describe('Collection Api', () => { try { await milvusClient.createCollection( - genCollectionParams({ collectionName: 'any', dim: [8] }) + genCollectionParams({ + collectionName: 'any', + vectorType: [DataType.BinaryVector], + dim: [6], + }) ); } catch (error) { expect(error.message).toEqual( From 6564a8c27ca10ffe0e0385529d69c376f8341eba Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 6 Nov 2024 15:32:08 +0800 Subject: [PATCH 15/15] update test Signed-off-by: ryjiang --- test/utils/Format.spec.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index 473b673a..dc0ad36f 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -557,10 +557,16 @@ describe('utils/format', () => { it('should return the JSON stringified value of the field for JSON type', () => { const row = { name: 'John', data: { age: 25, city: 'New York' } }; const field = { type: 'JSON', name: 'data' }; - expect(JSON.parse(buildFieldData(row, field as _Field)!.toString())).toEqual({ + expect( + JSON.parse(buildFieldData(row, field as _Field)!.toString()) + ).toEqual({ age: 25, city: 'New York', }); + + // if json field is not in the row, should return Buffer.alloc(0) + const row2 = { name: 'John' }; + expect(buildFieldData(row2, field as _Field)).toEqual(Buffer.alloc(0)); }); it('should recursively call buildFieldData for Array type', () => {