Add JSON data type

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

entity/columns_json.go

@@ -0,0 +1,84 @@
+package entity
+
+import (
+	"fmt"
+
+	"github.com/cockroachdb/errors"
+	schema "github.com/milvus-io/milvus-proto/go-api/schemapb"
+)
+
+var _ (Column) = (*ColumnJSONBytes)(nil)
+
+// ColumnJSONBytes column type for JSON.
+// all items are marshaled json bytes.
+type ColumnJSONBytes struct {
+	name   string
+	values [][]byte
+}
+
+// Name returns column name.
+func (c *ColumnJSONBytes) Name() string {
+	return c.name
+}
+
+// Type returns column FieldType.
+func (c *ColumnJSONBytes) Type() FieldType {
+	return FieldTypeJSON
+}
+
+// Len returns column values length.
+func (c *ColumnJSONBytes) Len() int {
+	return len(c.values)
+}
+
+// FieldData return column data mapped to schema.FieldData.
+func (c *ColumnJSONBytes) FieldData() *schema.FieldData {
+	fd := &schema.FieldData{
+		Type:      schema.DataType_JSON,
+		FieldName: c.name,
+	}
+
+	fd.Field = &schema.FieldData_Scalars{
+		Scalars: &schema.ScalarField{
+			Data: &schema.ScalarField_JsonData{
+				JsonData: &schema.JSONArray{
+					Data: c.values,
+				},
+			},
+		},
+	}
+
+	return fd
+}
+
+// ValueByIdx returns value of the provided index.
+func (c *ColumnJSONBytes) ValueByIdx(idx int) ([]byte, error) {
+	if idx < 0 || idx >= c.Len() {
+		return nil, errors.New("index out of range")
+	}
+	return c.values[idx], nil
+}
+
+// AppendValue append value into column.
+func (c *ColumnJSONBytes) AppendValue(i interface{}) error {
+	v, ok := i.([]byte)
+	if !ok {
+		return fmt.Errorf("invalid type, expected []byte, got %T", i)
+	}
+	c.values = append(c.values, v)
+
+	return nil
+}
+
+// Data returns column data.
+func (c *ColumnJSONBytes) Data() [][]byte {
+	return c.values
+}
+
+// NewColumnJSONBytes composes a Column with json bytes.
+func NewColumnJSONBytes(name string, values [][]byte) *ColumnJSONBytes {
+	return &ColumnJSONBytes{
+		name:   name,
+		values: values,
+	}
+}

entity/columns_json_test.go

@@ -0,0 +1,78 @@
+package entity
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type ColumnJSONBytesSuite struct {
+	suite.Suite
+}
+
+func (s *ColumnJSONBytesSuite) SetupSuite() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func (s *ColumnJSONBytesSuite) TestAttrMethods() {
+	columnName := fmt.Sprintf("column_jsonbs_%d", rand.Int())
+	columnLen := 8 + rand.Intn(10)
+
+	v := make([][]byte, columnLen)
+	column := NewColumnJSONBytes(columnName, v)
+
+	s.Run("test_meta", func() {
+		ft := FieldTypeJSON
+		s.Equal("JSON", ft.Name())
+		s.Equal("JSON", ft.String())
+		pbName, pbType := ft.PbFieldType()
+		s.Equal("JSON", pbName)
+		s.Equal("JSON", pbType)
+	})
+
+	s.Run("test_column_attribute", func() {
+		s.Equal(columnName, column.Name())
+		s.Equal(FieldTypeJSON, column.Type())
+		s.Equal(columnLen, column.Len())
+		s.EqualValues(v, column.Data())
+	})
+
+	s.Run("test_column_field_data", func() {
+		fd := column.FieldData()
+		s.NotNil(fd)
+		s.Equal(fd.GetFieldName(), columnName)
+	})
+
+	s.Run("test_column_valuer_by_idx", func() {
+		_, err := column.ValueByIdx(-1)
+		s.Error(err)
+		_, err = column.ValueByIdx(columnLen)
+		s.Error(err)
+		for i := 0; i < columnLen; i++ {
+			v, err := column.ValueByIdx(i)
+			s.NoError(err)
+			s.Equal(column.values[i], v)
+		}
+	})
+
+	s.Run("test_append_value", func() {
+		item := make([]byte, 10)
+		err := column.AppendValue(item)
+		s.NoError(err)
+		s.Equal(columnLen+1, column.Len())
+		val, err := column.ValueByIdx(columnLen)
+		s.NoError(err)
+		s.Equal(item, val)
+
+		err = column.AppendValue(1)
+		s.Error(err)
+	})
+
+}
+
+func TestColumnJSONBytes(t *testing.T) {
+	suite.Run(t, new(ColumnJSONBytesSuite))
+}

entity/schema.go

@@ -163,6 +163,8 @@ func (t FieldType) Name() string {
 		return "String"
 	case FieldTypeVarChar:
 		return "VarChar"
+	case FieldTypeJSON:
+		return "JSON"
 	case FieldTypeBinaryVector:
 		return "BinaryVector"
 	case FieldTypeFloatVector:
@@ -193,6 +195,8 @@ func (t FieldType) String() string {
 		return "string"
 	case FieldTypeVarChar:
 		return "string"
+	case FieldTypeJSON:
+		return "JSON"
 	case FieldTypeBinaryVector:
 		return "[]byte"
 	case FieldTypeFloatVector:
@@ -223,15 +227,15 @@ func (t FieldType) PbFieldType() (string, string) {
 		return "String", "string"
 	case FieldTypeVarChar:
 		return "VarChar", "string"
+	case FieldTypeJSON:
+		return "JSON", "JSON"
 	case FieldTypeBinaryVector:
 		return "[]byte", ""
 	case FieldTypeFloatVector:
 		return "[]float32", ""
 	default:
 		return "undefined", ""
-
 	}
-
 }
 
 // Match schema definition
@@ -256,6 +260,9 @@ const (
 	FieldTypeString FieldType = 20
 	// FieldTypeVarChar field type varchar
 	FieldTypeVarChar FieldType = 21 // variable-length strings with a specified maximum length
+	// FieldTypeArray FieldType = 22
+	// FieldTypeJSON field type JSON
+	FieldTypeJSON FieldType = 23
 	// FieldTypeBinaryVector field type binary vector
 	FieldTypeBinaryVector FieldType = 100
 	// FieldTypeFloatVector field type float vector