feat TransformSchema struct ,when is_ignore is True, sink_key and sin…

…k_name can be empty.
sunhailin-Leo · Dec 4, 2024 · 82c0f0c · 82c0f0c
1 parent b812c97
commit 82c0f0c
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 10 deletions.
diff --git a/docs/zh/transform/jsonpath.md b/docs/zh/transform/jsonpath.md
@@ -12,6 +12,8 @@ JsonPath
 | `Path`      | `string` | 否  | JSON 路径（使用 JSONPath） |
 | `DestField` | `string` | 否  | 目标字段名                |
 
+> 使用(GJSON Playground 在线试验语法)[https://gjson.dev/]
+
 ## 示例
 假设我们有以下 来自kafka 的 JSON 数据：
 ```json

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -16,17 +16,18 @@ type SourceConfig struct {
 }
 
 // TransformSchema transform unit config
+// when is_ignore is True, sink_key and sink_name can be empty.
 type TransformSchema struct {
-	SourceKey    string `json:"source_key" vd:"len($)>0"`  // source key
-	SinkKey      string `json:"sink_key" vd:"len($)>0"`    // sink key
-	Converter    string `json:"converter"`                 // Converter, Like: toInt, toFloat32, toString, etc.
-	IsIgnore     bool   `json:"is_ignore"`                 // is ignored key
-	IsStrictMode bool   `json:"is_strict_mode"`            // is strict mode
-	IsKeepKeys   bool   `json:"is_keep_keys"`              // key is keep origin key
-	IsExpand     bool   `json:"is_expand"`                 // is expanded col
-	ExpandValue  any    `json:"expand_value"`              // expand value
-	SourceName   string `json:"source_name" vd:"len($)>0"` // source alias name
-	SinkName     string `json:"sink_name" vd:"len($)>0"`   // sink alias name
+	SourceKey    string `json:"source_key" vd:"len($)>0"`                       // source key
+	SinkKey      string `json:"sink_key" vd:"len($)>0 || (IsIgnore)$ == true"`  // sink key
+	Converter    string `json:"converter"`                                      // Converter, Like: toInt, toFloat32, toString, etc.
+	IsIgnore     bool   `json:"is_ignore"`                                      // is ignored key
+	IsStrictMode bool   `json:"is_strict_mode"`                                 // is strict mode
+	IsKeepKeys   bool   `json:"is_keep_keys"`                                   // key is keep origin key
+	IsExpand     bool   `json:"is_expand"`                                      // is expanded col
+	ExpandValue  any    `json:"expand_value"`                                   // expand value
+	SourceName   string `json:"source_name" vd:"len($)>0"`                      // source alias name
+	SinkName     string `json:"sink_name" vd:"len($)>0 || (IsIgnore)$ == true"` // sink alias name
 }
 
 // TransformJsonPath transform json path config

diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -3,6 +3,7 @@ package config
 import (
 	"testing"
 
+	vd "github.com/bytedance/go-tagexpr/v2/validator"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -88,3 +89,66 @@ func TestStreamConfigGetSinkTagBySinkName(t *testing.T) {
 	tag = streamConfig.GetSinkTagBySinkName("sink3")
 	assert.Equal(t, "", tag)
 }
+
+func TestTransformSchema(t *testing.T) {
+	// Test cases
+	testCases := []struct {
+		TransformSchema
+		wantErr bool
+	}{
+		// IsIgnore is true, SinkKey can be empty
+		{
+			TransformSchema: TransformSchema{
+				SourceKey:  "sourceKey",
+				SinkKey:    "",
+				Converter:  "toInt",
+				IsIgnore:   true,
+				SourceName: "sourceName",
+				SinkName:   "sinkName",
+			},
+			wantErr: false,
+		},
+		// IsIgnore is true, SinkKey can have a value
+		{
+			TransformSchema: TransformSchema{
+				SourceKey:  "sourceKey",
+				SinkKey:    "sinkKey",
+				Converter:  "toInt",
+				IsIgnore:   true,
+				SourceName: "sourceName",
+				SinkName:   "sinkName",
+			},
+			wantErr: false,
+		},
+		// IsIgnore is false, SinkKey cannot be empty
+		{
+			TransformSchema: TransformSchema{
+				SourceKey:  "sourceKey",
+				SinkKey:    "",
+				Converter:  "toInt",
+				IsIgnore:   false,
+				SourceName: "sourceName",
+				SinkName:   "sinkName",
+			},
+			wantErr: true,
+		},
+		// IsIgnore is false, SinkKey having a value is valid
+		{
+			TransformSchema: TransformSchema{
+				SourceKey:  "sourceKey",
+				SinkKey:    "sinkKey",
+				Converter:  "toInt",
+				IsIgnore:   false,
+				SourceName: "sourceName",
+				SinkName:   "sinkName",
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		schema := tc.TransformSchema
+		err := vd.Validate(schema)
+		assert.Equal(t, tc.wantErr, err)
+	}
+}