diff --git a/docs/zh/transform/jsonpath.md b/docs/zh/transform/jsonpath.md index 4ab7ec6..ecbb841 100644 --- a/docs/zh/transform/jsonpath.md +++ b/docs/zh/transform/jsonpath.md @@ -12,6 +12,8 @@ JsonPath | `Path` | `string` | 否 | JSON 路径(使用 JSONPath) | | `DestField` | `string` | 否 | 目标字段名 | +> 使用(GJSON Playground 在线试验语法)[https://gjson.dev/] + ## 示例 假设我们有以下 来自kafka 的 JSON 数据: ```json diff --git a/pkg/config/config.go b/pkg/config/config.go index 65f2a62..551e64f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,17 +16,18 @@ type SourceConfig struct { } // TransformSchema transform unit config +// when is_ignore is True, sink_key and sink_name can be empty. type TransformSchema struct { - SourceKey string `json:"source_key" vd:"len($)>0"` // source key - SinkKey string `json:"sink_key" vd:"len($)>0"` // sink key - Converter string `json:"converter"` // Converter, Like: toInt, toFloat32, toString, etc. - IsIgnore bool `json:"is_ignore"` // is ignored key - IsStrictMode bool `json:"is_strict_mode"` // is strict mode - IsKeepKeys bool `json:"is_keep_keys"` // key is keep origin key - IsExpand bool `json:"is_expand"` // is expanded col - ExpandValue any `json:"expand_value"` // expand value - SourceName string `json:"source_name" vd:"len($)>0"` // source alias name - SinkName string `json:"sink_name" vd:"len($)>0"` // sink alias name + SourceKey string `json:"source_key" vd:"len($)>0"` // source key + SinkKey string `json:"sink_key" vd:"len($)>0 || (IsIgnore)$ == true"` // sink key + Converter string `json:"converter"` // Converter, Like: toInt, toFloat32, toString, etc. + IsIgnore bool `json:"is_ignore"` // is ignored key + IsStrictMode bool `json:"is_strict_mode"` // is strict mode + IsKeepKeys bool `json:"is_keep_keys"` // key is keep origin key + IsExpand bool `json:"is_expand"` // is expanded col + ExpandValue any `json:"expand_value"` // expand value + SourceName string `json:"source_name" vd:"len($)>0"` // source alias name + SinkName string `json:"sink_name" vd:"len($)>0 || (IsIgnore)$ == true"` // sink alias name } // TransformJsonPath transform json path config diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 31110c1..1031781 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "testing" + vd "github.com/bytedance/go-tagexpr/v2/validator" "github.com/stretchr/testify/assert" ) @@ -88,3 +89,66 @@ func TestStreamConfigGetSinkTagBySinkName(t *testing.T) { tag = streamConfig.GetSinkTagBySinkName("sink3") assert.Equal(t, "", tag) } + +func TestTransformSchema(t *testing.T) { + // Test cases + testCases := []struct { + TransformSchema + wantErr bool + }{ + // IsIgnore is true, SinkKey can be empty + { + TransformSchema: TransformSchema{ + SourceKey: "sourceKey", + SinkKey: "", + Converter: "toInt", + IsIgnore: true, + SourceName: "sourceName", + SinkName: "sinkName", + }, + wantErr: false, + }, + // IsIgnore is true, SinkKey can have a value + { + TransformSchema: TransformSchema{ + SourceKey: "sourceKey", + SinkKey: "sinkKey", + Converter: "toInt", + IsIgnore: true, + SourceName: "sourceName", + SinkName: "sinkName", + }, + wantErr: false, + }, + // IsIgnore is false, SinkKey cannot be empty + { + TransformSchema: TransformSchema{ + SourceKey: "sourceKey", + SinkKey: "", + Converter: "toInt", + IsIgnore: false, + SourceName: "sourceName", + SinkName: "sinkName", + }, + wantErr: true, + }, + // IsIgnore is false, SinkKey having a value is valid + { + TransformSchema: TransformSchema{ + SourceKey: "sourceKey", + SinkKey: "sinkKey", + Converter: "toInt", + IsIgnore: false, + SourceName: "sourceName", + SinkName: "sinkName", + }, + wantErr: false, + }, + } + + for _, tc := range testCases { + schema := tc.TransformSchema + err := vd.Validate(schema) + assert.Equal(t, tc.wantErr, err) + } +}