diff --git a/cdc/entry/mounter.go b/cdc/entry/mounter.go index eb14be764b1..794b3045691 100644 --- a/cdc/entry/mounter.go +++ b/cdc/entry/mounter.go @@ -310,20 +310,21 @@ func datum2Column(tableInfo *model.TableInfo, datums map[int64]types.Datum, fill colName := colInfo.Name.O colDatums, exist := datums[colInfo.ID] var colValue interface{} + if !exist && !fillWithDefaultValue { + continue + } + var err error + var warn string if exist { - var err error - var warn string colValue, warn, err = formatColVal(colDatums, colInfo.Tp) - if err != nil { - return nil, errors.Trace(err) - } - if warn != "" { - log.Warn(warn, zap.String("table", tableInfo.TableName.String()), zap.String("column", colInfo.Name.String())) - } } else if fillWithDefaultValue { - colValue = getDefaultOrZeroValue(colInfo) - } else { - continue + colValue, warn, err = getDefaultOrZeroValue(colInfo) + } + if err != nil { + return nil, errors.Trace(err) + } + if warn != "" { + log.Warn(warn, zap.String("table", tableInfo.TableName.String()), zap.String("column", colInfo.Name.String())) } cols[tableInfo.RowColumnsOffset[colInfo.ID]] = &model.Column{ Name: colName, @@ -441,35 +442,39 @@ func formatColVal(datum types.Datum, tp byte) (value interface{}, warn string, e } return v, warn, nil default: + // FIXME: GetValue() may return some types that go sql not support, which will cause sink DML fail + // Make specified convert upper if you need + // Go sql support type ref to: https://github.com/golang/go/blob/go1.17.4/src/database/sql/driver/types.go#L236 return datum.GetValue(), "", nil } } -func getDefaultOrZeroValue(col *timodel.ColumnInfo) interface{} { - // see https://github.com/pingcap/tidb/issues/9304 - // must use null if TiDB not write the column value when default value is null - // and the value is null +// getDefaultOrZeroValue return interface{} need to meet to require type in +// https://github.com/golang/go/blob/go1.17.4/src/database/sql/driver/types.go#L236 +// Supported type is: nil, basic type(Int, Int8,..., Float32, Float64, String), Slice(uint8), other types not support +func getDefaultOrZeroValue(col *timodel.ColumnInfo) (value interface{}, warn string, err error) { + var d types.Datum if !mysql.HasNotNullFlag(col.Flag) { - d := types.NewDatum(nil) - return d.GetValue() - } - - if col.GetDefaultValue() != nil { - d := types.NewDatum(col.GetDefaultValue()) - return d.GetValue() - } - switch col.Tp { - case mysql.TypeEnum: - // For enum type, if no default value and not null is set, - // the default value is the first element of the enum list - d := types.NewDatum(col.FieldType.Elems[0]) - return d.GetValue() - case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar: - return emptyBytes + // see https://github.com/pingcap/tidb/issues/9304 + // must use null if TiDB not write the column value when default value is null + // and the value is null + d = types.NewDatum(nil) + } else if col.GetDefaultValue() != nil { + d = types.NewDatum(col.GetDefaultValue()) + } else { + switch col.Tp { + case mysql.TypeEnum: + // For enum type, if no default value and not null is set, + // the default value is the first element of the enum list + d = types.NewDatum(col.FieldType.Elems[0]) + case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar: + return emptyBytes, "", nil + default: + d = table.GetZeroValue(col) + } } - d := table.GetZeroValue(col) - return d.GetValue() + return formatColVal(d, col.Tp) } func fetchHandleValue(tableInfo *model.TableInfo, recordID int64) (pkCoID int64, pkValue *types.Datum, err error) { diff --git a/cdc/entry/mounter_test.go b/cdc/entry/mounter_test.go index 374cfbcd1e4..ff9deae6769 100644 --- a/cdc/entry/mounter_test.go +++ b/cdc/entry/mounter_test.go @@ -16,9 +16,11 @@ package entry import ( "context" "strings" + "testing" "time" "github.com/pingcap/check" + timodel "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/ticdc/cdc/model" "github.com/pingcap/ticdc/pkg/regionspan" @@ -28,6 +30,8 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/testkit" + "github.com/pingcap/tidb/types" + "github.com/stretchr/testify/require" ) type mountTxnsSuite struct{} @@ -386,3 +390,307 @@ func walkTableSpanInStore(c *check.C, store tidbkv.Storage, tableID int64, f fun c.Assert(err, check.IsNil) } } + +// Check following MySQL type, ref to: +// https://github.com/pingcap/tidb/blob/master/parser/mysql/type.go +type columnInfoAndResult struct { + ColInfo timodel.ColumnInfo + Res interface{} +} + +func TestFormatColVal(t *testing.T) {} + +func TestGetDefaultZeroValue(t *testing.T) { + colAndRess := []columnInfoAndResult{ + // mysql flag null + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Flag: uint(0), + }, + }, + Res: nil, + }, + // mysql.TypeTiny + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeTiny, + Flag: mysql.NotNullFlag, + }, + }, + Res: int64(0), + }, + // mysql.TypeShort + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeShort, + Flag: mysql.NotNullFlag, + }, + }, + Res: int64(0), + }, + // mysql.TypeLong + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeLong, + Flag: mysql.NotNullFlag, + }, + }, + Res: int64(0), + }, + // mysql.TypeLonglong + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeLonglong, + Flag: mysql.NotNullFlag, + }, + }, + Res: int64(0), + }, + // mysql.TypeInt24 + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeInt24, + Flag: mysql.NotNullFlag, + }, + }, + Res: int64(0), + }, + // mysql.TypeFloat + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeFloat, + Flag: mysql.NotNullFlag, + }, + }, + Res: float64(0), + }, + // mysql.TypeDouble + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeDouble, + Flag: mysql.NotNullFlag, + }, + }, + Res: float64(0), + }, + // mysql.TypeNewDecimal + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeNewDecimal, + Flag: mysql.NotNullFlag, + Flen: 5, + Decimal: 2, + }, + }, + Res: "0", // related with Flen and Decimal, [TODO] need check default + }, + // mysql.TypeNull + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeNull, + Flag: mysql.NotNullFlag, + }, + }, + Res: nil, + }, + // mysql.TypeTimestamp + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeTimestamp, + Flag: mysql.NotNullFlag, + }, + }, + Res: "0000-00-00 00:00:00", + }, + // mysql.TypeDate + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeDate, + Flag: mysql.NotNullFlag, + }, + }, + Res: "0000-00-00", + }, + // mysql.TypeDuration + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeDuration, + Flag: mysql.NotNullFlag, + }, + }, + Res: "00:00:00", + }, + // mysql.TypeDatetime + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeDatetime, + Flag: mysql.NotNullFlag, + }, + }, + Res: "0000-00-00 00:00:00", + }, + // mysql.TypeYear + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeYear, + Flag: mysql.NotNullFlag, + }, + }, + Res: int64(0), + }, + // mysql.TypeNewDate + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeNewDate, + Flag: mysql.NotNullFlag, + }, + }, + Res: nil, // [TODO] seems not support by TiDB, need check + }, + // mysql.TypeVarchar + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeVarchar, + Flag: mysql.NotNullFlag, + }, + }, + Res: []byte{}, + }, + // mysql.TypeTinyBlob + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeTinyBlob, + Flag: mysql.NotNullFlag, + }, + }, + Res: []byte{}, + }, + // mysql.TypeMediumBlob + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeMediumBlob, + Flag: mysql.NotNullFlag, + }, + }, + Res: []byte{}, + }, + // mysql.TypeLongBlob + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeLongBlob, + Flag: mysql.NotNullFlag, + }, + }, + Res: []byte{}, + }, + // mysql.TypeBlob + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeBlob, + Flag: mysql.NotNullFlag, + }, + }, + Res: []byte{}, + }, + // mysql.TypeVarString + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeVarString, + Flag: mysql.NotNullFlag, + }, + }, + Res: []byte{}, + }, + // mysql.TypeString + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeString, + Flag: mysql.NotNullFlag, + }, + }, + Res: []byte{}, + }, + // mysql.TypeBit + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Flag: mysql.NotNullFlag, + Tp: mysql.TypeBit, + }, + }, + Res: uint64(0), + }, + // mysql.TypeJSON + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeJSON, + Flag: mysql.NotNullFlag, + }, + }, + Res: "null", + }, + // mysql.TypeEnum + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeEnum, + Flag: mysql.NotNullFlag, + Elems: []string{"e0", "e1"}, + }, + }, + Res: uint64(0), + }, + // mysql.TypeSet + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeSet, + Flag: mysql.NotNullFlag, + }, + }, + Res: uint64(0), + }, + // mysql.TypeGeometry + { + ColInfo: timodel.ColumnInfo{ + FieldType: types.FieldType{ + Tp: mysql.TypeGeometry, + Flag: mysql.NotNullFlag, + }, + }, + Res: nil, + }, + } + testGetDefaultZeroValue(t, colAndRess) +} + +func testGetDefaultZeroValue(t *testing.T, colAndRess []columnInfoAndResult) { + for _, colAndRes := range colAndRess { + val, _, _ := getDefaultOrZeroValue(&colAndRes.ColInfo) + require.Equal(t, colAndRes.Res, val) + } +}