From 83ceede82d64e609e4f8e98590bd82fad4468cfc Mon Sep 17 00:00:00 2001 From: Ling Jin Date: Mon, 16 Nov 2020 10:30:24 +0800 Subject: [PATCH 01/37] session, variables: Stream agg concurrency variable (#20960) --- executor/set_test.go | 7 +++++++ session/session.go | 1 + sessionctx/variable/session.go | 20 ++++++++++++++++++++ sessionctx/variable/sysvar.go | 1 + sessionctx/variable/tidb_vars.go | 5 +++++ sessionctx/variable/varsutil.go | 2 +- sessionctx/variable/varsutil_test.go | 12 ++++++++++++ 7 files changed, 47 insertions(+), 1 deletion(-) diff --git a/executor/set_test.go b/executor/set_test.go index acd45dfeb5614..7efac55965660 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -989,6 +989,7 @@ func (s *testSuite5) TestSetConcurrency(c *C) { tk.MustQuery("select @@tidb_hashagg_partial_concurrency;").Check(testkit.Rows(strconv.Itoa(variable.ConcurrencyUnset))) tk.MustQuery("select @@tidb_hashagg_final_concurrency;").Check(testkit.Rows(strconv.Itoa(variable.ConcurrencyUnset))) tk.MustQuery("select @@tidb_window_concurrency;").Check(testkit.Rows(strconv.Itoa(variable.ConcurrencyUnset))) + tk.MustQuery("select @@tidb_streamagg_concurrency;").Check(testkit.Rows(strconv.Itoa(variable.DefTiDBStreamAggConcurrency))) tk.MustQuery("select @@tidb_projection_concurrency;").Check(testkit.Rows(strconv.Itoa(variable.ConcurrencyUnset))) tk.MustQuery("select @@tidb_distsql_scan_concurrency;").Check(testkit.Rows(strconv.Itoa(variable.DefDistSQLScanConcurrency))) @@ -1002,6 +1003,7 @@ func (s *testSuite5) TestSetConcurrency(c *C) { c.Assert(vars.HashAggPartialConcurrency(), Equals, variable.DefExecutorConcurrency) c.Assert(vars.HashAggFinalConcurrency(), Equals, variable.DefExecutorConcurrency) c.Assert(vars.WindowConcurrency(), Equals, variable.DefExecutorConcurrency) + c.Assert(vars.StreamAggConcurrency(), Equals, variable.DefTiDBStreamAggConcurrency) c.Assert(vars.ProjectionConcurrency(), Equals, variable.DefExecutorConcurrency) c.Assert(vars.DistSQLScanConcurrency(), Equals, variable.DefDistSQLScanConcurrency) @@ -1037,6 +1039,9 @@ func (s *testSuite5) TestSetConcurrency(c *C) { checkSet(variable.TiDBWindowConcurrency) c.Assert(vars.WindowConcurrency(), Equals, 1) + checkSet(variable.TiDBStreamAggConcurrency) + c.Assert(vars.StreamAggConcurrency(), Equals, 1) + tk.MustExec(fmt.Sprintf("set @@%s=1;", variable.TiDBDistSQLScanConcurrency)) tk.MustQuery(fmt.Sprintf("select @@%s;", variable.TiDBDistSQLScanConcurrency)).Check(testkit.Rows("1")) c.Assert(vars.DistSQLScanConcurrency(), Equals, 1) @@ -1053,6 +1058,7 @@ func (s *testSuite5) TestSetConcurrency(c *C) { tk.MustExec("set @@tidb_hashagg_partial_concurrency=-1;") tk.MustExec("set @@tidb_hashagg_final_concurrency=-1;") tk.MustExec("set @@tidb_window_concurrency=-1;") + tk.MustExec("set @@tidb_streamagg_concurrency=-1;") tk.MustExec("set @@tidb_projection_concurrency=-1;") c.Assert(vars.IndexLookupConcurrency(), Equals, variable.DefExecutorConcurrency) @@ -1061,6 +1067,7 @@ func (s *testSuite5) TestSetConcurrency(c *C) { c.Assert(vars.HashAggPartialConcurrency(), Equals, variable.DefExecutorConcurrency) c.Assert(vars.HashAggFinalConcurrency(), Equals, variable.DefExecutorConcurrency) c.Assert(vars.WindowConcurrency(), Equals, variable.DefExecutorConcurrency) + c.Assert(vars.StreamAggConcurrency(), Equals, variable.DefExecutorConcurrency) c.Assert(vars.ProjectionConcurrency(), Equals, variable.DefExecutorConcurrency) _, err := tk.Exec("set @@tidb_executor_concurrency=-1;") diff --git a/session/session.go b/session/session.go index 3c3e5aeb28136..7d3d0044c8d53 100644 --- a/session/session.go +++ b/session/session.go @@ -2157,6 +2157,7 @@ var builtinGlobalVariable = []string{ variable.TiDBHashAggPartialConcurrency, variable.TiDBHashAggFinalConcurrency, variable.TiDBWindowConcurrency, + variable.TiDBStreamAggConcurrency, variable.TiDBExecutorConcurrency, variable.TiDBBackoffLockFast, variable.TiDBBackOffWeight, diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 0a28071b45d49..22be00689e456 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -873,6 +873,7 @@ func NewSessionVars() *SessionVars { hashAggPartialConcurrency: DefTiDBHashAggPartialConcurrency, hashAggFinalConcurrency: DefTiDBHashAggFinalConcurrency, windowConcurrency: DefTiDBWindowConcurrency, + streamAggConcurrency: DefTiDBStreamAggConcurrency, ExecutorConcurrency: DefExecutorConcurrency, } vars.MemQuota = MemQuota{ @@ -1327,6 +1328,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { s.hashAggFinalConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) case TiDBWindowConcurrency: s.windowConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) + case TiDBStreamAggConcurrency: + s.streamAggConcurrency = tidbOptPositiveInt32(val, ConcurrencyUnset) case TiDBDistSQLScanConcurrency: s.distSQLScanConcurrency = tidbOptPositiveInt32(val, DefDistSQLScanConcurrency) case TiDBIndexSerialScanConcurrency: @@ -1660,6 +1663,10 @@ type Concurrency struct { // windowConcurrency is deprecated, use ExecutorConcurrency instead. windowConcurrency int + // streamAggConcurrency is the number of concurrent stream aggregation worker. + // streamAggConcurrency is deprecated, use ExecutorConcurrency instead. + streamAggConcurrency int + // indexSerialScanConcurrency is the number of concurrent index serial scan worker. indexSerialScanConcurrency int @@ -1710,6 +1717,11 @@ func (c *Concurrency) SetWindowConcurrency(n int) { c.windowConcurrency = n } +// SetStreamAggConcurrency set the number of concurrent stream aggregation worker. +func (c *Concurrency) SetStreamAggConcurrency(n int) { + c.streamAggConcurrency = n +} + // SetIndexSerialScanConcurrency set the number of concurrent index serial scan worker. func (c *Concurrency) SetIndexSerialScanConcurrency(n int) { c.indexSerialScanConcurrency = n @@ -1776,6 +1788,14 @@ func (c *Concurrency) WindowConcurrency() int { return c.ExecutorConcurrency } +// StreamAggConcurrency return the number of concurrent stream aggregation worker. +func (c *Concurrency) StreamAggConcurrency() int { + if c.streamAggConcurrency != ConcurrencyUnset { + return c.streamAggConcurrency + } + return c.ExecutorConcurrency +} + // IndexSerialScanConcurrency return the number of concurrent index serial scan worker. // This option is not sync with ExecutorConcurrency since it's used by Analyze table. func (c *Concurrency) IndexSerialScanConcurrency() int { diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index efca5c483145a..d6e0e5f57e7f8 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1017,6 +1017,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashAggPartialConcurrency, Value: strconv.Itoa(DefTiDBHashAggPartialConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBHashAggFinalConcurrency, Value: strconv.Itoa(DefTiDBHashAggFinalConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBWindowConcurrency, Value: strconv.Itoa(DefTiDBWindowConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBStreamAggConcurrency, Value: strconv.Itoa(DefTiDBStreamAggConcurrency), Type: TypeInt, MinValue: 1, MaxValue: math.MaxInt64, AllowAutoValue: true}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableParallelApply, Value: BoolToOnOff(DefTiDBEnableParallelApply), Type: TypeBool}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBBackoffLockFast, Value: strconv.Itoa(kv.DefBackoffLockFast), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBBackOffWeight, Value: strconv.Itoa(kv.DefBackOffWeight), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64}, diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 4bb06d0d2b687..5e93fb1566511 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -312,6 +312,10 @@ const ( // tidb_window_concurrency is deprecated, use tidb_executor_concurrency instead. TiDBWindowConcurrency = "tidb_window_concurrency" + // tidb_stream_agg_concurrency is used for stream aggregation parallel executor. + // tidb_stream_agg_concurrency is deprecated, use tidb_executor_concurrency instead. + TiDBStreamAggConcurrency = "tidb_streamagg_concurrency" + // tidb_enable_parallel_apply is used for parallel apply. TiDBEnableParallelApply = "tidb_enable_parallel_apply" @@ -531,6 +535,7 @@ const ( DefTiDBHashAggPartialConcurrency = ConcurrencyUnset DefTiDBHashAggFinalConcurrency = ConcurrencyUnset DefTiDBWindowConcurrency = ConcurrencyUnset + DefTiDBStreamAggConcurrency = 1 DefTiDBForcePriority = mysql.NoPriority DefTiDBUseRadixJoin = false DefEnableWindowFunction = true diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index f049d01577670..8c757ab81f848 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -267,7 +267,7 @@ func CheckDeprecationSetSystemVar(s *SessionVars, name string) { switch name { case TiDBIndexLookupConcurrency, TiDBIndexLookupJoinConcurrency, TiDBHashJoinConcurrency, TiDBHashAggPartialConcurrency, TiDBHashAggFinalConcurrency, - TiDBProjectionConcurrency, TiDBWindowConcurrency: + TiDBProjectionConcurrency, TiDBWindowConcurrency, TiDBStreamAggConcurrency: s.StmtCtx.AppendWarning(errWarnDeprecatedSyntax.FastGenByArgs(name, TiDBExecutorConcurrency)) case TIDBMemQuotaHashJoin, TIDBMemQuotaMergeJoin, TIDBMemQuotaSort, TIDBMemQuotaTopn, diff --git a/sessionctx/variable/varsutil_test.go b/sessionctx/variable/varsutil_test.go index 14be2e8dd7306..b458d0787d74d 100644 --- a/sessionctx/variable/varsutil_test.go +++ b/sessionctx/variable/varsutil_test.go @@ -74,11 +74,13 @@ func (s *testVarsutilSuite) TestNewSessionVars(c *C) { c.Assert(vars.hashAggPartialConcurrency, Equals, ConcurrencyUnset) c.Assert(vars.hashAggFinalConcurrency, Equals, ConcurrencyUnset) c.Assert(vars.windowConcurrency, Equals, ConcurrencyUnset) + c.Assert(vars.streamAggConcurrency, Equals, DefTiDBStreamAggConcurrency) c.Assert(vars.distSQLScanConcurrency, Equals, DefDistSQLScanConcurrency) c.Assert(vars.ProjectionConcurrency(), Equals, DefExecutorConcurrency) c.Assert(vars.HashAggPartialConcurrency(), Equals, DefExecutorConcurrency) c.Assert(vars.HashAggFinalConcurrency(), Equals, DefExecutorConcurrency) c.Assert(vars.WindowConcurrency(), Equals, DefExecutorConcurrency) + c.Assert(vars.StreamAggConcurrency(), Equals, DefTiDBStreamAggConcurrency) c.Assert(vars.DistSQLScanConcurrency(), Equals, DefDistSQLScanConcurrency) c.Assert(vars.ExecutorConcurrency, Equals, DefExecutorConcurrency) c.Assert(vars.MaxChunkSize, Equals, DefMaxChunkSize) @@ -657,6 +659,14 @@ func (s *testVarsutilSuite) TestConcurrencyVariables(c *C) { c.Assert(vars.windowConcurrency, Equals, wdConcurrency) c.Assert(vars.WindowConcurrency(), Equals, wdConcurrency) + saConcurrency := 2 + c.Assert(vars.streamAggConcurrency, Equals, DefTiDBStreamAggConcurrency) + c.Assert(vars.StreamAggConcurrency(), Equals, DefTiDBStreamAggConcurrency) + err = SetSessionSystemVar(vars, TiDBStreamAggConcurrency, types.NewIntDatum(int64(saConcurrency))) + c.Assert(err, IsNil) + c.Assert(vars.streamAggConcurrency, Equals, saConcurrency) + c.Assert(vars.StreamAggConcurrency(), Equals, saConcurrency) + c.Assert(vars.indexLookupConcurrency, Equals, ConcurrencyUnset) c.Assert(vars.IndexLookupConcurrency(), Equals, DefExecutorConcurrency) exeConcurrency := DefExecutorConcurrency + 1 @@ -665,4 +675,6 @@ func (s *testVarsutilSuite) TestConcurrencyVariables(c *C) { c.Assert(vars.indexLookupConcurrency, Equals, ConcurrencyUnset) c.Assert(vars.IndexLookupConcurrency(), Equals, exeConcurrency) c.Assert(vars.WindowConcurrency(), Equals, wdConcurrency) + c.Assert(vars.StreamAggConcurrency(), Equals, saConcurrency) + } From e90aac77238965f269b2c11f53354acc84c14595 Mon Sep 17 00:00:00 2001 From: HuaiyuXu <391585975@qq.com> Date: Mon, 16 Nov 2020 11:42:27 +0800 Subject: [PATCH 02/37] plannr: build empty range for overflow predicate (#21042) --- expression/builtin_compare.go | 12 ++++++++++++ planner/core/integration_test.go | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 8f590a6fc5ad7..5564b1f00d11c 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -1371,6 +1371,18 @@ func (c *compareFunctionClass) refineArgsByUnsignedFlag(ctx sessionctx.Context, if err != nil || isNull || v > 0 { return args } + if mysql.HasUnsignedFlag(con.RetType.Flag) && !mysql.HasUnsignedFlag(col.RetType.Flag) { + op := c.op + if i == 1 { + op = symmetricOp[c.op] + } + if op == opcode.EQ || op == opcode.NullEQ { + if _, err := types.ConvertUintToInt(uint64(v), types.IntergerSignedUpperBound(col.RetType.Tp), col.RetType.Tp); err != nil { + args[i], args[1-i] = NewOne(), NewZero() + return args + } + } + } if mysql.HasUnsignedFlag(col.RetType.Flag) && mysql.HasNotNullFlag(col.RetType.Flag) && !mysql.HasUnsignedFlag(con.RetType.Flag) { op := c.op if i == 1 { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 1775bb2e171ce..63efdac7648bb 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1822,3 +1822,13 @@ func (s *testIntegrationSuite) TestPartitionUnionWithPPruningColumn(c *C) { "3290 LE1327_r5")) } + +func (s *testIntegrationSuite) TestIssue10448(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + + tk.MustExec("create table t(pk int(11) primary key)") + tk.MustExec("insert into t values(1),(2),(3)") + tk.MustQuery("select a from (select pk as a from t) t1 where a = 18446744073709551615").Check(testkit.Rows()) +} From 31ddba99da388d82c5fd55a21e87d15a6aefda09 Mon Sep 17 00:00:00 2001 From: pengdaqian2020 <71140907+pengdaqian2020@users.noreply.github.com> Date: Sun, 15 Nov 2020 22:07:28 -0600 Subject: [PATCH 03/37] executor: fix The JSON Data can not import to TiDB correctly by `load data` (#21043) --- executor/executor_pkg_test.go | 28 ++++++++++++++++++++++++++++ executor/load_data.go | 18 +++++++++--------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/executor/executor_pkg_test.go b/executor/executor_pkg_test.go index 780e2df38c74e..75868833ede04 100644 --- a/executor/executor_pkg_test.go +++ b/executor/executor_pkg_test.go @@ -246,6 +246,7 @@ func (s *testExecSuite) TestGetFieldsFromLine(c *C) { FieldsInfo: &ast.FieldsClause{ Enclosed: '"', Terminated: ",", + Escaped: '\\', }, } @@ -259,6 +260,33 @@ func (s *testExecSuite) TestGetFieldsFromLine(c *C) { c.Assert(err, IsNil) } +func (s *testExecSerialSuite) TestLoadDataWithDifferentEscapeChar(c *C) { + tests := []struct { + input string + escapeChar byte + expected []string + }{ + { + `"{""itemRangeType"":0,""itemContainType"":0,""shopRangeType"":1,""shopJson"":""[{\""id\"":\""A1234\"",\""shopName\"":\""AAAAAA\""}]""}"`, + byte(0), // escaped by '' + []string{`{"itemRangeType":0,"itemContainType":0,"shopRangeType":1,"shopJson":"[{\"id\":\"A1234\",\"shopName\":\"AAAAAA\"}]"}`}, + }, + } + + for _, test := range tests { + ldInfo := LoadDataInfo{ + FieldsInfo: &ast.FieldsClause{ + Enclosed: '"', + Terminated: ",", + Escaped: test.escapeChar, + }, + } + got, err := ldInfo.getFieldsFromLine([]byte(test.input)) + c.Assert(err, IsNil, Commentf("failed: %s", test.input)) + assertEqualStrings(c, got, test.expected) + } +} + func assertEqualStrings(c *C, got []field, expect []string) { c.Assert(len(got), Equals, len(expect)) for i := 0; i < len(got); i++ { diff --git a/executor/load_data.go b/executor/load_data.go index ce953f4230ad4..429b9f3d48cd3 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -627,17 +627,19 @@ type fieldWriter struct { term string enclosedChar byte fieldTermChar byte + escapeChar byte isEnclosed bool isLineStart bool isFieldStart bool } -func (w *fieldWriter) Init(enclosedChar byte, fieldTermChar byte, readBuf []byte, term string) { +func (w *fieldWriter) Init(enclosedChar, escapeChar, fieldTermChar byte, readBuf []byte, term string) { w.isEnclosed = false w.isLineStart = true w.isFieldStart = true w.ReadBuf = readBuf w.enclosedChar = enclosedChar + w.escapeChar = escapeChar w.fieldTermChar = fieldTermChar w.term = term } @@ -743,13 +745,12 @@ func (w *fieldWriter) GetField() (bool, field) { w.OutputBuf = append(w.OutputBuf, w.enclosedChar) w.putback() } - } else if ch == '\\' { - // TODO: escape only support '\' + } else if ch == w.escapeChar { // When the escaped character is interpreted as if // it was not escaped, backslash is ignored. flag, ch = w.getChar() if flag { - w.OutputBuf = append(w.OutputBuf, '\\') + w.OutputBuf = append(w.OutputBuf, w.escapeChar) w.OutputBuf = append(w.OutputBuf, ch) } } else { @@ -771,10 +772,10 @@ func (e *LoadDataInfo) getFieldsFromLine(line []byte) ([]field, error) { return fields, nil } - reader.Init(e.FieldsInfo.Enclosed, e.FieldsInfo.Terminated[0], line, e.FieldsInfo.Terminated) + reader.Init(e.FieldsInfo.Enclosed, e.FieldsInfo.Escaped, e.FieldsInfo.Terminated[0], line, e.FieldsInfo.Terminated) for { eol, f := reader.GetField() - f = f.escape() + f = f.escape(reader.escapeChar) if bytes.Equal(f.str, null) && !f.enclosed { f.str = []byte{'N'} f.maybeNull = true @@ -789,12 +790,11 @@ func (e *LoadDataInfo) getFieldsFromLine(line []byte) ([]field, error) { // escape handles escape characters when running load data statement. // See http://dev.mysql.com/doc/refman/5.7/en/load-data.html -// TODO: escape only support '\' as the `ESCAPED BY` character, it should support specify characters. -func (f *field) escape() field { +func (f *field) escape(escapeChar byte) field { pos := 0 for i := 0; i < len(f.str); i++ { c := f.str[i] - if i+1 < len(f.str) && f.str[i] == '\\' { + if i+1 < len(f.str) && f.str[i] == escapeChar { c = f.escapeChar(f.str[i+1]) i++ } From dfc3bbdee6c4779ee5c44421bd6c819797562560 Mon Sep 17 00:00:00 2001 From: HuaiyuXu <391585975@qq.com> Date: Mon, 16 Nov 2020 14:51:26 +0800 Subject: [PATCH 04/37] util, executor: parse set value correctly in HashChunkRow (#21048) --- executor/join_test.go | 13 +++++++++++++ util/codec/codec.go | 16 ++++++++++------ util/codec/codec_test.go | 1 + 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/executor/join_test.go b/executor/join_test.go index 09dad2121a292..e83c019d0ba09 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -2473,3 +2473,16 @@ func (s *testSuiteJoinSerial) TestIssue20779(c *C) { _, err = session.GetRows4Test(context.Background(), nil, rs) c.Assert(err.Error(), Matches, "testIssue20779") } + +func (s *testSuiteJoinSerial) TestIssue20219(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t,s ") + tk.MustExec("CREATE TABLE `t` ( `a` set('a','b','c','d','e','f','g','h','i','j') DEFAULT NULL );") + tk.MustExec("insert into t values('i'), ('j');") + tk.MustExec("CREATE TABLE `s` ( `a` char(1) DEFAULT NULL, KEY `a` (`a`) )") + tk.MustExec("insert into s values('i'), ('j');") + tk.MustQuery("select /*+ inl_hash_join(s)*/ t.a from t left join s on t.a = s.a;").Check(testkit.Rows("i", "j")) + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustQuery("select /*+ inl_join(s)*/ t.a from t left join s on t.a = s.a;").Check(testkit.Rows("i", "j")) + tk.MustQuery("show warnings").Check(testkit.Rows()) +} diff --git a/util/codec/codec.go b/util/codec/codec.go index 38a99b53b85dc..8ede0bb67bf27 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -363,9 +363,11 @@ func encodeHashChunkRowIdx(sc *stmtctx.StatementContext, row chunk.Row, tp *type b = ConvertByCollation(hack.Slice(str), tp) case mysql.TypeSet: flag = compactBytesFlag - v := uint64(row.GetSet(idx).ToNumber()) - str := tp.Elems[v-1] - b = ConvertByCollation(hack.Slice(str), tp) + s, err := types.ParseSetValue(tp.Elems, row.GetSet(idx).Value) + if err != nil { + return 0, nil, err + } + b = ConvertByCollation(hack.Slice(s.Name), tp) case mysql.TypeBit: // We don't need to handle errors here since the literal is ensured to be able to store in uint64 in convertToMysqlBit. flag = uvarintFlag @@ -588,9 +590,11 @@ func HashChunkSelected(sc *stmtctx.StatementContext, h []hash.Hash64, chk *chunk isNull[i] = !ignoreNull } else { buf[0] = compactBytesFlag - v := uint64(column.GetSet(i).ToNumber()) - str := tp.Elems[v-1] - b = ConvertByCollation(hack.Slice(str), tp) + s, err := types.ParseSetValue(tp.Elems, column.GetSet(i).Value) + if err != nil { + return err + } + b = ConvertByCollation(hack.Slice(s.Name), tp) } // As the golang doc described, `Hash.Write` never returns an error. diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index eb0fff931be78..8649aa3f0fa46 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -1034,6 +1034,7 @@ func datumsForTest(sc *stmtctx.StatementContext) ([]types.Datum, []*types.FieldT {types.Duration{Duration: time.Second, Fsp: 1}, types.NewFieldType(mysql.TypeDuration)}, {types.Enum{Name: "a", Value: 1}, &types.FieldType{Tp: mysql.TypeEnum, Elems: []string{"a"}}}, {types.Set{Name: "a", Value: 1}, &types.FieldType{Tp: mysql.TypeSet, Elems: []string{"a"}}}, + {types.Set{Name: "f", Value: 32}, &types.FieldType{Tp: mysql.TypeSet, Elems: []string{"a", "b", "c", "d", "e", "f"}}}, {types.BinaryLiteral{100}, &types.FieldType{Tp: mysql.TypeBit, Flen: 8}}, {json.CreateBinary("abc"), types.NewFieldType(mysql.TypeJSON)}, {int64(1), types.NewFieldType(mysql.TypeYear)}, From 41786cb976f43df6ec10cba549c7f845c8531d88 Mon Sep 17 00:00:00 2001 From: HuaiyuXu <391585975@qq.com> Date: Mon, 16 Nov 2020 16:24:51 +0800 Subject: [PATCH 05/37] *: seperate hash keys from join keys in IndexJoin (#20761) --- cmd/explaintest/r/explain_join_stats.result | 2 +- cmd/explaintest/r/explain_union_scan.result | 2 +- cmd/explaintest/r/generated_columns.result | 2 +- cmd/explaintest/r/index_join.result | 6 +- cmd/explaintest/r/topn_push_down.result | 6 +- cmd/explaintest/r/tpch.result | 12 +-- executor/benchmark_test.go | 8 ++ executor/builder.go | 13 ++- executor/index_lookup_hash_join.go | 10 +-- executor/index_lookup_join.go | 42 +++++---- executor/join_test.go | 38 ++++++-- expression/testdata/partition_pruner_out.json | 4 +- planner/core/exhaust_physical_plans.go | 43 ++++++++- planner/core/explain.go | 14 +++ planner/core/integration_test.go | 25 ++++++ planner/core/physical_plans.go | 6 ++ planner/core/resolve_indices.go | 11 +++ planner/core/testdata/analyze_suite_out.json | 2 +- .../testdata/integration_serial_suite_in.json | 11 +++ .../integration_serial_suite_out.json | 87 +++++++++++++++++++ .../core/testdata/integration_suite_out.json | 26 +++--- .../testdata/plan_normalized_suite_out.json | 4 +- planner/core/testdata/plan_suite_out.json | 2 +- 23 files changed, 307 insertions(+), 69 deletions(-) diff --git a/cmd/explaintest/r/explain_join_stats.result b/cmd/explaintest/r/explain_join_stats.result index 73bb6e2671ba4..723df63732dbe 100644 --- a/cmd/explaintest/r/explain_join_stats.result +++ b/cmd/explaintest/r/explain_join_stats.result @@ -17,7 +17,7 @@ StreamAgg_13 1.00 root funcs:count(1)->Column#5 explain select /*+ TIDB_INLJ(e) */ count(*) from e, lo where lo.a=e.a and e.b=22336; id estRows task access object operator info StreamAgg_12 1.00 root funcs:count(1)->Column#5 -└─IndexJoin_56 19977.00 root inner join, inner:IndexLookUp_55, outer key:test.lo.a, inner key:test.e.a +└─IndexJoin_56 19977.00 root inner join, inner:IndexLookUp_55, outer key:test.lo.a, inner key:test.e.a, equal cond:eq(test.lo.a, test.e.a) ├─TableReader_40(Build) 250.00 root data:TableFullScan_39 │ └─TableFullScan_39 250.00 cop[tikv] table:lo keep order:false └─IndexLookUp_55(Probe) 79.91 root diff --git a/cmd/explaintest/r/explain_union_scan.result b/cmd/explaintest/r/explain_union_scan.result index 087f0d62efa21..edd9d414f6fcb 100644 --- a/cmd/explaintest/r/explain_union_scan.result +++ b/cmd/explaintest/r/explain_union_scan.result @@ -15,7 +15,7 @@ id estRows task access object operator info Limit_20 10.00 root offset:0, count:10 └─HashJoin_22 10.00 root left outer join, equal:[eq(test.city.province_id, test.city.province_id)] ├─Limit_25(Build) 10.00 root offset:0, count:10 - │ └─IndexJoin_38 10.00 root inner join, inner:UnionScan_37, outer key:test.city.id, inner key:test.city.id + │ └─IndexJoin_38 10.00 root inner join, inner:UnionScan_37, outer key:test.city.id, inner key:test.city.id, equal cond:eq(test.city.id, test.city.id) │ ├─UnionScan_47(Build) 10.00 root │ │ └─TableReader_49 10.00 root data:TableFullScan_48 │ │ └─TableFullScan_48 10.00 cop[tikv] table:t2 keep order:false diff --git a/cmd/explaintest/r/generated_columns.result b/cmd/explaintest/r/generated_columns.result index 590d5141307d3..dad75f5aeca9e 100644 --- a/cmd/explaintest/r/generated_columns.result +++ b/cmd/explaintest/r/generated_columns.result @@ -72,7 +72,7 @@ VALUES ('{"a": 1}', '{"1": "1"}'); ANALYZE TABLE sgc1, sgc2; EXPLAIN SELECT /*+ TIDB_INLJ(sgc1, sgc2) */ * from sgc1 join sgc2 on sgc1.a=sgc2.a; id estRows task access object operator info -IndexJoin_26 5.00 root inner join, inner:IndexLookUp_25, outer key:test.sgc2.a, inner key:test.sgc1.a +IndexJoin_26 5.00 root inner join, inner:IndexLookUp_25, outer key:test.sgc2.a, inner key:test.sgc1.a, equal cond:eq(test.sgc2.a, test.sgc1.a) ├─TableReader_47(Build) 1.00 root data:Selection_46 │ └─Selection_46 1.00 cop[tikv] not(isnull(test.sgc2.a)) │ └─TableFullScan_45 1.00 cop[tikv] table:sgc2 keep order:false diff --git a/cmd/explaintest/r/index_join.result b/cmd/explaintest/r/index_join.result index 851d74e672c43..418d5b7e6fb98 100644 --- a/cmd/explaintest/r/index_join.result +++ b/cmd/explaintest/r/index_join.result @@ -8,7 +8,7 @@ set session tidb_hashagg_partial_concurrency = 1; set session tidb_hashagg_final_concurrency = 1; explain select /*+ TIDB_INLJ(t1, t2) */ * from t1 join t2 on t1.a=t2.a; id estRows task access object operator info -IndexJoin_25 5.00 root inner join, inner:IndexLookUp_24, outer key:test.t2.a, inner key:test.t1.a +IndexJoin_25 5.00 root inner join, inner:IndexLookUp_24, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a) ├─TableReader_43(Build) 1.00 root data:Selection_42 │ └─Selection_42 1.00 cop[tikv] not(isnull(test.t2.a)) │ └─TableFullScan_41 1.00 cop[tikv] table:t2 keep order:false @@ -32,7 +32,7 @@ create table t2(a int not null, b int not null, key a(a)); set @@tidb_opt_insubq_to_join_and_agg=0; explain select /*+ TIDB_INLJ(t2@sel_2) */ * from t1 where t1.a in (select t2.a from t2); id estRows task access object operator info -IndexJoin_10 8000.00 root semi join, inner:IndexReader_9, outer key:test.t1.a, inner key:test.t2.a +IndexJoin_10 8000.00 root semi join, inner:IndexReader_9, outer key:test.t1.a, inner key:test.t2.a, equal cond:eq(test.t1.a, test.t2.a) ├─TableReader_18(Build) 10000.00 root data:TableFullScan_17 │ └─TableFullScan_17 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo └─IndexReader_9(Probe) 1.25 root index:IndexRangeScan_8 @@ -45,7 +45,7 @@ create table t1(a int not null, b int not null, key a(a)); create table t2(a int not null, b int not null, key a(a)); explain select /*+ TIDB_INLJ(t1) */ * from t1 where t1.a in (select t2.a from t2); id estRows task access object operator info -IndexJoin_13 10000.00 root inner join, inner:IndexLookUp_12, outer key:test.t2.a, inner key:test.t1.a +IndexJoin_13 10000.00 root inner join, inner:IndexLookUp_12, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a) ├─StreamAgg_26(Build) 8000.00 root group by:test.t2.a, funcs:firstrow(test.t2.a)->test.t2.a │ └─IndexReader_39 10000.00 root index:IndexFullScan_38 │ └─IndexFullScan_38 10000.00 cop[tikv] table:t2, index:a(a) keep order:true, stats:pseudo diff --git a/cmd/explaintest/r/topn_push_down.result b/cmd/explaintest/r/topn_push_down.result index 03f5ad84692b0..c98a4085ec755 100644 --- a/cmd/explaintest/r/topn_push_down.result +++ b/cmd/explaintest/r/topn_push_down.result @@ -168,7 +168,7 @@ ORDER BY te.expect_time asc LIMIT 0, 5; id estRows task access object operator info Limit_19 0.00 root offset:0, count:5 -└─IndexJoin_118 0.00 root left outer join, inner:IndexReader_117, outer key:test.tr.id, inner key:test.p.relate_id +└─IndexJoin_118 0.00 root left outer join, inner:IndexReader_117, outer key:test.tr.id, inner key:test.p.relate_id, equal cond:eq(test.tr.id, test.p.relate_id) ├─TopN_127(Build) 0.00 root test.te.expect_time, offset:0, count:5 │ └─IndexMergeJoin_53 0.00 root inner join, inner:Projection_51, outer key:test.tr.id, inner key:test.te.trade_id │ ├─IndexLookUp_98(Build) 0.00 root @@ -221,7 +221,7 @@ create table t(a int not null, index idx(a)); explain select /*+ TIDB_INLJ(t2) */ * from t t1 join t t2 on t1.a = t2.a limit 5; id estRows task access object operator info Limit_11 5.00 root offset:0, count:5 -└─IndexJoin_15 5.00 root inner join, inner:IndexReader_14, outer key:test.t.a, inner key:test.t.a +└─IndexJoin_15 5.00 root inner join, inner:IndexReader_14, outer key:test.t.a, inner key:test.t.a, equal cond:eq(test.t.a, test.t.a) ├─TableReader_23(Build) 4.00 root data:TableFullScan_22 │ └─TableFullScan_22 4.00 cop[tikv] table:t1 keep order:false, stats:pseudo └─IndexReader_14(Probe) 1.25 root index:IndexRangeScan_13 @@ -230,7 +230,7 @@ explain select /*+ TIDB_INLJ(t2) */ * from t t1 left join t t2 on t1.a = t2.a wh id estRows task access object operator info Limit_12 5.00 root offset:0, count:5 └─Selection_13 5.00 root isnull(test.t.a) - └─IndexJoin_17 5.00 root left outer join, inner:IndexReader_16, outer key:test.t.a, inner key:test.t.a + └─IndexJoin_17 5.00 root left outer join, inner:IndexReader_16, outer key:test.t.a, inner key:test.t.a, equal cond:eq(test.t.a, test.t.a) ├─TableReader_25(Build) 4.00 root data:TableFullScan_24 │ └─TableFullScan_24 4.00 cop[tikv] table:t1 keep order:false, stats:pseudo └─IndexReader_16(Probe) 1.25 root index:IndexRangeScan_15 diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index 611030fafd436..0282fc5e9be67 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -297,7 +297,7 @@ id estRows task access object operator info Sort_10 1.00 root tpch.orders.o_orderpriority └─Projection_12 1.00 root tpch.orders.o_orderpriority, Column#27 └─HashAgg_15 1.00 root group by:tpch.orders.o_orderpriority, funcs:count(1)->Column#27, funcs:firstrow(tpch.orders.o_orderpriority)->tpch.orders.o_orderpriority - └─IndexHashJoin_23 2340750.00 root semi join, inner:IndexLookUp_20, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey + └─IndexHashJoin_23 2340750.00 root semi join, inner:IndexLookUp_20, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey, equal cond:eq(tpch.orders.o_orderkey, tpch.lineitem.l_orderkey) ├─TableReader_42(Build) 2925937.50 root data:Selection_41 │ └─Selection_41 2925937.50 cop[tikv] ge(tpch.orders.o_orderdate, 1995-01-01 00:00:00.000000), lt(tpch.orders.o_orderdate, 1995-04-01) │ └─TableFullScan_40 75000000.00 cop[tikv] table:orders keep order:false @@ -530,7 +530,7 @@ Sort_29 719.02 root Column#62 ├─TableReader_117(Build) 61674.00 root data:Selection_116 │ └─Selection_116 61674.00 cop[tikv] eq(tpch.part.p_type, "SMALL PLATED COPPER") │ └─TableFullScan_115 10000000.00 cop[tikv] table:part keep order:false - └─IndexHashJoin_77(Probe) 90788402.51 root inner join, inner:IndexLookUp_74, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey + └─IndexHashJoin_77(Probe) 90788402.51 root inner join, inner:IndexLookUp_74, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey, equal cond:eq(tpch.orders.o_orderkey, tpch.lineitem.l_orderkey) ├─HashJoin_87(Build) 22413367.93 root inner join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] │ ├─HashJoin_89(Build) 1500000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.customer.c_nationkey)] │ │ ├─HashJoin_102(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] @@ -661,7 +661,7 @@ Projection_17 20.00 root tpch.customer.c_custkey, tpch.customer.c_name, Column# └─TopN_20 20.00 root Column#39:desc, offset:0, count:20 └─HashAgg_26 3017307.69 root group by:Column#53, Column#54, Column#55, Column#56, Column#57, Column#58, Column#59, funcs:sum(Column#45)->Column#39, funcs:firstrow(Column#46)->tpch.customer.c_custkey, funcs:firstrow(Column#47)->tpch.customer.c_name, funcs:firstrow(Column#48)->tpch.customer.c_address, funcs:firstrow(Column#49)->tpch.customer.c_phone, funcs:firstrow(Column#50)->tpch.customer.c_acctbal, funcs:firstrow(Column#51)->tpch.customer.c_comment, funcs:firstrow(Column#52)->tpch.nation.n_name └─Projection_67 12222016.17 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#45, tpch.customer.c_custkey, tpch.customer.c_name, tpch.customer.c_address, tpch.customer.c_phone, tpch.customer.c_acctbal, tpch.customer.c_comment, tpch.nation.n_name, tpch.customer.c_custkey, tpch.customer.c_name, tpch.customer.c_acctbal, tpch.customer.c_phone, tpch.nation.n_name, tpch.customer.c_address, tpch.customer.c_comment - └─IndexHashJoin_34 12222016.17 root inner join, inner:IndexLookUp_31, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey + └─IndexHashJoin_34 12222016.17 root inner join, inner:IndexLookUp_31, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey, equal cond:eq(tpch.orders.o_orderkey, tpch.lineitem.l_orderkey) ├─HashJoin_44(Build) 3017307.69 root inner join, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] │ ├─TableReader_63(Build) 3017307.69 root data:Selection_62 │ │ └─Selection_62 3017307.69 cop[tikv] ge(tpch.orders.o_orderdate, 1993-08-01 00:00:00.000000), lt(tpch.orders.o_orderdate, 1993-11-01) @@ -1156,7 +1156,7 @@ Sort_28 20000.00 root tpch.supplier.s_name └─Selection_55 257492.04 root gt(cast(tpch.partsupp.ps_availqty), mul(0.5, Column#44)) └─HashAgg_58 321865.05 root group by:tpch.partsupp.ps_partkey, tpch.partsupp.ps_suppkey, funcs:firstrow(tpch.partsupp.ps_suppkey)->tpch.partsupp.ps_suppkey, funcs:firstrow(tpch.partsupp.ps_availqty)->tpch.partsupp.ps_availqty, funcs:sum(tpch.lineitem.l_quantity)->Column#44 └─HashJoin_62 9711455.06 root left outer join, equal:[eq(tpch.partsupp.ps_partkey, tpch.lineitem.l_partkey) eq(tpch.partsupp.ps_suppkey, tpch.lineitem.l_suppkey)] - ├─IndexHashJoin_69(Build) 321865.05 root inner join, inner:IndexLookUp_66, outer key:tpch.part.p_partkey, inner key:tpch.partsupp.ps_partkey + ├─IndexHashJoin_69(Build) 321865.05 root inner join, inner:IndexLookUp_66, outer key:tpch.part.p_partkey, inner key:tpch.partsupp.ps_partkey, equal cond:eq(tpch.part.p_partkey, tpch.partsupp.ps_partkey) │ ├─TableReader_98(Build) 80007.93 root data:Selection_97 │ │ └─Selection_97 80007.93 cop[tikv] like(tpch.part.p_name, "green%", 92) │ │ └─TableFullScan_96 10000000.00 cop[tikv] table:part keep order:false @@ -1218,8 +1218,8 @@ id estRows task access object operator info Projection_25 100.00 root tpch.supplier.s_name, Column#72 └─TopN_28 100.00 root Column#72:desc, tpch.supplier.s_name, offset:0, count:100 └─HashAgg_34 12800.00 root group by:tpch.supplier.s_name, funcs:count(1)->Column#72, funcs:firstrow(tpch.supplier.s_name)->tpch.supplier.s_name - └─IndexHashJoin_42 7828961.66 root anti semi join, inner:IndexLookUp_39, outer key:tpch.lineitem.l_orderkey, inner key:tpch.lineitem.l_orderkey, other cond:ne(tpch.lineitem.l_suppkey, tpch.lineitem.l_suppkey) - ├─IndexHashJoin_82(Build) 9786202.08 root semi join, inner:IndexLookUp_79, outer key:tpch.lineitem.l_orderkey, inner key:tpch.lineitem.l_orderkey, other cond:ne(tpch.lineitem.l_suppkey, tpch.lineitem.l_suppkey), ne(tpch.lineitem.l_suppkey, tpch.supplier.s_suppkey) + └─IndexHashJoin_42 7828961.66 root anti semi join, inner:IndexLookUp_39, outer key:tpch.lineitem.l_orderkey, inner key:tpch.lineitem.l_orderkey, equal cond:eq(tpch.lineitem.l_orderkey, tpch.lineitem.l_orderkey), other cond:ne(tpch.lineitem.l_suppkey, tpch.lineitem.l_suppkey) + ├─IndexHashJoin_82(Build) 9786202.08 root semi join, inner:IndexLookUp_79, outer key:tpch.lineitem.l_orderkey, inner key:tpch.lineitem.l_orderkey, equal cond:eq(tpch.lineitem.l_orderkey, tpch.lineitem.l_orderkey), other cond:ne(tpch.lineitem.l_suppkey, tpch.lineitem.l_suppkey), ne(tpch.lineitem.l_suppkey, tpch.supplier.s_suppkey) │ ├─IndexMergeJoin_101(Build) 12232752.60 root inner join, inner:TableReader_96, outer key:tpch.lineitem.l_orderkey, inner key:tpch.orders.o_orderkey │ │ ├─HashJoin_105(Build) 12232752.60 root inner join, equal:[eq(tpch.supplier.s_suppkey, tpch.lineitem.l_suppkey)] │ │ │ ├─HashJoin_118(Build) 20000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] diff --git a/executor/benchmark_test.go b/executor/benchmark_test.go index 7b8b00ae04cbb..d710deb59b93e 100644 --- a/executor/benchmark_test.go +++ b/executor/benchmark_test.go @@ -1140,6 +1140,8 @@ type indexJoinTestCase struct { ctx sessionctx.Context outerJoinKeyIdx []int innerJoinKeyIdx []int + outerHashKeyIdx []int + innerHashKeyIdx []int innerIdx []int needOuterSort bool rawData string @@ -1167,6 +1169,8 @@ func defaultIndexJoinTestCase() *indexJoinTestCase { ctx: ctx, outerJoinKeyIdx: []int{0, 1}, innerJoinKeyIdx: []int{0, 1}, + outerHashKeyIdx: []int{0, 1}, + innerHashKeyIdx: []int{0, 1}, innerIdx: []int{0, 1}, rawData: wideString, } @@ -1216,12 +1220,14 @@ func prepare4IndexInnerHashJoin(tc *indexJoinTestCase, outerDS *mockDataSource, outerCtx: outerCtx{ rowTypes: leftTypes, keyCols: tc.outerJoinKeyIdx, + hashCols: tc.outerHashKeyIdx, }, innerCtx: innerCtx{ readerBuilder: &dataReaderBuilder{Plan: &mockPhysicalIndexReader{e: innerDS}, executorBuilder: newExecutorBuilder(tc.ctx, nil)}, rowTypes: rightTypes, colLens: colLens, keyCols: tc.innerJoinKeyIdx, + hashCols: tc.innerHashKeyIdx, }, workerWg: new(sync.WaitGroup), joiner: newJoiner(tc.ctx, 0, false, defaultValues, nil, leftTypes, rightTypes, nil), @@ -1484,6 +1490,8 @@ func newMergeJoinBenchmark(numOuterRows, numInnerDup, numInnerRedundant int) (tc ctx: ctx, outerJoinKeyIdx: []int{0, 1}, innerJoinKeyIdx: []int{0, 1}, + outerHashKeyIdx: []int{0, 1}, + innerHashKeyIdx: []int{0, 1}, innerIdx: []int{0, 1}, rawData: wideString, } diff --git a/executor/builder.go b/executor/builder.go index 9b514f50d8384..ad128c559b484 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -2315,12 +2315,23 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) for i := 0; i < len(v.OuterJoinKeys); i++ { outerKeyCols[i] = v.OuterJoinKeys[i].Index } - e.outerCtx.keyCols = outerKeyCols innerKeyCols := make([]int, len(v.InnerJoinKeys)) for i := 0; i < len(v.InnerJoinKeys); i++ { innerKeyCols[i] = v.InnerJoinKeys[i].Index } + e.outerCtx.keyCols = outerKeyCols e.innerCtx.keyCols = innerKeyCols + + outerHashCols, innerHashCols := make([]int, len(v.OuterHashKeys)), make([]int, len(v.InnerHashKeys)) + for i := 0; i < len(v.OuterHashKeys); i++ { + outerHashCols[i] = v.OuterHashKeys[i].Index + } + for i := 0; i < len(v.InnerHashKeys); i++ { + innerHashCols[i] = v.InnerHashKeys[i].Index + } + e.outerCtx.hashCols = outerHashCols + e.innerCtx.hashCols = innerHashCols + e.joinResult = newFirstChunk(e) executorCounterIndexLookUpJoin.Inc() return e diff --git a/executor/index_lookup_hash_join.go b/executor/index_lookup_hash_join.go index 0f886cffc9212..62fa460c39111 100644 --- a/executor/index_lookup_hash_join.go +++ b/executor/index_lookup_hash_join.go @@ -546,14 +546,14 @@ func (iw *indexHashJoinInnerWorker) buildHashTableForOuterResult(ctx context.Con continue } row := chk.GetRow(rowIdx) - keyColIdx := iw.outerCtx.keyCols - for _, i := range keyColIdx { + hashColIdx := iw.outerCtx.hashCols + for _, i := range hashColIdx { if row.IsNull(i) { continue OUTER } } h.Reset() - err := codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, row, iw.outerCtx.rowTypes, keyColIdx, buf) + err := codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, row, iw.outerCtx.rowTypes, hashColIdx, buf) failpoint.Inject("testIndexHashJoinBuildErr", func() { err = errors.New("mockIndexHashJoinBuildErr") }) @@ -644,7 +644,7 @@ func (iw *indexHashJoinInnerWorker) doJoinUnordered(ctx context.Context, task *i func (iw *indexHashJoinInnerWorker) getMatchedOuterRows(innerRow chunk.Row, task *indexHashJoinTask, h hash.Hash64, buf []byte) (matchedRows []chunk.Row, matchedRowPtr []chunk.RowPtr, err error) { h.Reset() - err = codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, innerRow, iw.rowTypes, iw.keyCols, buf) + err = codec.HashChunkRow(iw.ctx.GetSessionVars().StmtCtx, h, innerRow, iw.rowTypes, iw.hashCols, buf) if err != nil { return nil, nil, err } @@ -658,7 +658,7 @@ func (iw *indexHashJoinInnerWorker) getMatchedOuterRows(innerRow chunk.Row, task matchedRowPtr = make([]chunk.RowPtr, 0, len(iw.matchedOuterPtrs)) for _, ptr := range iw.matchedOuterPtrs { outerRow := task.outerResult.GetRow(ptr) - ok, err := codec.EqualChunkRow(iw.ctx.GetSessionVars().StmtCtx, innerRow, iw.rowTypes, iw.keyCols, outerRow, iw.outerCtx.rowTypes, iw.outerCtx.keyCols) + ok, err := codec.EqualChunkRow(iw.ctx.GetSessionVars().StmtCtx, innerRow, iw.rowTypes, iw.keyCols, outerRow, iw.outerCtx.rowTypes, iw.outerCtx.hashCols) if err != nil { return nil, nil, err } diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go index 88af9acad2ef7..9d901d369633f 100644 --- a/executor/index_lookup_join.go +++ b/executor/index_lookup_join.go @@ -87,6 +87,7 @@ type IndexLookUpJoin struct { type outerCtx struct { rowTypes []*types.FieldType keyCols []int + hashCols []int filter expression.CNFExprs } @@ -94,6 +95,7 @@ type innerCtx struct { readerBuilder *dataReaderBuilder rowTypes []*types.FieldType keyCols []int + hashCols []int colLens []int hasPrefixCol bool } @@ -511,17 +513,17 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi chk := task.outerResult.GetChunk(chkIdx) numRows := chk.NumRows() for rowIdx := 0; rowIdx < numRows; rowIdx++ { - dLookUpKey, err := iw.constructDatumLookupKey(task, chkIdx, rowIdx) + dLookUpKey, dHashKey, err := iw.constructDatumLookupKey(task, chkIdx, rowIdx) if err != nil { return nil, err } - if dLookUpKey == nil { + if dHashKey == nil { // Append null to make looUpKeys the same length as outer Result. task.encodedLookUpKeys[chkIdx].AppendNull(0) continue } keyBuf = keyBuf[:0] - keyBuf, err = codec.EncodeKey(iw.ctx.GetSessionVars().StmtCtx, keyBuf, dLookUpKey...) + keyBuf, err = codec.EncodeKey(iw.ctx.GetSessionVars().StmtCtx, keyBuf, dHashKey...) if err != nil { return nil, err } @@ -548,45 +550,49 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi return lookUpContents, nil } -func (iw *innerWorker) constructDatumLookupKey(task *lookUpJoinTask, chkIdx, rowIdx int) ([]types.Datum, error) { +func (iw *innerWorker) constructDatumLookupKey(task *lookUpJoinTask, chkIdx, rowIdx int) ([]types.Datum, []types.Datum, error) { if task.outerMatch != nil && !task.outerMatch[chkIdx][rowIdx] { - return nil, nil + return nil, nil, nil } outerRow := task.outerResult.GetChunk(chkIdx).GetRow(rowIdx) sc := iw.ctx.GetSessionVars().StmtCtx keyLen := len(iw.keyCols) dLookupKey := make([]types.Datum, 0, keyLen) - for i, keyCol := range iw.outerCtx.keyCols { - outerValue := outerRow.GetDatum(keyCol, iw.outerCtx.rowTypes[keyCol]) + dHashKey := make([]types.Datum, 0, len(iw.hashCols)) + for i, hashCol := range iw.outerCtx.hashCols { + outerValue := outerRow.GetDatum(hashCol, iw.outerCtx.rowTypes[hashCol]) // Join-on-condition can be promised to be equal-condition in // IndexNestedLoopJoin, thus the filter will always be false if // outerValue is null, and we don't need to lookup it. if outerValue.IsNull() { - return nil, nil + return nil, nil, nil } - innerColType := iw.rowTypes[iw.keyCols[i]] + innerColType := iw.rowTypes[iw.hashCols[i]] innerValue, err := outerValue.ConvertTo(sc, innerColType) if err != nil { // If the converted outerValue overflows, we don't need to lookup it. if terror.ErrorEqual(err, types.ErrOverflow) { - return nil, nil + return nil, nil, nil } if terror.ErrorEqual(err, types.ErrTruncated) && (innerColType.Tp == mysql.TypeSet || innerColType.Tp == mysql.TypeEnum) { - return nil, nil + return nil, nil, nil } - return nil, err + return nil, nil, err } cmp, err := outerValue.CompareDatum(sc, &innerValue) if err != nil { - return nil, err + return nil, nil, err } if cmp != 0 { // If the converted outerValue is not equal to the origin outerValue, we don't need to lookup it. - return nil, nil + return nil, nil, nil + } + if i < keyLen { + dLookupKey = append(dLookupKey, innerValue) } - dLookupKey = append(dLookupKey, innerValue) + dHashKey = append(dHashKey, innerValue) } - return dLookupKey, nil + return dLookupKey, dHashKey, nil } func (iw *innerWorker) sortAndDedupLookUpContents(lookUpContents []*indexJoinLookUpContent) []*indexJoinLookUpContent { @@ -681,7 +687,7 @@ func (iw *innerWorker) buildLookUpMap(task *lookUpJoinTask) error { } keyBuf = keyBuf[:0] - for _, keyCol := range iw.keyCols { + for _, keyCol := range iw.hashCols { d := innerRow.GetDatum(keyCol, iw.rowTypes[keyCol]) var err error keyBuf, err = codec.EncodeKey(iw.ctx.GetSessionVars().StmtCtx, keyBuf, d) @@ -698,7 +704,7 @@ func (iw *innerWorker) buildLookUpMap(task *lookUpJoinTask) error { } func (iw *innerWorker) hasNullInJoinKey(row chunk.Row) bool { - for _, ordinal := range iw.keyCols { + for _, ordinal := range iw.hashCols { if row.IsNull(ordinal) { return true } diff --git a/executor/join_test.go b/executor/join_test.go index e83c019d0ba09..402b6458f9034 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -1281,7 +1281,7 @@ func (s *testSuiteJoin1) TestIndexLookupJoin(c *C) { tk.MustQuery("desc select /*+ TIDB_INLJ(s) */ count(*) from t join s use index(idx) on s.a = t.a and s.b < t.b").Check(testkit.Rows( "HashAgg_9 1.00 root funcs:count(1)->Column#6", - "└─IndexJoin_16 64.00 root inner join, inner:IndexReader_15, outer key:test.t.a, inner key:test.s.a, other cond:lt(test.s.b, test.t.b)", + "└─IndexJoin_16 64.00 root inner join, inner:IndexReader_15, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), other cond:lt(test.s.b, test.t.b)", " ├─TableReader_26(Build) 64.00 root data:Selection_25", " │ └─Selection_25 64.00 cop[tikv] not(isnull(test.t.b))", " │ └─TableFullScan_24 64.00 cop[tikv] table:t keep order:false", @@ -1308,7 +1308,7 @@ func (s *testSuiteJoin1) TestIndexLookupJoin(c *C) { tk.MustQuery("desc select /*+ INL_HASH_JOIN(s) */ count(*) from t join s use index(idx) on s.a = t.a and s.b < t.b").Check(testkit.Rows( "HashAgg_9 1.00 root funcs:count(1)->Column#6", - "└─IndexHashJoin_18 64.00 root inner join, inner:IndexReader_15, outer key:test.t.a, inner key:test.s.a, other cond:lt(test.s.b, test.t.b)", + "└─IndexHashJoin_18 64.00 root inner join, inner:IndexReader_15, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), other cond:lt(test.s.b, test.t.b)", " ├─TableReader_26(Build) 64.00 root data:Selection_25", " │ └─Selection_25 64.00 cop[tikv] not(isnull(test.t.b))", " │ └─TableFullScan_24 64.00 cop[tikv] table:t keep order:false", @@ -1354,7 +1354,7 @@ func (s *testSuiteJoinSerial) TestIndexNestedLoopHashJoin(c *C) { tk.MustExec("analyze table s") // Test IndexNestedLoopHashJoin keepOrder. tk.MustQuery("explain select /*+ INL_HASH_JOIN(s) */ * from t left join s on t.a=s.a order by t.pk").Check(testkit.Rows( - "IndexHashJoin_27 100.00 root left outer join, inner:TableReader_22, outer key:test.t.a, inner key:test.s.a", + "IndexHashJoin_27 100.00 root left outer join, inner:TableReader_22, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a)", "├─TableReader_30(Build) 100.00 root data:TableFullScan_29", "│ └─TableFullScan_29 100.00 cop[tikv] table:t keep order:true", "└─TableReader_22(Probe) 1.00 root data:TableRangeScan_21", @@ -1390,7 +1390,7 @@ func (s *testSuiteJoinSerial) TestIndexNestedLoopHashJoin(c *C) { tk.MustExec("set @@tidb_index_join_batch_size=2") tk.MustQuery("desc select * from t l1 where exists ( select * from t l2 where l2.l_orderkey = l1.l_orderkey and l2.l_suppkey <> l1.l_suppkey ) order by `l_orderkey`,`l_linenumber`;").Check(testkit.Rows( "Sort_9 7.20 root test.t.l_orderkey, test.t.l_linenumber", - "└─IndexHashJoin_17 7.20 root semi join, inner:IndexLookUp_15, outer key:test.t.l_orderkey, inner key:test.t.l_orderkey, other cond:ne(test.t.l_suppkey, test.t.l_suppkey)", + "└─IndexHashJoin_17 7.20 root semi join, inner:IndexLookUp_15, outer key:test.t.l_orderkey, inner key:test.t.l_orderkey, equal cond:eq(test.t.l_orderkey, test.t.l_orderkey), other cond:ne(test.t.l_suppkey, test.t.l_suppkey)", " ├─TableReader_20(Build) 9.00 root data:Selection_19", " │ └─Selection_19 9.00 cop[tikv] not(isnull(test.t.l_suppkey))", " │ └─TableFullScan_18 9.00 cop[tikv] table:l1 keep order:false", @@ -1401,7 +1401,7 @@ func (s *testSuiteJoinSerial) TestIndexNestedLoopHashJoin(c *C) { tk.MustQuery("select * from t l1 where exists ( select * from t l2 where l2.l_orderkey = l1.l_orderkey and l2.l_suppkey <> l1.l_suppkey )order by `l_orderkey`,`l_linenumber`;").Check(testkit.Rows("0 0 0 0", "0 1 0 1", "0 2 0 0", "1 0 1 0", "1 1 1 1", "1 2 1 0", "2 0 0 0", "2 1 0 1", "2 2 0 0")) tk.MustQuery("desc select count(*) from t l1 where exists ( select * from t l2 where l2.l_orderkey = l1.l_orderkey and l2.l_suppkey <> l1.l_suppkey );").Check(testkit.Rows( "StreamAgg_14 1.00 root funcs:count(1)->Column#11", - "└─IndexHashJoin_29 7.20 root semi join, inner:IndexLookUp_27, outer key:test.t.l_orderkey, inner key:test.t.l_orderkey, other cond:ne(test.t.l_suppkey, test.t.l_suppkey)", + "└─IndexHashJoin_29 7.20 root semi join, inner:IndexLookUp_27, outer key:test.t.l_orderkey, inner key:test.t.l_orderkey, equal cond:eq(test.t.l_orderkey, test.t.l_orderkey), other cond:ne(test.t.l_suppkey, test.t.l_suppkey)", " ├─TableReader_23(Build) 9.00 root data:Selection_22", " │ └─Selection_22 9.00 cop[tikv] not(isnull(test.t.l_suppkey))", " │ └─TableFullScan_21 9.00 cop[tikv] table:l1 keep order:false", @@ -1497,7 +1497,7 @@ func (s *testSuiteJoin3) TestIssue13449(c *C) { tk.MustExec("set @@tidb_index_join_batch_size=32;") tk.MustQuery("desc select /*+ INL_HASH_JOIN(s) */ * from t join s on t.a=s.a order by t.a;").Check(testkit.Rows( - "IndexHashJoin_30 12487.50 root inner join, inner:IndexReader_27, outer key:test.t.a, inner key:test.s.a", + "IndexHashJoin_30 12487.50 root inner join, inner:IndexReader_27, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a)", "├─IndexReader_37(Build) 9990.00 root index:IndexFullScan_36", "│ └─IndexFullScan_36 9990.00 cop[tikv] table:t, index:a(a) keep order:true, stats:pseudo", "└─IndexReader_27(Probe) 1.25 root index:Selection_26", @@ -2085,7 +2085,7 @@ func (s *testSuiteJoinSerial) TestOuterTableBuildHashTableIsuse13933(c *C) { tk.MustExec("Insert into t values (11,2),(1,2),(5,2)") tk.MustQuery("select /*+ INL_HASH_JOIN(s)*/ * from t left join s on s.b=t.b and s.a < t.a;").Sort().Check(testkit.Rows("1 2 ", "11 2 1 2", "5 2 1 2")) tk.MustQuery("explain select /*+ INL_HASH_JOIN(s)*/ * from t left join s on s.b=t.b and s.a < t.a;").Check(testkit.Rows( - "IndexHashJoin_14 12475.01 root left outer join, inner:IndexLookUp_11, outer key:test.t.b, inner key:test.s.b, other cond:lt(test.s.a, test.t.a)", + "IndexHashJoin_14 12475.01 root left outer join, inner:IndexLookUp_11, outer key:test.t.b, inner key:test.s.b, equal cond:eq(test.t.b, test.s.b), other cond:lt(test.s.a, test.t.a)", "├─TableReader_24(Build) 10000.00 root data:TableFullScan_23", "│ └─TableFullScan_23 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─IndexLookUp_11(Probe) 1.25 root ", @@ -2174,6 +2174,7 @@ func (s *testSuiteJoinSerial) TestInlineProjection4HashJoinIssue15316(c *C) { defer func() { plannercore.ForcedHashLeftJoin4Test = false }() tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") + tk.MustExec("drop table if exists S, T") tk.MustExec("create table S (a int not null, b int, c int);") tk.MustExec("create table T (a int not null, b int, c int);") tk.MustExec("insert into S values (0,1,2),(0,1,null),(0,1,2);") @@ -2456,6 +2457,29 @@ func (s *testSuiteJoinSerial) TestIssue20270(c *C) { failpoint.Disable("github.com/pingcap/tidb/executor/killedInJoin2ChunkForOuterHashJoin") } +func (s *testSuiteJoinSerial) TestIssue20710(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t;") + tk.MustExec("drop table if exists s;") + tk.MustExec("create table t(a int, b int)") + tk.MustExec("create table s(a int, b int, index(a))") + tk.MustExec("insert into t values(1,1),(1,2),(2,2)") + tk.MustExec("insert into s values(1,1),(2,2),(2,1)") + tk.MustQuery("select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.b = s.b").Sort().Check(testkit.Rows("1 1 1 1", "2 2 2 2")) + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustQuery("select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.b = s.a").Sort().Check(testkit.Rows("1 1 1 1", "2 2 2 1", "2 2 2 2")) + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustQuery("select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.a = s.b").Sort().Check(testkit.Rows("1 1 1 1", "1 2 1 1", "2 2 2 2")) + tk.MustQuery("show warnings").Check(testkit.Rows()) + + tk.MustQuery("select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.b = s.b").Sort().Check(testkit.Rows("1 1 1 1", "2 2 2 2")) + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustQuery("select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.b = s.a").Sort().Check(testkit.Rows("1 1 1 1", "2 2 2 1", "2 2 2 2")) + tk.MustQuery("show warnings").Check(testkit.Rows()) + tk.MustQuery("select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.a = s.b").Sort().Check(testkit.Rows("1 1 1 1", "1 2 1 1", "2 2 2 2")) + tk.MustQuery("show warnings").Check(testkit.Rows()) +} + func (s *testSuiteJoinSerial) TestIssue20779(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("drop table if exists t1") diff --git a/expression/testdata/partition_pruner_out.json b/expression/testdata/partition_pruner_out.json index 83bb20f469aaa..c7d56352a64d6 100644 --- a/expression/testdata/partition_pruner_out.json +++ b/expression/testdata/partition_pruner_out.json @@ -765,7 +765,7 @@ ], "Plan": [ "StreamAgg_12 1.00 root funcs:count(1)->Column#9", - "└─IndexJoin_34 4.00 root inner join, inner:IndexReader_33, outer key:test_partition.t6.b, inner key:test_partition.t5.b", + "└─IndexJoin_34 4.00 root inner join, inner:IndexReader_33, outer key:test_partition.t6.b, inner key:test_partition.t5.b, equal cond:eq(test_partition.t6.b, test_partition.t5.b)", " ├─IndexReader_27(Build) 4.00 root partition:p0 index:Selection_26", " │ └─Selection_26 4.00 cop[tikv] not(isnull(test_partition.t6.b))", " │ └─IndexRangeScan_25 4.00 cop[tikv] table:t6, index:a(a, b) range:[1 1,1 1], [1 6,1 6], [2 1,2 1], [2 6,2 6], keep order:false, stats:pseudo", @@ -781,7 +781,7 @@ ], "Plan": [ "StreamAgg_12 1.00 root funcs:count(1)->Column#9", - "└─IndexHashJoin_36 4.00 root inner join, inner:IndexReader_33, outer key:test_partition.t6.b, inner key:test_partition.t5.b", + "└─IndexHashJoin_36 4.00 root inner join, inner:IndexReader_33, outer key:test_partition.t6.b, inner key:test_partition.t5.b, equal cond:eq(test_partition.t6.b, test_partition.t5.b)", " ├─IndexReader_27(Build) 4.00 root partition:p0 index:Selection_26", " │ └─Selection_26 4.00 cop[tikv] not(isnull(test_partition.t6.b))", " │ └─IndexRangeScan_25 4.00 cop[tikv] table:t6, index:a(a, b) range:[1 1,1 1], [1 6,1 6], [2 1,2 1], [2 6,2 6], keep order:false, stats:pseudo", diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index dfd9d78b7b774..b3ca49f4b0820 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -391,6 +391,7 @@ func (p *LogicalJoin) getHashJoin(prop *property.PhysicalProperty, innerIdx int, // When inner plan is TableReader, the parameter `ranges` will be nil. Because pk only have one column. So all of its range // is generated during execution time. func (p *LogicalJoin) constructIndexJoin( + joinTP string, prop *property.PhysicalProperty, outerIdx int, innerTask task, @@ -437,6 +438,38 @@ func (p *LogicalJoin) constructIndexJoin( newIsNullEQ = append(newIsNullEQ, isNullEQ[keyOff]) newKeyOff = append(newKeyOff, idxOff) } + + var outerHashKeys, innerHashKeys []*expression.Column + // HashKey is only used for IndexJoin and IndexHashJoin since they need to + // build hash tables. + if joinTP != plancodec.TypeIndexMergeJoin { + outerHashKeys, innerHashKeys = make([]*expression.Column, len(newOuterKeys)), make([]*expression.Column, len(newInnerKeys)) + copy(outerHashKeys, newOuterKeys) + copy(innerHashKeys, newInnerKeys) + // we can use the `col col` in `OtherCondition` to build the hashtable to avoid the unnecessary calculating. + for i := len(newOtherConds) - 1; i >= 0; i = i - 1 { + switch c := newOtherConds[i].(type) { + case *expression.ScalarFunction: + if c.FuncName.L == ast.EQ { + lhs, ok1 := c.GetArgs()[0].(*expression.Column) + rhs, ok2 := c.GetArgs()[1].(*expression.Column) + if ok1 && ok2 { + outerSchema, innerSchema := p.Children()[outerIdx].Schema(), p.Children()[1-outerIdx].Schema() + if outerSchema.Contains(lhs) && innerSchema.Contains(rhs) { + outerHashKeys = append(outerHashKeys, lhs) + innerHashKeys = append(innerHashKeys, rhs) + } else if innerSchema.Contains(lhs) && outerSchema.Contains(rhs) { + outerHashKeys = append(outerHashKeys, rhs) + innerHashKeys = append(innerHashKeys, lhs) + } + newOtherConds = append(newOtherConds[:i], newOtherConds[i+1:]...) + } + } + default: + continue + } + } + } baseJoin := basePhysicalJoin{ InnerChildIdx: 1 - outerIdx, LeftConditions: p.LeftConditions, @@ -454,6 +487,8 @@ func (p *LogicalJoin) constructIndexJoin( KeyOff2IdxOff: newKeyOff, Ranges: ranges, CompareFilters: compareFilters, + OuterHashKeys: outerHashKeys, + InnerHashKeys: innerHashKeys, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), p.blockOffset, chReqProps...) if path != nil { join.IdxColLens = path.IdxColLens @@ -471,7 +506,7 @@ func (p *LogicalJoin) constructIndexMergeJoin( path *util.AccessPath, compareFilters *ColWithCmpFuncManager, ) []PhysicalPlan { - indexJoins := p.constructIndexJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) + indexJoins := p.constructIndexJoin(plancodec.TypeIndexMergeJoin, prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) indexMergeJoins := make([]PhysicalPlan, 0, len(indexJoins)) for _, plan := range indexJoins { join := plan.(*PhysicalIndexJoin) @@ -556,7 +591,7 @@ func (p *LogicalJoin) constructIndexHashJoin( path *util.AccessPath, compareFilters *ColWithCmpFuncManager, ) []PhysicalPlan { - indexJoins := p.constructIndexJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) + indexJoins := p.constructIndexJoin(plancodec.TypeIndexHashJoin, prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) indexHashJoins := make([]PhysicalPlan, 0, len(indexJoins)) for _, plan := range indexJoins { join := plan.(*PhysicalIndexJoin) @@ -724,7 +759,7 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( failpoint.Return(p.constructIndexHashJoin(prop, outerIdx, innerTask, nil, keyOff2IdxOff, nil, nil)) } }) - joins = append(joins, p.constructIndexJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, nil, nil)...) + joins = append(joins, p.constructIndexJoin(plancodec.TypeIndexJoin, prop, outerIdx, innerTask, ranges, keyOff2IdxOff, nil, nil)...) // We can reuse the `innerTask` here since index nested loop hash join // do not need the inner child to promise the order. joins = append(joins, p.constructIndexHashJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, nil, nil)...) @@ -759,7 +794,7 @@ func (p *LogicalJoin) buildIndexJoinInner2IndexScan( failpoint.Return(p.constructIndexHashJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)) } }) - joins = append(joins, p.constructIndexJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)...) + joins = append(joins, p.constructIndexJoin(plancodec.TypeIndexJoin, prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)...) // We can reuse the `innerTask` here since index nested loop hash join // do not need the inner child to promise the order. joins = append(joins, p.constructIndexHashJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)...) diff --git a/planner/core/explain.go b/planner/core/explain.go index bdbb65d4b7087..79a5569b4f6b1 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" "github.com/pingcap/tidb/infoschema" @@ -28,6 +29,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/stringutil" ) @@ -510,6 +512,18 @@ func (p *PhysicalIndexJoin) explainInfo(normalized bool) string { fmt.Fprintf(buffer, ", inner key:%s", expression.ExplainColumnList(p.InnerJoinKeys)) } + + if len(p.OuterHashKeys) > 0 { + exprs := make([]expression.Expression, 0, len(p.OuterHashKeys)) + for i := range p.OuterHashKeys { + expr, err := expression.NewFunctionBase(MockContext(), ast.EQ, types.NewFieldType(mysql.TypeLonglong), p.OuterHashKeys[i], p.InnerHashKeys[i]) + if err != nil { + } + exprs = append(exprs, expr) + } + fmt.Fprintf(buffer, ", equal cond:%s", + sortedExplainExpressionList(exprs)) + } if len(p.LeftConditions) > 0 { fmt.Fprintf(buffer, ", left cond:%s", sortedExplainExpressionList(p.LeftConditions)) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 63efdac7648bb..826cbbfbb8475 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1823,6 +1823,31 @@ func (s *testIntegrationSuite) TestPartitionUnionWithPPruningColumn(c *C) { } +func (s *testIntegrationSerialSuite) TestIssue20710(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t;") + tk.MustExec("drop table if exists s;") + tk.MustExec("create table t(a int, b int)") + tk.MustExec("create table s(a int, b int, index(a))") + tk.MustExec("insert into t values(1,1),(1,2),(2,2)") + tk.MustExec("insert into s values(1,1),(2,2),(2,1)") + + var input []string + var output []struct { + SQL string + Plan []string + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + s.testData.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = s.testData.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} + func (s *testIntegrationSuite) TestIssue10448(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 832e9a452de05..1e68984b6e4f7 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -796,6 +796,12 @@ type PhysicalIndexJoin struct { // need to be evaluated after we fetch the data of t1. // This struct stores them and evaluate them to ranges. CompareFilters *ColWithCmpFuncManager + // OuterHashKeys indicates the outer keys used to build hash table during + // execution. OuterJoinKeys is the prefix of OuterHashKeys. + OuterHashKeys []*expression.Column + // InnerHashKeys indicates the inner keys used to build hash table during + // execution. InnerJoinKeys is the prefix of InnerHashKeys. + InnerHashKeys []*expression.Column } // PhysicalIndexMergeJoin represents the plan of index look up merge join. diff --git a/planner/core/resolve_indices.go b/planner/core/resolve_indices.go index fc2c85e0fc0d2..8f4a35f2211bd 100644 --- a/planner/core/resolve_indices.go +++ b/planner/core/resolve_indices.go @@ -251,6 +251,17 @@ func (p *PhysicalIndexJoin) ResolveIndices() (err error) { p.CompareFilters.affectedColSchema.Columns[i] = resolvedCol.(*expression.Column) } } + for i := range p.OuterHashKeys { + outerKey, err := p.OuterHashKeys[i].ResolveIndices(p.children[1-p.InnerChildIdx].Schema()) + if err != nil { + return err + } + innerKey, err := p.InnerHashKeys[i].ResolveIndices(p.children[p.InnerChildIdx].Schema()) + if err != nil { + return err + } + p.OuterHashKeys[i], p.InnerHashKeys[i] = outerKey.(*expression.Column), innerKey.(*expression.Column) + } return } diff --git a/planner/core/testdata/analyze_suite_out.json b/planner/core/testdata/analyze_suite_out.json index e7d6df463ef1c..ef7a4b7303c05 100644 --- a/planner/core/testdata/analyze_suite_out.json +++ b/planner/core/testdata/analyze_suite_out.json @@ -142,7 +142,7 @@ "explain select /*+ TIDB_INLJ(t2) */ * from t1 join t2 on t2.a=t1.a and t2.b>t1.b-1 and t2.b1;", "Plan": [ - " IndexJoin root inner join, inner:TableReader, outer key:test.t1.a, inner key:test.t2.a", + " IndexJoin root inner join, inner:TableReader, outer key:test.t1.a, inner key:test.t2.a, equal cond:eq(test.t1.a, test.t2.a)", " ├─TableReader root ", " │ └─Selection cop gt(test.t1.c, ?)", " │ └─TableScan cop table:t1, range:[?,?], keep order:false", @@ -123,7 +123,7 @@ { "SQL": "SELECT /*+ TIDB_INLJ(t1, t2) */ * from t1, t2 where t1.a = t2.a and t1.c>1;", "Plan": [ - " IndexJoin root inner join, inner:TableReader, outer key:test.t1.a, inner key:test.t2.a", + " IndexJoin root inner join, inner:TableReader, outer key:test.t1.a, inner key:test.t2.a, equal cond:eq(test.t1.a, test.t2.a)", " ├─TableReader root ", " │ └─Selection cop gt(test.t1.c, ?)", " │ └─TableScan cop table:t1, range:[?,?], keep order:false", diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 4b23877d0bd01..074fcc6ba52ee 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -1327,7 +1327,7 @@ "explain select /*+ TIDB_INLJ(t2) */ * from t t1, t t2 where t1.a = t2.a" ], "Plan": [ - "IndexJoin_12 12500.00 root inner join, inner:UnionScan_11, outer key:test.t.a, inner key:test.t.a", + "IndexJoin_12 12500.00 root inner join, inner:UnionScan_11, outer key:test.t.a, inner key:test.t.a, equal cond:eq(test.t.a, test.t.a)", "├─UnionScan_15(Build) 10000.00 root ", "│ └─TableReader_17 10000.00 root data:TableFullScan_16", "│ └─TableFullScan_16 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", From f0bef40d6bb83bca960c3b84a8f3e1b34af61016 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Mon, 16 Nov 2020 17:25:27 +0800 Subject: [PATCH 06/37] dep: update etcd and other dependencies for lightning and CDC (#21076) --- executor/brie.go | 1 + go.mod | 27 ++++++---- go.sum | 136 ++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 148 insertions(+), 16 deletions(-) diff --git a/executor/brie.go b/executor/brie.go index 4dd86aa9fdb1b..1852a7d23421d 100644 --- a/executor/brie.go +++ b/executor/brie.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" + _ "github.com/pingcap/tidb-lightning/lightning" // stub that update go.mod filter "github.com/pingcap/tidb-tools/pkg/table-filter" pd "github.com/tikv/pd/client" diff --git a/go.mod b/go.mod index 4079101896b74..b39702a72a1d4 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,15 @@ module github.com/pingcap/tidb require ( - cloud.google.com/go v0.51.0 // indirect github.com/BurntSushi/toml v0.3.1 + github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect github.com/Jeffail/gabs/v2 v2.5.1 github.com/aws/aws-sdk-go v1.30.24 // indirect github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d + github.com/carlmjohnson/flagext v0.20.2 // indirect github.com/cheggaaa/pb/v3 v3.0.4 // indirect + github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b // indirect github.com/codahale/hdrhistogram v0.9.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect @@ -21,7 +23,7 @@ require ( github.com/go-sql-driver/mysql v1.5.0 github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.3.4 - github.com/golang/snappy v0.0.1 + github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf github.com/google/btree v1.0.0 github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 github.com/google/uuid v1.1.1 @@ -30,12 +32,15 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 + github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 // indirect github.com/klauspost/cpuid v1.2.1 + github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.6 // indirect github.com/montanaflynn/stats v0.5.0 // indirect github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef github.com/ngaut/unistore v0.0.0-20201026053041-4333e8d852f1 + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/basictracer-go v1.0.0 @@ -53,18 +58,19 @@ require ( github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 github.com/pingcap/parser v0.0.0-20201112065012-c9380f220ff9 github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 + github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 - github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/shirou/gopsutil v2.20.3+incompatible github.com/sirupsen/logrus v1.6.0 github.com/soheilhy/cmux v0.1.4 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cobra v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.6.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09 @@ -72,26 +78,27 @@ require ( github.com/uber-go/atomic v1.4.0 github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect + github.com/xitongsys/parquet-go v1.5.4 // indirect github.com/zhangjinpeng1987/raft v0.0.0-20200819064223-df31bb68a018 // indirect go.etcd.io/bbolt v1.3.4 // indirect - go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 + go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b go.uber.org/atomic v1.7.0 go.uber.org/automaxprocs v1.2.0 go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.16.0 + golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200904194848-62affa334b73 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20200819171115-d785dc25833f golang.org/x/text v0.3.4 - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect golang.org/x/tools v0.0.0-20200820010801-b793a1359eac - google.golang.org/api v0.15.1 // indirect - google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f // indirect - google.golang.org/grpc v1.26.0 + google.golang.org/grpc v1.27.1 + gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v2 v2.3.0 // indirect honnef.co/go/tools v0.0.1-2020.1.6 // indirect + modernc.org/mathutil v1.1.1 // indirect sigs.k8s.io/yaml v1.2.0 // indirect sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 diff --git a/go.sum b/go.sum index c999e41d85b47..3e71d8fca7de8 100644 --- a/go.sum +++ b/go.sum @@ -7,23 +7,34 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0 h1:0E3eE8MX426vUOs7aHfI7aN1BrIzzzf4ccKCSfSjGmc= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0 h1:9/vpR43S4aJaROxqQHQ3nH9lfyKKV0dC3vOmnw8ebQQ= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg= github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= github.com/Jeffail/gabs/v2 v2.5.1 h1:ANfZYjpMlfTTKebycu4X1AgkVWumFVDYQl7JwOr4mDk= @@ -44,9 +55,13 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= +github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.30.24 h1:y3JPD51VuEmVqN3BEDVm4amGpDma2cKJcDPuAU1OR58= github.com/aws/aws-sdk-go v1.30.24/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -58,8 +73,12 @@ github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQ github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= +github.com/carlmjohnson/flagext v0.20.2 h1:qvpMM+TytSrlh3+EIVn/pzOwwq9y13hXZab6Y4Gvqpo= +github.com/carlmjohnson/flagext v0.20.2/go.mod h1:Eenv0epIUAr4NuedNmkzI8WmBmjIxZC239XcKxYS2ac= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -72,8 +91,16 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b h1:YHjo2xnqFCeFa0CdxEccHfUY1/DnXPAZdZt0+s/Mvdg= +github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b/go.mod h1:crLnbSFbwAcQNs9FPfI1avHb5BqVgqZcr4r+IzpJ5FM= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codahale/hdrhistogram v0.9.0 h1:9GjrtRI+mLEFPtTfR/AZhcxp+Ii8NZYWq5104FbZQY0= github.com/codahale/hdrhistogram v0.9.0/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2c1ceQ0ec6MkLQyDp+IGdVM0D8XM= @@ -96,6 +123,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= @@ -139,6 +167,9 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.17.0 h1:OeH75kBZcZa3ZE+zz/mFdJ2btt9FgqfjI7gIh9+5fvk= github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -150,6 +181,7 @@ github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmC github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -191,18 +223,28 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -218,6 +260,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 h1:K+lX49/3eURCE1IjlaZN//u6c+9nfDAMnyQ9E2dsJbY= github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -249,6 +292,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -260,6 +304,7 @@ github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73t github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= @@ -267,6 +312,8 @@ github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 h1:3Jr6Mtili6DsXSF0RwRlAqpOUWXcSVUxdOm5kFPb3xY= +github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/joomcode/errorx v1.0.1/go.mod h1:kgco15ekB6cs+4Xjzo7SPeXzx38PbJzBwbnu9qfVNHQ= @@ -286,6 +333,9 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc= +github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -299,6 +349,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -349,6 +401,8 @@ github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo github.com/ngaut/unistore v0.0.0-20201026053041-4333e8d852f1 h1:WHACEcI4Om4RVVUCssGTUwvJeF2CTuHHLmxr2h3joWQ= github.com/ngaut/unistore v0.0.0-20201026053041-4333e8d852f1/go.mod h1:ZR3NH+HzqfiYetwdoAivApnIy8iefPZHTMLfrFNm8g4= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= @@ -365,7 +419,9 @@ github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKw github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.3.0 h1:e5+lF2E4Y2WCIxBefVowBuB0iHrUH4HZ8q+6mGF7fJc= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= @@ -415,6 +471,8 @@ github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NM github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 h1:b2G/eqDeywtdJF3w9nIUdqMmXChsmpLvf4FzUxJ9Vmk= github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= +github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible h1:y6cwuJJo8tLeHjgjODpwCSqNAv1g+9WUtcsFOHu/7ag= +github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible/go.mod h1:UrN2l06neiEfkass87Tjvv5JFIMsOMb2DT7P7s+fwMc= github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible h1:qPppnsXVh3KswqRZdSAShGLLPd7dB+5w4lXDnpYn0SQ= github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f h1:J+0TAI+7Hvebz4bM4GnRCRT4MpjYnUxbyi9ky5ZQUsU= @@ -458,6 +516,7 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -467,6 +526,7 @@ github.com/shirou/gopsutil v2.20.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= @@ -481,6 +541,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -501,6 +562,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= @@ -544,6 +607,12 @@ github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oT github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= +github.com/xitongsys/parquet-go v1.5.4 h1:zsdMNZcCv9t3YnlOfysMI78vBw+cN65jQznQlizVtqE= +github.com/xitongsys/parquet-go v1.5.4/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= +github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -557,10 +626,14 @@ go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b h1:3kC4J3eQF6p1UEfQTkC67eEeb3rTk+shQqdX6tFyq9Q= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -588,6 +661,7 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -600,6 +674,8 @@ golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -608,6 +684,11 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -619,6 +700,7 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -626,6 +708,7 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -651,7 +734,9 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -672,6 +757,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -694,21 +781,27 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f h1:KJuwZVtZBVzDmEDtB2zro9CXkD9O0dpCv4o2LHbQIAw= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -745,9 +838,16 @@ golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -764,8 +864,9 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.1 h1:5mMS6mYvK5LVB8+ujVBC33Y8gltBo/kT6HBm6kU80G4= -google.golang.org/api v0.15.1/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0 h1:TgDr+1inK2XVUKZx3BYAqQg/GwucGdBkzZjWaTg/I+A= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -786,8 +887,12 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f h1:2wh8dWY8959cBGQvk1RD+/eQBgRYYDaZ+hT0/zsARoA= -google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63 h1:YzfoEYWbODU5Fbt37+h7X16BWQbad7Q4S6gclTKFXM8= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -799,6 +904,9 @@ google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRn google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= @@ -806,6 +914,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -813,6 +923,11 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -827,15 +942,24 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc= honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From eb8c3d1deb40c67e238846ffbfb5b797e5462920 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Mon, 16 Nov 2020 18:02:56 +0800 Subject: [PATCH 07/37] util: avoid to create tmp directory if memory-alarm is closed. (#20851) --- util/disk/tempDir.go | 21 ++++++++++++--------- util/expensivequery/memory_usage_alarm.go | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/util/disk/tempDir.go b/util/disk/tempDir.go index 1b05d4c47bf1a..2d42d21d82d39 100644 --- a/util/disk/tempDir.go +++ b/util/disk/tempDir.go @@ -81,15 +81,6 @@ func InitializeTempDir() error { return err } - // Create dir for MemoryUsageAlarmRecord. - _, err = os.Stat(filepath.Join(tempDir, "record")) - if err != nil && !os.IsExist(err) { - err = os.MkdirAll(filepath.Join(tempDir, "record"), 0755) - if err != nil { - return err - } - } - subDirs, err := ioutil.ReadDir(tempDir) if err != nil { return err @@ -122,3 +113,15 @@ func CleanUp() { terror.Log(errors.Trace(err)) } } + +// CheckAndCreateDir check whether the directory is existed. If not, then create it. +func CheckAndCreateDir(path string) error { + _, err := os.Stat(path) + if err != nil && !os.IsExist(err) { + err = os.MkdirAll(path, 0755) + if err != nil { + return err + } + } + return nil +} diff --git a/util/expensivequery/memory_usage_alarm.go b/util/expensivequery/memory_usage_alarm.go index 16914542e5fca..d8f86b80eeee5 100644 --- a/util/expensivequery/memory_usage_alarm.go +++ b/util/expensivequery/memory_usage_alarm.go @@ -65,6 +65,9 @@ func initMemoryUsageAlarmRecord() (record *memoryUsageAlarm) { } record.lastCheckTime = time.Time{} record.tmpDir = filepath.Join(config.GetGlobalConfig().TempStoragePath, "record") + if record.err = disk.CheckAndCreateDir(record.tmpDir); record.err != nil { + return + } record.lastProfileFileName = make([][]string, 2) // Read last records files, err := ioutil.ReadDir(record.tmpDir) @@ -131,7 +134,7 @@ func (record *memoryUsageAlarm) doRecord(memUsage uint64, instanceMemoryUsage ui logutil.BgLogger().Warn("tidb-server has the risk of OOM. Running SQLs and heap profile will be recorded in record path", fields...) - if record.err = disk.CheckAndInitTempDir(); record.err != nil { + if record.err = disk.CheckAndCreateDir(record.tmpDir); record.err != nil { return } record.recordSQL(sm) @@ -143,7 +146,6 @@ func (record *memoryUsageAlarm) doRecord(memUsage uint64, instanceMemoryUsage ui err := os.Remove((*filename)[0]) if err != nil { logutil.BgLogger().Error("remove temp files failed", zap.Error(err)) - return } *filename = (*filename)[1:] } From 7c88ca7974048491f0754668f3a0fca0ef78391f Mon Sep 17 00:00:00 2001 From: xiongjiwei Date: Mon, 16 Nov 2020 19:03:26 +0800 Subject: [PATCH 08/37] ddl, session: set charset and coordinate collation (#21034) --- ddl/ddl_api.go | 4 +-- executor/set_test.go | 41 ++++++++++++++++++++++++ executor/union_scan_test.go | 2 +- expression/integration_test.go | 11 +++++++ sessionctx/variable/session.go | 58 +++++++++++++++++++++++----------- sessionctx/variable/sysvar.go | 16 +++++++--- 6 files changed, 106 insertions(+), 26 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index a5e082a0117b6..ad67066ee8b3f 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1909,10 +1909,10 @@ func (d *ddl) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err err tblCharset := "" tblCollate := "" - if v, ok := ctx.GetSessionVars().GetSystemVar("character_set_client"); ok { + if v, ok := ctx.GetSessionVars().GetSystemVar(variable.CharacterSetConnection); ok { tblCharset = v } - if v, ok := ctx.GetSessionVars().GetSystemVar("collation_connection"); ok { + if v, ok := ctx.GetSessionVars().GetSystemVar(variable.CollationConnection); ok { tblCollate = v } diff --git a/executor/set_test.go b/executor/set_test.go index 7efac55965660..e68d31c9f2cd8 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -599,6 +599,47 @@ func (s *testSuite5) TestSetCharset(c *C) { ) } +func (s *testSuite5) TestSetCollationAndCharset(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + ctx := tk.Se.(sessionctx.Context) + sessionVars := ctx.GetSessionVars() + + cases := []struct { + charset string + collation string + expectCharset string + expectCollation string + }{ + {variable.CharacterSetConnection, variable.CollationConnection, "utf8", "utf8_bin"}, + {variable.CharsetDatabase, variable.CollationDatabase, "utf8", "utf8_bin"}, + {variable.CharacterSetServer, variable.CollationServer, "utf8", "utf8_bin"}, + } + + for _, t := range cases { + tk.MustExec(fmt.Sprintf("set %s = %s;", t.charset, t.expectCharset)) + sVar, ok := sessionVars.GetSystemVar(t.charset) + c.Assert(ok, IsTrue) + c.Assert(sVar, Equals, t.expectCharset) + sVar, ok = sessionVars.GetSystemVar(t.collation) + c.Assert(ok, IsTrue) + c.Assert(sVar, Equals, t.expectCollation) + } + + tk = testkit.NewTestKitWithInit(c, s.store) + ctx = tk.Se.(sessionctx.Context) + sessionVars = ctx.GetSessionVars() + + for _, t := range cases { + tk.MustExec(fmt.Sprintf("set %s = %s;", t.collation, t.expectCollation)) + sVar, ok := sessionVars.GetSystemVar(t.charset) + c.Assert(ok, IsTrue) + c.Assert(sVar, Equals, t.expectCharset) + sVar, ok = sessionVars.GetSystemVar(t.collation) + c.Assert(ok, IsTrue) + c.Assert(sVar, Equals, t.expectCollation) + } +} + func (s *testSuite5) TestValidateSetVar(c *C) { tk := testkit.NewTestKit(c, s.store) diff --git a/executor/union_scan_test.go b/executor/union_scan_test.go index 235fc0e1c6616..48a5768a10be8 100644 --- a/executor/union_scan_test.go +++ b/executor/union_scan_test.go @@ -383,7 +383,7 @@ func (s *testSuite7) TestForApplyAndUnionScan(c *C) { // See https://github.com/pingcap/tidb/issues/19431 tk.MustExec("DROP TABLE IF EXISTS `t`") - tk.MustExec("CREATE TABLE `t` ( `c_int` int(11) NOT NULL, `c_str` varchar(40) NOT NULL, `c_datetime` datetime NOT NULL, PRIMARY KEY (`c_int`,`c_str`,`c_datetime`), KEY `c_str` (`c_str`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; /*!40101 SET character_set_client = @saved_cs_client */") + tk.MustExec("CREATE TABLE `t` ( `c_int` int(11) NOT NULL, `c_str` varchar(40) NOT NULL, `c_datetime` datetime NOT NULL, PRIMARY KEY (`c_int`,`c_str`,`c_datetime`), KEY `c_str` (`c_str`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;") tk.MustExec("INSERT INTO `t` VALUES (1,'cool pasteur','2020-04-21 19:01:04'),(3,'friendly stonebraker','2020-06-09 18:58:00'),(5,'happy shannon','2020-02-29 21:39:08'),(6,'competent torvalds','2020-05-24 04:18:45'),(7,'fervent kapitsa','2020-05-21 16:58:12'),(8,'quirky jennings','2020-03-12 12:52:58'),(9,'adoring swartz','2020-04-19 02:20:32'),(14,'intelligent keller','2020-01-08 09:47:42'),(15,'vibrant zhukovsky','2020-04-15 15:15:55'),(18,'keen chatterjee','2020-02-09 06:39:31'),(20,'elastic gauss','2020-03-01 13:34:06'),(21,'affectionate margulis','2020-06-20 10:20:29'),(27,'busy keldysh','2020-05-21 09:10:45'),(31,'flamboyant banach','2020-03-04 21:28:44'),(39,'keen banach','2020-06-09 03:07:57'),(41,'nervous gagarin','2020-06-12 23:43:04'),(47,'wonderful chebyshev','2020-04-15 14:51:17'),(50,'reverent brahmagupta','2020-06-25 21:50:52'),(52,'suspicious elbakyan','2020-05-28 04:55:34'),(55,'epic lichterman','2020-05-16 19:24:09'),(57,'determined taussig','2020-06-18 22:51:37')") tk.MustExec("DROP TABLE IF EXISTS `t1`") tk.MustExec("CREATE TABLE `t1` ( `c_int` int(11) DEFAULT NULL, `c_str` varchar(40) NOT NULL, `c_datetime` datetime DEFAULT NULL, PRIMARY KEY (`c_str`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin") diff --git a/expression/integration_test.go b/expression/integration_test.go index a5a554cd1b29e..086f6162e0c94 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -7575,6 +7575,17 @@ func (s *testIntegrationSerialSuite) TestIssue19804(c *C) { tk.MustGetErrMsg(`alter table t change a a set('a', 'b', 'c', 'e', 'f');`, "[ddl:8200]Unsupported modify column: cannot modify set column value d to e, and tidb_enable_change_column_type is false") } +func (s *testIntegrationSerialSuite) TestIssue20209(c *C) { + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) + + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test;`) + tk.MustExec(`set @@character_set_client=utf8mb4;`) + tk.MustExec(`set @@collation_connection=utf8_bin;`) + tk.MustExec("CREATE VIEW tview_1 AS SELECT 'a' AS `id`;") +} + func (s *testIntegrationSerialSuite) TestIssue18949(c *C) { collate.SetNewCollationEnabledForTest(true) defer collate.SetNewCollationEnabledForTest(false) diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 22be00689e456..81aa506a71074 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1469,26 +1469,48 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { case TiDBMetricSchemaRangeDuration: s.MetricSchemaRangeDuration = tidbOptInt64(val, DefTiDBMetricSchemaRangeDuration) case CollationConnection, CollationDatabase, CollationServer: - if _, err := collate.GetCollationByName(val); err != nil { - var ok bool - var charsetVal string - var err2 error - if name == CollationConnection { - charsetVal, ok = s.systems[CharacterSetConnection] - } else if name == CollationDatabase { - charsetVal, ok = s.systems[CharsetDatabase] - } else { - // CollationServer - charsetVal, ok = s.systems[CharacterSetServer] - } - if !ok { - return err - } - val, err2 = charset.GetDefaultCollation(charsetVal) - if err2 != nil { - return err2 + coll, err := collate.GetCollationByName(val) + if err != nil { + logutil.BgLogger().Warn(err.Error()) + coll, err = collate.GetCollationByName(charset.CollationUTF8MB4) + } + switch name { + case CollationConnection: + s.systems[CollationConnection] = coll.Name + s.systems[CharacterSetConnection] = coll.CharsetName + case CollationDatabase: + s.systems[CollationDatabase] = coll.Name + s.systems[CharsetDatabase] = coll.CharsetName + case CollationServer: + s.systems[CollationServer] = coll.Name + s.systems[CharacterSetServer] = coll.CharsetName + } + case CharacterSetSystem, CharacterSetConnection, CharacterSetClient, CharacterSetResults, + CharacterSetServer, CharsetDatabase, CharacterSetFilesystem: + if val == "" { + if name == CharacterSetResults { + s.systems[CharacterSetResults] = "" + return nil } + return ErrWrongValueForVar.GenWithStackByArgs(name, "NULL") + } + cht, coll, err := charset.GetCharsetInfo(val) + if err != nil { logutil.BgLogger().Warn(err.Error()) + cht, coll = charset.GetDefaultCharsetAndCollate() + } + switch name { + case CharacterSetConnection: + s.systems[CollationConnection] = coll + s.systems[CharacterSetConnection] = cht + case CharsetDatabase: + s.systems[CollationDatabase] = coll + s.systems[CharsetDatabase] = cht + case CharacterSetServer: + s.systems[CollationServer] = coll + s.systems[CharacterSetServer] = cht + default: + s.systems[name] = cht } case TiDBSlowLogThreshold: atomic.StoreUint64(&config.GetGlobalConfig().Log.SlowThreshold, uint64(tidbOptInt64(val, logutil.DefaultSlowThreshold))) diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index d6e0e5f57e7f8..0bc6e7db8ccda 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1165,15 +1165,15 @@ func initSynonymsSysVariables() { // SetNamesVariables is the system variable names related to set names statements. var SetNamesVariables = []string{ - "character_set_client", - "character_set_connection", - "character_set_results", + CharacterSetClient, + CharacterSetConnection, + CharacterSetResults, } // SetCharsetVariables is the system variable names related to set charset statements. var SetCharsetVariables = []string{ - "character_set_client", - "character_set_results", + CharacterSetClient, + CharacterSetResults, } const ( @@ -1185,6 +1185,12 @@ const ( CharsetDatabase = "character_set_database" // CollationDatabase is the name for collation_database system variable. CollationDatabase = "collation_database" + // CharacterSetFilesystem is the name for character_set_filesystem system variable. + CharacterSetFilesystem = "character_set_filesystem" + // CharacterSetClient is the name for character_set_client system variable. + CharacterSetClient = "character_set_client" + // CharacterSetSystem is the name for character_set_system system variable. + CharacterSetSystem = "character_set_system" // GeneralLog is the name for 'general_log' system variable. GeneralLog = "general_log" // AvoidTemporalUpgrade is the name for 'avoid_temporal_upgrade' system variable. From e136429d8dc5d70f43cd3f94179b0b9f47595097 Mon Sep 17 00:00:00 2001 From: Rain Li Date: Mon, 16 Nov 2020 19:58:27 +0800 Subject: [PATCH 09/37] *: refactor parser initialize method and add tidb_enable_strict_double_type_check variable (#20967) --- ddl/db_integration_test.go | 13 +++++++++++++ executor/prepared.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- planner/cascades/transformation_rules_test.go | 2 +- planner/core/logical_plan_builder.go | 2 +- planner/core/logical_plan_test.go | 2 +- planner/core/physical_plan_test.go | 2 +- planner/core/preprocess.go | 9 ++++----- planner/core/preprocess_test.go | 1 - planner/core/stats_test.go | 2 +- session/bootstrap.go | 13 +++++++++++++ session/session.go | 5 +++-- session/tidb.go | 2 +- sessionctx/variable/session.go | 14 ++++++++++++++ sessionctx/variable/sysvar.go | 1 + sessionctx/variable/tidb_vars.go | 4 ++++ 17 files changed, 62 insertions(+), 18 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index fe20bc54c028f..db37d2f4e0d1e 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -2577,3 +2577,16 @@ func (s *testIntegrationSuite3) TestEnumAndSetDefaultValue(c *C) { c.Assert(tbl.Meta().Columns[0].DefaultValue, Equals, "a") c.Assert(tbl.Meta().Columns[1].DefaultValue, Equals, "a") } + +func (s *testIntegrationSuite3) TestStrictDoubleTypeCheck(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("set @@tidb_enable_strict_double_type_check = 'ON'") + sql := "create table double_type_check(id int, c double(10));" + _, err := tk.Exec(sql) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[parser:1149]You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use") + tk.MustExec("set @@tidb_enable_strict_double_type_check = 'OFF'") + defer tk.MustExec("set @@tidb_enable_strict_double_type_check = 'ON'") + tk.MustExec(sql) +} diff --git a/executor/prepared.go b/executor/prepared.go index 278fd84d4dad0..82020fa42e8cd 100644 --- a/executor/prepared.go +++ b/executor/prepared.go @@ -119,7 +119,7 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { stmts, err = sqlParser.ParseSQL(e.sqlText, charset, collation) } else { p := parser.New() - p.EnableWindowFunc(vars.EnableWindowFunction) + p.SetParserConfig(vars.BuildParserConfig()) var warns []error stmts, warns, err = p.Parse(e.sqlText, charset, collation) for _, warn := range warns { diff --git a/go.mod b/go.mod index b39702a72a1d4..745a4d157407e 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9 github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 - github.com/pingcap/parser v0.0.0-20201112065012-c9380f220ff9 + github.com/pingcap/parser v0.0.0-20201112105242-773b8b74f44e github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible diff --git a/go.sum b/go.sum index 3e71d8fca7de8..b9c3b55dcc79c 100644 --- a/go.sum +++ b/go.sum @@ -465,8 +465,8 @@ github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIf github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 h1:Jboj+s4jSCp5E1WDgmRUv5rIFKFHaaSWuSZ4wMwXIcc= github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20201112065012-c9380f220ff9 h1:/Vd4G/b+sifGUe14+GsxyXlWvJBJwJlTPcyV9IcQYdU= -github.com/pingcap/parser v0.0.0-20201112065012-c9380f220ff9/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE= +github.com/pingcap/parser v0.0.0-20201112105242-773b8b74f44e h1:TCSZlqvNP2Ipt0xUwZ5Z6ehmKo1dlk5KCU1SOLXUYL4= +github.com/pingcap/parser v0.0.0-20201112105242-773b8b74f44e/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 h1:b2G/eqDeywtdJF3w9nIUdqMmXChsmpLvf4FzUxJ9Vmk= diff --git a/planner/cascades/transformation_rules_test.go b/planner/cascades/transformation_rules_test.go index 421f815313945..feae5eca355d1 100644 --- a/planner/cascades/transformation_rules_test.go +++ b/planner/cascades/transformation_rules_test.go @@ -44,7 +44,7 @@ func (s *testTransformationRuleSuite) SetUpSuite(c *C) { var err error s.testData, err = testutil.LoadTestSuiteData("testdata", "transformation_rules_suite") c.Assert(err, IsNil) - s.Parser.EnableWindowFunc(true) + s.Parser.SetParserConfig(parser.ParserConfig{EnableWindowFunction: true, EnableStrictDoubleTypeCheck: true}) } func (s *testTransformationRuleSuite) TearDownSuite(c *C) { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 14f32499e1098..c9b51ac5bb67f 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -3323,7 +3323,7 @@ func (b *PlanBuilder) BuildDataSourceFromView(ctx context.Context, dbName model. charset, collation := b.ctx.GetSessionVars().GetCharsetInfo() viewParser := parser.New() - viewParser.EnableWindowFunc(b.ctx.GetSessionVars().EnableWindowFunction) + viewParser.SetParserConfig(b.ctx.GetSessionVars().BuildParserConfig()) selectNode, err := viewParser.ParseOneStmt(tableInfo.View.SelectStmt, charset, collation) if err != nil { return nil, err diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 6e8f6e0ef033f..ee8d39b1017a6 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -61,7 +61,7 @@ func (s *testPlanSuite) SetUpSuite(c *C) { s.ctx = MockContext() s.ctx.GetSessionVars().EnableWindowFunction = true s.Parser = parser.New() - s.Parser.EnableWindowFunc(true) + s.Parser.SetParserConfig(parser.ParserConfig{EnableWindowFunction: true, EnableStrictDoubleTypeCheck: true}) var err error s.testData, err = testutil.LoadTestSuiteData("testdata", "plan_suite_unexported") diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 62519e84ba530..07e9e6abc1396 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -48,7 +48,7 @@ type testPlanSuiteBase struct { func (s *testPlanSuiteBase) SetUpSuite(c *C) { s.is = infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable(), core.MockUnsignedTable()}) s.Parser = parser.New() - s.Parser.EnableWindowFunc(true) + s.Parser.SetParserConfig(parser.ParserConfig{EnableWindowFunction: true, EnableStrictDoubleTypeCheck: true}) } type testPlanSerialSuite struct { diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index 97b436435cfbb..11b44d83e038d 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -830,11 +830,10 @@ func checkColumn(colDef *ast.ColumnDef) error { // For FLOAT, the SQL standard permits an optional specification of the precision. // https://dev.mysql.com/doc/refman/8.0/en/floating-point-types.html if tp.Decimal == -1 { - if tp.Tp == mysql.TypeDouble { - if tp.Flen != -1 { - return types.ErrSyntax.GenWithStackByArgs() - } - } else { + switch tp.Tp { + case mysql.TypeDouble: + // For Double type Flen and Decimal check is moved to parser component + default: if tp.Flen > mysql.MaxDoublePrecisionLength { return types.ErrWrongFieldSpec.GenWithStackByArgs(colDef.Name.Name.O) } diff --git a/planner/core/preprocess_test.go b/planner/core/preprocess_test.go index 442dcee025b30..4f6d90422a391 100644 --- a/planner/core/preprocess_test.go +++ b/planner/core/preprocess_test.go @@ -227,7 +227,6 @@ func (s *testValidatorSuite) TestValidator(c *C) { {"CREATE TABLE t (a float(53))", true, nil}, {"CREATE TABLE t (a float(54))", false, types.ErrWrongFieldSpec}, {"CREATE TABLE t (a double)", true, nil}, - {"CREATE TABLE t (a double(54))", false, types.ErrSyntax}, // FIXME: temporary 'not implemented yet' test for 'CREATE TABLE ... SELECT' (issue 4754) {"CREATE TABLE t SELECT * FROM u", false, errors.New("'CREATE TABLE ... SELECT' is not implemented yet")}, diff --git a/planner/core/stats_test.go b/planner/core/stats_test.go index 0b3176eb87b60..2239d27a5f75e 100644 --- a/planner/core/stats_test.go +++ b/planner/core/stats_test.go @@ -34,7 +34,7 @@ type testStatsSuite struct { func (s *testStatsSuite) SetUpSuite(c *C) { s.Parser = parser.New() - s.Parser.EnableWindowFunc(true) + s.Parser.SetParserConfig(parser.ParserConfig{EnableWindowFunction: true, EnableStrictDoubleTypeCheck: true}) var err error s.testData, err = testutil.LoadTestSuiteData("testdata", "stats_suite") diff --git a/session/bootstrap.go b/session/bootstrap.go index a6a55b52b4daa..6d77e864c4051 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -427,6 +427,8 @@ const ( version51 = 51 // version52 change mysql.stats_histograms cm_sketch column from blob to blob(6291456) version52 = 52 + // version53 introduce Global variable tidb_enable_strict_double_type_check + version53 = 53 ) var ( @@ -482,6 +484,7 @@ var ( upgradeToVer50, upgradeToVer51, upgradeToVer52, + upgradeToVer53, } ) @@ -1190,6 +1193,16 @@ func upgradeToVer52(s Session, ver int64) { doReentrantDDL(s, "ALTER TABLE mysql.stats_histograms MODIFY cm_sketch BLOB(6291456)") } +func upgradeToVer53(s Session, ver int64) { + if ver >= version53 { + return + } + // when upgrade from old tidb and no `tidb_enable_strict_double_type_check` in GLOBAL_VARIABLES, init it with 1` + sql := fmt.Sprintf("INSERT IGNORE INTO %s.%s (`VARIABLE_NAME`, `VARIABLE_VALUE`) VALUES ('%s', '%d')", + mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBEnableStrictDoubleTypeCheck, 0) + mustExecute(s, sql) +} + // updateBootstrapVer updates bootstrap version variable in mysql.TiDB table. func updateBootstrapVer(s Session) { // Update bootstrap version. diff --git a/session/session.go b/session/session.go index 7d3d0044c8d53..740a394251899 100644 --- a/session/session.go +++ b/session/session.go @@ -1077,7 +1077,7 @@ func (s *session) ParseSQL(ctx context.Context, sql, charset, collation string) } defer trace.StartRegion(ctx, "ParseSQL").End() s.parser.SetSQLMode(s.sessionVars.SQLMode) - s.parser.EnableWindowFunc(s.sessionVars.EnableWindowFunction) + s.parser.SetParserConfig(s.sessionVars.BuildParserConfig()) return s.parser.Parse(sql, charset, collation) } @@ -2067,7 +2067,7 @@ func CreateSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, er const ( notBootstrapped = 0 - currentBootstrapVersion = version52 + currentBootstrapVersion = version53 ) func getStoreBootstrapVersion(store kv.Storage) int64 { @@ -2184,6 +2184,7 @@ var builtinGlobalVariable = []string{ variable.TiDBRetryLimit, variable.TiDBDisableTxnAutoRetry, variable.TiDBEnableWindowFunction, + variable.TiDBEnableStrictDoubleTypeCheck, variable.TiDBEnableTablePartition, variable.TiDBEnableVectorizedExpression, variable.TiDBEnableFastAnalyze, diff --git a/session/tidb.go b/session/tidb.go index d281c4cc0ce3d..18945e8183f95 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -172,7 +172,7 @@ func Parse(ctx sessionctx.Context, src string) ([]ast.StmtNode, error) { logutil.BgLogger().Debug("compiling", zap.String("source", src)) charset, collation := ctx.GetSessionVars().GetCharsetInfo() p := parser.New() - p.EnableWindowFunc(ctx.GetSessionVars().EnableWindowFunction) + p.SetParserConfig(ctx.GetSessionVars().BuildParserConfig()) p.SetSQLMode(ctx.GetSessionVars().SQLMode) stmts, warns, err := p.Parse(src, charset, collation) for _, warn := range warns { diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 81aa506a71074..dcc26f40a217e 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -30,6 +30,7 @@ import ( "github.com/klauspost/cpuid" "github.com/pingcap/errors" + "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" "github.com/pingcap/parser/charset" @@ -552,6 +553,9 @@ type SessionVars struct { // EnableWindowFunction enables the window function. EnableWindowFunction bool + // EnableStrictDoubleTypeCheck enables table field double type check. + EnableStrictDoubleTypeCheck bool + // EnableVectorizedExpression enables the vectorized expression evaluation. EnableVectorizedExpression bool @@ -741,6 +745,14 @@ func (s *SessionVars) UseDynamicPartitionPrune() bool { return PartitionPruneMode(s.PartitionPruneMode.Load()) == DynamicOnly } +// BuildParserConfig generate parser.ParserConfig for initial parser +func (s *SessionVars) BuildParserConfig() parser.ParserConfig { + return parser.ParserConfig{ + EnableWindowFunction: s.EnableWindowFunction, + EnableStrictDoubleTypeCheck: s.EnableStrictDoubleTypeCheck, + } +} + // PartitionPruneMode presents the prune mode used. type PartitionPruneMode string @@ -1404,6 +1416,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { s.EnableRadixJoin = TiDBOptOn(val) case TiDBEnableWindowFunction: s.EnableWindowFunction = TiDBOptOn(val) + case TiDBEnableStrictDoubleTypeCheck: + s.EnableStrictDoubleTypeCheck = TiDBOptOn(val) case TiDBEnableVectorizedExpression: s.EnableVectorizedExpression = TiDBOptOn(val) case TiDBOptJoinReorderThreshold: diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 0bc6e7db8ccda..84daed5f8dada 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1028,6 +1028,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: TiDBRowFormatVersion, Value: strconv.Itoa(DefTiDBRowFormatV1), Type: TypeUnsigned, MinValue: 1, MaxValue: 2}, {Scope: ScopeSession, Name: TiDBOptimizerSelectivityLevel, Value: strconv.Itoa(DefTiDBOptimizerSelectivityLevel), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxUint64}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableWindowFunction, Value: BoolToOnOff(DefEnableWindowFunction), Type: TypeBool}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableStrictDoubleTypeCheck, Value: BoolToOnOff(DefEnableStrictDoubleTypeCheck), Type: TypeBool}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableVectorizedExpression, Value: BoolToOnOff(DefEnableVectorizedExpression), Type: TypeBool}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableFastAnalyze, Value: BoolToOnOff(DefTiDBUseFastAnalyze), Type: TypeBool}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBSkipIsolationLevelCheck, Value: BoolToOnOff(DefTiDBSkipIsolationLevelCheck), Type: TypeBool}, diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 5e93fb1566511..352d009387735 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -372,6 +372,9 @@ const ( // tidb_enable_window_function is used to control whether to enable the window function. TiDBEnableWindowFunction = "tidb_enable_window_function" + // tidb_enable_strict_double_type_check is used to control table field double type syntax check. + TiDBEnableStrictDoubleTypeCheck = "tidb_enable_strict_double_type_check" + // tidb_enable_vectorized_expression is used to control whether to enable the vectorized expression evaluation. TiDBEnableVectorizedExpression = "tidb_enable_vectorized_expression" @@ -539,6 +542,7 @@ const ( DefTiDBForcePriority = mysql.NoPriority DefTiDBUseRadixJoin = false DefEnableWindowFunction = true + DefEnableStrictDoubleTypeCheck = true DefEnableVectorizedExpression = true DefTiDBOptJoinReorderThreshold = 0 DefTiDBDDLSlowOprThreshold = 300 From d5684efe4903f34464219d96f9e3fff13039de32 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 17 Nov 2020 10:21:26 +0800 Subject: [PATCH 10/37] executor: fix a potential DATA RACE in TestFastAnalyze (#21023) --- executor/analyze.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 23b61ad5f12b6..9eb5987292d92 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -1131,9 +1131,9 @@ func (e *AnalyzeFastExec) buildStats() (hists []*statistics.Histogram, cms []*st // To set rand seed, it's for unit test. // To ensure that random sequences are different in non-test environments, RandSeed must be set time.Now(). if RandSeed == 1 { - e.randSeed = time.Now().UnixNano() + atomic.StoreInt64(&e.randSeed, time.Now().UnixNano()) } else { - e.randSeed = RandSeed + atomic.StoreInt64(&e.randSeed, RandSeed) } err = e.buildSampTask() From 9c3b93750a5493673957730a93a36cbd72aeba68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B7=B7=E6=B2=8CDM?= Date: Tue, 17 Nov 2020 11:18:56 +0800 Subject: [PATCH 11/37] pd: rename GetMemberInfo to GetAllMembers (#20218) Signed-off-by: Zheng Xiangsheng --- go.mod | 2 +- go.sum | 12 +++++++----- store/mockstore/mocktikv/pd.go | 2 +- store/mockstore/unistore/pd.go | 2 +- store/tikv/backoff.go | 2 +- store/tikv/kv.go | 4 ++-- store/tikv/store_test.go | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 745a4d157407e..322918607a90a 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/stretchr/testify v1.6.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09 + github.com/tikv/pd v1.1.0-beta.0.20200925080742-6d0aa719fa17 github.com/twmb/murmur3 v1.1.3 github.com/uber-go/atomic v1.4.0 github.com/uber/jaeger-client-go v2.22.1+incompatible diff --git a/go.sum b/go.sum index b9c3b55dcc79c..6309d344b7d06 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,7 @@ github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -430,7 +431,7 @@ github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5e github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200908071351-a715a95c7de2/go.mod h1:X3r7/4Wr9fSC5KlsfezBh/5noeWGEJNQuSvjgS2rvdI= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200921100341-0e148dfc0029/go.mod h1:X3r7/4Wr9fSC5KlsfezBh/5noeWGEJNQuSvjgS2rvdI= github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19 h1:IXpGy7y9HyoShAFmzW2OPF0xCA5EOoSTyZHwsgYk9Ro= github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= github.com/pingcap/br v4.0.0-beta.2.0.20201014031603-5676c8fdad1a+incompatible h1:RMx7D+dQtUTUZjelJyV0WwYr3GFxrjsSarFXhV2SWzI= @@ -444,7 +445,7 @@ github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSq github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= +github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 h1:wQKuKP2HUtej2gSvx1cZmY4DENUH6tlOxRkfvPT8EBU= github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= @@ -457,7 +458,7 @@ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17Xtb github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200810113304-6157337686b1/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200916031750-f9473f2c5379/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9 h1:o/MpH2TJ2qJk0EwSI1O8Fw/dzx/ImSSKp2gXEUDvH6w= github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= @@ -562,6 +563,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= @@ -578,8 +580,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfK github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09 h1:5NsHTjk0O7C3/d8vfl/cWu9L6db+8YGvEj7XBGbMTbY= -github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09/go.mod h1:Z+EQXV6FyfpH7olLqXH0zvYOnFcCNGJmzm+MN4W1/RE= +github.com/tikv/pd v1.1.0-beta.0.20200925080742-6d0aa719fa17 h1:xI/CKdl55/WeoIvMS3040I9vJ52bIPkFK2Z3HbbOgO0= +github.com/tikv/pd v1.1.0-beta.0.20200925080742-6d0aa719fa17/go.mod h1:U5MWRAfLdLeeUQxs6/+3HdiOT2g/EXxHRs1+i6BDHbU= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/store/mockstore/mocktikv/pd.go b/store/mockstore/mocktikv/pd.go index f4d28932b53cf..cf653978085b2 100644 --- a/store/mockstore/mocktikv/pd.go +++ b/store/mockstore/mocktikv/pd.go @@ -175,7 +175,7 @@ func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetO return &pdpb.GetOperatorResponse{Status: pdpb.OperatorStatus_SUCCESS}, nil } -func (c *pdClient) GetMemberInfo(ctx context.Context) ([]*pdpb.Member, error) { +func (c *pdClient) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { return nil, nil } diff --git a/store/mockstore/unistore/pd.go b/store/mockstore/unistore/pd.go index fd935036a7893..11f5e1f32ea78 100644 --- a/store/mockstore/unistore/pd.go +++ b/store/mockstore/unistore/pd.go @@ -97,6 +97,6 @@ func (c *pdClient) ScatterRegionWithOption(ctx context.Context, regionID uint64, return nil } -func (c *pdClient) GetMemberInfo(ctx context.Context) ([]*pdpb.Member, error) { +func (c *pdClient) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { return nil, nil } diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index 02f01c4bc4210..0364909122492 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -213,7 +213,7 @@ func (t backoffType) TError() error { // Maximum total sleep time(in ms) for kv/cop commands. const ( - GetMemberInfoBackoff = 5000 + GetAllMembersBackoff = 5000 copBuildTaskMaxBackoff = 5000 tsoMaxBackoff = 15000 scannerNextMaxBackoff = 20000 diff --git a/store/tikv/kv.go b/store/tikv/kv.go index a553403af6f5f..53dae50d13c04 100644 --- a/store/tikv/kv.go +++ b/store/tikv/kv.go @@ -255,14 +255,14 @@ func (s *tikvStore) EtcdAddrs() ([]string, error) { } ctx := context.Background() - bo := NewBackoffer(ctx, GetMemberInfoBackoff) + bo := NewBackoffer(ctx, GetAllMembersBackoff) etcdAddrs := make([]string, 0) pdClient := s.pdClient if pdClient == nil { return nil, errors.New("Etcd client not found") } for { - members, err := pdClient.GetMemberInfo(ctx) + members, err := pdClient.GetAllMembers(ctx) if err != nil { err := bo.Backoff(BoRegionMiss, err) if err != nil { diff --git a/store/tikv/store_test.go b/store/tikv/store_test.go index f6163cf0b2d63..2f07d28db6120 100644 --- a/store/tikv/store_test.go +++ b/store/tikv/store_test.go @@ -117,7 +117,7 @@ func (c *mockPDClient) disable() { c.stop = true } -func (c *mockPDClient) GetMemberInfo(ctx context.Context) ([]*pdpb.Member, error) { +func (c *mockPDClient) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { return nil, nil } From ee5d026855e0504fe8792430438d7462e0b8cdb9 Mon Sep 17 00:00:00 2001 From: Zhuomin Liu Date: Tue, 17 Nov 2020 12:02:27 +0800 Subject: [PATCH 12/37] executor: fix unstable test of TestIntegrationCopCache (#21071) --- executor/executor_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 68c197e5b07c6..5338d9dfbdeca 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -6664,7 +6664,11 @@ func (s *testCoprCache) TestIntegrationCopCache(c *C) { rows = tk.MustQuery("explain analyze select * from t").Rows() c.Assert(rows[0][2], Equals, "12") c.Assert(strings.Contains(rows[0][5].(string), "cop_task: {num: 6"), Equals, true) - c.Assert(strings.Contains(rows[0][5].(string), "copr_cache_hit_ratio: 0.67"), Equals, true) + hitRatioIdx := strings.Index(rows[0][5].(string), "copr_cache_hit_ratio:") + len("copr_cache_hit_ratio: ") + c.Assert(hitRatioIdx >= len("copr_cache_hit_ratio: "), Equals, true) + hitRatio, err := strconv.ParseFloat(rows[0][5].(string)[hitRatioIdx:hitRatioIdx+4], 64) + c.Assert(err, IsNil) + c.Assert(hitRatio > 0, Equals, true) } func (s *testSerialSuite) TestCoprocessorOOMAction(c *C) { From c20369d0750bc6120f361671963013bc7edad573 Mon Sep 17 00:00:00 2001 From: Yilin Chen Date: Tue, 17 Nov 2020 13:23:57 +0800 Subject: [PATCH 13/37] tikv: fix race at minCommitTS (#21032) Signed-off-by: Yilin Chen --- store/tikv/prewrite.go | 6 ++-- store/tikv/prewrite_test.go | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 store/tikv/prewrite_test.go diff --git a/store/tikv/prewrite.go b/store/tikv/prewrite.go index df3a22bbad6b5..9703a43882f4f 100644 --- a/store/tikv/prewrite.go +++ b/store/tikv/prewrite.go @@ -56,7 +56,9 @@ func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchMutations, txnSize u } isPessimisticLock[i] = m.IsPessimisticLock(i) } - var minCommitTS = c.minCommitTS + c.mu.Lock() + minCommitTS := c.minCommitTS + c.mu.Unlock() if c.forUpdateTS > 0 && c.forUpdateTS >= minCommitTS { minCommitTS = c.forUpdateTS + 1 } else if c.startTS >= minCommitTS { @@ -87,8 +89,6 @@ func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchMutations, txnSize u req.Secondaries = c.asyncSecondaries() } req.UseAsyncCommit = true - // The async commit can not be used for large transactions, and the commit ts can't be pushed. - req.MinCommitTs = 0 } if c.isOnePC() { diff --git a/store/tikv/prewrite_test.go b/store/tikv/prewrite_test.go new file mode 100644 index 0000000000000..48336eeb8cfab --- /dev/null +++ b/store/tikv/prewrite_test.go @@ -0,0 +1,67 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tikv + +import ( + . "github.com/pingcap/check" + pb "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/tidb/store/mockstore/unistore" +) + +type testPrewriteSuite struct { + store *tikvStore +} + +var _ = Suite(&testPrewriteSuite{}) + +func (s *testPrewriteSuite) SetUpTest(c *C) { + client, pdClient, cluster, err := unistore.New("") + c.Assert(err, IsNil) + unistore.BootstrapWithSingleStore(cluster) + store, err := NewTestTiKVStore(client, pdClient, nil, nil, 0) + c.Assert(err, IsNil) + s.store = store.(*tikvStore) +} + +func (s *testPrewriteSuite) TestSetMinCommitTSInAsyncCommit(c *C) { + t, err := s.store.Begin() + c.Assert(err, IsNil) + txn := t.(*tikvTxn) + err = txn.Set([]byte("k"), []byte("v")) + c.Assert(err, IsNil) + committer, err := newTwoPhaseCommitterWithInit(txn, 1) + c.Assert(err, IsNil) + committer.useAsyncCommit = 1 + + buildRequest := func() *pb.PrewriteRequest { + batch := batchMutations{mutations: committer.mutations} + req := committer.buildPrewriteRequest(batch, 1) + return req.Req.(*pb.PrewriteRequest) + } + + // no forUpdateTS + req := buildRequest() + c.Assert(req.MinCommitTs, Equals, txn.startTS+1) + + // forUpdateTS is set + committer.forUpdateTS = txn.startTS + (5 << 18) + req = buildRequest() + c.Assert(req.MinCommitTs, Equals, committer.forUpdateTS+1) + + // minCommitTS is set + committer.minCommitTS = txn.startTS + (10 << 18) + req = buildRequest() + c.Assert(req.MinCommitTs, Equals, committer.minCommitTS) + +} From 7e0821f12b3ba1eec3e05bfc43299034f7bb53c4 Mon Sep 17 00:00:00 2001 From: Rain Li Date: Tue, 17 Nov 2020 14:09:27 +0800 Subject: [PATCH 14/37] planner: fix point get and batch point get should not use invisible index (#21088) --- planner/core/point_get_plan.go | 7 ++----- planner/core/point_get_plan_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index fd3a8afa25d16..77efe8f5d5a67 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -540,7 +540,7 @@ func newBatchPointGetPlan( var matchIdxInfo *model.IndexInfo permutations := make([]int, len(whereColNames)) for _, idxInfo := range tbl.Indices { - if !idxInfo.Unique || idxInfo.State != model.StatePublic { + if !idxInfo.Unique || idxInfo.State != model.StatePublic || idxInfo.Invisible { continue } if len(idxInfo.Columns) != len(whereColNames) || idxInfo.HasPrefixIndex() { @@ -799,10 +799,7 @@ func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt) *PointGetP } for _, idxInfo := range tbl.Indices { - if !idxInfo.Unique { - continue - } - if idxInfo.State != model.StatePublic { + if !idxInfo.Unique || idxInfo.State != model.StatePublic || idxInfo.Invisible { continue } if isTableDual { diff --git a/planner/core/point_get_plan_test.go b/planner/core/point_get_plan_test.go index c74a1140fc4a2..91c2ff0c5cc5a 100644 --- a/planner/core/point_get_plan_test.go +++ b/planner/core/point_get_plan_test.go @@ -536,3 +536,29 @@ func (s *testPointGetSuite) TestIssue20692(c *C) { tk3.MustExec("commit;") tk3.MustQuery("select * from t;").Check(testkit.Rows("10 20 30 40")) } + +func (s *testPointGetSuite) TestPointGetWithInvisibleIndex(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (c1 int, unique(c1))") + tk.MustExec("alter table t alter index c1 invisible") + tk.MustQuery("explain select * from t where c1 = 10").Check(testkit.Rows( + "TableReader_7 1.00 root data:Selection_6", + "└─Selection_6 1.00 cop[tikv] eq(test.t.c1, 10)", + " └─TableFullScan_5 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + )) +} + +func (s *testPointGetSuite) TestBatchPointGetWithInvisibleIndex(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (c1 int, unique(c1))") + tk.MustExec("alter table t alter index c1 invisible") + tk.MustQuery("explain select * from t where c1 in (10, 20)").Check(testkit.Rows( + "TableReader_7 2.00 root data:Selection_6", + "└─Selection_6 2.00 cop[tikv] in(test.t.c1, 10, 20)", + " └─TableFullScan_5 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + )) +} From 863117ba052e567f446813efb1ecf9494d34070e Mon Sep 17 00:00:00 2001 From: Song Gao Date: Tue, 17 Nov 2020 14:28:27 +0800 Subject: [PATCH 15/37] session: Check the partition placement constraint of local transactions (#21039) Signed-off-by: Song Gao --- ddl/error.go | 3 + ddl/placement/const.go | 4 + ddl/placement_sql_test.go | 142 +++++++++++++++++++++++++++++++++ errno/errcode.go | 1 + errno/errname.go | 1 + errors.toml | 5 ++ session/session.go | 44 +++++++++- sessionctx/variable/session.go | 3 + 8 files changed, 201 insertions(+), 2 deletions(-) diff --git a/ddl/error.go b/ddl/error.go index e66ca06a4d95d..749c752d33526 100644 --- a/ddl/error.go +++ b/ddl/error.go @@ -251,6 +251,9 @@ var ( // ErrInvalidPlacementSpec is returned when add/alter an invalid placement rule ErrInvalidPlacementSpec = dbterror.ClassDDL.NewStd(mysql.ErrInvalidPlacementSpec) + // ErrInvalidPlacementPolicyCheck is returned when txn_scope and commit data changing do not meet the placement policy + ErrInvalidPlacementPolicyCheck = dbterror.ClassDDL.NewStd(mysql.ErrPlacementPolicyCheck) + // ErrMultipleDefConstInListPart returns multiple definition of same constant in list partitioning. ErrMultipleDefConstInListPart = dbterror.ClassDDL.NewStd(mysql.ErrMultipleDefConstInListPart) diff --git a/ddl/placement/const.go b/ddl/placement/const.go index 9c171dcbf582b..d90763d91e97e 100644 --- a/ddl/placement/const.go +++ b/ddl/placement/const.go @@ -28,3 +28,7 @@ const ( // RuleIndexIndex is the index for a rule of index. RuleIndexIndex ) + +// DCLabelKey indicates the key of label which represents the dc for Store. +// FIXME: currently we assumes "zone" is the dcLabel key in Store +const DCLabelKey = "zone" diff --git a/ddl/placement_sql_test.go b/ddl/placement_sql_test.go index 3d1e0503585a6..83c79cb203d2c 100644 --- a/ddl/placement_sql_test.go +++ b/ddl/placement_sql_test.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/ddl/placement" + "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/util/testkit" ) @@ -326,3 +327,144 @@ func (s *testDBSuite1) TestPlacementPolicyCache(c *C) { tk.MustExec("truncate table t1") tk.MustQuery("select * from information_schema.placement_policy").Check(testkit.Rows()) } + +func (s *testSerialDBSuite) TestTxnScopeConstraint(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1") + defer tk.MustExec("drop table if exists t1") + + tk.MustExec(`create table t1 (c int) +PARTITION BY RANGE (c) ( + PARTITION p0 VALUES LESS THAN (6), + PARTITION p1 VALUES LESS THAN (11), + PARTITION p2 VALUES LESS THAN (16), + PARTITION p3 VALUES LESS THAN (21) +);`) + + bundles := make(map[string]*placement.Bundle) + is := s.dom.InfoSchema() + is.MockBundles(bundles) + + tb, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t1")) + c.Assert(err, IsNil) + partDefs := tb.Meta().GetPartitionInfo().Definitions + + for _, def := range partDefs { + if def.Name.String() == "p0" { + groupID := placement.GroupID(def.ID) + bundles[groupID] = &placement.Bundle{ + ID: groupID, + Rules: []*placement.Rule{ + { + GroupID: groupID, + Role: placement.Leader, + Count: 1, + LabelConstraints: []placement.LabelConstraint{ + { + Key: placement.DCLabelKey, + Op: placement.In, + Values: []string{"sh"}, + }, + }, + }, + }, + } + } else if def.Name.String() == "p2" { + groupID := placement.GroupID(def.ID) + bundles[groupID] = &placement.Bundle{ + ID: groupID, + Rules: []*placement.Rule{ + { + GroupID: groupID, + Role: placement.Follower, + Count: 3, + LabelConstraints: []placement.LabelConstraint{ + { + Key: placement.DCLabelKey, + Op: placement.In, + Values: []string{"sh"}, + }, + }, + }, + }, + } + + } + } + + testCases := []struct { + name string + sql string + txnScope string + disableAutoCommit bool + err error + }{ + { + name: "Insert into PARTITION p0 with global txnScope", + sql: "insert into t1 (c) values (1)", + txnScope: "global", + err: nil, + }, + { + name: "insert into PARTITION p0 with wrong txnScope", + sql: "insert into t1 (c) values (1)", + txnScope: "bj", + err: fmt.Errorf(".*out of txn_scope.*"), + }, + { + name: "insert into PARTITION p1 with local txnScope", + sql: "insert into t1 (c) values (10)", + txnScope: "bj", + err: fmt.Errorf(".*don't have placement policies with txn_scope.*"), + }, + { + name: "insert into PARTITION p1 with global txnScope", + sql: "insert into t1 (c) values (10)", + txnScope: "global", + err: nil, + }, + { + name: "insert into PARTITION p2 with local txnScope", + sql: "insert into t1 (c) values (15)", + txnScope: "bj", + err: fmt.Errorf(".*leader placement policy is not defined.*"), + }, + { + name: "insert into PARTITION p2 with global txnScope", + sql: "insert into t1 (c) values (15)", + txnScope: "global", + err: nil, + }, + { + name: "insert into PARTITION p0 with wrong txnScope and autocommit off", + sql: "insert into t1 (c) values (1)", + txnScope: "bj", + disableAutoCommit: true, + err: fmt.Errorf(".*out of txn_scope.*"), + }, + } + + for _, testcase := range testCases { + c.Log(testcase.name) + se, err := session.CreateSession4Test(s.store) + c.Check(err, IsNil) + tk.Se = se + tk.MustExec("use test") + tk.MustExec(fmt.Sprintf("set @@txn_scope = %v", testcase.txnScope)) + if testcase.disableAutoCommit { + tk.MustExec("set @@autocommit = 0") + tk.MustExec(testcase.sql) + _, err = tk.Exec("commit") + } else { + _, err = tk.Exec(testcase.sql) + } + if testcase.err == nil { + c.Assert(err, IsNil) + } else { + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, testcase.err.Error()) + fmt.Println(err.Error()) + } + } +} diff --git a/errno/errcode.go b/errno/errcode.go index 24ae800753ef1..d07358216dc19 100644 --- a/errno/errcode.go +++ b/errno/errcode.go @@ -1023,6 +1023,7 @@ const ( ErrTableOptionInsertMethodUnsupported = 8233 ErrInvalidPlacementSpec = 8234 ErrDDLReorgElementNotExist = 8235 + ErrPlacementPolicyCheck = 8236 // TiKV/PD errors. ErrPDServerTimeout = 9001 diff --git a/errno/errname.go b/errno/errname.go index d97af3d80a432..3d6b299837e04 100644 --- a/errno/errname.go +++ b/errno/errname.go @@ -1028,6 +1028,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{ ErrBRIEExportFailed: mysql.Message("Export failed: %s", nil), ErrInvalidPlacementSpec: mysql.Message("Invalid placement policy '%s': %s", nil), + ErrPlacementPolicyCheck: mysql.Message("Placement policy didn't meet the constraint, reason: %s", nil), // TiKV/PD errors. ErrPDServerTimeout: mysql.Message("PD server timeout", nil), diff --git a/errors.toml b/errors.toml index 937b08bf73834..0b4e353f97fff 100644 --- a/errors.toml +++ b/errors.toml @@ -431,6 +431,11 @@ error = ''' Invalid placement policy '%s': %s ''' +["ddl:8236"] +error = ''' +Placement policy didn't meet the constraint, reason: %s +''' + ["domain:8027"] error = ''' Information schema is out of date: schema failed to update in 1 lease, please make sure TiDB can connect to TiKV diff --git a/session/session.go b/session/session.go index 740a394251899..efde542c10a38 100644 --- a/session/session.go +++ b/session/session.go @@ -43,6 +43,8 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/bindinfo" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/ddl" + "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" @@ -473,6 +475,11 @@ func (s *session) doCommitWithRetry(ctx context.Context) error { // If the transaction is invalid, maybe it has already been rolled back by the client. return nil } + var err error + err = s.checkPlacementPolicyBeforeCommit() + if err != nil { + return err + } txnSize := s.txn.Size() isPessimistic := s.txn.IsPessimistic() if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { @@ -480,7 +487,7 @@ func (s *session) doCommitWithRetry(ctx context.Context) error { defer span1.Finish() ctx = opentracing.ContextWithSpan(ctx, span1) } - err := s.doCommit(ctx) + err = s.doCommit(ctx) if err != nil { commitRetryLimit := s.sessionVars.RetryLimit if !s.sessionVars.TxnCtx.CouldRetry { @@ -550,7 +557,6 @@ func (s *session) CommitTxn(ctx context.Context) error { failpoint.Return(err) } }) - s.sessionVars.TxnCtx.Cleanup() return err } @@ -1624,6 +1630,7 @@ func (s *session) NewTxn(ctx context.Context) error { CreateTime: time.Now(), StartTS: txn.StartTS(), ShardStep: int(s.sessionVars.ShardAllocateStep), + TxnScope: s.GetSessionVars().TxnScope, } return nil } @@ -2298,6 +2305,7 @@ func (s *session) PrepareTxnCtx(ctx context.Context) { SchemaVersion: is.SchemaMetaVersion(), CreateTime: time.Now(), ShardStep: int(s.sessionVars.ShardAllocateStep), + TxnScope: s.GetSessionVars().TxnScope, } if !s.sessionVars.IsAutocommit() || s.sessionVars.RetryInfo.Retrying { if s.sessionVars.TxnMode == ast.Pessimistic { @@ -2433,3 +2441,35 @@ func (s *session) recordOnTransactionExecution(err error, counter int, duration } } } + +func (s *session) checkPlacementPolicyBeforeCommit() error { + var err error + txnScope := s.GetSessionVars().TxnCtx.TxnScope + if txnScope == "" { + txnScope = config.DefTxnScope + } + if txnScope != config.DefTxnScope { + is := infoschema.GetInfoSchema(s) + for physicalTableID := range s.GetSessionVars().TxnCtx.TableDeltaMap { + bundle, ok := is.BundleByName(placement.GroupID(physicalTableID)) + if !ok { + err = ddl.ErrInvalidPlacementPolicyCheck.GenWithStackByArgs( + fmt.Sprintf("table or partition %v don't have placement policies with txn_scope %v", + physicalTableID, txnScope)) + break + } + dcLocation, ok := placement.GetLeaderDCByBundle(bundle, placement.DCLabelKey) + if !ok { + err = ddl.ErrInvalidPlacementPolicyCheck.GenWithStackByArgs( + fmt.Sprintf("table or partition %v's leader placement policy is not defined", physicalTableID)) + break + } + if dcLocation != txnScope { + err = ddl.ErrInvalidPlacementPolicyCheck.GenWithStackByArgs( + fmt.Sprintf("table or partition %v's leader location %v is out of txn_scope %v", physicalTableID, dcLocation, txnScope)) + break + } + } + } + return err +} diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index dcc26f40a217e..0ae6d2e4f4f95 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -170,6 +170,9 @@ type TransactionContext struct { Isolation string LockExpire uint32 ForUpdate uint32 + + // TxnScope stores the value of 'txn_scope'. + TxnScope string } // GetShard returns the shard prefix for the next `count` rowids. From f72cec77b6cb8c9f4f5bbebf76da807ceb57ef85 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Tue, 17 Nov 2020 15:19:04 +0800 Subject: [PATCH 16/37] *: support SQL bind for Update / Delete / Insert / Replace (#20686) --- bindinfo/bind_test.go | 291 ++++++++++++++++++++- bindinfo/handle.go | 42 ++- executor/compiler.go | 27 +- planner/core/hints.go | 6 + planner/core/preprocess.go | 55 +++- planner/core/preprocess_test.go | 2 +- planner/core/testdata/plan_suite_out.json | 2 +- planner/optimize.go | 30 ++- session/session_test.go | 4 +- util/hint/hint_processor.go | 63 ++++- util/stmtsummary/statement_summary.go | 6 +- util/stmtsummary/statement_summary_test.go | 10 +- 12 files changed, 486 insertions(+), 52 deletions(-) diff --git a/bindinfo/bind_test.go b/bindinfo/bind_test.go index f510761b9bad7..85f726cdfcafd 100644 --- a/bindinfo/bind_test.go +++ b/bindinfo/bind_test.go @@ -159,6 +159,27 @@ func (s *testSuite) TestBindParse(c *C) { tk.MustExec(`create binding for (select * from t) union all (select * from t) using (select * from t use index(index_t)) union all (select * from t use index())`) tk.MustExec(`drop binding for (select * from t) union all (select * from t) using (select * from t use index(index_t)) union all (select * from t use index())`) + // Test Update / Delete. + tk.MustExec("create table t1(a int, b int, c int, key(b), key(c))") + tk.MustExec("create table t2(a int, b int, c int, key(b), key(c))") + tk.MustExec("create binding for delete from t1 where b = 1 and c > 1 using delete /*+ use_index(t1, c) */ from t1 where b = 1 and c > 1") + tk.MustExec("drop binding for delete from t1 where b = 1 and c > 1 using delete /*+ use_index(t1, c) */ from t1 where b = 1 and c > 1") + tk.MustExec("create binding for delete t1, t2 from t1 inner join t2 on t1.b = t2.b where t1.c = 1 using delete /*+ hash_join(t1, t2), use_index(t1, c) */ t1, t2 from t1 inner join t2 on t1.b = t2.b where t1.c = 1") + tk.MustExec("drop binding for delete t1, t2 from t1 inner join t2 on t1.b = t2.b where t1.c = 1 using delete /*+ hash_join(t1, t2), use_index(t1, c) */ t1, t2 from t1 inner join t2 on t1.b = t2.b where t1.c = 1") + tk.MustExec("create binding for update t1 set a = 1 where b = 1 and c > 1 using update /*+ use_index(t1, c) */ t1 set a = 1 where b = 1 and c > 1") + tk.MustExec("drop binding for update t1 set a = 1 where b = 1 and c > 1 using update /*+ use_index(t1, c) */ t1 set a = 1 where b = 1 and c > 1") + tk.MustExec("create binding for update t1, t2 set t1.a = 1 where t1.b = t2.b using update /*+ inl_join(t1) */ t1, t2 set t1.a = 1 where t1.b = t2.b") + tk.MustExec("drop binding for update t1, t2 set t1.a = 1 where t1.b = t2.b using update /*+ inl_join(t1) */ t1, t2 set t1.a = 1 where t1.b = t2.b") + // Test Insert / Replace. + tk.MustExec("create binding for insert into t1 select * from t2 where t2.b = 1 and t2.c > 1 using insert into t1 select /*+ use_index(t2,c) */ * from t2 where t2.b = 1 and t2.c > 1") + tk.MustExec("drop binding for insert into t1 select * from t2 where t2.b = 1 and t2.c > 1 using insert into t1 select /*+ use_index(t2,c) */ * from t2 where t2.b = 1 and t2.c > 1") + tk.MustExec("create binding for replace into t1 select * from t2 where t2.b = 1 and t2.c > 1 using replace into t1 select /*+ use_index(t2,c) */ * from t2 where t2.b = 1 and t2.c > 1") + tk.MustExec("drop binding for replace into t1 select * from t2 where t2.b = 1 and t2.c > 1 using replace into t1 select /*+ use_index(t2,c) */ * from t2 where t2.b = 1 and t2.c > 1") + err = tk.ExecToErr("create binding for insert into t1 values(1,1,1) using insert into t1 values(1,1,1)") + c.Assert(err.Error(), Equals, "create binding only supports INSERT / REPLACE INTO SELECT") + err = tk.ExecToErr("create binding for replace into t1 values(1,1,1) using replace into t1 values(1,1,1)") + c.Assert(err.Error(), Equals, "create binding only supports INSERT / REPLACE INTO SELECT") + // Test errors. tk.MustExec(`drop table if exists t1`) tk.MustExec("create table t1(i int, s varchar(20))") @@ -220,6 +241,60 @@ var testSQLs = []struct { dropSQL: "binding for select * from t except select * from t", memoryUsage: float64(132), }, + { + createSQL: "binding for delete from t where i = 1 using delete /*+ use_index(t,index_t) */ from t where i = 1", + overlaySQL: "", + querySQL: "delete from t where i = 2", + originSQL: "delete from t where i = ?", + bindSQL: "delete /*+ use_index(t,index_t) */ from t where i = 1", + dropSQL: "binding for delete from t where i = 1", + memoryUsage: float64(103), + }, + { + createSQL: "binding for delete t, t1 from t inner join t1 on t.s = t1.s where t.i = 1 using delete /*+ use_index(t,index_t), hash_join(t,t1) */ t, t1 from t inner join t1 on t.s = t1.s where t.i = 1", + overlaySQL: "", + querySQL: "delete t, t1 from t inner join t1 on t.s = t1.s where t.i = 2", + originSQL: "delete t , t1 from t inner join t1 on t . s = t1 . s where t . i = ?", + bindSQL: "delete /*+ use_index(t,index_t), hash_join(t,t1) */ t, t1 from t inner join t1 on t.s = t1.s where t.i = 1", + dropSQL: "binding for delete t, t1 from t inner join t1 on t.s = t1.s where t.i = 1", + memoryUsage: float64(199), + }, + { + createSQL: "binding for update t set s = 'a' where i = 1 using update /*+ use_index(t,index_t) */ t set s = 'a' where i = 1", + overlaySQL: "", + querySQL: "update t set s='b' where i=2", + originSQL: "update t set s = ? where i = ?", + bindSQL: "update /*+ use_index(t,index_t) */ t set s = 'a' where i = 1", + dropSQL: "binding for update t set s = 'a' where i = 1", + memoryUsage: float64(115), + }, + { + createSQL: "binding for update t, t1 set t.s = 'a' where t.i = t1.i using update /*+ inl_join(t1) */ t, t1 set t.s = 'a' where t.i = t1.i", + overlaySQL: "", + querySQL: "update t , t1 set t.s='b' where t.i=t1.i", + originSQL: "update t , t1 set t . s = ? where t . i = t1 . i", + bindSQL: "update /*+ inl_join(t1) */ t, t1 set t.s = 'a' where t.i = t1.i", + dropSQL: "binding for update t, t1 set t.s = 'a' where t.i = t1.i", + memoryUsage: float64(136), + }, + { + createSQL: "binding for insert into t1 select * from t where t.i = 1 using insert into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", + overlaySQL: "", + querySQL: "insert into t1 select * from t where t.i = 2", + originSQL: "insert into t1 select * from t where t . i = ?", + bindSQL: "insert into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", + dropSQL: "binding for insert into t1 select * from t where t.i = 1", + memoryUsage: float64(143), + }, + { + createSQL: "binding for replace into t1 select * from t where t.i = 1 using replace into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", + overlaySQL: "", + querySQL: "replace into t1 select * from t where t.i = 2", + originSQL: "replace into t1 select * from t where t . i = ?", + bindSQL: "replace into t1 select /*+ use_index(t,index_t) */ * from t where t.i = 1", + dropSQL: "binding for replace into t1 select * from t where t.i = 1", + memoryUsage: float64(145), + }, } func (s *testSuite) TestGlobalBinding(c *C) { @@ -523,6 +598,60 @@ func (s *testSuite) TestBindingSymbolList(c *C) { c.Check(bind.UpdateTime, NotNil) } +func (s *testSuite) TestDMLSQLBind(c *C) { + tk := testkit.NewTestKit(c, s.store) + s.cleanBindingEnv(tk) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t1(a int, b int, c int, key idx_b(b), key idx_c(c))") + tk.MustExec("create table t2(a int, b int, c int, key idx_b(b), key idx_c(c))") + + tk.MustExec("delete from t1 where b = 1 and c > 1") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t1:idx_b") + c.Assert(tk.MustUseIndex("delete from t1 where b = 1 and c > 1", "idx_b(b)"), IsTrue) + tk.MustExec("create global binding for delete from t1 where b = 1 and c > 1 using delete /*+ use_index(t1,idx_c) */ from t1 where b = 1 and c > 1") + tk.MustExec("delete from t1 where b = 1 and c > 1") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t1:idx_c") + c.Assert(tk.MustUseIndex("delete from t1 where b = 1 and c > 1", "idx_c(c)"), IsTrue) + + c.Assert(tk.HasPlan("delete t1, t2 from t1 inner join t2 on t1.b = t2.b", "HashJoin"), IsTrue) + tk.MustExec("create global binding for delete t1, t2 from t1 inner join t2 on t1.b = t2.b using delete /*+ inl_join(t1) */ t1, t2 from t1 inner join t2 on t1.b = t2.b") + c.Assert(tk.HasPlan("delete t1, t2 from t1 inner join t2 on t1.b = t2.b", "IndexJoin"), IsTrue) + + tk.MustExec("update t1 set a = 1 where b = 1 and c > 1") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t1:idx_b") + c.Assert(tk.MustUseIndex("update t1 set a = 1 where b = 1 and c > 1", "idx_b(b)"), IsTrue) + tk.MustExec("create global binding for update t1 set a = 1 where b = 1 and c > 1 using update /*+ use_index(t1,idx_c) */ t1 set a = 1 where b = 1 and c > 1") + tk.MustExec("delete from t1 where b = 1 and c > 1") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t1:idx_c") + c.Assert(tk.MustUseIndex("update t1 set a = 1 where b = 1 and c > 1", "idx_c(c)"), IsTrue) + + c.Assert(tk.HasPlan("update t1, t2 set t1.a = 1 where t1.b = t2.b", "HashJoin"), IsTrue) + tk.MustExec("create global binding for update t1, t2 set t1.a = 1 where t1.b = t2.b using update /*+ inl_join(t1) */ t1, t2 set t1.a = 1 where t1.b = t2.b") + c.Assert(tk.HasPlan("update t1, t2 set t1.a = 1 where t1.b = t2.b", "IndexJoin"), IsTrue) + + tk.MustExec("insert into t1 select * from t2 where t2.b = 2 and t2.c > 2") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t2:idx_b") + c.Assert(tk.MustUseIndex("insert into t1 select * from t2 where t2.b = 2 and t2.c > 2", "idx_b(b)"), IsTrue) + tk.MustExec("create global binding for insert into t1 select * from t2 where t2.b = 1 and t2.c > 1 using insert /*+ use_index(t2,idx_c) */ into t1 select * from t2 where t2.b = 1 and t2.c > 1") + tk.MustExec("insert into t1 select * from t2 where t2.b = 2 and t2.c > 2") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t2:idx_b") + c.Assert(tk.MustUseIndex("insert into t1 select * from t2 where t2.b = 2 and t2.c > 2", "idx_b(b)"), IsTrue) + tk.MustExec("drop global binding for insert into t1 select * from t2 where t2.b = 1 and t2.c > 1") + tk.MustExec("create global binding for insert into t1 select * from t2 where t2.b = 1 and t2.c > 1 using insert into t1 select /*+ use_index(t2,idx_c) */ * from t2 where t2.b = 1 and t2.c > 1") + tk.MustExec("insert into t1 select * from t2 where t2.b = 2 and t2.c > 2") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t2:idx_c") + c.Assert(tk.MustUseIndex("insert into t1 select * from t2 where t2.b = 2 and t2.c > 2", "idx_c(c)"), IsTrue) + + tk.MustExec("replace into t1 select * from t2 where t2.b = 2 and t2.c > 2") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t2:idx_b") + c.Assert(tk.MustUseIndex("replace into t1 select * from t2 where t2.b = 2 and t2.c > 2", "idx_b(b)"), IsTrue) + tk.MustExec("create global binding for replace into t1 select * from t2 where t2.b = 1 and t2.c > 1 using replace into t1 select /*+ use_index(t2,idx_c) */ * from t2 where t2.b = 1 and t2.c > 1") + tk.MustExec("replace into t1 select * from t2 where t2.b = 2 and t2.c > 2") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t2:idx_c") + c.Assert(tk.MustUseIndex("replace into t1 select * from t2 where t2.b = 2 and t2.c > 2", "idx_c(c)"), IsTrue) +} + func (s *testSuite) TestBestPlanInBaselines(c *C) { tk := testkit.NewTestKit(c, s.store) s.cleanBindingEnv(tk) @@ -627,6 +756,104 @@ func (s *testSuite) TestPreparedStmt(c *C) { tk.MustExec("drop binding for select * from t") tk.MustExec("execute stmt1") c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 0) + + tk.MustExec("drop table t") + tk.MustExec("create table t(a int, b int, c int, index idx_b(b), index idx_c(c))") + tk.MustExec("set @p = 1") + + tk.MustExec("prepare stmt from 'delete from t where b = ? and c > ?'") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_b") + tk.MustExec("create binding for delete from t where b = 2 and c > 2 using delete /*+ use_index(t,idx_c) */ from t where b = 2 and c > 2") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") + + tk.MustExec("prepare stmt from 'update t set a = 1 where b = ? and c > ?'") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_b") + tk.MustExec("create binding for update t set a = 2 where b = 2 and c > 2 using update /*+ use_index(t,idx_c) */ t set a = 2 where b = 2 and c > 2") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 like t") + tk.MustExec("prepare stmt from 'insert into t1 select * from t where t.b = ? and t.c > ?'") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_b") + tk.MustExec("create binding for insert into t1 select * from t where t.b = 2 and t.c > 2 using insert into t1 select /*+ use_index(t,idx_c) */ * from t where t.b = 2 and t.c > 2") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") + + tk.MustExec("prepare stmt from 'replace into t1 select * from t where t.b = ? and t.c > ?'") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_b") + tk.MustExec("create binding for replace into t1 select * from t where t.b = 2 and t.c > 2 using replace into t1 select /*+ use_index(t,idx_c) */ * from t where t.b = 2 and t.c > 2") + tk.MustExec("execute stmt using @p,@p") + c.Assert(len(tk.Se.GetSessionVars().StmtCtx.IndexNames), Equals, 1) + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") +} + +func (s *testSuite) TestDMLCapturePlanBaseline(c *C) { + tk := testkit.NewTestKit(c, s.store) + s.cleanBindingEnv(tk) + stmtsummary.StmtSummaryByDigestMap.Clear() + tk.MustExec(" set @@tidb_capture_plan_baselines = on") + defer func() { + tk.MustExec(" set @@tidb_capture_plan_baselines = off") + }() + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int, c int, key idx_b(b), key idx_c(c))") + tk.MustExec("create table t1 like t") + s.domain.BindHandle().CaptureBaselines() + tk.MustQuery("show global bindings").Check(testkit.Rows()) + tk.MustExec("delete from t where b = 1 and c > 1") + tk.MustExec("delete from t where b = 1 and c > 1") + tk.MustExec("update t set a = 1 where b = 1 and c > 1") + tk.MustExec("update t set a = 1 where b = 1 and c > 1") + tk.MustExec("insert into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("insert into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("replace into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("replace into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("insert into t1 values(1,1,1)") + tk.MustExec("insert into t1 values(1,1,1)") + tk.MustExec("replace into t1 values(1,1,1)") + tk.MustExec("replace into t1 values(1,1,1)") + tk.MustExec("admin capture bindings") + rows := tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 0) + + c.Assert(tk.Se.Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil), IsTrue) + tk.MustExec("delete from t where b = 1 and c > 1") + tk.MustExec("delete from t where b = 1 and c > 1") + tk.MustExec("update t set a = 1 where b = 1 and c > 1") + tk.MustExec("update t set a = 1 where b = 1 and c > 1") + tk.MustExec("insert into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("insert into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("replace into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("replace into t1 select * from t where t.b = 1 and t.c > 1") + tk.MustExec("insert into t1 values(1,1,1)") + tk.MustExec("insert into t1 values(1,1,1)") + tk.MustExec("replace into t1 values(1,1,1)") + tk.MustExec("replace into t1 values(1,1,1)") + tk.MustExec("admin capture bindings") + rows = tk.MustQuery("show global bindings").Sort().Rows() + c.Assert(len(rows), Equals, 4) + c.Assert(rows[0][0], Equals, "delete from t where b = ? and c > ?") + c.Assert(rows[0][1], Equals, "DELETE /*+ use_index(@`del_1` `test`.`t` `idx_b`)*/ FROM `t` WHERE `b`=1 AND `c`>1") + c.Assert(rows[1][0], Equals, "insert into t1 select * from t where t . b = ? and t . c > ?") + c.Assert(rows[1][1], Equals, "INSERT INTO `t1` SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_b`)*/ * FROM `t` WHERE `t`.`b`=1 AND `t`.`c`>1") + c.Assert(rows[2][0], Equals, "replace into t1 select * from t where t . b = ? and t . c > ?") + c.Assert(rows[2][1], Equals, "REPLACE INTO `t1` SELECT /*+ use_index(@`sel_1` `test`.`t` `idx_b`)*/ * FROM `t` WHERE `t`.`b`=1 AND `t`.`c`>1") + c.Assert(rows[3][0], Equals, "update t set a = ? where b = ? and c > ?") + c.Assert(rows[3][1], Equals, "UPDATE /*+ use_index(@`upd_1` `test`.`t` `idx_b`)*/ `t` SET `a`=1 WHERE `b`=1 AND `c`>1") } func (s *testSuite) TestCapturePlanBaseline(c *C) { @@ -638,7 +865,7 @@ func (s *testSuite) TestCapturePlanBaseline(c *C) { tk.MustExec(" set @@tidb_capture_plan_baselines = off") }() tk.MustExec("use test") - tk.MustExec("drop table if exists t, t1") + tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") s.domain.BindHandle().CaptureBaselines() tk.MustQuery("show global bindings").Check(testkit.Rows()) @@ -728,6 +955,66 @@ func (s *testSuite) TestDropSingleBindings(c *C) { c.Assert(len(rows), Equals, 0) } +func (s *testSuite) TestDMLEvolveBaselines(c *C) { + tk := testkit.NewTestKit(c, s.store) + s.cleanBindingEnv(tk) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int, c int, index idx_b(b), index idx_c(c))") + tk.MustExec("insert into t values (1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5)") + tk.MustExec("analyze table t") + tk.MustExec("set @@tidb_evolve_plan_baselines=1") + + tk.MustExec("create global binding for delete from t where b = 1 and c > 1 using delete /*+ use_index(t,idx_c) */ from t where b = 1 and c > 1") + rows := tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 1) + tk.MustExec("delete /*+ use_index(t,idx_b) */ from t where b = 2 and c > 1") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") + tk.MustExec("admin flush bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 1) + tk.MustExec("admin evolve bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 1) + + tk.MustExec("create global binding for update t set a = 1 where b = 1 and c > 1 using update /*+ use_index(t,idx_c) */ t set a = 1 where b = 1 and c > 1") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 2) + tk.MustExec("update /*+ use_index(t,idx_b) */ t set a = 2 where b = 2 and c > 1") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") + tk.MustExec("admin flush bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 2) + tk.MustExec("admin evolve bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 2) + + tk.MustExec("create table t1 like t") + tk.MustExec("create global binding for insert into t1 select * from t where t.b = 1 and t.c > 1 using insert into t1 select /*+ use_index(t,idx_c) */ * from t where t.b = 1 and t.c > 1") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 3) + tk.MustExec("insert into t1 select /*+ use_index(t,idx_b) */ * from t where t.b = 2 and t.c > 2") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") + tk.MustExec("admin flush bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 3) + tk.MustExec("admin evolve bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 3) + + tk.MustExec("create global binding for replace into t1 select * from t where t.b = 1 and t.c > 1 using replace into t1 select /*+ use_index(t,idx_c) */ * from t where t.b = 1 and t.c > 1") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 4) + tk.MustExec("replace into t1 select /*+ use_index(t,idx_b) */ * from t where t.b = 2 and t.c > 2") + c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_c") + tk.MustExec("admin flush bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 4) + tk.MustExec("admin evolve bindings") + rows = tk.MustQuery("show global bindings").Rows() + c.Assert(len(rows), Equals, 4) +} + func (s *testSuite) TestAddEvolveTasks(c *C) { tk := testkit.NewTestKit(c, s.store) s.cleanBindingEnv(tk) @@ -1267,7 +1554,7 @@ func (s *testSuite) TestInvisibleIndex(c *C) { tk.MustExec("drop binding for select * from t") } -func (s *testSuite) TestbindingSource(c *C) { +func (s *testSuite) TestBindingSource(c *C) { tk := testkit.NewTestKit(c, s.store) s.cleanBindingEnv(tk) tk.MustExec("use test") diff --git a/bindinfo/handle.go b/bindinfo/handle.go index 03757e3da113d..3cc58be349e4b 100644 --- a/bindinfo/handle.go +++ b/bindinfo/handle.go @@ -596,16 +596,19 @@ func (h *BindHandle) logicalDeleteBindInfoSQL(originalSQL, db string, updateTs t // CaptureBaselines is used to automatically capture plan baselines. func (h *BindHandle) CaptureBaselines() { parser4Capture := parser.New() - schemas, sqls := stmtsummary.StmtSummaryByDigestMap.GetMoreThanOnceSelect() + schemas, sqls := stmtsummary.StmtSummaryByDigestMap.GetMoreThanOnceBindableStmt() for i := range sqls { stmt, err := parser4Capture.ParseOneStmt(sqls[i], "", "") if err != nil { logutil.BgLogger().Debug("parse SQL failed", zap.String("SQL", sqls[i]), zap.Error(err)) continue } - normalizedSQL, digiest := parser.NormalizeDigest(sqls[i]) + if insertStmt, ok := stmt.(*ast.InsertStmt); ok && insertStmt.Select == nil { + continue + } + normalizedSQL, digest := parser.NormalizeDigest(sqls[i]) dbName := utilparser.GetDefaultDB(stmt, schemas[i]) - if r := h.GetBindRecord(digiest, normalizedSQL, dbName); r != nil && r.HasUsingBinding() { + if r := h.GetBindRecord(digest, normalizedSQL, dbName); r != nil && r.HasUsingBinding() { continue } h.sctx.Lock() @@ -682,10 +685,35 @@ func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string logutil.Logger(ctx).Warn("Restore SQL failed", zap.Error(err)) } bindSQL := sb.String() - selectIdx := strings.Index(bindSQL, "SELECT") - // Remove possible `explain` prefix. - bindSQL = bindSQL[selectIdx:] - return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) + switch n := stmtNode.(type) { + case *ast.DeleteStmt: + deleteIdx := strings.Index(bindSQL, "DELETE") + // Remove possible `explain` prefix. + bindSQL = bindSQL[deleteIdx:] + return strings.Replace(bindSQL, "DELETE", fmt.Sprintf("DELETE /*+ %s*/", planHint), 1) + case *ast.UpdateStmt: + updateIdx := strings.Index(bindSQL, "UPDATE") + // Remove possible `explain` prefix. + bindSQL = bindSQL[updateIdx:] + return strings.Replace(bindSQL, "UPDATE", fmt.Sprintf("UPDATE /*+ %s*/", planHint), 1) + case *ast.SelectStmt: + selectIdx := strings.Index(bindSQL, "SELECT") + // Remove possible `explain` prefix. + bindSQL = bindSQL[selectIdx:] + return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) + case *ast.InsertStmt: + insertIdx := int(0) + if n.IsReplace { + insertIdx = strings.Index(bindSQL, "REPLACE") + } else { + insertIdx = strings.Index(bindSQL, "INSERT") + } + // Remove possible `explain` prefix. + bindSQL = bindSQL[insertIdx:] + return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1) + } + logutil.Logger(ctx).Warn("Unexpected statement type") + return "" } type paramMarkerChecker struct { diff --git a/executor/compiler.go b/executor/compiler.go index 49ae9e3e844da..0ae89ed95b785 100644 --- a/executor/compiler.go +++ b/executor/compiler.go @@ -218,17 +218,36 @@ func getStmtDbLabel(stmtNode ast.StmtNode) map[string]struct{} { } } case *ast.CreateBindingStmt: + var resNode ast.ResultSetNode if x.OriginNode != nil { - originSelect := x.OriginNode.(*ast.SelectStmt) - dbLabels := getDbFromResultNode(originSelect.From.TableRefs) + switch n := x.OriginNode.(type) { + case *ast.SelectStmt: + resNode = n.From.TableRefs + case *ast.DeleteStmt: + resNode = n.TableRefs.TableRefs + case *ast.UpdateStmt: + resNode = n.TableRefs.TableRefs + case *ast.InsertStmt: + resNode = n.Table.TableRefs + } + dbLabels := getDbFromResultNode(resNode) for _, db := range dbLabels { dbLabelSet[db] = struct{}{} } } if len(dbLabelSet) == 0 && x.HintedNode != nil { - hintedSelect := x.HintedNode.(*ast.SelectStmt) - dbLabels := getDbFromResultNode(hintedSelect.From.TableRefs) + switch n := x.HintedNode.(type) { + case *ast.SelectStmt: + resNode = n.From.TableRefs + case *ast.DeleteStmt: + resNode = n.TableRefs.TableRefs + case *ast.UpdateStmt: + resNode = n.TableRefs.TableRefs + case *ast.InsertStmt: + resNode = n.Table.TableRefs + } + dbLabels := getDbFromResultNode(resNode) for _, db := range dbLabels { dbLabelSet[db] = struct{}{} } diff --git a/planner/core/hints.go b/planner/core/hints.go index dc7dc370f8dc3..e9c3e32d8c5ac 100644 --- a/planner/core/hints.go +++ b/planner/core/hints.go @@ -31,6 +31,9 @@ func GenHintsFromPhysicalPlan(p Plan) []*ast.TableOptimizerHint { hints = genHintsFromPhysicalPlan(pp.SelectPlan, utilhint.TypeUpdate) case *Delete: hints = genHintsFromPhysicalPlan(pp.SelectPlan, utilhint.TypeDelete) + // For Insert, we only generate hints that would be used in select query block. + case *Insert: + hints = genHintsFromPhysicalPlan(pp.SelectPlan, utilhint.TypeSelect) case PhysicalPlan: hints = genHintsFromPhysicalPlan(pp, utilhint.TypeSelect) } @@ -104,6 +107,9 @@ func getJoinHints(sctx sessionctx.Context, joinType string, parentOffset int, no } func genHintsFromPhysicalPlan(p PhysicalPlan, nodeType utilhint.NodeType) (res []*ast.TableOptimizerHint) { + if p == nil { + return res + } for _, child := range p.Children() { res = append(res, genHintsFromPhysicalPlan(child, nodeType)...) } diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index 11b44d83e038d..2bdec8654a4e9 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -209,18 +209,57 @@ func EraseLastSemicolon(stmt ast.StmtNode) { } } -func (p *preprocessor) checkBindGrammar(originNode, hintedNode ast.StmtNode) { - var originSQL, hintedSQL string - switch node := originNode.(type) { +const ( + // TypeInvalid for unexpected types. + TypeInvalid byte = iota + // TypeSelect for SelectStmt. + TypeSelect + // TypeSetOpr for SetOprStmt. + TypeSetOpr + // TypeDelete for DeleteStmt. + TypeDelete + // TypeUpdate for UpdateStmt. + TypeUpdate + // TypeInsert for InsertStmt. + TypeInsert +) + +func bindableStmtType(node ast.StmtNode) byte { + switch node.(type) { case *ast.SelectStmt: - originSQL = parser.Normalize(node.Text()) - hintedSQL = parser.Normalize(hintedNode.(*ast.SelectStmt).Text()) + return TypeSelect case *ast.SetOprStmt: - originSQL = parser.Normalize(node.Text()) - hintedSQL = parser.Normalize(hintedNode.(*ast.SetOprStmt).Text()) - default: + return TypeSetOpr + case *ast.DeleteStmt: + return TypeDelete + case *ast.UpdateStmt: + return TypeUpdate + case *ast.InsertStmt: + return TypeInsert + } + return TypeInvalid +} + +func (p *preprocessor) checkBindGrammar(originNode, hintedNode ast.StmtNode) { + origTp := bindableStmtType(originNode) + hintedTp := bindableStmtType(hintedNode) + if origTp == TypeInvalid || hintedTp == TypeInvalid { p.err = errors.Errorf("create binding doesn't support this type of query") + return + } + if origTp != hintedTp { + p.err = errors.Errorf("hinted sql and original sql have different query types") + return + } + if origTp == TypeInsert { + origInsert, hintedInsert := originNode.(*ast.InsertStmt), hintedNode.(*ast.InsertStmt) + if origInsert.Select == nil || hintedInsert.Select == nil { + p.err = errors.Errorf("create binding only supports INSERT / REPLACE INTO SELECT") + return + } } + originSQL := parser.Normalize(originNode.Text()) + hintedSQL := parser.Normalize(hintedNode.Text()) if originSQL != hintedSQL { p.err = errors.Errorf("hinted sql and origin sql don't match when hinted sql erase the hint info, after erase hint info, originSQL:%s, hintedSQL:%s", originSQL, hintedSQL) } diff --git a/planner/core/preprocess_test.go b/planner/core/preprocess_test.go index 4f6d90422a391..c453fc5d012bb 100644 --- a/planner/core/preprocess_test.go +++ b/planner/core/preprocess_test.go @@ -58,7 +58,7 @@ func (s *testValidatorSuite) SetUpTest(c *C) { func (s *testValidatorSuite) runSQL(c *C, sql string, inPrepare bool, terr error) { stmts, err1 := session.Parse(s.ctx, sql) - c.Assert(err1, IsNil) + c.Assert(err1, IsNil, Commentf("sql: %s", sql)) c.Assert(stmts, HasLen, 1) stmt := stmts[0] var opts []core.PreprocessOpt diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 074fcc6ba52ee..5a60d84223e14 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -712,7 +712,7 @@ { "SQL": "insert into t select * from t where b < 1 order by d limit 1", "Best": "TableReader(Table(t)->Sel([lt(test.t.b, 1)])->TopN([test.t.d],0,1))->TopN([test.t.d],0,1)->Insert", - "Hints": "" + "Hints": "use_index(@`sel_1` `test`.`t` )" }, { "SQL": "insert into t (a, b, c, e, f, g) values(0,0,0,0,0,0)", diff --git a/planner/optimize.go b/planner/optimize.go index 312c319c6a1f1..c435033817154 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -193,10 +193,12 @@ func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is in bestPlanAmongHints = plan } } - // 1. If there is already a evolution task, we do not need to handle it again. - // 2. If the origin binding contain `read_from_storage` hint, we should ignore the evolve task. - // 3. If the best plan contain TiFlash hint, we should ignore the evolve task. - if sctx.GetSessionVars().EvolvePlanBaselines && binding == nil && + // 1. If it is a select query. + // 2. If there is already a evolution task, we do not need to handle it again. + // 3. If the origin binding contain `read_from_storage` hint, we should ignore the evolve task. + // 4. If the best plan contain TiFlash hint, we should ignore the evolve task. + if _, ok := stmtNode.(*ast.SelectStmt); ok && + sctx.GetSessionVars().EvolvePlanBaselines && binding == nil && !originHints.ContainTableHint(plannercore.HintReadFromStorage) && !bindRecord.Bindings[0].Hint.ContainTableHint(plannercore.HintReadFromStorage) { handleEvolveTasks(ctx, sctx, bindRecord, stmtNode, bestPlanHintStr) @@ -271,10 +273,24 @@ func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode) (ast.StmtNode, strin switch x := stmtNode.(type) { case *ast.ExplainStmt: switch x.Stmt.(type) { - case *ast.SelectStmt: + case *ast.SelectStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt: plannercore.EraseLastSemicolon(x) normalizeExplainSQL := parser.Normalize(x.Text()) - idx := strings.Index(normalizeExplainSQL, "select") + idx := int(0) + switch n := x.Stmt.(type) { + case *ast.SelectStmt: + idx = strings.Index(normalizeExplainSQL, "select") + case *ast.DeleteStmt: + idx = strings.Index(normalizeExplainSQL, "delete") + case *ast.UpdateStmt: + idx = strings.Index(normalizeExplainSQL, "update") + case *ast.InsertStmt: + if n.IsReplace { + idx = strings.Index(normalizeExplainSQL, "replace") + } else { + idx = strings.Index(normalizeExplainSQL, "insert") + } + } normalizeSQL := normalizeExplainSQL[idx:] hash := parser.DigestNormalized(normalizeSQL) return x.Stmt, normalizeSQL, hash @@ -290,7 +306,7 @@ func extractSelectAndNormalizeDigest(stmtNode ast.StmtNode) (ast.StmtNode, strin hash := parser.DigestNormalized(normalizeSQL) return x.Stmt, normalizeSQL, hash } - case *ast.SelectStmt, *ast.SetOprStmt: + case *ast.SelectStmt, *ast.SetOprStmt, *ast.DeleteStmt, *ast.UpdateStmt, *ast.InsertStmt: plannercore.EraseLastSemicolon(x) normalizedSQL, hash := parser.NormalizeDigest(x.Text()) return x, normalizedSQL, hash diff --git a/session/session_test.go b/session/session_test.go index 827bc7d45230c..0d547b39f018e 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -1780,12 +1780,12 @@ func (s *testSessionSuite3) TestUnique(c *C) { c.Assert(err, NotNil) // Check error type and error message c.Assert(terror.ErrorEqual(err, kv.ErrKeyExists), IsTrue, Commentf("err %v", err)) - c.Assert(err.Error(), Equals, "previous statement: insert into test(id, val) values(1, 1);: [kv:1062]Duplicate entry '1' for key 'PRIMARY'") + c.Assert(err.Error(), Equals, "previous statement: insert into test(id, val) values(1, 1): [kv:1062]Duplicate entry '1' for key 'PRIMARY'") _, err = tk1.Exec("commit") c.Assert(err, NotNil) c.Assert(terror.ErrorEqual(err, kv.ErrKeyExists), IsTrue, Commentf("err %v", err)) - c.Assert(err.Error(), Equals, "previous statement: insert into test(id, val) values(2, 2);: [kv:1062]Duplicate entry '2' for key 'val'") + c.Assert(err.Error(), Equals, "previous statement: insert into test(id, val) values(2, 2): [kv:1062]Duplicate entry '2' for key 'val'") // Test for https://github.com/pingcap/tidb/issues/463 tk.MustExec("drop table test;") diff --git a/util/hint/hint_processor.go b/util/hint/hint_processor.go index 86804acee6ab1..b0a1854dbd0a0 100644 --- a/util/hint/hint_processor.go +++ b/util/hint/hint_processor.go @@ -62,6 +62,18 @@ func (hs *HintsSet) ContainTableHint(hint string) bool { return false } +// setTableHints4StmtNode sets table hints for select/update/delete. +func setTableHints4StmtNode(node ast.Node, hints []*ast.TableOptimizerHint) { + switch x := node.(type) { + case *ast.SelectStmt: + x.TableHints = hints + case *ast.UpdateStmt: + x.TableHints = hints + case *ast.DeleteStmt: + x.TableHints = hints + } +} + // ExtractTableHintsFromStmtNode extracts table hints from this node. func ExtractTableHintsFromStmtNode(node ast.Node, sctx sessionctx.Context) []*ast.TableOptimizerHint { switch x := node.(type) { @@ -177,25 +189,32 @@ func (hs *HintsSet) Restore() (string, error) { type hintProcessor struct { *HintsSet // bindHint2Ast indicates the behavior of the processor, `true` for bind hint to ast, `false` for extract hint from ast. - bindHint2Ast bool - tableCounter int - indexCounter int + bindHint2Ast bool + tableCounter int + indexCounter int + selectCounter int } func (hp *hintProcessor) Enter(in ast.Node) (ast.Node, bool) { switch v := in.(type) { - case *ast.SelectStmt: + case *ast.SelectStmt, *ast.UpdateStmt, *ast.DeleteStmt: if hp.bindHint2Ast { if hp.tableCounter < len(hp.tableHints) { - v.TableHints = hp.tableHints[hp.tableCounter] + setTableHints4StmtNode(in, hp.tableHints[hp.tableCounter]) } else { - v.TableHints = nil + setTableHints4StmtNode(in, nil) } hp.tableCounter++ } else { - hp.tableHints = append(hp.tableHints, v.TableHints) + hp.tableHints = append(hp.tableHints, ExtractTableHintsFromStmtNode(in, nil)) + } + if _, ok := in.(*ast.SelectStmt); ok { + hp.selectCounter++ } case *ast.TableName: + if hp.selectCounter == 0 { + return in, false + } if hp.bindHint2Ast { if hp.indexCounter < len(hp.indexHints) { v.IndexHints = hp.indexHints[hp.indexCounter] @@ -211,6 +230,9 @@ func (hp *hintProcessor) Enter(in ast.Node) (ast.Node, bool) { } func (hp *hintProcessor) Leave(in ast.Node) (ast.Node, bool) { + if _, ok := in.(*ast.SelectStmt); ok { + hp.selectCounter-- + } return in, true } @@ -240,18 +262,19 @@ func ParseHintsSet(p *parser.Parser, sql, charset, collation, db string) (*Hints hs := CollectHint(stmtNodes[0]) processor := &BlockHintProcessor{} stmtNodes[0].Accept(processor) + hintNodeType := nodeType4Stmt(stmtNodes[0]) for i, tblHints := range hs.tableHints { newHints := make([]*ast.TableOptimizerHint, 0, len(tblHints)) for _, tblHint := range tblHints { if tblHint.HintName.L == hintQBName { continue } - offset := processor.GetHintOffset(tblHint.QBName, TypeSelect, i+1) - if offset < 0 || !processor.checkTableQBName(tblHint.Tables, TypeSelect) { + offset := processor.GetHintOffset(tblHint.QBName, hintNodeType, i+1) + if offset < 0 || !processor.checkTableQBName(tblHint.Tables, hintNodeType) { hintStr := RestoreTableOptimizerHint(tblHint) return nil, nil, errors.New(fmt.Sprintf("Unknown query block name in hint %s", hintStr)) } - tblHint.QBName = GenerateQBName(TypeSelect, offset) + tblHint.QBName = GenerateQBName(hintNodeType, offset) for i, tbl := range tblHint.Tables { if tbl.DBName.String() == "" { tblHint.Tables[i].DBName = model.NewCIStr(db) @@ -360,8 +383,24 @@ const ( TypeDelete // TypeSelect for SELECT. TypeSelect + // TypeInvalid for unexpected statements. + TypeInvalid ) +// nodeType4Stmt returns the NodeType for a statement. The type is used for SQL bind. +func nodeType4Stmt(node ast.StmtNode) NodeType { + switch node.(type) { + // This type is used by SQL bind, we only handle SQL bind for INSERT INTO SELECT, so we treat InsertStmt as TypeSelect. + case *ast.SelectStmt, *ast.InsertStmt: + return TypeSelect + case *ast.UpdateStmt: + return TypeUpdate + case *ast.DeleteStmt: + return TypeDelete + } + return TypeInvalid +} + // getBlockName finds the offset of query block name. It use 0 as offset for top level update or delete, // -1 for invalid block name. func (p *BlockHintProcessor) getBlockOffset(blockName model.CIStr, nodeType NodeType) int { @@ -428,9 +467,9 @@ func (p *BlockHintProcessor) GetCurrentStmtHints(hints []*ast.TableOptimizerHint // GenerateQBName builds QBName from offset. func GenerateQBName(nodeType NodeType, blockOffset int) model.CIStr { - if nodeType == TypeDelete && blockOffset == 0 { + if nodeType == TypeDelete && (blockOffset == 0 || blockOffset == 1) { return model.NewCIStr(defaultDeleteBlockName) - } else if nodeType == TypeUpdate && blockOffset == 0 { + } else if nodeType == TypeUpdate && (blockOffset == 0 || blockOffset == 1) { return model.NewCIStr(defaultUpdateBlockName) } return model.NewCIStr(fmt.Sprintf("%s%d", defaultSelectBlockPrefix, blockOffset)) diff --git a/util/stmtsummary/statement_summary.go b/util/stmtsummary/statement_summary.go index b7f600c1fd99e..1e73359f6048e 100644 --- a/util/stmtsummary/statement_summary.go +++ b/util/stmtsummary/statement_summary.go @@ -348,8 +348,8 @@ func (ssMap *stmtSummaryByDigestMap) ToHistoryDatum(user *auth.UserIdentity, isS return rows } -// GetMoreThanOnceSelect gets users' select SQLs that occurred more than once. -func (ssMap *stmtSummaryByDigestMap) GetMoreThanOnceSelect() ([]string, []string) { +// GetMoreThanOnceBindableStmt gets users' select/update/delete SQLs that occurred more than once. +func (ssMap *stmtSummaryByDigestMap) GetMoreThanOnceBindableStmt() ([]string, []string) { ssMap.Lock() values := ssMap.summaryMap.Values() ssMap.Unlock() @@ -361,7 +361,7 @@ func (ssMap *stmtSummaryByDigestMap) GetMoreThanOnceSelect() ([]string, []string func() { ssbd.Lock() defer ssbd.Unlock() - if ssbd.initialized && ssbd.stmtType == "Select" { + if ssbd.initialized && (ssbd.stmtType == "Select" || ssbd.stmtType == "Delete" || ssbd.stmtType == "Update" || ssbd.stmtType == "Insert" || ssbd.stmtType == "Replace") { if ssbd.history.Len() > 0 { ssElement := ssbd.history.Back().Value.(*stmtSummaryByDigestElement) ssElement.Lock() diff --git a/util/stmtsummary/statement_summary_test.go b/util/stmtsummary/statement_summary_test.go index 726683ed11dd2..c20b77bcbd072 100644 --- a/util/stmtsummary/statement_summary_test.go +++ b/util/stmtsummary/statement_summary_test.go @@ -912,8 +912,8 @@ func (s *testStmtSummarySuite) TestEnableSummaryParallel(c *C) { c.Assert(s.ssMap.Enabled(), IsTrue) } -// Test GetMoreThanOnceSelect. -func (s *testStmtSummarySuite) TestGetMoreThanOnceSelect(c *C) { +// Test GetMoreThanOnceBindableStmt. +func (s *testStmtSummarySuite) TestGetMoreThanOnceBindableStmt(c *C) { s.ssMap.Clear() stmtExecInfo1 := generateAnyExecInfo() @@ -921,7 +921,7 @@ func (s *testStmtSummarySuite) TestGetMoreThanOnceSelect(c *C) { stmtExecInfo1.NormalizedSQL = "insert ?" stmtExecInfo1.StmtCtx.StmtType = "Insert" s.ssMap.AddStatement(stmtExecInfo1) - schemas, sqls := s.ssMap.GetMoreThanOnceSelect() + schemas, sqls := s.ssMap.GetMoreThanOnceBindableStmt() c.Assert(len(schemas), Equals, 0) c.Assert(len(sqls), Equals, 0) @@ -929,12 +929,12 @@ func (s *testStmtSummarySuite) TestGetMoreThanOnceSelect(c *C) { stmtExecInfo1.Digest = "digest1" stmtExecInfo1.StmtCtx.StmtType = "Select" s.ssMap.AddStatement(stmtExecInfo1) - schemas, sqls = s.ssMap.GetMoreThanOnceSelect() + schemas, sqls = s.ssMap.GetMoreThanOnceBindableStmt() c.Assert(len(schemas), Equals, 0) c.Assert(len(sqls), Equals, 0) s.ssMap.AddStatement(stmtExecInfo1) - schemas, sqls = s.ssMap.GetMoreThanOnceSelect() + schemas, sqls = s.ssMap.GetMoreThanOnceBindableStmt() c.Assert(len(schemas), Equals, 1) c.Assert(len(sqls), Equals, 1) } From f1a0a28af46e0aa0bfb639881d6a3ba45effa965 Mon Sep 17 00:00:00 2001 From: HuaiyuXu <391585975@qq.com> Date: Tue, 17 Nov 2020 15:36:57 +0800 Subject: [PATCH 17/37] executor, planner: do not coalesce column for update/delete natural_using_join (#21085) --- executor/join_test.go | 5 +++++ planner/core/logical_plan_builder.go | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/executor/join_test.go b/executor/join_test.go index 402b6458f9034..c7b023b58d4b2 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -628,6 +628,11 @@ func (s *testSuiteJoin1) TestUsing(c *C) { tk.MustExec("insert into t1 (a) values(1)") tk.MustQuery("select t1.*, t2.* from t1 join t1 t2 using(a)").Check(testkit.Rows("1 1")) tk.MustQuery("select * from t1 join t1 t2 using(a)").Check(testkit.Rows("1")) + + // For issue18992 + tk.MustExec("drop table t") + tk.MustExec("CREATE TABLE t ( a varchar(55) NOT NULL, b varchar(55) NOT NULL, c int(11) DEFAULT NULL, d int(11) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;") + tk.MustExec("update t t1 join t t2 using(a,b) set t1.c=t2.d;") } func (s *testSuiteJoin1) TestNaturalJoin(c *C) { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index c9b51ac5bb67f..c9c8ebf1100bc 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -724,7 +724,16 @@ func (b *PlanBuilder) buildUsingClause(p *LogicalJoin, leftPlan, rightPlan Logic for _, col := range join.Using { filter[col.Name.L] = true } - return b.coalesceCommonColumns(p, leftPlan, rightPlan, join.Tp, filter) + err := b.coalesceCommonColumns(p, leftPlan, rightPlan, join.Tp, filter) + if err != nil { + return err + } + // We do not need to coalesce columns for update and delete. + if b.inUpdateStmt || b.inDeleteStmt { + p.setSchemaAndNames(expression.MergeSchema(p.Children()[0].Schema(), p.Children()[1].Schema()), + append(p.Children()[0].OutputNames(), p.Children()[1].OutputNames()...)) + } + return nil } // buildNaturalJoin builds natural join output schema. It finds out all the common columns @@ -734,7 +743,16 @@ func (b *PlanBuilder) buildUsingClause(p *LogicalJoin, leftPlan, rightPlan Logic // Every column in the first (left) table that is not a common column // Every column in the second (right) table that is not a common column func (b *PlanBuilder) buildNaturalJoin(p *LogicalJoin, leftPlan, rightPlan LogicalPlan, join *ast.Join) error { - return b.coalesceCommonColumns(p, leftPlan, rightPlan, join.Tp, nil) + err := b.coalesceCommonColumns(p, leftPlan, rightPlan, join.Tp, nil) + if err != nil { + return err + } + // We do not need to coalesce columns for update and delete. + if b.inUpdateStmt || b.inDeleteStmt { + p.setSchemaAndNames(expression.MergeSchema(p.Children()[0].Schema(), p.Children()[1].Schema()), + append(p.Children()[0].OutputNames(), p.Children()[1].OutputNames()...)) + } + return nil } // coalesceCommonColumns is used by buildUsingClause and buildNaturalJoin. The filter is used by buildUsingClause. From c23394e95488797b0a0175b531dc66ddf1a21962 Mon Sep 17 00:00:00 2001 From: xufei Date: Tue, 17 Nov 2020 16:05:58 +0800 Subject: [PATCH 18/37] test: migrate tiflash mock test from mocktikv to unistore (#21038) --- executor/tiflash_test.go | 9 +-- go.mod | 2 +- go.sum | 4 +- store/mockstore/cluster/cluster.go | 2 +- store/mockstore/mocktikv/cluster.go | 4 +- .../unistore/cophandler/closure_exec.go | 66 ++++++++++++++-- .../unistore/cophandler/cop_handler.go | 6 +- store/mockstore/unistore/rpc.go | 76 ++++++++++++++++++- store/tikv/batch_coprocessor_test.go | 19 +++-- 9 files changed, 159 insertions(+), 29 deletions(-) diff --git a/executor/tiflash_test.go b/executor/tiflash_test.go index fdc77266c3b8b..7a2184c295672 100644 --- a/executor/tiflash_test.go +++ b/executor/tiflash_test.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/mockstore/cluster" - "github.com/pingcap/tidb/store/mockstore/mocktikv" + "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/testkit" ) @@ -41,20 +41,19 @@ func (s *tiflashTestSuite) SetUpSuite(c *C) { var err error s.store, err = mockstore.NewMockStore( mockstore.WithClusterInspector(func(c cluster.Cluster) { - mockCluster := c.(*mocktikv.Cluster) + mockCluster := c.(*unistore.Cluster) _, _, region1 := mockstore.BootstrapWithSingleStore(c) tiflashIdx := 0 for tiflashIdx < 2 { store2 := c.AllocID() peer2 := c.AllocID() addr2 := fmt.Sprintf("tiflash%d", tiflashIdx) - mockCluster.AddStore(store2, addr2) - mockCluster.UpdateStoreAddr(store2, addr2, &metapb.StoreLabel{Key: "engine", Value: "tiflash"}) + mockCluster.AddStore(store2, addr2, &metapb.StoreLabel{Key: "engine", Value: "tiflash"}) mockCluster.AddPeer(region1, store2, peer2) tiflashIdx++ } }), - mockstore.WithStoreType(mockstore.MockTiKV), + mockstore.WithStoreType(mockstore.EmbedUnistore), ) c.Assert(err, IsNil) diff --git a/go.mod b/go.mod index 322918607a90a..0bc5aa87e1d8e 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/montanaflynn/stats v0.5.0 // indirect github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef - github.com/ngaut/unistore v0.0.0-20201026053041-4333e8d852f1 + github.com/ngaut/unistore v0.0.0-20201113064408-907e3fcf8e7d github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect diff --git a/go.sum b/go.sum index 6309d344b7d06..10e8e7b78372d 100644 --- a/go.sum +++ b/go.sum @@ -399,8 +399,8 @@ github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdc github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= -github.com/ngaut/unistore v0.0.0-20201026053041-4333e8d852f1 h1:WHACEcI4Om4RVVUCssGTUwvJeF2CTuHHLmxr2h3joWQ= -github.com/ngaut/unistore v0.0.0-20201026053041-4333e8d852f1/go.mod h1:ZR3NH+HzqfiYetwdoAivApnIy8iefPZHTMLfrFNm8g4= +github.com/ngaut/unistore v0.0.0-20201113064408-907e3fcf8e7d h1:hh0yCo0UtCuakNdkiRPaLHqzfgxacwUk6/pb9iJyJKU= +github.com/ngaut/unistore v0.0.0-20201113064408-907e3fcf8e7d/go.mod h1:ZR3NH+HzqfiYetwdoAivApnIy8iefPZHTMLfrFNm8g4= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= diff --git a/store/mockstore/cluster/cluster.go b/store/mockstore/cluster/cluster.go index 569bb3717095a..d18fe22de959b 100644 --- a/store/mockstore/cluster/cluster.go +++ b/store/mockstore/cluster/cluster.go @@ -43,7 +43,7 @@ type Cluster interface { // SplitKeys evenly splits the start, end key into "count" regions. SplitKeys(start, end kv.Key, count int) // AddStore adds a new Store to the cluster. - AddStore(storeID uint64, addr string) + AddStore(storeID uint64, addr string, labels ...*metapb.StoreLabel) // RemoveStore removes a Store from the cluster. RemoveStore(storeID uint64) } diff --git a/store/mockstore/mocktikv/cluster.go b/store/mockstore/mocktikv/cluster.go index d7131647a69b6..9c14e52aeb35a 100644 --- a/store/mockstore/mocktikv/cluster.go +++ b/store/mockstore/mocktikv/cluster.go @@ -198,11 +198,11 @@ func (c *Cluster) GetAndCheckStoreByAddr(addr string) (*metapb.Store, error) { } // AddStore add a new Store to the cluster. -func (c *Cluster) AddStore(storeID uint64, addr string) { +func (c *Cluster) AddStore(storeID uint64, addr string, labels ...*metapb.StoreLabel) { c.Lock() defer c.Unlock() - c.stores[storeID] = newStore(storeID, addr) + c.stores[storeID] = newStore(storeID, addr, labels...) } // RemoveStore removes a Store from the cluster. diff --git a/store/mockstore/unistore/cophandler/closure_exec.go b/store/mockstore/unistore/cophandler/closure_exec.go index 0b0ee64d052f6..f3f8a999ea0ad 100644 --- a/store/mockstore/unistore/cophandler/closure_exec.go +++ b/store/mockstore/unistore/cophandler/closure_exec.go @@ -62,6 +62,33 @@ func mapPkStatusToHandleStatus(pkStatus int) tablecodec.HandleStatus { return tablecodec.HandleDefault } +func getExecutorList(dagReq *tipb.DAGRequest) ([]*tipb.Executor, error) { + if len(dagReq.Executors) > 0 { + return dagReq.Executors, nil + } + // convert TiFlash executors tree to executor list + executors := make([]*tipb.Executor, 0, 3) + currentExec := dagReq.RootExecutor + for currentExec.Tp != tipb.ExecType_TypeTableScan { + executors = append(executors, currentExec) + switch currentExec.Tp { + case tipb.ExecType_TypeTopN: + currentExec = currentExec.TopN.Child + case tipb.ExecType_TypeStreamAgg, tipb.ExecType_TypeAggregation: + currentExec = currentExec.Aggregation.Child + case tipb.ExecType_TypeLimit: + currentExec = currentExec.Limit.Child + default: + return nil, errors.New("unsupported executor type " + currentExec.Tp.String()) + } + } + executors = append(executors, currentExec) + for i, j := 0, len(executors)-1; i < j; i, j = i+1, j-1 { + executors[i], executors[j] = executors[j], executors[i] + } + return executors, nil +} + // buildClosureExecutor build a closureExecutor for the DAGRequest. // Currently the composition of executors are: // tableScan|indexScan [selection] [topN | limit | agg] @@ -70,7 +97,10 @@ func buildClosureExecutor(dagCtx *dagContext, dagReq *tipb.DAGRequest) (*closure if err != nil { return nil, errors.Trace(err) } - executors := dagReq.Executors + executors, err1 := getExecutorList(dagReq) + if err1 != nil { + return nil, err1 + } scanExec := executors[0] if scanExec.Tp == tipb.ExecType_TypeTableScan { ce.processor = &tableScanProcessor{closureExecutor: ce} @@ -121,6 +151,28 @@ func convertToExprs(sc *stmtctx.StatementContext, fieldTps []*types.FieldType, p return exprs, nil } +func getScanExec(dagReq *tipb.DAGRequest) (*tipb.Executor, error) { + if len(dagReq.Executors) > 0 { + return dagReq.Executors[0], nil + } + currentExec := dagReq.RootExecutor + for currentExec.Tp != tipb.ExecType_TypeTableScan { + switch currentExec.Tp { + case tipb.ExecType_TypeAggregation, tipb.ExecType_TypeStreamAgg: + currentExec = currentExec.Aggregation.Child + case tipb.ExecType_TypeLimit: + currentExec = currentExec.Limit.Child + case tipb.ExecType_TypeSelection: + currentExec = currentExec.Selection.Child + case tipb.ExecType_TypeTopN: + currentExec = currentExec.TopN.Child + default: + return nil, errors.New("Unsupported DAG request") + } + } + return currentExec, nil +} + func newClosureExecutor(dagCtx *dagContext, dagReq *tipb.DAGRequest) (*closureExecutor, error) { e := &closureExecutor{ dagContext: dagCtx, @@ -131,20 +183,22 @@ func newClosureExecutor(dagCtx *dagContext, dagReq *tipb.DAGRequest) (*closureEx seCtx := mockpkg.NewContext() seCtx.GetSessionVars().StmtCtx = e.sc e.seCtx = seCtx - executors := dagReq.Executors - scanExec := executors[0] + scanExec, err := getScanExec(dagReq) + if err != nil { + return nil, err + } switch scanExec.Tp { case tipb.ExecType_TypeTableScan: - tblScan := executors[0].TblScan + tblScan := scanExec.TblScan e.unique = true e.scanCtx.desc = tblScan.Desc case tipb.ExecType_TypeIndexScan: - idxScan := executors[0].IdxScan + idxScan := scanExec.IdxScan e.unique = idxScan.GetUnique() e.scanCtx.desc = idxScan.Desc e.initIdxScanCtx(idxScan) default: - panic(fmt.Sprintf("unknown first executor type %s", executors[0].Tp)) + panic(fmt.Sprintf("unknown first executor type %s", scanExec.Tp)) } ranges, err := extractKVRanges(dagCtx.dbReader.StartKey, dagCtx.dbReader.EndKey, dagCtx.keyRanges, e.scanCtx.desc) if err != nil { diff --git a/store/mockstore/unistore/cophandler/cop_handler.go b/store/mockstore/unistore/cophandler/cop_handler.go index 00067b13d617d..d34ba9f35ea1a 100644 --- a/store/mockstore/unistore/cophandler/cop_handler.go +++ b/store/mockstore/unistore/cophandler/cop_handler.go @@ -124,7 +124,11 @@ func buildDAG(reader *dbreader.DBReader, lockStore *lockstore.MemStore, req *cop startTS: req.StartTs, resolvedLocks: req.Context.ResolvedLocks, } - scanExec := dagReq.Executors[0] + var scanExec *tipb.Executor = nil + scanExec, err = getScanExec(dagReq) + if err != nil { + return nil, nil, err + } if scanExec.Tp == tipb.ExecType_TypeTableScan { ctx.setColumnInfo(scanExec.TblScan.Columns) ctx.primaryCols = scanExec.TblScan.PrimaryColumnIds diff --git a/store/mockstore/unistore/rpc.go b/store/mockstore/unistore/rpc.go index 8ef39ed5b6c5f..bf8c65f183dc6 100644 --- a/store/mockstore/unistore/rpc.go +++ b/store/mockstore/unistore/rpc.go @@ -177,6 +177,19 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R resp.Resp, err = c.usSvr.Coprocessor(ctx, req.Cop()) case tikvrpc.CmdCopStream: resp.Resp, err = c.handleCopStream(ctx, req.Cop()) + case tikvrpc.CmdBatchCop: + failpoint.Inject("BatchCopCancelled", func(value failpoint.Value) { + if value.(bool) { + failpoint.Return(nil, context.Canceled) + } + }) + + failpoint.Inject("BatchCopRpcErr"+addr, func(value failpoint.Value) { + if value.(string) == addr { + failpoint.Return(nil, errors.New("rpc error")) + } + }) + resp.Resp, err = c.handleBatchCop(ctx, req.BatchCop(), timeout) case tikvrpc.CmdMvccGetByKey: resp.Resp, err = c.usSvr.MvccGetByKey(ctx, req.MvccGetByKey()) case tikvrpc.CmdMvccGetByStartTs: @@ -192,7 +205,10 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R if err != nil { return nil, err } - regErr, err := resp.GetRegionError() + var regErr *errorpb.Error = nil + if req.Type != tikvrpc.CmdBatchCop { + regErr, err = resp.GetRegionError() + } if err != nil { return nil, err } @@ -217,6 +233,25 @@ func (c *RPCClient) handleCopStream(ctx context.Context, req *coprocessor.Reques }, nil } +func (c *RPCClient) handleBatchCop(ctx context.Context, r *coprocessor.BatchRequest, timeout time.Duration) (*tikvrpc.BatchCopStreamResponse, error) { + mockBatchCopServer := &mockBatchCoprocessorStreamServer{} + err := c.usSvr.BatchCoprocessor(r, mockBatchCopServer) + if err != nil { + return nil, err + } + var mockBatchCopClient = mockBatchCopClient{batchResponses: mockBatchCopServer.batchResponses, idx: 0} + batchResp := &tikvrpc.BatchCopStreamResponse{Tikv_BatchCoprocessorClient: &mockBatchCopClient} + _, cancel := context.WithCancel(ctx) + batchResp.Lease.Cancel = cancel + batchResp.Timeout = timeout + first, err := batchResp.Recv() + if err != nil { + return nil, errors.Trace(err) + } + batchResp.BatchResponse = first + return batchResp, nil +} + func (c *RPCClient) handleDebugGetRegionProperties(ctx context.Context, req *debugpb.GetRegionPropertiesRequest) (*debugpb.GetRegionPropertiesResponse, error) { region := c.cluster.GetRegion(req.RegionId) _, start, err := codec.DecodeBytes(region.StartKey, nil) @@ -324,3 +359,42 @@ type mockCopStreamClient struct { func (mock *mockCopStreamClient) Recv() (*coprocessor.Response, error) { return nil, io.EOF } + +type mockBatchCopClient struct { + mockClientStream + batchResponses []*coprocessor.BatchResponse + idx int +} + +func (mock *mockBatchCopClient) Recv() (*coprocessor.BatchResponse, error) { + if mock.idx < len(mock.batchResponses) { + ret := mock.batchResponses[mock.idx] + mock.idx++ + var err error = nil + if len(ret.OtherError) > 0 { + err = errors.New(ret.OtherError) + ret = nil + } + return ret, err + } + return nil, io.EOF +} + +type mockServerStream struct{} + +func (mockServerStream) SetHeader(metadata.MD) error { return nil } +func (mockServerStream) SendHeader(metadata.MD) error { return nil } +func (mockServerStream) SetTrailer(metadata.MD) {} +func (mockServerStream) Context() context.Context { return nil } +func (mockServerStream) SendMsg(interface{}) error { return nil } +func (mockServerStream) RecvMsg(interface{}) error { return nil } + +type mockBatchCoprocessorStreamServer struct { + mockServerStream + batchResponses []*coprocessor.BatchResponse +} + +func (mockBatchCopServer *mockBatchCoprocessorStreamServer) Send(response *coprocessor.BatchResponse) error { + mockBatchCopServer.batchResponses = append(mockBatchCopServer.batchResponses, response) + return nil +} diff --git a/store/tikv/batch_coprocessor_test.go b/store/tikv/batch_coprocessor_test.go index 11fffd6c330df..2039cefb3311c 100644 --- a/store/tikv/batch_coprocessor_test.go +++ b/store/tikv/batch_coprocessor_test.go @@ -28,7 +28,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/mockstore/cluster" - "github.com/pingcap/tidb/store/mockstore/mocktikv" + "github.com/pingcap/tidb/store/mockstore/unistore" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/util/testkit" ) @@ -41,20 +41,19 @@ var _ = SerialSuites(&testBatchCopSuite{}) func newStoreWithBootstrap(tiflashNum int) (kv.Storage, *domain.Domain, error) { store, err := mockstore.NewMockStore( mockstore.WithClusterInspector(func(c cluster.Cluster) { - mockCluster := c.(*mocktikv.Cluster) + mockCluster := c.(*unistore.Cluster) _, _, region1 := mockstore.BootstrapWithSingleStore(c) tiflashIdx := 0 for tiflashIdx < tiflashNum { store2 := c.AllocID() peer2 := c.AllocID() addr2 := fmt.Sprintf("tiflash%d", tiflashIdx) - mockCluster.AddStore(store2, addr2) - mockCluster.UpdateStoreAddr(store2, addr2, &metapb.StoreLabel{Key: "engine", Value: "tiflash"}) + mockCluster.AddStore(store2, addr2, &metapb.StoreLabel{Key: "engine", Value: "tiflash"}) mockCluster.AddPeer(region1, store2, peer2) tiflashIdx++ } }), - mockstore.WithStoreType(mockstore.MockTiKV), + mockstore.WithStoreType(mockstore.EmbedUnistore), ) if err != nil { @@ -104,16 +103,16 @@ func (s *testBatchCopSuite) TestStoreErr(c *C) { tk.MustExec("insert into t values(1,0)") tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/BatchCopCancelled", "1*return(true)"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/BatchCopCancelled", "1*return(true)"), IsNil) err = tk.QueryToErr("select count(*) from t") c.Assert(errors.Cause(err), Equals, context.Canceled) - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/BatchCopRpcErrtiflash0", "1*return(\"tiflash0\")"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/BatchCopRpcErrtiflash0", "1*return(\"tiflash0\")"), IsNil) tk.MustQuery("select count(*) from t").Check(testkit.Rows("1")) - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/BatchCopRpcErrtiflash0", "return(\"tiflash0\")"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/BatchCopRpcErrtiflash0", "return(\"tiflash0\")"), IsNil) err = tk.QueryToErr("select count(*) from t") c.Assert(err, NotNil) } @@ -139,11 +138,11 @@ func (s *testBatchCopSuite) TestStoreSwitchPeer(c *C) { tk.MustExec("insert into t values(1,0)") tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"") - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/BatchCopRpcErrtiflash0", "return(\"tiflash0\")"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/BatchCopRpcErrtiflash0", "return(\"tiflash0\")"), IsNil) tk.MustQuery("select count(*) from t").Check(testkit.Rows("1")) - c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/mocktikv/BatchCopRpcErrtiflash1", "return(\"tiflash1\")"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/BatchCopRpcErrtiflash1", "return(\"tiflash1\")"), IsNil) err = tk.QueryToErr("select count(*) from t") c.Assert(err, NotNil) From de75e604ff2a9f9942c57d870105c5ceee84a55c Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Tue, 17 Nov 2020 17:03:41 +0800 Subject: [PATCH 19/37] *: differentiate types for user variables (#18973) Co-authored-by: lzmhhh123 --- executor/executor_test.go | 12 + executor/explainfor_test.go | 61 ++++ executor/prepared_test.go | 112 +++++- executor/set.go | 13 +- executor/show_test.go | 4 +- executor/testdata/prepare_suite_in.json | 58 +++ executor/testdata/prepare_suite_out.json | 208 +++++++++++ expression/bench_test.go | 14 + expression/builtin.go | 1 - expression/builtin_other.go | 336 +++++++++++++++--- expression/builtin_other_test.go | 40 ++- expression/builtin_other_vec.go | 230 +++++++++++- expression/builtin_other_vec_test.go | 3 + expression/distsql_builtin.go | 4 +- expression/integration_test.go | 3 - expression/scalar_function.go | 5 +- expression/util_test.go | 3 +- .../transformation_rules_suite_out.json | 10 +- planner/core/cache.go | 24 +- planner/core/common_plans.go | 87 +++-- planner/core/expression_rewriter.go | 22 +- planner/core/logical_plan_builder.go | 48 +++ planner/core/planbuilder.go | 7 +- sessionctx/variable/session.go | 4 + 24 files changed, 1176 insertions(+), 133 deletions(-) create mode 100644 executor/testdata/prepare_suite_in.json create mode 100644 executor/testdata/prepare_suite_out.json diff --git a/executor/executor_test.go b/executor/executor_test.go index 5338d9dfbdeca..1987bde05c43b 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -135,6 +135,7 @@ var _ = SerialSuites(&tiflashTestSuite{}) var _ = SerialSuites(&globalIndexSuite{&baseTestSuite{}}) var _ = SerialSuites(&testSerialSuite{&baseTestSuite{}}) var _ = SerialSuites(&testCoprCache{}) +var _ = SerialSuites(&testPrepareSuite{}) type testSuite struct{ *baseTestSuite } type testSuiteP1 struct{ *baseTestSuite } @@ -153,6 +154,7 @@ type testCoprCache struct { dom *domain.Domain cls cluster.Cluster } +type testPrepareSuite struct{ testData testutil.TestData } type baseTestSuite struct { cluster cluster.Cluster @@ -201,6 +203,16 @@ func (s *testSuiteWithData) TearDownSuite(c *C) { c.Assert(s.testData.GenerateOutputIfNeeded(), IsNil) } +func (s *testPrepareSuite) SetUpSuite(c *C) { + var err error + s.testData, err = testutil.LoadTestSuiteData("testdata", "prepare_suite") + c.Assert(err, IsNil) +} + +func (s *testPrepareSuite) TearDownSuite(c *C) { + c.Assert(s.testData.GenerateOutputIfNeeded(), IsNil) +} + func (s *baseTestSuite) TearDownSuite(c *C) { s.domain.Close() s.store.Close() diff --git a/executor/explainfor_test.go b/executor/explainfor_test.go index 2d420fc64013c..9e32abc37a79d 100644 --- a/executor/explainfor_test.go +++ b/executor/explainfor_test.go @@ -426,3 +426,64 @@ func (s *testSuite) TestExplainTiFlashSystemTables(c *C) { tk.MustQuery(fmt.Sprintf("desc select * from information_schema.TIFLASH_SEGMENTS where TIFLASH_INSTANCE = '%s' and TIDB_DATABASE = '%s' and TIDB_TABLE = '%s'", tiflashInstance, database, table)).Check(testkit.Rows( fmt.Sprintf("MemTableScan_5 10000.00 root table:TIFLASH_SEGMENTS tiflash_instances:[\"%s\"], tidb_databases:[\"%s\"], tidb_tables:[\"%s\"]", tiflashInstance, database, table))) } + +func (s *testPrepareSerialSuite) TestPointGetUserVarPlanCache(c *C) { + orgEnable := core.PreparedPlanCacheEnabled() + defer func() { + core.SetPreparedPlanCache(orgEnable) + }() + core.SetPreparedPlanCache(true) + + tk := testkit.NewTestKitWithInit(c, s.store) + tk.Se.Auth(&auth.UserIdentity{Username: "root", Hostname: "localhost", CurrentUser: true, AuthUsername: "root", AuthHostname: "%"}, nil, []byte("012345678901234567890")) + + tk.MustExec("use test") + tk.MustExec("set @@tidb_enable_collect_execution_info=0;") + tk.MustExec("drop table if exists t1") + tk.MustExec("CREATE TABLE t1 (a BIGINT, b VARCHAR(40), PRIMARY KEY (a, b))") + tk.MustExec("INSERT INTO t1 VALUES (1,'3'),(2,'4')") + tk.MustExec("drop table if exists t2") + tk.MustExec("CREATE TABLE t2 (a BIGINT, b VARCHAR(40), UNIQUE KEY idx_a (a))") + tk.MustExec("INSERT INTO t2 VALUES (1,'1'),(2,'2')") + tk.MustExec("prepare stmt from 'select * from t1, t2 where t1.a = t2.a and t2.a = ?'") + tk.MustExec("set @a=1") + tk.MustQuery("execute stmt using @a").Check(testkit.Rows( + "1 3 1 1", + )) + tkProcess := tk.Se.ShowProcess() + ps := []*util.ProcessInfo{tkProcess} + tk.Se.SetSessionManager(&mockSessionManager1{PS: ps}) + // t2 should use PointGet. + tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows( + "Projection_7 1.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b", + "└─IndexMergeJoin_19 1.00 root inner join, inner:TableReader_14, outer key:test.t2.a, inner key:test.t1.a", + " ├─Selection_41(Build) 0.80 root not(isnull(test.t2.a))", + " │ └─Point_Get_40 1.00 root table:t2, index:idx_a(a) ", + " └─TableReader_14(Probe) 0.00 root data:Selection_13", + " └─Selection_13 0.00 cop[tikv] eq(test.t1.a, 1)", + " └─TableRangeScan_12 1.00 cop[tikv] table:t1 range: decided by [test.t2.a], keep order:true, stats:pseudo", + )) + tk.MustExec("set @a=2") + tk.MustQuery("execute stmt using @a").Check(testkit.Rows( + "2 4 2 2", + )) + tkProcess = tk.Se.ShowProcess() + ps = []*util.ProcessInfo{tkProcess} + tk.Se.SetSessionManager(&mockSessionManager1{PS: ps}) + // t2 should use PointGet, range is changed to [2,2]. + tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows( + "Projection_7 1.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b", + "└─IndexMergeJoin_19 1.00 root inner join, inner:TableReader_14, outer key:test.t2.a, inner key:test.t1.a", + " ├─Selection_41(Build) 0.80 root not(isnull(test.t2.a))", + " │ └─Point_Get_40 1.00 root table:t2, index:idx_a(a) ", + " └─TableReader_14(Probe) 0.00 root data:Selection_13", + " └─Selection_13 0.00 cop[tikv] eq(test.t1.a, 2)", + " └─TableRangeScan_12 1.00 cop[tikv] table:t1 range: decided by [test.t2.a], keep order:true, stats:pseudo", + )) + tk.MustQuery("execute stmt using @a").Check(testkit.Rows( + "2 4 2 2", + )) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows( + "1", + )) +} diff --git a/executor/prepared_test.go b/executor/prepared_test.go index c43150deff391..1dd5beafde5bf 100644 --- a/executor/prepared_test.go +++ b/executor/prepared_test.go @@ -196,16 +196,16 @@ func (s *testSerialSuite) TestPlanCacheClusterIndex(c *C) { // For table scan tk.MustExec(`prepare stmt1 from "select * from t1 where t1.a = ? and t1.b > ?"`) - tk.MustExec("set @v1 = 1") - tk.MustExec("set @v2 = 0") + tk.MustExec("set @v1 = '1'") + tk.MustExec("set @v2 = '0'") tk.MustQuery("execute stmt1 using @v1,@v2").Check(testkit.Rows("1 1 111")) tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) - tk.MustExec("set @v1 = 2") - tk.MustExec("set @v2 = 1") + tk.MustExec("set @v1 = '2'") + tk.MustExec("set @v2 = '1'") tk.MustQuery("execute stmt1 using @v1,@v2").Check(testkit.Rows("2 2 222")) tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) - tk.MustExec("set @v1 = 3") - tk.MustExec("set @v2 = 2") + tk.MustExec("set @v1 = '3'") + tk.MustExec("set @v2 = '2'") tk.MustQuery("execute stmt1 using @v1,@v2").Check(testkit.Rows("3 3 333")) tkProcess := tk.Se.ShowProcess() ps := []*util.ProcessInfo{tkProcess} @@ -215,16 +215,16 @@ func (s *testSerialSuite) TestPlanCacheClusterIndex(c *C) { // For point get tk.MustExec(`prepare stmt2 from "select * from t1 where t1.a = ? and t1.b = ?"`) - tk.MustExec("set @v1 = 1") - tk.MustExec("set @v2 = 1") + tk.MustExec("set @v1 = '1'") + tk.MustExec("set @v2 = '1'") tk.MustQuery("execute stmt2 using @v1,@v2").Check(testkit.Rows("1 1 111")) tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) - tk.MustExec("set @v1 = 2") - tk.MustExec("set @v2 = 2") + tk.MustExec("set @v1 = '2'") + tk.MustExec("set @v2 = '2'") tk.MustQuery("execute stmt2 using @v1,@v2").Check(testkit.Rows("2 2 222")) tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) - tk.MustExec("set @v1 = 3") - tk.MustExec("set @v2 = 3") + tk.MustExec("set @v1 = '3'") + tk.MustExec("set @v2 = '3'") tk.MustQuery("execute stmt2 using @v1,@v2").Check(testkit.Rows("3 3 333")) tkProcess = tk.Se.ShowProcess() ps = []*util.ProcessInfo{tkProcess} @@ -297,3 +297,91 @@ func (s *testSerialSuite) TestPlanCacheClusterIndex(c *C) { tk.MustQuery(`execute stmt2 using @v1,@v2,@v3,@v4`).Check(testkit.Rows("2 2 222", "3 3 333")) tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) } + +func (s *testPrepareSuite) TestPlanCacheWithDifferentVariableTypes(c *C) { + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + orgEnable := plannercore.PreparedPlanCacheEnabled() + defer func() { + plannercore.SetPreparedPlanCache(orgEnable) + }() + plannercore.SetPreparedPlanCache(true) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("set @@tidb_enable_collect_execution_info=0;") + tk.MustExec("create table t1(a varchar(20), b int, c float, key(b, a))") + tk.MustExec("insert into t1 values('1',1,1.1),('2',2,222),('3',3,333)") + tk.MustExec("create table t2(a varchar(20), b int, c float, key(b, a))") + tk.MustExec("insert into t2 values('3',3,3.3),('2',2,222),('3',3,333)") + + var input []struct { + PrepareStmt string + Executes []struct { + Vars []struct { + Name string + Value string + } + ExecuteSQL string + } + } + var output []struct { + PrepareStmt string + Executes []struct { + SQL string + Vars []struct { + Name string + Value string + } + Plan []string + LastPlanUseCache string + Result []string + } + } + s.testData.GetTestCases(c, &input, &output) + for i, tt := range input { + tk.MustExec(tt.PrepareStmt) + s.testData.OnRecord(func() { + output[i].PrepareStmt = tt.PrepareStmt + output[i].Executes = make([]struct { + SQL string + Vars []struct { + Name string + Value string + } + Plan []string + LastPlanUseCache string + Result []string + }, len(tt.Executes)) + }) + c.Assert(output[i].PrepareStmt, Equals, tt.PrepareStmt) + for j, exec := range tt.Executes { + for _, v := range exec.Vars { + tk.MustExec(fmt.Sprintf(`set @%s = %s`, v.Name, v.Value)) + } + res := tk.MustQuery(exec.ExecuteSQL) + lastPlanUseCache := tk.MustQuery("select @@last_plan_from_cache").Rows()[0][0] + tk.MustQuery(exec.ExecuteSQL) + tkProcess := tk.Se.ShowProcess() + ps := []*util.ProcessInfo{tkProcess} + tk.Se.SetSessionManager(&mockSessionManager1{PS: ps}) + plan := tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)) + s.testData.OnRecord(func() { + output[i].Executes[j].SQL = exec.ExecuteSQL + output[i].Executes[j].Plan = s.testData.ConvertRowsToStrings(plan.Rows()) + output[i].Executes[j].Vars = exec.Vars + output[i].Executes[j].LastPlanUseCache = lastPlanUseCache.(string) + output[i].Executes[j].Result = s.testData.ConvertRowsToStrings(res.Rows()) + }) + c.Assert(output[i].Executes[j].SQL, Equals, exec.ExecuteSQL) + plan.Check(testkit.Rows(output[i].Executes[j].Plan...)) + c.Assert(output[i].Executes[j].Vars, DeepEquals, exec.Vars) + c.Assert(output[i].Executes[j].LastPlanUseCache, Equals, lastPlanUseCache.(string)) + res.Check(testkit.Rows(output[i].Executes[j].Result...)) + } + } +} diff --git a/executor/set.go b/executor/set.go index d6c357e9a7080..d057bb0256ad7 100644 --- a/executor/set.go +++ b/executor/set.go @@ -33,7 +33,6 @@ import ( "github.com/pingcap/tidb/util/gcutil" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/stmtsummary" - "github.com/pingcap/tidb/util/stringutil" "go.uber.org/zap" ) @@ -91,17 +90,15 @@ func (e *SetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { if err != nil { return err } - + sessionVars.UsersLock.Lock() if value.IsNull() { delete(sessionVars.Users, name) + delete(sessionVars.UserVarTypes, name) } else { - svalue, err1 := value.ToString() - if err1 != nil { - return err1 - } - - sessionVars.SetUserVar(name, stringutil.Copy(svalue), value.Collation()) + sessionVars.Users[name] = value + sessionVars.UserVarTypes[name] = v.Expr.GetType() } + sessionVars.UsersLock.Unlock() continue } diff --git a/executor/show_test.go b/executor/show_test.go index 9803bd0c49179..1c6aeb1fa1ab3 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -1066,9 +1066,9 @@ func (s *testSuite5) TestShowBuiltin(c *C) { res := tk.MustQuery("show builtins;") c.Assert(res, NotNil) rows := res.Rows() - c.Assert(268, Equals, len(rows)) + c.Assert(267, Equals, len(rows)) c.Assert("abs", Equals, rows[0][0].(string)) - c.Assert("yearweek", Equals, rows[267][0].(string)) + c.Assert("yearweek", Equals, rows[266][0].(string)) } func (s *testSuite5) TestShowClusterConfig(c *C) { diff --git a/executor/testdata/prepare_suite_in.json b/executor/testdata/prepare_suite_in.json new file mode 100644 index 0000000000000..a592b6ec327c0 --- /dev/null +++ b/executor/testdata/prepare_suite_in.json @@ -0,0 +1,58 @@ +[ + { + "name": "TestPlanCacheWithDifferentVariableTypes", + "cases": [ + { + "PrepareStmt": "prepare stmt from \"select ?, ?\"", + "Executes": [ + { + "Vars": [{"Name": "v1", "Value": "1"}, {"Name": "v2", "Value": "2"}], + "ExecuteSQL": "execute stmt using @v1, @v2" + }, + { + "Vars": [{"Name": "v1", "Value": "1"}, {"Name": "v2", "Value": "\"abc\""}], + "ExecuteSQL": "execute stmt using @v1, @v2" + }, + { + "Vars": [{"Name": "v1", "Value": "10"}, {"Name": "v2", "Value": "\"cba\""}], + "ExecuteSQL": "execute stmt using @v1, @v2" + } + ] + }, + { + "PrepareStmt": "prepare stmt from \"select a from t1 where t1.b = ?\"", + "Executes": [ + { + "Vars": [{"Name": "v1", "Value": "3"}], + "ExecuteSQL": "execute stmt using @v1" + }, + { + "Vars": [{"Name": "v1", "Value": "2"}], + "ExecuteSQL": "execute stmt using @v1" + }, + { + "Vars": [{"Name": "v1", "Value": "\"abc\""}], + "ExecuteSQL": "execute stmt using @v1" + } + ] + }, + { + "PrepareStmt": "prepare stmt from \"select t1.c, t2.c from t1 join t2 on t1.b = t2.b and t1.a = t2.a where t1.b = ?\"", + "Executes": [ + { + "Vars": [{"Name": "v1", "Value": "1"}], + "ExecuteSQL": "execute stmt using @v1" + }, + { + "Vars": [{"Name": "v1", "Value": "2"}], + "ExecuteSQL": "execute stmt using @v1" + }, + { + "Vars": [{"Name": "v1", "Value": "\"abc\""}], + "ExecuteSQL": "execute stmt using @v1" + } + ] + } + ] + } +] diff --git a/executor/testdata/prepare_suite_out.json b/executor/testdata/prepare_suite_out.json new file mode 100644 index 0000000000000..5caa86bded46b --- /dev/null +++ b/executor/testdata/prepare_suite_out.json @@ -0,0 +1,208 @@ +[ + { + "Name": "TestPlanCacheWithDifferentVariableTypes", + "Cases": [ + { + "PrepareStmt": "prepare stmt from \"select ?, ?\"", + "Executes": [ + { + "SQL": "execute stmt using @v1, @v2", + "Vars": [ + { + "Name": "v1", + "Value": "1" + }, + { + "Name": "v2", + "Value": "2" + } + ], + "Plan": [ + "Projection_3 1.00 root 1->Column#1, 2->Column#2", + "└─TableDual_4 1.00 root rows:1" + ], + "LastPlanUseCache": "0", + "Result": [ + "1 2" + ] + }, + { + "SQL": "execute stmt using @v1, @v2", + "Vars": [ + { + "Name": "v1", + "Value": "1" + }, + { + "Name": "v2", + "Value": "\"abc\"" + } + ], + "Plan": [ + "Projection_3 1.00 root 1->Column#1, abc->Column#2", + "└─TableDual_4 1.00 root rows:1" + ], + "LastPlanUseCache": "0", + "Result": [ + "1 abc" + ] + }, + { + "SQL": "execute stmt using @v1, @v2", + "Vars": [ + { + "Name": "v1", + "Value": "10" + }, + { + "Name": "v2", + "Value": "\"cba\"" + } + ], + "Plan": [ + "Projection_3 1.00 root 10->Column#1, cba->Column#2", + "└─TableDual_4 1.00 root rows:1" + ], + "LastPlanUseCache": "0", + "Result": [ + "10 cba" + ] + } + ] + }, + { + "PrepareStmt": "prepare stmt from \"select a from t1 where t1.b = ?\"", + "Executes": [ + { + "SQL": "execute stmt using @v1", + "Vars": [ + { + "Name": "v1", + "Value": "3" + } + ], + "Plan": [ + "Projection_4 10.00 root test.t1.a", + "└─IndexReader_6 10.00 root index:IndexRangeScan_5", + " └─IndexRangeScan_5 10.00 cop[tikv] table:t1, index:b(b, a) range:[3,3], keep order:false, stats:pseudo" + ], + "LastPlanUseCache": "0", + "Result": [ + "3" + ] + }, + { + "SQL": "execute stmt using @v1", + "Vars": [ + { + "Name": "v1", + "Value": "2" + } + ], + "Plan": [ + "Projection_4 10.00 root test.t1.a", + "└─IndexReader_6 10.00 root index:IndexRangeScan_5", + " └─IndexRangeScan_5 10.00 cop[tikv] table:t1, index:b(b, a) range:[2,2], keep order:false, stats:pseudo" + ], + "LastPlanUseCache": "1", + "Result": [ + "2" + ] + }, + { + "SQL": "execute stmt using @v1", + "Vars": [ + { + "Name": "v1", + "Value": "\"abc\"" + } + ], + "Plan": [ + "Projection_4 8000.00 root test.t1.a", + "└─IndexReader_10 8000.00 root index:Selection_9", + " └─Selection_9 8000.00 cop[tikv] eq(cast(test.t1.b), 0)", + " └─IndexFullScan_8 10000.00 cop[tikv] table:t1, index:b(b, a) keep order:false, stats:pseudo" + ], + "LastPlanUseCache": "0", + "Result": null + } + ] + }, + { + "PrepareStmt": "prepare stmt from \"select t1.c, t2.c from t1 join t2 on t1.b = t2.b and t1.a = t2.a where t1.b = ?\"", + "Executes": [ + { + "SQL": "execute stmt using @v1", + "Vars": [ + { + "Name": "v1", + "Value": "1" + } + ], + "Plan": [ + "MergeJoin_9 6387.21 root inner join, left key:test.t1.a, right key:test.t2.a, other cond:eq(test.t1.b, test.t2.b)", + "├─Projection_59(Build) 99.80 root test.t2.a, test.t2.b, test.t2.c", + "│ └─IndexLookUp_58 99.80 root ", + "│ ├─Selection_57(Build) 99.80 cop[tikv] not(isnull(test.t2.b))", + "│ │ └─IndexRangeScan_55 99.90 cop[tikv] table:t2, index:b(b, a) range:[1 -inf,1 +inf], keep order:true, stats:pseudo", + "│ └─TableRowIDScan_56(Probe) 99.80 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─Projection_54(Probe) 99.80 root test.t1.a, test.t1.b, test.t1.c", + " └─IndexLookUp_53 99.80 root ", + " ├─Selection_52(Build) 99.80 cop[tikv] not(isnull(test.t1.b))", + " │ └─IndexRangeScan_50 99.90 cop[tikv] table:t1, index:b(b, a) range:[1 -inf,1 +inf], keep order:true, stats:pseudo", + " └─TableRowIDScan_51(Probe) 99.80 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "LastPlanUseCache": "0", + "Result": null + }, + { + "SQL": "execute stmt using @v1", + "Vars": [ + { + "Name": "v1", + "Value": "2" + } + ], + "Plan": [ + "MergeJoin_9 6387.21 root inner join, left key:test.t1.a, right key:test.t2.a, other cond:eq(test.t1.b, test.t2.b)", + "├─Projection_59(Build) 99.80 root test.t2.a, test.t2.b, test.t2.c", + "│ └─IndexLookUp_58 99.80 root ", + "│ ├─Selection_57(Build) 99.80 cop[tikv] not(isnull(test.t2.b))", + "│ │ └─IndexRangeScan_55 99.90 cop[tikv] table:t2, index:b(b, a) range:[2 -inf,2 +inf], keep order:true, stats:pseudo", + "│ └─TableRowIDScan_56(Probe) 99.80 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─Projection_54(Probe) 99.80 root test.t1.a, test.t1.b, test.t1.c", + " └─IndexLookUp_53 99.80 root ", + " ├─Selection_52(Build) 99.80 cop[tikv] not(isnull(test.t1.b))", + " │ └─IndexRangeScan_50 99.90 cop[tikv] table:t1, index:b(b, a) range:[2 -inf,2 +inf], keep order:true, stats:pseudo", + " └─TableRowIDScan_51(Probe) 99.80 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "LastPlanUseCache": "1", + "Result": [ + "222 222" + ] + }, + { + "SQL": "execute stmt using @v1", + "Vars": [ + { + "Name": "v1", + "Value": "\"abc\"" + } + ], + "Plan": [ + "HashJoin_37 63744383.74 root inner join, equal:[eq(test.t1.b, test.t2.b) eq(test.t1.a, test.t2.a)]", + "├─TableReader_58(Build) 7984.01 root data:Selection_57", + "│ └─Selection_57 7984.01 cop[tikv] eq(cast(test.t2.b), 0), not(isnull(test.t2.a)), not(isnull(test.t2.b))", + "│ └─TableFullScan_56 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─TableReader_51(Probe) 7984.01 root data:Selection_50", + " └─Selection_50 7984.01 cop[tikv] eq(cast(test.t1.b), 0), not(isnull(test.t1.a)), not(isnull(test.t1.b))", + " └─TableFullScan_49 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" + ], + "LastPlanUseCache": "0", + "Result": null + } + ] + } + ] + } +] diff --git a/expression/bench_test.go b/expression/bench_test.go index edccc58a82f03..9f7c9fab154f1 100644 --- a/expression/bench_test.go +++ b/expression/bench_test.go @@ -1244,6 +1244,20 @@ func genVecBuiltinFuncBenchCase(ctx sessionctx.Context, funcName string, testCas fc = &castAsStringFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp} } baseFunc, err = fc.getFunction(ctx, cols) + } else if funcName == ast.GetVar { + var fc functionClass + tp := eType2FieldType(testCase.retEvalType) + switch testCase.retEvalType { + case types.ETInt: + fc = &getIntVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, tp}} + case types.ETDecimal: + fc = &getDecimalVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, tp}} + case types.ETReal: + fc = &getRealVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, tp}} + default: + fc = &getStringVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, tp}} + } + baseFunc, err = fc.getFunction(ctx, cols) } else { baseFunc, err = funcs[funcName].getFunction(ctx, cols) } diff --git a/expression/builtin.go b/expression/builtin.go index 8df6a37587b20..d632af718269c 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -808,7 +808,6 @@ var funcs = map[string]functionClass{ ast.Case: &caseWhenFunctionClass{baseFunctionClass{ast.Case, 1, -1}}, ast.RowFunc: &rowFunctionClass{baseFunctionClass{ast.RowFunc, 2, -1}}, ast.SetVar: &setVarFunctionClass{baseFunctionClass{ast.SetVar, 2, 2}}, - ast.GetVar: &getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}}, ast.BitCount: &bitCountFunctionClass{baseFunctionClass{ast.BitCount, 1, 1}}, ast.GetParam: &getParamFunctionClass{baseFunctionClass{ast.GetParam, 1, 1}}, diff --git a/expression/builtin_other.go b/expression/builtin_other.go index 6574054964b8d..b2ecf84036590 100644 --- a/expression/builtin_other.go +++ b/expression/builtin_other.go @@ -18,6 +18,8 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" @@ -33,7 +35,10 @@ var ( _ functionClass = &inFunctionClass{} _ functionClass = &rowFunctionClass{} _ functionClass = &setVarFunctionClass{} - _ functionClass = &getVarFunctionClass{} + _ functionClass = &getIntVarFunctionClass{} + _ functionClass = &getRealVarFunctionClass{} + _ functionClass = &getDecimalVarFunctionClass{} + _ functionClass = &getStringVarFunctionClass{} _ functionClass = &lockFunctionClass{} _ functionClass = &releaseLockFunctionClass{} _ functionClass = &valuesFunctionClass{} @@ -51,8 +56,14 @@ var ( _ builtinFunc = &builtinInDurationSig{} _ builtinFunc = &builtinInJSONSig{} _ builtinFunc = &builtinRowSig{} - _ builtinFunc = &builtinSetVarSig{} - _ builtinFunc = &builtinGetVarSig{} + _ builtinFunc = &builtinSetStringVarSig{} + _ builtinFunc = &builtinSetIntVarSig{} + _ builtinFunc = &builtinSetRealVarSig{} + _ builtinFunc = &builtinSetDecimalVarSig{} + _ builtinFunc = &builtinGetStringVarSig{} + _ builtinFunc = &builtinGetIntVarSig{} + _ builtinFunc = &builtinGetRealVarSig{} + _ builtinFunc = &builtinGetDecimalVarSig{} _ builtinFunc = &builtinLockSig{} _ builtinFunc = &builtinReleaseLockSig{} _ builtinFunc = &builtinValuesIntSig{} @@ -673,45 +684,56 @@ func (c *setVarFunctionClass) getFunction(ctx sessionctx.Context, args []Express if err = c.verifyArgs(args); err != nil { return nil, err } - bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, types.ETString, types.ETString) + argTp := args[1].GetType().EvalType() + if argTp == types.ETTimestamp || argTp == types.ETDatetime || argTp == types.ETDuration || argTp == types.ETJson { + argTp = types.ETString + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, argTp, types.ETString, argTp) if err != nil { return nil, err } bf.tp.Flen = args[1].GetType().Flen - // TODO: we should consider the type of the argument, but not take it as string for all situations. - sig = &builtinSetVarSig{bf} - return sig, err + switch argTp { + case types.ETString: + sig = &builtinSetStringVarSig{bf} + case types.ETReal: + sig = &builtinSetRealVarSig{bf} + case types.ETDecimal: + sig = &builtinSetDecimalVarSig{bf} + case types.ETInt: + sig = &builtinSetIntVarSig{bf} + default: + return nil, errors.Errorf("unexpected types.EvalType %v", argTp) + } + return sig, nil } -type builtinSetVarSig struct { +type builtinSetStringVarSig struct { baseBuiltinFunc } -func (b *builtinSetVarSig) Clone() builtinFunc { - newSig := &builtinSetVarSig{} +func (b *builtinSetStringVarSig) Clone() builtinFunc { + newSig := &builtinSetStringVarSig{} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } -func (b *builtinSetVarSig) evalString(row chunk.Row) (res string, isNull bool, err error) { +func (b *builtinSetStringVarSig) evalString(row chunk.Row) (res string, isNull bool, err error) { var varName string sessionVars := b.ctx.GetSessionVars() varName, isNull, err = b.args[0].EvalString(b.ctx, row) if isNull || err != nil { return "", isNull, err } - datum, err := b.args[1].Eval(row) isNull = datum.IsNull() if isNull || err != nil { return "", isNull, err } - res, err = datum.ToString() if err != nil { return "", isNull, err } - varName = strings.ToLower(varName) sessionVars.UsersLock.Lock() sessionVars.SetUserVar(varName, stringutil.Copy(res), datum.Collation()) @@ -719,72 +741,298 @@ func (b *builtinSetVarSig) evalString(row chunk.Row) (res string, isNull bool, e return res, false, nil } +type builtinSetRealVarSig struct { + baseBuiltinFunc +} + +func (b *builtinSetRealVarSig) Clone() builtinFunc { + newSig := &builtinSetRealVarSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinSetRealVarSig) evalReal(row chunk.Row) (res float64, isNull bool, err error) { + var varName string + sessionVars := b.ctx.GetSessionVars() + varName, isNull, err = b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return 0, isNull, err + } + datum, err := b.args[1].Eval(row) + isNull = datum.IsNull() + if isNull || err != nil { + return 0, isNull, err + } + res = datum.GetFloat64() + varName = strings.ToLower(varName) + sessionVars.UsersLock.Lock() + sessionVars.Users[varName] = datum + sessionVars.UsersLock.Unlock() + return res, false, nil +} + +type builtinSetDecimalVarSig struct { + baseBuiltinFunc +} + +func (b *builtinSetDecimalVarSig) Clone() builtinFunc { + newSig := &builtinSetDecimalVarSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinSetDecimalVarSig) evalDecimal(row chunk.Row) (*types.MyDecimal, bool, error) { + sessionVars := b.ctx.GetSessionVars() + varName, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return nil, isNull, err + } + datum, err := b.args[1].Eval(row) + isNull = datum.IsNull() + if isNull || err != nil { + return nil, isNull, err + } + res := datum.GetMysqlDecimal() + varName = strings.ToLower(varName) + sessionVars.UsersLock.Lock() + sessionVars.Users[varName] = datum + sessionVars.UsersLock.Unlock() + return res, false, nil +} + +type builtinSetIntVarSig struct { + baseBuiltinFunc +} + +func (b *builtinSetIntVarSig) Clone() builtinFunc { + newSig := &builtinSetIntVarSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinSetIntVarSig) evalInt(row chunk.Row) (int64, bool, error) { + sessionVars := b.ctx.GetSessionVars() + varName, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return 0, isNull, err + } + datum, err := b.args[1].Eval(row) + isNull = datum.IsNull() + if isNull || err != nil { + return 0, isNull, err + } + res := datum.GetInt64() + varName = strings.ToLower(varName) + sessionVars.UsersLock.Lock() + sessionVars.Users[varName] = datum + sessionVars.UsersLock.Unlock() + return res, false, nil +} + +// BuildGetVarFunction builds a GetVar ScalarFunction from the Expression. +func BuildGetVarFunction(ctx sessionctx.Context, expr Expression, retType *types.FieldType) (Expression, error) { + var fc functionClass + switch retType.EvalType() { + case types.ETInt: + fc = &getIntVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, retType}} + case types.ETDecimal: + fc = &getDecimalVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, retType}} + case types.ETReal: + fc = &getRealVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, retType}} + default: + fc = &getStringVarFunctionClass{getVarFunctionClass{baseFunctionClass{ast.GetVar, 1, 1}, retType}} + } + f, err := fc.getFunction(ctx, []Expression{expr}) + if err != nil { + return nil, err + } + if builtinRetTp := f.getRetTp(); builtinRetTp.Tp != mysql.TypeUnspecified || retType.Tp == mysql.TypeUnspecified { + retType = builtinRetTp + } + return &ScalarFunction{ + FuncName: model.NewCIStr(ast.GetVar), + RetType: retType, + Function: f, + }, nil +} + type getVarFunctionClass struct { baseFunctionClass + + tp *types.FieldType } -func (c *getVarFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) { +type getStringVarFunctionClass struct { + getVarFunctionClass +} + +func (c *getStringVarFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) { if err = c.verifyArgs(args); err != nil { return nil, err } - // TODO: we should consider the type of the argument, but not take it as string for all situations. bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, types.ETString) if err != nil { return nil, err } - bf.tp.Flen = mysql.MaxFieldVarCharLength - if err := c.resolveCollation(ctx, args, &bf); err != nil { + bf.tp.Flen = c.tp.Flen + if len(c.tp.Charset) > 0 { + bf.tp.Charset = c.tp.Charset + bf.tp.Collate = c.tp.Collate + } + sig = &builtinGetStringVarSig{bf} + return sig, nil +} + +type builtinGetStringVarSig struct { + baseBuiltinFunc +} + +func (b *builtinGetStringVarSig) Clone() builtinFunc { + newSig := &builtinGetStringVarSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinGetStringVarSig) evalString(row chunk.Row) (string, bool, error) { + sessionVars := b.ctx.GetSessionVars() + varName, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return "", isNull, err + } + varName = strings.ToLower(varName) + sessionVars.UsersLock.RLock() + defer sessionVars.UsersLock.RUnlock() + if v, ok := sessionVars.Users[varName]; ok { + return v.GetString(), false, nil + } + return "", true, nil +} + +type getIntVarFunctionClass struct { + getVarFunctionClass +} + +func (c *getIntVarFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) { + if err = c.verifyArgs(args); err != nil { + return nil, err + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETInt, types.ETString) + if err != nil { return nil, err } - sig = &builtinGetVarSig{bf} + bf.tp.Flen = c.tp.Flen + bf.tp.Flag = c.tp.Flag + sig = &builtinGetIntVarSig{bf} return sig, nil } -func (c *getVarFunctionClass) resolveCollation(ctx sessionctx.Context, args []Expression, bf *baseBuiltinFunc) (err error) { - if constant, ok := args[0].(*Constant); ok { - varName, err := constant.Value.ToString() - if err != nil { - return err - } - varName = strings.ToLower(varName) - ctx.GetSessionVars().UsersLock.RLock() - defer ctx.GetSessionVars().UsersLock.RUnlock() - if v, ok := ctx.GetSessionVars().Users[varName]; ok { - bf.tp.Collate = v.Collation() - if len(bf.tp.Charset) <= 0 { - charset, _ := ctx.GetSessionVars().GetCharsetInfo() - bf.tp.Charset = charset - } - return nil - } +type builtinGetIntVarSig struct { + baseBuiltinFunc +} + +func (b *builtinGetIntVarSig) Clone() builtinFunc { + newSig := &builtinGetIntVarSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinGetIntVarSig) evalInt(row chunk.Row) (int64, bool, error) { + sessionVars := b.ctx.GetSessionVars() + varName, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return 0, isNull, err + } + varName = strings.ToLower(varName) + sessionVars.UsersLock.RLock() + defer sessionVars.UsersLock.RUnlock() + if v, ok := sessionVars.Users[varName]; ok { + return v.GetInt64(), false, nil } + return 0, true, nil +} - return nil +type getRealVarFunctionClass struct { + getVarFunctionClass } -type builtinGetVarSig struct { +func (c *getRealVarFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) { + if err = c.verifyArgs(args); err != nil { + return nil, err + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETReal, types.ETString) + if err != nil { + return nil, err + } + bf.tp.Flen = c.tp.Flen + sig = &builtinGetRealVarSig{bf} + return sig, nil +} + +type builtinGetRealVarSig struct { baseBuiltinFunc } -func (b *builtinGetVarSig) Clone() builtinFunc { - newSig := &builtinGetVarSig{} +func (b *builtinGetRealVarSig) Clone() builtinFunc { + newSig := &builtinGetRealVarSig{} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } -func (b *builtinGetVarSig) evalString(row chunk.Row) (string, bool, error) { +func (b *builtinGetRealVarSig) evalReal(row chunk.Row) (float64, bool, error) { sessionVars := b.ctx.GetSessionVars() varName, isNull, err := b.args[0].EvalString(b.ctx, row) if isNull || err != nil { - return "", isNull, err + return 0, isNull, err } varName = strings.ToLower(varName) sessionVars.UsersLock.RLock() defer sessionVars.UsersLock.RUnlock() if v, ok := sessionVars.Users[varName]; ok { - return v.GetString(), false, nil + return v.GetFloat64(), false, nil } - return "", true, nil + return 0, true, nil +} + +type getDecimalVarFunctionClass struct { + getVarFunctionClass +} + +func (c *getDecimalVarFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (sig builtinFunc, err error) { + if err = c.verifyArgs(args); err != nil { + return nil, err + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETDecimal, types.ETString) + if err != nil { + return nil, err + } + bf.tp.Flen = c.tp.Flen + sig = &builtinGetDecimalVarSig{bf} + return sig, nil +} + +type builtinGetDecimalVarSig struct { + baseBuiltinFunc +} + +func (b *builtinGetDecimalVarSig) Clone() builtinFunc { + newSig := &builtinGetDecimalVarSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinGetDecimalVarSig) evalDecimal(row chunk.Row) (*types.MyDecimal, bool, error) { + sessionVars := b.ctx.GetSessionVars() + varName, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return nil, isNull, err + } + varName = strings.ToLower(varName) + sessionVars.UsersLock.RLock() + defer sessionVars.UsersLock.RUnlock() + if v, ok := sessionVars.Users[varName]; ok { + return v.GetMysqlDecimal(), false, nil + } + return nil, true, nil } type valuesFunctionClass struct { diff --git a/expression/builtin_other_test.go b/expression/builtin_other_test.go index 05e921216d367..b635f845b2944 100644 --- a/expression/builtin_other_test.go +++ b/expression/builtin_other_test.go @@ -151,47 +151,54 @@ func (s *testEvaluatorSuite) TestRowFunc(c *C) { func (s *testEvaluatorSuite) TestSetVar(c *C) { fc := funcs[ast.SetVar] + dec := types.NewDecFromInt(5) testCases := []struct { args []interface{} res interface{} }{ {[]interface{}{"a", "12"}, "12"}, {[]interface{}{"b", "34"}, "34"}, - {[]interface{}{"c", nil}, ""}, + {[]interface{}{"c", nil}, nil}, {[]interface{}{"c", "ABC"}, "ABC"}, {[]interface{}{"c", "dEf"}, "dEf"}, + {[]interface{}{"d", int64(3)}, int64(3)}, + {[]interface{}{"e", float64(2.5)}, float64(2.5)}, + {[]interface{}{"f", dec}, dec}, } for _, tc := range testCases { fn, err := fc.getFunction(s.ctx, s.datumsToConstants(types.MakeDatums(tc.args...))) c.Assert(err, IsNil) d, err := evalBuiltinFunc(fn, chunk.MutRowFromDatums(types.MakeDatums(tc.args...)).ToRow()) c.Assert(err, IsNil) - c.Assert(d.GetString(), Equals, tc.res) + c.Assert(d.GetValue(), Equals, tc.res) if tc.args[1] != nil { key, ok := tc.args[0].(string) c.Assert(ok, Equals, true) - val, ok := tc.res.(string) - c.Assert(ok, Equals, true) sessionVar, ok := s.ctx.GetSessionVars().Users[key] c.Assert(ok, Equals, true) - c.Assert(sessionVar.GetString(), Equals, val) + c.Assert(sessionVar.GetValue(), Equals, tc.res) } } } func (s *testEvaluatorSuite) TestGetVar(c *C) { - fc := funcs[ast.GetVar] - + dec := types.NewDecFromInt(5) sessionVars := []struct { key string - val string + val interface{} }{ {"a", "中"}, {"b", "文字符chuan"}, {"c", ""}, + {"e", int64(3)}, + {"f", float64(2.5)}, + {"g", dec}, } for _, kv := range sessionVars { - s.ctx.GetSessionVars().Users[kv.key] = types.NewStringDatum(kv.val) + s.ctx.GetSessionVars().Users[kv.key] = types.NewDatum(kv.val) + tp := types.NewFieldType(mysql.TypeVarString) + types.DefaultParamTypeForValue(kv.val, tp) + s.ctx.GetSessionVars().UserVarTypes[kv.key] = tp } testCases := []struct { @@ -201,14 +208,21 @@ func (s *testEvaluatorSuite) TestGetVar(c *C) { {[]interface{}{"a"}, "中"}, {[]interface{}{"b"}, "文字符chuan"}, {[]interface{}{"c"}, ""}, - {[]interface{}{"d"}, ""}, + {[]interface{}{"d"}, nil}, + {[]interface{}{"e"}, int64(3)}, + {[]interface{}{"f"}, float64(2.5)}, + {[]interface{}{"g"}, dec}, } for _, tc := range testCases { - fn, err := fc.getFunction(s.ctx, s.datumsToConstants(types.MakeDatums(tc.args...))) + tp, ok := s.ctx.GetSessionVars().UserVarTypes[tc.args[0].(string)] + if !ok { + tp = types.NewFieldType(mysql.TypeVarString) + } + fn, err := BuildGetVarFunction(s.ctx, s.datumsToConstants(types.MakeDatums(tc.args...))[0], tp) c.Assert(err, IsNil) - d, err := evalBuiltinFunc(fn, chunk.MutRowFromDatums(types.MakeDatums(tc.args...)).ToRow()) + d, err := fn.Eval(chunk.Row{}) c.Assert(err, IsNil) - c.Assert(d.GetString(), Equals, tc.res) + c.Assert(d.GetValue(), Equals, tc.res) } } diff --git a/expression/builtin_other_vec.go b/expression/builtin_other_vec.go index 7bbbd8260857e..557f8ac87ae2e 100644 --- a/expression/builtin_other_vec.go +++ b/expression/builtin_other_vec.go @@ -155,11 +155,11 @@ func (b *builtinGetParamStringSig) vecEvalString(input *chunk.Chunk, result *chu return nil } -func (b *builtinSetVarSig) vectorized() bool { +func (b *builtinSetStringVarSig) vectorized() bool { return true } -func (b *builtinSetVarSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { +func (b *builtinSetStringVarSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { n := input.NumRows() buf0, err := b.bufAllocator.get(types.ETString, n) if err != nil { @@ -195,6 +195,126 @@ func (b *builtinSetVarSig) vecEvalString(input *chunk.Chunk, result *chunk.Colum return nil } +func (b *builtinSetIntVarSig) vectorized() bool { + return true +} + +func (b *builtinSetIntVarSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf0, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err := b.args[0].VecEvalString(b.ctx, input, buf0); err != nil { + return err + } + buf1, err := b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf1) + if err := b.args[1].VecEvalInt(b.ctx, input, buf1); err != nil { + return err + } + result.ResizeInt64(n, false) + i64s := result.Int64s() + sessionVars := b.ctx.GetSessionVars() + sessionVars.UsersLock.Lock() + defer sessionVars.UsersLock.Unlock() + for i := 0; i < n; i++ { + if buf0.IsNull(i) || buf1.IsNull(i) { + result.SetNull(i, true) + continue + } + varName := strings.ToLower(buf0.GetString(i)) + res := buf1.GetInt64(i) + sessionVars.Users[varName] = types.NewIntDatum(res) + i64s[i] = res + } + return nil +} + +func (b *builtinSetRealVarSig) vectorized() bool { + return true +} + +func (b *builtinSetRealVarSig) vecEvalReal(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf0, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err := b.args[0].VecEvalString(b.ctx, input, buf0); err != nil { + return err + } + buf1, err := b.bufAllocator.get(types.ETReal, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf1) + if err := b.args[1].VecEvalReal(b.ctx, input, buf1); err != nil { + return err + } + result.ResizeFloat64(n, false) + f64s := result.Float64s() + sessionVars := b.ctx.GetSessionVars() + sessionVars.UsersLock.Lock() + defer sessionVars.UsersLock.Unlock() + for i := 0; i < n; i++ { + if buf0.IsNull(i) || buf1.IsNull(i) { + result.SetNull(i, true) + continue + } + varName := strings.ToLower(buf0.GetString(i)) + res := buf1.GetFloat64(i) + sessionVars.Users[varName] = types.NewFloat64Datum(res) + f64s[i] = res + } + return nil +} + +func (b *builtinSetDecimalVarSig) vectorized() bool { + return true +} + +func (b *builtinSetDecimalVarSig) vecEvalDecimal(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf0, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err := b.args[0].VecEvalString(b.ctx, input, buf0); err != nil { + return err + } + buf1, err := b.bufAllocator.get(types.ETDecimal, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf1) + if err := b.args[1].VecEvalDecimal(b.ctx, input, buf1); err != nil { + return err + } + result.ResizeDecimal(n, false) + decs := result.Decimals() + sessionVars := b.ctx.GetSessionVars() + sessionVars.UsersLock.Lock() + defer sessionVars.UsersLock.Unlock() + for i := 0; i < n; i++ { + if buf0.IsNull(i) || buf1.IsNull(i) { + result.SetNull(i, true) + continue + } + varName := strings.ToLower(buf0.GetString(i)) + res := buf1.GetDecimal(i) + sessionVars.Users[varName] = types.NewDecimalDatum(res) + decs[i] = *res + } + return nil +} + func (b *builtinValuesDecimalSig) vectorized() bool { return false } @@ -203,11 +323,11 @@ func (b *builtinValuesDecimalSig) vecEvalDecimal(input *chunk.Chunk, result *chu return errors.Errorf("not implemented") } -func (b *builtinGetVarSig) vectorized() bool { +func (b *builtinGetStringVarSig) vectorized() bool { return true } -func (b *builtinGetVarSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { +func (b *builtinGetStringVarSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { n := input.NumRows() buf0, err := b.bufAllocator.get(types.ETString, n) if err != nil { @@ -235,3 +355,105 @@ func (b *builtinGetVarSig) vecEvalString(input *chunk.Chunk, result *chunk.Colum } return nil } + +func (b *builtinGetIntVarSig) vectorized() bool { + return true +} + +func (b *builtinGetIntVarSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf0, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err := b.args[0].VecEvalString(b.ctx, input, buf0); err != nil { + return err + } + result.ResizeInt64(n, false) + result.MergeNulls(buf0) + i64s := result.Int64s() + sessionVars := b.ctx.GetSessionVars() + sessionVars.UsersLock.Lock() + defer sessionVars.UsersLock.Unlock() + for i := 0; i < n; i++ { + if result.IsNull(i) { + continue + } + varName := strings.ToLower(buf0.GetString(i)) + if v, ok := sessionVars.Users[varName]; ok { + i64s[i] = v.GetInt64() + continue + } + result.SetNull(i, true) + } + return nil +} + +func (b *builtinGetRealVarSig) vectorized() bool { + return true +} + +func (b *builtinGetRealVarSig) vecEvalReal(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf0, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err := b.args[0].VecEvalString(b.ctx, input, buf0); err != nil { + return err + } + result.ResizeFloat64(n, false) + result.MergeNulls(buf0) + f64s := result.Float64s() + sessionVars := b.ctx.GetSessionVars() + sessionVars.UsersLock.Lock() + defer sessionVars.UsersLock.Unlock() + for i := 0; i < n; i++ { + if result.IsNull(i) { + continue + } + varName := strings.ToLower(buf0.GetString(i)) + if v, ok := sessionVars.Users[varName]; ok { + f64s[i] = v.GetFloat64() + continue + } + result.SetNull(i, true) + } + return nil +} + +func (b *builtinGetDecimalVarSig) vectorized() bool { + return true +} + +func (b *builtinGetDecimalVarSig) vecEvalDecimal(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf0, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf0) + if err := b.args[0].VecEvalString(b.ctx, input, buf0); err != nil { + return err + } + result.ResizeDecimal(n, false) + result.MergeNulls(buf0) + decs := result.Decimals() + sessionVars := b.ctx.GetSessionVars() + sessionVars.UsersLock.Lock() + defer sessionVars.UsersLock.Unlock() + for i := 0; i < n; i++ { + if result.IsNull(i) { + continue + } + varName := strings.ToLower(buf0.GetString(i)) + if v, ok := sessionVars.Users[varName]; ok { + decs[i] = *v.GetMysqlDecimal() + continue + } + result.SetNull(i, true) + } + return nil +} diff --git a/expression/builtin_other_vec_test.go b/expression/builtin_other_vec_test.go index 55541186822d0..b042de216115d 100644 --- a/expression/builtin_other_vec_test.go +++ b/expression/builtin_other_vec_test.go @@ -36,6 +36,9 @@ func dateTimeFromString(s string) types.Time { var vecBuiltinOtherCases = map[string][]vecExprBenchCase{ ast.SetVar: { {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString, types.ETString}}, + {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETString, types.ETInt}}, + {retEvalType: types.ETReal, childrenTypes: []types.EvalType{types.ETString, types.ETReal}}, + {retEvalType: types.ETDecimal, childrenTypes: []types.EvalType{types.ETString, types.ETDecimal}}, }, ast.GetVar: { {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString}}, diff --git a/expression/distsql_builtin.go b/expression/distsql_builtin.go index a5e707cadf29e..97f199885fd27 100644 --- a/expression/distsql_builtin.go +++ b/expression/distsql_builtin.go @@ -487,10 +487,10 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti case tipb.ScalarFuncSig_GetParamString: f = &builtinGetParamStringSig{base} case tipb.ScalarFuncSig_GetVar: - f = &builtinGetVarSig{base} + f = &builtinGetStringVarSig{base} //case tipb.ScalarFuncSig_RowSig: case tipb.ScalarFuncSig_SetVar: - f = &builtinSetVarSig{base} + f = &builtinSetStringVarSig{base} //case tipb.ScalarFuncSig_ValuesDecimal: // f = &builtinValuesDecimalSig{base} //case tipb.ScalarFuncSig_ValuesDuration: diff --git a/expression/integration_test.go b/expression/integration_test.go index 086f6162e0c94..b9c15e8691b1f 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -5971,9 +5971,6 @@ func (s *testIntegrationSuite) TestCollation(c *C) { tk.MustExec("set names utf8mb4 collate utf8mb4_general_ci") tk.MustExec("set @test_collate_var = 'a'") tk.MustQuery("select collation(@test_collate_var)").Check(testkit.Rows("utf8mb4_general_ci")) - tk.MustExec("set names utf8mb4 collate utf8mb4_general_ci") - tk.MustExec("set @test_collate_var = 1") - tk.MustQuery("select collation(@test_collate_var)").Check(testkit.Rows("utf8mb4_general_ci")) tk.MustExec("set @test_collate_var = concat(\"a\", \"b\" collate utf8mb4_bin)") tk.MustQuery("select collation(@test_collate_var)").Check(testkit.Rows("utf8mb4_bin")) } diff --git a/expression/scalar_function.go b/expression/scalar_function.go index e3d692bd506d8..9283e1123c858 100644 --- a/expression/scalar_function.go +++ b/expression/scalar_function.go @@ -181,8 +181,11 @@ func newFunctionImpl(ctx sessionctx.Context, fold int, funcName string, retType if retType == nil { return nil, errors.Errorf("RetType cannot be nil for ScalarFunction.") } - if funcName == ast.Cast { + switch funcName { + case ast.Cast: return BuildCastFunction(ctx, args[0], retType), nil + case ast.GetVar: + return BuildGetVarFunction(ctx, args[0], retType) } fc, ok := funcs[funcName] if !ok { diff --git a/expression/util_test.go b/expression/util_test.go index 172a66e59d1e0..0d7b3f15e6b1f 100644 --- a/expression/util_test.go +++ b/expression/util_test.go @@ -124,7 +124,8 @@ func (s *testUtilSuite) TestClone(c *check.C) { &builtinUnaryMinusIntSig{}, &builtinDecimalIsNullSig{}, &builtinDurationIsNullSig{}, &builtinIntIsNullSig{}, &builtinRealIsNullSig{}, &builtinStringIsNullSig{}, &builtinTimeIsNullSig{}, &builtinUnaryNotRealSig{}, &builtinUnaryNotDecimalSig{}, &builtinUnaryNotIntSig{}, &builtinSleepSig{}, &builtinInIntSig{}, &builtinInStringSig{}, &builtinInDecimalSig{}, &builtinInRealSig{}, &builtinInTimeSig{}, &builtinInDurationSig{}, - &builtinInJSONSig{}, &builtinRowSig{}, &builtinSetVarSig{}, &builtinGetVarSig{}, &builtinLockSig{}, + &builtinInJSONSig{}, &builtinRowSig{}, &builtinSetStringVarSig{}, &builtinSetIntVarSig{}, &builtinSetRealVarSig{}, &builtinSetDecimalVarSig{}, + &builtinGetIntVarSig{}, &builtinGetRealVarSig{}, &builtinGetDecimalVarSig{}, &builtinGetStringVarSig{}, &builtinLockSig{}, &builtinReleaseLockSig{}, &builtinValuesIntSig{}, &builtinValuesRealSig{}, &builtinValuesDecimalSig{}, &builtinValuesStringSig{}, &builtinValuesTimeSig{}, &builtinValuesDurationSig{}, &builtinValuesJSONSig{}, &builtinBitCountSig{}, &builtinGetParamStringSig{}, &builtinLengthSig{}, &builtinASCIISig{}, &builtinConcatSig{}, &builtinConcatWSSig{}, &builtinLeftSig{}, diff --git a/planner/cascades/testdata/transformation_rules_suite_out.json b/planner/cascades/testdata/transformation_rules_suite_out.json index 4d0c3cad6d735..7f4d933048652 100644 --- a/planner/cascades/testdata/transformation_rules_suite_out.json +++ b/planner/cascades/testdata/transformation_rules_suite_out.json @@ -53,9 +53,9 @@ "SQL": "select b, @i:=@i+1 as ii from (select b, @i:=0 from t as t1) as t2 where @i < 10", "Result": [ "Group#0 Schema:[test.t.b,Column#14]", - " Projection_4 input:[Group#1], test.t.b, setvar(i, cast(plus(cast(getvar(i), double BINARY), 1), var_string(5)))->Column#14", + " Projection_4 input:[Group#1], test.t.b, setvar(i, plus(getvar(i), 1))->Column#14", "Group#1 Schema:[test.t.b,Column#13]", - " Selection_3 input:[Group#2], lt(cast(getvar(\"i\")), 10)", + " Selection_3 input:[Group#2], lt(getvar(\"i\"), 10)", "Group#2 Schema:[test.t.b,Column#13]", " Projection_2 input:[Group#3], test.t.b, setvar(i, 0)->Column#13", "Group#3 Schema:[test.t.b]", @@ -68,9 +68,9 @@ "SQL": "select b, @i:=@i+1 as ii from (select a, b, @i:=0 from t as t1) as t2 where @i < 10 and a > 10", "Result": [ "Group#0 Schema:[test.t.b,Column#14]", - " Projection_4 input:[Group#1], test.t.b, setvar(i, cast(plus(cast(getvar(i), double BINARY), 1), var_string(5)))->Column#14", + " Projection_4 input:[Group#1], test.t.b, setvar(i, plus(getvar(i), 1))->Column#14", "Group#1 Schema:[test.t.a,test.t.b,Column#13]", - " Selection_6 input:[Group#2], lt(cast(getvar(\"i\")), 10)", + " Selection_6 input:[Group#2], lt(getvar(\"i\"), 10)", "Group#2 Schema:[test.t.a,test.t.b,Column#13]", " Projection_2 input:[Group#3], test.t.a, test.t.b, setvar(i, 0)->Column#13", "Group#3 Schema:[test.t.a,test.t.b]", @@ -495,7 +495,7 @@ "Group#1 Schema:[Column#13,Column#14]", " TopN_7 input:[Group#2], Column#13:desc, offset:0, count:1", "Group#2 Schema:[Column#13,Column#14]", - " Projection_2 input:[Group#3], getvar(i)->Column#13, setvar(i, cast(plus(cast(getvar(i), double BINARY), 1), var_string(5)))->Column#14", + " Projection_2 input:[Group#3], getvar(i)->Column#13, setvar(i, plus(getvar(i), 1))->Column#14", "Group#3 Schema:[test.t.a]", " TiKVSingleGather_9 input:[Group#4], table:t", " TiKVSingleGather_21 input:[Group#5], table:t, index:e_d_c_str_prefix", diff --git a/planner/core/cache.go b/planner/core/cache.go index 8f37a59a8c9b3..d2c8a0b2ee7a3 100644 --- a/planner/core/cache.go +++ b/planner/core/cache.go @@ -146,23 +146,45 @@ func NewPSTMTPlanCacheKey(sessionVars *variable.SessionVars, pstmtID uint32, sch return key } +// FieldSlice is the slice of the types.FieldType +type FieldSlice []types.FieldType + +// Equal compares FieldSlice with []*types.FieldType +func (s FieldSlice) Equal(tps []*types.FieldType) bool { + if len(s) != len(tps) { + return false + } + for i := range tps { + if !s[i].Equal(tps[i]) { + return false + } + } + return true +} + // PSTMTPlanCacheValue stores the cached Statement and StmtNode. type PSTMTPlanCacheValue struct { Plan Plan OutPutNames []*types.FieldName TblInfo2UnionScan map[*model.TableInfo]bool + UserVarTypes FieldSlice } // NewPSTMTPlanCacheValue creates a SQLCacheValue. -func NewPSTMTPlanCacheValue(plan Plan, names []*types.FieldName, srcMap map[*model.TableInfo]bool) *PSTMTPlanCacheValue { +func NewPSTMTPlanCacheValue(plan Plan, names []*types.FieldName, srcMap map[*model.TableInfo]bool, userVarTps []*types.FieldType) *PSTMTPlanCacheValue { dstMap := make(map[*model.TableInfo]bool) for k, v := range srcMap { dstMap[k] = v } + userVarTypes := make([]types.FieldType, len(userVarTps)) + for i, tp := range userVarTps { + userVarTypes[i] = *tp + } return &PSTMTPlanCacheValue{ Plan: plan, OutPutNames: names, TblInfo2UnionScan: dstMap, + UserVarTypes: userVarTypes, } } diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 107eea6352269..7640b23cfc744 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -282,6 +282,14 @@ func (e *Execute) getPhysicalPlan(ctx context.Context, sctx sessionctx.Context, if prepared.UseCache { cacheKey = NewPSTMTPlanCacheKey(sctx.GetSessionVars(), e.ExecID, prepared.SchemaVersion) } + tps := make([]*types.FieldType, len(e.UsingVars)) + for i, param := range e.UsingVars { + name := param.(*expression.ScalarFunction).GetArgs()[0].String() + tps[i] = sctx.GetSessionVars().UserVarTypes[name] + if tps[i] == nil { + tps[i] = types.NewFieldType(mysql.TypeNull) + } + } if prepared.CachedPlan != nil { // Rewriting the expression in the select.where condition will convert its // type from "paramMarker" to "Constant".When Point Select queries are executed, @@ -313,36 +321,42 @@ func (e *Execute) getPhysicalPlan(ctx context.Context, sctx sessionctx.Context, if err := e.checkPreparedPriv(ctx, sctx, preparedStmt, is); err != nil { return err } - cachedVal := cacheValue.(*PSTMTPlanCacheValue) - planValid := true - for tblInfo, unionScan := range cachedVal.TblInfo2UnionScan { - if !unionScan && tableHasDirtyContent(sctx, tblInfo) { - planValid = false - // TODO we can inject UnionScan into cached plan to avoid invalidating it, though - // rebuilding the filters in UnionScan is pretty trivial. - sctx.PreparedPlanCache().Delete(cacheKey) - break - } - } - if planValid { - err := e.rebuildRange(cachedVal.Plan) - if err != nil { - logutil.BgLogger().Debug("rebuild range failed", zap.Error(err)) - goto REBUILD + cachedVals := cacheValue.([]*PSTMTPlanCacheValue) + for _, cachedVal := range cachedVals { + if !cachedVal.UserVarTypes.Equal(tps) { + continue } - err = e.setFoundInPlanCache(sctx, true) - if err != nil { - return err + planValid := true + for tblInfo, unionScan := range cachedVal.TblInfo2UnionScan { + if !unionScan && tableHasDirtyContent(sctx, tblInfo) { + planValid = false + // TODO we can inject UnionScan into cached plan to avoid invalidating it, though + // rebuilding the filters in UnionScan is pretty trivial. + sctx.PreparedPlanCache().Delete(cacheKey) + break + } } - if metrics.ResettablePlanCacheCounterFortTest { - metrics.PlanCacheCounter.WithLabelValues("prepare").Inc() - } else { - planCacheCounter.Inc() + if planValid { + err := e.rebuildRange(cachedVal.Plan) + if err != nil { + logutil.BgLogger().Debug("rebuild range failed", zap.Error(err)) + goto REBUILD + } + err = e.setFoundInPlanCache(sctx, true) + if err != nil { + return err + } + if metrics.ResettablePlanCacheCounterFortTest { + metrics.PlanCacheCounter.WithLabelValues("prepare").Inc() + } else { + planCacheCounter.Inc() + } + e.names = cachedVal.OutPutNames + e.Plan = cachedVal.Plan + stmtCtx.SetPlanDigest(preparedStmt.NormalizedPlan, preparedStmt.PlanDigest) + return nil } - e.names = cachedVal.OutPutNames - e.Plan = cachedVal.Plan - stmtCtx.SetPlanDigest(preparedStmt.NormalizedPlan, preparedStmt.PlanDigest) - return nil + break } } } @@ -361,10 +375,25 @@ REBUILD: e.Plan = p _, isTableDual := p.(*PhysicalTableDual) if !isTableDual && prepared.UseCache { - cached := NewPSTMTPlanCacheValue(p, names, stmtCtx.TblInfo2UnionScan) + cached := NewPSTMTPlanCacheValue(p, names, stmtCtx.TblInfo2UnionScan, tps) preparedStmt.NormalizedPlan, preparedStmt.PlanDigest = NormalizePlan(p) stmtCtx.SetPlanDigest(preparedStmt.NormalizedPlan, preparedStmt.PlanDigest) - sctx.PreparedPlanCache().Put(cacheKey, cached) + if cacheVals, exists := sctx.PreparedPlanCache().Get(cacheKey); exists { + hitVal := false + for i, cacheVal := range cacheVals.([]*PSTMTPlanCacheValue) { + if cacheVal.UserVarTypes.Equal(tps) { + hitVal = true + cacheVals.([]*PSTMTPlanCacheValue)[i] = cached + break + } + } + if !hitVal { + cacheVals = append(cacheVals.([]*PSTMTPlanCacheValue), cached) + } + sctx.PreparedPlanCache().Put(cacheKey, cacheVals) + } else { + sctx.PreparedPlanCache().Put(cacheKey, []*PSTMTPlanCacheValue{cached}) + } } err = e.setFoundInPlanCache(sctx, false) return err diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index a8e04c2999bd2..de881e482be53 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -1158,17 +1158,27 @@ func (er *expressionRewriter) rewriteVariable(v *ast.VariableExpr) { sessionVars := er.b.ctx.GetSessionVars() if !v.IsSystem { if v.Value != nil { - er.ctxStack[stkLen-1], er.err = er.newFunction(ast.SetVar, - er.ctxStack[stkLen-1].GetType(), + tp := er.ctxStack[stkLen-1].GetType() + er.ctxStack[stkLen-1], er.err = er.newFunction(ast.SetVar, tp, expression.DatumToConstant(types.NewDatum(name), mysql.TypeString), er.ctxStack[stkLen-1]) er.ctxNameStk[stkLen-1] = types.EmptyName + // Store the field type of the variable into SessionVars.UserVarTypes. + // Normally we can infer the type from SessionVars.User, but we need SessionVars.UserVarTypes when + // GetVar has not been executed to fill the SessionVars.Users. + sessionVars.UsersLock.Lock() + sessionVars.UserVarTypes[name] = tp + sessionVars.UsersLock.Unlock() return } - f, err := er.newFunction(ast.GetVar, - // TODO: Here is wrong, the sessionVars should store a name -> Datum map. Will fix it later. - types.NewFieldType(mysql.TypeString), - expression.DatumToConstant(types.NewStringDatum(name), mysql.TypeString)) + sessionVars.UsersLock.RLock() + tp, ok := sessionVars.UserVarTypes[name] + sessionVars.UsersLock.RUnlock() + if !ok { + tp = types.NewFieldType(mysql.TypeVarString) + tp.Flen = mysql.MaxFieldVarCharLength + } + f, err := er.newFunction(ast.GetVar, tp, expression.DatumToConstant(types.NewStringDatum(name), mysql.TypeString)) if err != nil { er.err = err return diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index c9c8ebf1100bc..b7043dea2f630 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -1021,6 +1021,50 @@ func (b *PlanBuilder) buildProjectionField(ctx context.Context, p LogicalPlan, f return newCol, name, nil } +type userVarTypeProcessor struct { + ctx context.Context + plan LogicalPlan + builder *PlanBuilder + mapper map[*ast.AggregateFuncExpr]int + err error +} + +func (p *userVarTypeProcessor) Enter(in ast.Node) (ast.Node, bool) { + v, ok := in.(*ast.VariableExpr) + if !ok { + return in, false + } + if v.IsSystem || v.Value == nil { + return in, true + } + _, p.plan, p.err = p.builder.rewrite(p.ctx, v, p.plan, p.mapper, true) + return in, true +} + +func (p *userVarTypeProcessor) Leave(in ast.Node) (ast.Node, bool) { + return in, p.err == nil +} + +func (b *PlanBuilder) preprocessUserVarTypes(ctx context.Context, p LogicalPlan, fields []*ast.SelectField, mapper map[*ast.AggregateFuncExpr]int) error { + aggMapper := make(map[*ast.AggregateFuncExpr]int) + for agg, i := range mapper { + aggMapper[agg] = i + } + processor := userVarTypeProcessor{ + ctx: ctx, + plan: p, + builder: b, + mapper: aggMapper, + } + for _, field := range fields { + field.Expr.Accept(&processor) + if processor.err != nil { + return processor.err + } + } + return nil +} + // findColFromNaturalUsingJoin is used to recursively find the column from the // underlying natural-using-join. // e.g. For SQL like `select t2.a from t1 join t2 using(a) where t2.a > 0`, the @@ -1042,6 +1086,10 @@ func findColFromNaturalUsingJoin(p LogicalPlan, col *expression.Column) (name *t // buildProjection returns a Projection plan and non-aux columns length. func (b *PlanBuilder) buildProjection(ctx context.Context, p LogicalPlan, fields []*ast.SelectField, mapper map[*ast.AggregateFuncExpr]int, windowMapper map[*ast.WindowFuncExpr]int, considerWindow bool, expandGenerateColumn bool) (LogicalPlan, int, error) { + err := b.preprocessUserVarTypes(ctx, p, fields, mapper) + if err != nil { + return nil, 0, err + } b.optFlag |= flagEliminateProjection b.curClause = fieldList proj := LogicalProjection{Exprs: make([]expression.Expression, 0, len(fields))}.Init(b.ctx, b.getSelectOffset()) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 7156a6f562d53..5c1705f33eb34 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1002,7 +1002,12 @@ func (b *PlanBuilder) buildPrepare(x *ast.PrepareStmt) Plan { } if x.SQLVar != nil { if v, ok := b.ctx.GetSessionVars().Users[strings.ToLower(x.SQLVar.Name)]; ok { - p.SQLText = v.GetString() + var err error + p.SQLText, err = v.ToString() + if err != nil { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(err) + p.SQLText = "NULL" + } } else { p.SQLText = "NULL" } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 0ae6d2e4f4f95..790f6e036c1ed 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -370,6 +370,9 @@ type SessionVars struct { UsersLock sync.RWMutex // Users are user defined variables. Users map[string]types.Datum + // UserVarTypes stores the FieldType for user variables, it cannot be inferred from Users when Users have not been set yet. + // It is read/write protected by UsersLock. + UserVarTypes map[string]*types.FieldType // systems variables, don't modify it directly, use GetSystemVar/SetSystemVar method. systems map[string]string // stmtVars variables are temporarily set by SET_VAR hint @@ -812,6 +815,7 @@ type ConnectionInfo struct { func NewSessionVars() *SessionVars { vars := &SessionVars{ Users: make(map[string]types.Datum), + UserVarTypes: make(map[string]*types.FieldType), systems: make(map[string]string), stmtVars: make(map[string]string), PreparedStmts: make(map[uint32]interface{}), From a426a0e5ff45300bed9b7e569d2004c3401ec401 Mon Sep 17 00:00:00 2001 From: dy <34701401+dyzsr@users.noreply.github.com> Date: Tue, 17 Nov 2020 18:33:53 +0800 Subject: [PATCH 20/37] planner: disallow multi-updates on primary key (#20603) --- errors.toml | 5 ++++ executor/executor_test.go | 8 +++--- planner/core/errors.go | 1 + planner/core/integration_test.go | 35 +++++++++++++++++++++++ planner/core/logical_plan_builder.go | 16 +++++++++++ planner/core/testdata/plan_suite_in.json | 4 +-- planner/core/testdata/plan_suite_out.json | 4 +-- 7 files changed, 65 insertions(+), 8 deletions(-) diff --git a/errors.toml b/errors.toml index 0b4e353f97fff..cef0539d0ae67 100644 --- a/errors.toml +++ b/errors.toml @@ -891,6 +891,11 @@ error = ''' `%-.192s`.`%-.192s` contains view recursion ''' +["planner:1706"] +error = ''' +Primary key/partition key update is not allowed since the table is updated both as '%-.192s' and '%-.192s'. +''' + ["planner:1747"] error = ''' PARTITION () clause on non partitioned table diff --git a/executor/executor_test.go b/executor/executor_test.go index 1987bde05c43b..a84876b2ffba9 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -1984,16 +1984,16 @@ func (s *testSuiteP1) TestGeneratedColumnRead(c *C) { result.Check(testkit.Rows(`0 `, `1 3 2 6`, `3 7 12 14`, `8 16 64 32`)) // Test multi-update on generated columns. - tk.MustExec(`UPDATE test_gc_read m, test_gc_read n SET m.a = m.a + 10, n.a = n.a + 10`) + tk.MustExec(`UPDATE test_gc_read m, test_gc_read n SET m.b = m.b + 10, n.b = n.b + 10`) result = tk.MustQuery(`SELECT * FROM test_gc_read ORDER BY a`) - result.Check(testkit.Rows(`10 `, `11 2 13 22 26`, `13 4 17 52 34`, `18 8 26 144 52`)) + result.Check(testkit.Rows(`0 `, `1 12 13 12 26`, `3 14 17 42 34`, `8 18 26 144 52`)) tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") - tk.MustExec("insert into t values(18)") + tk.MustExec("insert into t values(8)") tk.MustExec("update test_gc_read set a = a+1 where a in (select a from t)") result = tk.MustQuery("select * from test_gc_read order by a") - result.Check(testkit.Rows(`10 `, `11 2 13 22 26`, `13 4 17 52 34`, `19 8 27 152 54`)) + result.Check(testkit.Rows(`0 `, `1 12 13 12 26`, `3 14 17 42 34`, `9 18 27 162 54`)) // Test different types between generation expression and generated column. tk.MustExec(`CREATE TABLE test_gc_read_cast(a VARCHAR(255), b VARCHAR(255), c INT AS (JSON_EXTRACT(a, b)), d INT AS (JSON_EXTRACT(a, b)) STORED)`) diff --git a/planner/core/errors.go b/planner/core/errors.go index 52338a23b096c..8231a0932c151 100644 --- a/planner/core/errors.go +++ b/planner/core/errors.go @@ -45,6 +45,7 @@ var ( ErrWrongGroupField = dbterror.ClassOptimizer.NewStd(mysql.ErrWrongGroupField) ErrDupFieldName = dbterror.ClassOptimizer.NewStd(mysql.ErrDupFieldName) ErrNonUpdatableTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNonUpdatableTable) + ErrMultiUpdateKeyConflict = dbterror.ClassOptimizer.NewStd(mysql.ErrMultiUpdateKeyConflict) ErrInternal = dbterror.ClassOptimizer.NewStd(mysql.ErrInternal) ErrNonUniqTable = dbterror.ClassOptimizer.NewStd(mysql.ErrNonuniqTable) ErrWindowInvalidWindowFuncUse = dbterror.ClassOptimizer.NewStd(mysql.ErrWindowInvalidWindowFuncUse) diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 826cbbfbb8475..656943ec7a569 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -1857,3 +1857,38 @@ func (s *testIntegrationSuite) TestIssue10448(c *C) { tk.MustExec("insert into t values(1),(2),(3)") tk.MustQuery("select a from (select pk as a from t) t1 where a = 18446744073709551615").Check(testkit.Rows()) } + +func (s *testIntegrationSuite) TestUpdateMultiUpdatePK(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int not null primary key)") + tk.MustExec("insert into t values (1)") + tk.MustGetErrMsg(`UPDATE t m, t n SET m.a = m.a + 10, n.a = n.a + 10`, + `[planner:1706]Primary key/partition key update is not allowed since the table is updated both as 'm' and 'n'.`) + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a varchar(10) not null primary key)") + tk.MustExec("insert into t values ('abc')") + tk.MustGetErrMsg(`UPDATE t m, t n SET m.a = 'def', n.a = 'xyz'`, + `[planner:1706]Primary key/partition key update is not allowed since the table is updated both as 'm' and 'n'.`) + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int, primary key (a, b))") + tk.MustExec("insert into t values (1, 2)") + tk.MustGetErrMsg(`UPDATE t m, t n SET m.a = m.a + 10, n.b = n.b + 10`, + `[planner:1706]Primary key/partition key update is not allowed since the table is updated both as 'm' and 'n'.`) + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int primary key, b int)") + tk.MustExec("insert into t values (1, 2)") + tk.MustGetErrMsg(`UPDATE t m, t n SET m.a = m.a + 10, n.a = n.a + 10`, + `[planner:1706]Primary key/partition key update is not allowed since the table is updated both as 'm' and 'n'.`) + + tk.MustExec(`UPDATE t m, t n SET m.b = m.b + 10, n.b = n.b + 10`) + tk.MustQuery("SELECT * FROM t").Check(testkit.Rows("1 12")) + + tk.MustExec(`UPDATE t m, t n SET m.a = m.a + 1, n.b = n.b + 10`) + tk.MustQuery("SELECT * FROM t").Check(testkit.Rows("2 12")) +} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index b7043dea2f630..f263a5ae58238 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -3800,13 +3800,29 @@ func checkUpdateList(ctx sessionctx.Context, tblID2table map[int64]table.Table, if err != nil { return err } + isPKUpdated := make(map[int64]model.CIStr) for _, content := range updt.TblColPosInfos { tbl := tblID2table[content.TblID] flags := assignFlags[content.Start:content.End] + var updatePK bool for i, col := range tbl.WritableCols() { if flags[i] && col.State != model.StatePublic { return ErrUnknownColumn.GenWithStackByArgs(col.Name, clauseMsg[fieldList]) } + // Check for multi-updates on primary key, + // see https://dev.mysql.com/doc/mysql-errors/5.7/en/server-error-reference.html#error_er_multi_update_key_conflict + if !flags[i] { + continue + } + if col.IsPKHandleColumn(tbl.Meta()) || col.IsCommonHandleColumn(tbl.Meta()) { + updatePK = true + } + } + if updatePK { + if otherTblName, ok := isPKUpdated[tbl.Meta().ID]; ok { + return ErrMultiUpdateKeyConflict.GenWithStackByArgs(otherTblName.O, updt.names[content.Start].TblName.O) + } + isPKUpdated[tbl.Meta().ID] = updt.names[content.Start].TblName } } return nil diff --git a/planner/core/testdata/plan_suite_in.json b/planner/core/testdata/plan_suite_in.json index 22e77d4f6e792..e2a62fb8112cb 100644 --- a/planner/core/testdata/plan_suite_in.json +++ b/planner/core/testdata/plan_suite_in.json @@ -250,8 +250,8 @@ // Test join hint for delete and update "delete /*+ TIDB_INLJ(t1, t2) */ t1 from t t1, t t2 where t1.c=t2.c", "delete /*+ TIDB_SMJ(t1, t2) */ from t1 using t t1, t t2 where t1.c=t2.c", - "update /*+ TIDB_SMJ(t1, t2) */ t t1, t t2 set t1.a=1, t2.a=1 where t1.a=t2.a", - "update /*+ TIDB_HJ(t1, t2) */ t t1, t t2 set t1.a=1, t2.a=1 where t1.a=t2.a", + "update /*+ TIDB_SMJ(t1, t2) */ t t1, t t2 set t1.c=1, t2.a=1 where t1.a=t2.a", + "update /*+ TIDB_HJ(t1, t2) */ t t1, t t2 set t1.c=1, t2.a=1 where t1.a=t2.a", // Test complex delete. "delete from t where b < 1 order by d limit 1", // Test simple delete. diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 5a60d84223e14..75ffa492dc74a 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -685,12 +685,12 @@ "Hints": "use_index(@`del_1` `test`.`t1` `c_d_e`), use_index(@`del_1` `test`.`t2` `c_d_e`), merge_join(@`del_1` `test`.`t1`)" }, { - "SQL": "update /*+ TIDB_SMJ(t1, t2) */ t t1, t t2 set t1.a=1, t2.a=1 where t1.a=t2.a", + "SQL": "update /*+ TIDB_SMJ(t1, t2) */ t t1, t t2 set t1.c=1, t2.a=1 where t1.a=t2.a", "Best": "MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t.a,test.t.a)->Update", "Hints": "use_index(@`upd_1` `test`.`t1` ), use_index(@`upd_1` `test`.`t2` ), merge_join(@`upd_1` `test`.`t1`)" }, { - "SQL": "update /*+ TIDB_HJ(t1, t2) */ t t1, t t2 set t1.a=1, t2.a=1 where t1.a=t2.a", + "SQL": "update /*+ TIDB_HJ(t1, t2) */ t t1, t t2 set t1.c=1, t2.a=1 where t1.a=t2.a", "Best": "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t.a,test.t.a)->Update", "Hints": "use_index(@`upd_1` `test`.`t1` ), use_index(@`upd_1` `test`.`t2` ), hash_join(@`upd_1` `test`.`t1`)" }, From f55aa047136800846fc433553a81482882cef2c5 Mon Sep 17 00:00:00 2001 From: Rain Li Date: Wed, 18 Nov 2020 19:27:52 +0800 Subject: [PATCH 21/37] ddl: fix partition value type check (#21016) --- ddl/db_partition_test.go | 93 +++++++++++++++++++++++++++++++++++++++- ddl/ddl_api.go | 24 ++++++++++- 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 09a28bb9f94b0..d9e325e931c25 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -825,7 +825,7 @@ func (s *testIntegrationSuite5) TestAlterTableAddPartitionByListColumns(c *C) { ddl.ErrMultipleDefConstInListPart, }, {"alter table t add partition (partition p6 values in (('a','a')))", - ddl.ErrNotAllowedTypeInPartition, + ddl.ErrWrongTypeColumnValue, }, } @@ -2866,3 +2866,94 @@ func (s *testIntegrationSuite7) TestCommitWhenSchemaChange(c *C) { tk.MustExec("admin check table nt") tk.MustQuery("select * from nt").Check(testkit.Rows()) } + +func (s *testIntegrationSuite7) TestCreatePartitionTableWithWrongType(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + var err error + _, err = tk.Exec(`create table t( + b int(10) + ) partition by range columns (b) ( + partition p0 values less than (0x10), + partition p3 values less than (0x20) + )`) + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec(`create table t( + b int(10) + ) partition by range columns (b) ( + partition p0 values less than ('g'), + partition p3 values less than ('k') + )`) + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec(`create table t( + b char(10) + ) partition by range columns (b) ( + partition p0 values less than (30), + partition p3 values less than (60) + )`) + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec(`create table t( + b datetime + ) partition by range columns (b) ( + partition p0 values less than ('g'), + partition p3 values less than ('m') + )`) + c.Assert(err, NotNil) +} + +func (s *testIntegrationSuite7) TestAddPartitionForTableWithWrongType(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop tables if exists t_int, t_char, t_date") + tk.MustExec(`create table t_int(b int(10)) + partition by range columns (b) ( + partition p0 values less than (10) + )`) + + tk.MustExec(`create table t_char(b char(10)) + partition by range columns (b) ( + partition p0 values less than ('a') + )`) + + tk.MustExec(`create table t_date(b datetime) + partition by range columns (b) ( + partition p0 values less than ('2020-09-01') + )`) + + var err error + + _, err = tk.Exec("alter table t_int add partition (partition p1 values less than ('g'))") + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec("alter table t_int add partition (partition p1 values less than (0x20))") + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec("alter table t_char add partition (partition p1 values less than (0x20))") + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec("alter table t_char add partition (partition p1 values less than (10))") + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec("alter table t_date add partition (partition p1 values less than ('m'))") + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec("alter table t_date add partition (partition p1 values less than (0x20))") + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) + + _, err = tk.Exec("alter table t_date add partition (partition p1 values less than (20))") + c.Assert(err, NotNil) + c.Assert(ddl.ErrWrongTypeColumnValue.Equal(err), IsTrue) +} diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index ad67066ee8b3f..74a53702eb7f8 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -5288,10 +5288,32 @@ func checkColumnsTypeAndValuesMatch(ctx sessionctx.Context, meta *model.TableInf return err } // Check val.ConvertTo(colType) doesn't work, so we need this case by case check. + vkind := val.Kind() switch colType.Tp { case mysql.TypeDate, mysql.TypeDatetime: - switch val.Kind() { + switch vkind { case types.KindString, types.KindBytes: + if _, err := val.ConvertTo(ctx.GetSessionVars().StmtCtx, colType); err != nil { + return ErrWrongTypeColumnValue.GenWithStackByArgs() + } + default: + return ErrWrongTypeColumnValue.GenWithStackByArgs() + } + case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong: + switch vkind { + case types.KindInt64, types.KindUint64, types.KindNull: + default: + return ErrWrongTypeColumnValue.GenWithStackByArgs() + } + case mysql.TypeFloat, mysql.TypeDouble: + switch vkind { + case types.KindFloat32, types.KindFloat64, types.KindNull: + default: + return ErrWrongTypeColumnValue.GenWithStackByArgs() + } + case mysql.TypeString, mysql.TypeVarString: + switch vkind { + case types.KindString, types.KindBytes, types.KindNull: default: return ErrWrongTypeColumnValue.GenWithStackByArgs() } From e6e894dcc409e6414487f837cdfe16db301289db Mon Sep 17 00:00:00 2001 From: Rain Li Date: Thu, 19 Nov 2020 10:58:45 +0800 Subject: [PATCH 22/37] executor: add missing update table delta for TxnCtx (#20982) --- executor/batch_point_get.go | 9 +- executor/executor.go | 18 ++++ executor/executor_test.go | 150 +++++++++++++++++++++++++++++++++ executor/point_get.go | 3 + sessionctx/variable/session.go | 9 ++ 5 files changed, 188 insertions(+), 1 deletion(-) diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index 3b7e1694101fd..826a88d3aacdf 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -139,6 +139,9 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error { if err := e.initialize(ctx); err != nil { return err } + if e.lock { + e.updateDeltaForTableID(e.tblInfo.ID) + } } if e.index >= len(e.values) { @@ -232,7 +235,11 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { } e.handles = append(e.handles, handle) if e.tblInfo.Partition != nil { - e.physIDs = append(e.physIDs, tablecodec.DecodeTableID(key)) + pid := tablecodec.DecodeTableID(key) + e.physIDs = append(e.physIDs, pid) + if e.lock { + e.updateDeltaForTableID(pid) + } } } diff --git a/executor/executor.go b/executor/executor.go index 2102f3ca1fabf..8f211050cc251 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -203,6 +203,12 @@ func (e *baseExecutor) Next(ctx context.Context, req *chunk.Chunk) error { return nil } +func (e *baseExecutor) updateDeltaForTableID(id int64) { + txnCtx := e.ctx.GetSessionVars().TxnCtx + udpp := e.ctx.GetSessionVars().UseDynamicPartitionPrune() + txnCtx.UpdateDeltaForTable(id, id, 0, 0, map[int64]int64{}, udpp) +} + func newBaseExecutor(ctx sessionctx.Context, schema *expression.Schema, id int, children ...Executor) baseExecutor { e := baseExecutor{ children: children, @@ -943,6 +949,18 @@ func (e *SelectLockExec) Next(ctx context.Context, req *chunk.Chunk) error { lockWaitTime = int64(e.Lock.WaitSec) * 1000 } + if len(e.tblID2Handle) > 0 { + for id := range e.tblID2Handle { + e.updateDeltaForTableID(id) + } + } + if len(e.partitionedTable) > 0 { + for _, p := range e.partitionedTable { + pid := p.Meta().ID + e.updateDeltaForTableID(pid) + } + } + return doLockKeys(ctx, e.ctx, newLockCtx(e.ctx.GetSessionVars(), lockWaitTime), e.keys...) } diff --git a/executor/executor_test.go b/executor/executor_test.go index a84876b2ffba9..f36cd4a021246 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -6865,3 +6865,153 @@ func (s *testSuite) TestIssue19667(c *C) { tk.MustExec("INSERT INTO t VALUES('1988-04-17 01:59:59')") tk.MustQuery(`SELECT DATE_ADD(a, INTERVAL 1 SECOND) FROM t`).Check(testkit.Rows("1988-04-17 02:00:00")) } + +func issue20975Prepare(c *C, store kv.Storage) (*testkit.TestKit, *testkit.TestKit) { + tk1 := testkit.NewTestKit(c, store) + tk2 := testkit.NewTestKit(c, store) + tk1.MustExec("use test") + tk1.MustExec("drop table if exists t1, t2") + tk2.MustExec("use test") + tk1.MustExec("create table t1(id int primary key, c int)") + tk1.MustExec("insert into t1 values(1, 10), (2, 20)") + return tk1, tk2 +} + +func (s *testSuite) TestIssue20975UpdateNoChange(c *C) { + tk1, tk2 := issue20975Prepare(c, s.store) + tk1.MustExec("begin pessimistic") + tk1.MustExec("update t1 set c=c") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") +} + +func (s *testSuite) TestIssue20975SelectForUpdate(c *C) { + tk1, tk2 := issue20975Prepare(c, s.store) + tk1.MustExec("begin") + tk1.MustExec("select * from t1 for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") +} + +func (s *testSuite) TestIssue20975SelectForUpdatePointGet(c *C) { + tk1, tk2 := issue20975Prepare(c, s.store) + tk1.MustExec("begin") + tk1.MustExec("select * from t1 where id=1 for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 where id=1 for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") +} + +func (s *testSuite) TestIssue20975SelectForUpdateBatchPointGet(c *C) { + tk1, tk2 := issue20975Prepare(c, s.store) + tk1.MustExec("begin") + tk1.MustExec("select * from t1 where id in (1, 2) for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 where id in (1, 2) for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") +} + +func issue20975PreparePartitionTable(c *C, store kv.Storage) (*testkit.TestKit, *testkit.TestKit) { + tk1 := testkit.NewTestKit(c, store) + tk2 := testkit.NewTestKit(c, store) + tk1.MustExec("use test") + tk1.MustExec("drop table if exists t1, t2") + tk2.MustExec("use test") + tk1.MustExec(`create table t1(id int primary key, c int) partition by range (id) ( + partition p1 values less than (10), + partition p2 values less than (20) + )`) + tk1.MustExec("insert into t1 values(1, 10), (2, 20), (11, 30), (12, 40)") + return tk1, tk2 +} + +func (s *testSuite) TestIssue20975UpdateNoChangeWithPartitionTable(c *C) { + tk1, tk2 := issue20975PreparePartitionTable(c, s.store) + tk1.MustExec("begin pessimistic") + tk1.MustExec("update t1 set c=c") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") +} + +func (s *testSuite) TestIssue20975SelectForUpdateWithPartitionTable(c *C) { + tk1, tk2 := issue20975PreparePartitionTable(c, s.store) + tk1.MustExec("begin") + tk1.MustExec("select * from t1 for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") +} + +func (s *testSuite) TestIssue20975SelectForUpdatePointGetWithPartitionTable(c *C) { + tk1, tk2 := issue20975PreparePartitionTable(c, s.store) + tk1.MustExec("begin") + tk1.MustExec("select * from t1 where id=1 for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin") + tk1.MustExec("select * from t1 where id=12 for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 where id=1 for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 where id=12 for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") +} + +func (s *testSuite) TestIssue20975SelectForUpdateBatchPointGetWithPartitionTable(c *C) { + tk1, tk2 := issue20975PreparePartitionTable(c, s.store) + tk1.MustExec("begin") + tk1.MustExec("select * from t1 where id in (1, 2) for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin") + tk1.MustExec("select * from t1 where id in (11, 12) for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") + + tk1.MustExec("begin") + tk1.MustExec("select * from t1 where id in (1, 11) for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 where id in (1, 2) for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 where id in (11, 12) for update") + tk2.MustExec("create table t2(a int)") + tk1.MustExec("commit") + + tk1.MustExec("begin pessimistic") + tk1.MustExec("select * from t1 where id in (1, 11) for update") + tk2.MustExec("drop table t2") + tk1.MustExec("commit") + +} diff --git a/executor/point_get.go b/executor/point_get.go index 1dab17890b9ec..0dd73eec70b33 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -180,6 +180,9 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { } else { tblID = e.tblInfo.ID } + if e.lock { + e.updateDeltaForTableID(tblID) + } if e.idxInfo != nil { if isCommonHandleRead(e.tblInfo, e.idxInfo) { handleBytes, err := EncodeUniqueIndexValuesForKey(e.ctx, e.tblInfo, e.idxInfo, e.idxVals) diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 790f6e036c1ed..34e198237d5bc 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -171,6 +171,9 @@ type TransactionContext struct { LockExpire uint32 ForUpdate uint32 + // TableDeltaMap lock to prevent potential data race + tdmLock sync.Mutex + // TxnScope stores the value of 'txn_scope'. TxnScope string } @@ -221,6 +224,8 @@ func (tc *TransactionContext) CollectUnchangedRowKeys(buf []kv.Key) []kv.Key { // UpdateDeltaForTable updates the delta info for some table. func (tc *TransactionContext) UpdateDeltaForTable(logicalTableID, physicalTableID int64, delta int64, count int64, colSize map[int64]int64, saveAsLogicalTblID bool) { + tc.tdmLock.Lock() + defer tc.tdmLock.Unlock() if tc.TableDeltaMap == nil { tc.TableDeltaMap = make(map[int64]TableDelta) } @@ -265,13 +270,17 @@ func (tc *TransactionContext) Cleanup() { // tc.InfoSchema = nil; we cannot do it now, because some operation like handleFieldList depend on this. tc.Binlog = nil tc.History = nil + tc.tdmLock.Lock() tc.TableDeltaMap = nil + tc.tdmLock.Unlock() tc.pessimisticLockCache = nil } // ClearDelta clears the delta map. func (tc *TransactionContext) ClearDelta() { + tc.tdmLock.Lock() tc.TableDeltaMap = nil + tc.tdmLock.Unlock() } // GetForUpdateTS returns the ts for update. From c5caca14c57fe351f392f8ec17d55dec8b231ad6 Mon Sep 17 00:00:00 2001 From: Zejun Li Date: Thu, 19 Nov 2020 11:14:43 +0800 Subject: [PATCH 23/37] tikv: add size limit to batch get cache (#21015) Co-authored-by: ti-srebot <66930949+ti-srebot@users.noreply.github.com> --- store/tikv/snapshot.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index 3c34b47a79011..f13e45076e682 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -75,9 +75,10 @@ type tikvSnapshot struct { // It's OK as long as there are no zero-byte values in the protocol. mu struct { sync.RWMutex - hitCnt int64 - cached map[string][]byte - stats *SnapshotRuntimeStats + hitCnt int64 + cached map[string][]byte + cachedSize int + stats *SnapshotRuntimeStats } sampleStep uint32 } @@ -174,7 +175,23 @@ func (s *tikvSnapshot) BatchGet(ctx context.Context, keys []kv.Key) (map[string] s.mu.cached = make(map[string][]byte, len(m)) } for _, key := range keys { - s.mu.cached[string(key)] = m[string(key)] + val := m[string(key)] + s.mu.cachedSize += len(key) + len(val) + s.mu.cached[string(key)] = val + } + + const cachedSizeLimit = 10 << 30 + if s.mu.cachedSize >= cachedSizeLimit { + for k, v := range s.mu.cached { + if _, needed := m[k]; needed { + continue + } + delete(s.mu.cached, k) + s.mu.cachedSize -= len(k) + len(v) + if s.mu.cachedSize < cachedSizeLimit { + break + } + } } s.mu.Unlock() From 7475d92393724e5e9763ad0f3073204d7ba1d87b Mon Sep 17 00:00:00 2001 From: HuaiyuXu <391585975@qq.com> Date: Thu, 19 Nov 2020 11:27:53 +0800 Subject: [PATCH 24/37] *: inject projection for the plan pushed to TiDB (#21090) --- executor/coprocessor.go | 1 + infoschema/tables_test.go | 1 + planner/core/optimizer.go | 2 +- planner/core/pb_to_plan.go | 17 ++++++++++------- planner/core/rule_inject_extra_projection.go | 8 ++++++-- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/executor/coprocessor.go b/executor/coprocessor.go index cf11d38512177..32eb421d041d0 100644 --- a/executor/coprocessor.go +++ b/executor/coprocessor.go @@ -166,6 +166,7 @@ func (h *CoprocessorDAGHandler) buildDAGExecutor(req *coprocessor.Request) (Exec if err != nil { return nil, errors.Trace(err) } + plan = core.InjectExtraProjection(plan) // Build executor. b := newExecutorBuilder(h.sctx, is) return b.build(plan), nil diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 11e1a0de9e170..5d141d01cb03d 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -811,6 +811,7 @@ func (s *testClusterTableSuite) TestSelectClusterTable(c *C) { tk.MustQuery("select query_time, conn_id from `CLUSTER_SLOW_QUERY` order by time limit 1").Check(testkit.Rows("4.895492 6")) tk.MustQuery("select count(*) from `CLUSTER_SLOW_QUERY` group by digest").Check(testkit.Rows("1")) tk.MustQuery("select digest, count(*) from `CLUSTER_SLOW_QUERY` group by digest").Check(testkit.Rows("42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772 1")) + tk.MustQuery(`select length(query) as l,time from information_schema.cluster_slow_query where time > "2019-02-12 19:33:56" order by abs(l) desc limit 10;`).Check(testkit.Rows("21 2019-02-12 19:33:56.571953")) tk.MustQuery("select count(*) from `CLUSTER_SLOW_QUERY` where time > now() group by digest").Check(testkit.Rows()) re := tk.MustQuery("select * from `CLUSTER_statements_summary`") c.Assert(re, NotNil) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index ff6c64b9a84ad..983713c36acaa 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -146,7 +146,7 @@ func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic func postOptimize(sctx sessionctx.Context, plan PhysicalPlan) PhysicalPlan { plan = eliminatePhysicalProjection(plan) - plan = injectExtraProjection(plan) + plan = InjectExtraProjection(plan) plan = eliminateUnionScanAndLock(sctx, plan) plan = enableParallelApply(sctx, plan) return plan diff --git a/planner/core/pb_to_plan.go b/planner/core/pb_to_plan.go index 6556edd0fb85e..e624bb96374c8 100644 --- a/planner/core/pb_to_plan.go +++ b/planner/core/pb_to_plan.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/planner/util" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" @@ -48,7 +49,9 @@ func (b *PBPlanBuilder) Build(executors []*tipb.Executor) (p PhysicalPlan, err e if err != nil { return nil, errors.Trace(err) } - curr.SetChildren(src) + if src != nil { + curr.SetChildren(src) + } src = curr } _, src = b.predicatePushDown(src, nil) @@ -101,7 +104,7 @@ func (b *PBPlanBuilder) pbToTableScan(e *tipb.Executor) (PhysicalPlan, error) { DBName: dbInfo.Name, Table: tbl.Meta(), Columns: columns, - }.Init(b.sctx, nil, 0) + }.Init(b.sctx, &property.StatsInfo{}, 0) p.SetSchema(schema) if strings.ToUpper(p.Table.Name.O) == infoschema.ClusterTableSlowLog { p.Extractor = &SlowQueryExtractor{} @@ -134,7 +137,7 @@ func (b *PBPlanBuilder) pbToSelection(e *tipb.Executor) (PhysicalPlan, error) { } p := PhysicalSelection{ Conditions: conds, - }.Init(b.sctx, nil, 0) + }.Init(b.sctx, &property.StatsInfo{}, 0, &property.PhysicalProperty{}) return p, nil } @@ -152,14 +155,14 @@ func (b *PBPlanBuilder) pbToTopN(e *tipb.Executor) (PhysicalPlan, error) { p := PhysicalTopN{ ByItems: byItems, Count: topN.Limit, - }.Init(b.sctx, nil, 0) + }.Init(b.sctx, &property.StatsInfo{}, 0, &property.PhysicalProperty{}) return p, nil } func (b *PBPlanBuilder) pbToLimit(e *tipb.Executor) (PhysicalPlan, error) { p := PhysicalLimit{ Count: e.Limit.Limit, - }.Init(b.sctx, nil, 0) + }.Init(b.sctx, &property.StatsInfo{}, 0, &property.PhysicalProperty{}) return p, nil } @@ -176,9 +179,9 @@ func (b *PBPlanBuilder) pbToAgg(e *tipb.Executor, isStreamAgg bool) (PhysicalPla baseAgg.schema = schema var partialAgg PhysicalPlan if isStreamAgg { - partialAgg = baseAgg.initForStream(b.sctx, nil, 0) + partialAgg = baseAgg.initForStream(b.sctx, &property.StatsInfo{}, 0, &property.PhysicalProperty{}) } else { - partialAgg = baseAgg.initForHash(b.sctx, nil, 0) + partialAgg = baseAgg.initForHash(b.sctx, &property.StatsInfo{}, 0, &property.PhysicalProperty{}) } return partialAgg, nil } diff --git a/planner/core/rule_inject_extra_projection.go b/planner/core/rule_inject_extra_projection.go index 977c73761f2fe..62d6b17cbcd28 100644 --- a/planner/core/rule_inject_extra_projection.go +++ b/planner/core/rule_inject_extra_projection.go @@ -20,11 +20,15 @@ import ( "github.com/pingcap/tidb/sessionctx" ) -// injectExtraProjection is used to extract the expressions of specific +// InjectExtraProjection is used to extract the expressions of specific // operators into a physical Projection operator and inject the Projection below // the operators. Thus we can accelerate the expression evaluation by eager // evaluation. -func injectExtraProjection(plan PhysicalPlan) PhysicalPlan { +// This function will be called in two situations: +// 1. In postOptimize. +// 2. TiDB can be used as a coprocessor, when a plan tree been pushed down to +// TiDB, we need to inject extra projections for the plan tree as well. +func InjectExtraProjection(plan PhysicalPlan) PhysicalPlan { return NewProjInjector().inject(plan) } From 6a175f1036934b4067fd4b94f63a12ff9a64082b Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 19 Nov 2020 11:43:43 +0800 Subject: [PATCH 25/37] distsql: fix issue of table reader runtime stats display wrong result. (#21072) --- distsql/distsql_test.go | 12 ++++++++++++ distsql/select_result.go | 8 +++++--- store/tikv/region_request.go | 12 ++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/distsql/distsql_test.go b/distsql/distsql_test.go index 6942b7c32d60b..53b43c52d8da9 100644 --- a/distsql/distsql_test.go +++ b/distsql/distsql_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" @@ -179,6 +180,17 @@ func (s *testSuite) TestSelectResultRuntimeStats(c *C) { c.Assert(stats.String(), Equals, expect) // Test for idempotence. c.Assert(stats.String(), Equals, expect) + + s1.rpcStat.Stats[tikvrpc.CmdCop] = &tikv.RPCRuntimeStats{ + Count: 1, + Consume: int64(time.Second), + } + stmtStats.RegisterStats(2, s1) + stats = stmtStats.GetRootStats(2) + expect = "cop_task: {num: 2, max: 1s, min: 1ms, avg: 500.5ms, p95: 1s, max_proc_keys: 200, p95_proc_keys: 200, tot_proc: 1s, tot_wait: 1s, rpc_num: 1, rpc_time: 1s, copr_cache_hit_ratio: 0.00}, backoff{RegionMiss: 1ms}" + c.Assert(stats.String(), Equals, expect) + // Test for idempotence. + c.Assert(stats.String(), Equals, expect) } func (s *testSuite) createSelectStreaming(batch, totalRows int, c *C) (*streamResult, []*types.FieldType) { diff --git a/distsql/select_result.go b/distsql/select_result.go index 3247d65c76304..a316a4fba7ad6 100644 --- a/distsql/select_result.go +++ b/distsql/select_result.go @@ -403,6 +403,7 @@ func (s *selectResultRuntimeStats) Merge(rs execdetails.RuntimeStats) { func (s *selectResultRuntimeStats) String() string { buf := bytes.NewBuffer(nil) + rpcStat := s.rpcStat if len(s.copRespTime) > 0 { size := len(s.copRespTime) if size == 1 { @@ -440,9 +441,10 @@ func (s *selectResultRuntimeStats) String() string { } } } - copRPC := s.rpcStat.Stats[tikvrpc.CmdCop] + copRPC := rpcStat.Stats[tikvrpc.CmdCop] if copRPC != nil && copRPC.Count > 0 { - delete(s.rpcStat.Stats, tikvrpc.CmdCop) + rpcStat = rpcStat.Clone() + delete(rpcStat.Stats, tikvrpc.CmdCop) buf.WriteString(", rpc_num: ") buf.WriteString(strconv.FormatInt(copRPC.Count, 10)) buf.WriteString(", rpc_time: ") @@ -453,7 +455,7 @@ func (s *selectResultRuntimeStats) String() string { buf.WriteString("}") } - rpcStatsStr := s.rpcStat.String() + rpcStatsStr := rpcStat.String() if len(rpcStatsStr) > 0 { buf.WriteString(", ") buf.WriteString(rpcStatsStr) diff --git a/store/tikv/region_request.go b/store/tikv/region_request.go index e7de51d1ef703..8c3cf2cad7a1c 100644 --- a/store/tikv/region_request.go +++ b/store/tikv/region_request.go @@ -99,6 +99,18 @@ func (r *RegionRequestRuntimeStats) String() string { return buf.String() } +// Clone returns a copy of itself. +func (r *RegionRequestRuntimeStats) Clone() RegionRequestRuntimeStats { + newRs := NewRegionRequestRuntimeStats() + for cmd, v := range r.Stats { + newRs.Stats[cmd] = &RPCRuntimeStats{ + Count: v.Count, + Consume: v.Consume, + } + } + return newRs +} + // Merge merges other RegionRequestRuntimeStats. func (r *RegionRequestRuntimeStats) Merge(rs RegionRequestRuntimeStats) { for cmd, v := range rs.Stats { From 27c59c7363f74f9de5ffd4582f748041cf08b512 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Thu, 19 Nov 2020 12:03:47 +0800 Subject: [PATCH 26/37] server: make dump binary time more compatible with MySQL (#21127) --- server/util.go | 14 +++++++++++++- server/util_test.go | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/server/util.go b/server/util.go index 51df865a8907e..c4a70676f6c3a 100644 --- a/server/util.go +++ b/server/util.go @@ -199,12 +199,24 @@ func dumpBinaryDateTime(data []byte, t types.Time) []byte { switch t.Type() { case mysql.TypeTimestamp, mysql.TypeDatetime: if t.IsZero() { + // All zero. data = append(data, 0) - } else { + } else if t.Microsecond() != 0 { + // Has micro seconds. data = append(data, 11) data = dumpUint16(data, uint16(year)) data = append(data, byte(mon), byte(day), byte(t.Hour()), byte(t.Minute()), byte(t.Second())) data = dumpUint32(data, uint32(t.Microsecond())) + } else if t.Hour() != 0 || t.Minute() != 0 || t.Second() != 0 { + // Has HH:MM:SS + data = append(data, 7) + data = dumpUint16(data, uint16(year)) + data = append(data, byte(mon), byte(day), byte(t.Hour()), byte(t.Minute()), byte(t.Second())) + } else { + // Only YY:MM:DD + data = append(data, 4) + data = dumpUint16(data, uint16(year)) + data = append(data, byte(mon), byte(day)) } case mysql.TypeDate: if t.IsZero() { diff --git a/server/util_test.go b/server/util_test.go index 029734167dc9f..de209d2a644e2 100644 --- a/server/util_test.go +++ b/server/util_test.go @@ -85,7 +85,7 @@ func (s *testUtilSuite) TestDumpBinaryTime(c *C) { c.Assert(err, IsNil) d = dumpBinaryDateTime(nil, t) // 201 & 7 composed to uint16 1993 (litter-endian) - c.Assert(d, DeepEquals, []byte{11, 201, 7, 7, 13, 1, 1, 1, 0, 0, 0, 0}) + c.Assert(d, DeepEquals, []byte{7, 201, 7, 7, 13, 1, 1, 1}) t, err = types.ParseDate(nil, "0000-00-00") c.Assert(err, IsNil) From 629ef07597bd35586260003af298f97dd7f74cc7 Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Thu, 19 Nov 2020 12:24:23 +0800 Subject: [PATCH 27/37] plan: fix the IsSelectForUpdateLockType check (#21125) --- planner/core/point_get_plan.go | 3 ++- session/session_test.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index 77efe8f5d5a67..c6cf3b25260a9 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -453,11 +453,12 @@ func TryFastPlan(ctx sessionctx.Context, node ast.Node) (p Plan) { // IsSelectForUpdateLockType checks if the select lock type is for update type. func IsSelectForUpdateLockType(lockType ast.SelectLockType) bool { if lockType == ast.SelectLockForUpdate || + lockType == ast.SelectLockInShareMode || lockType == ast.SelectLockForUpdateNoWait || lockType == ast.SelectLockForUpdateWaitN { return true } - return true + return false } func getLockWaitTime(ctx sessionctx.Context, lockInfo *ast.SelectLockInfo) (lock bool, waitTime int64) { diff --git a/session/session_test.go b/session/session_test.go index 0d547b39f018e..6c9914f644def 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3516,3 +3516,15 @@ func (s *testBackupRestoreSuite) TestBackupAndRestore(c *C) { tk.MustExec("drop database br02") } } + +func (s *testSessionSuite2) TestSelectLockInShare(c *C) { + tk1 := testkit.NewTestKitWithInit(c, s.store) + tk1.MustExec("DROP TABLE IF EXISTS t_sel_in_share") + tk1.MustExec("CREATE TABLE t_sel_in_share (id int DEFAULT NULL)") + tk1.MustExec("insert into t_sel_in_share values (11)") + err := tk1.ExecToErr("select * from t_sel_in_share lock in share mode") + c.Assert(err, NotNil) + tk1.MustExec("set @@tidb_enable_noop_functions = 1") + tk1.MustQuery("select * from t_sel_in_share lock in share mode").Check(testkit.Rows("11")) + tk1.MustExec("DROP TABLE t_sel_in_share") +} From 2b363fb80d749a72ae13fa309866880508df4cd8 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Thu, 19 Nov 2020 13:40:31 +0800 Subject: [PATCH 28/37] *: support to modify config server-memory-quota-ratio dynamically (#20927) --- session/session.go | 1 + session/session_test.go | 17 ++++++++++++ sessionctx/variable/session.go | 2 ++ sessionctx/variable/sysvar.go | 1 + sessionctx/variable/tidb_vars.go | 5 ++++ sessionctx/variable/varsutil.go | 2 ++ util/expensivequery/expensivequery.go | 4 ++- util/expensivequery/memory_usage_alarm.go | 32 ++++++++++++++--------- 8 files changed, 50 insertions(+), 14 deletions(-) diff --git a/session/session.go b/session/session.go index efde542c10a38..6d73e87fe402e 100644 --- a/session/session.go +++ b/session/session.go @@ -2223,6 +2223,7 @@ var builtinGlobalVariable = []string{ variable.TiDBShardAllocateStep, variable.TiDBEnableChangeColumnType, variable.TiDBEnableAmendPessimisticTxn, + variable.TiDBMemoryUsageAlarmRatio, } var ( diff --git a/session/session_test.go b/session/session_test.go index 6c9914f644def..fc3e99cd6ddaa 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -3517,6 +3517,23 @@ func (s *testBackupRestoreSuite) TestBackupAndRestore(c *C) { } } +func (s *testSessionSuite2) TestMemoryUsageAlarmVariable(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + + tk.MustExec("set @@session.tidb_memory_usage_alarm_ratio=1") + tk.MustQuery("select @@session.tidb_memory_usage_alarm_ratio").Check(testkit.Rows("1")) + tk.MustExec("set @@session.tidb_memory_usage_alarm_ratio=0") + tk.MustQuery("select @@session.tidb_memory_usage_alarm_ratio").Check(testkit.Rows("0")) + tk.MustExec("set @@session.tidb_memory_usage_alarm_ratio=0.7") + tk.MustQuery("select @@session.tidb_memory_usage_alarm_ratio").Check(testkit.Rows("0.7")) + err := tk.ExecToErr("set @@session.tidb_memory_usage_alarm_ratio=1.1") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_memory_usage_alarm_ratio' can't be set to the value of '1.1'") + err = tk.ExecToErr("set @@session.tidb_memory_usage_alarm_ratio=-1") + c.Assert(err.Error(), Equals, "[variable:1231]Variable 'tidb_memory_usage_alarm_ratio' can't be set to the value of '-1'") + err = tk.ExecToErr("set @@global.tidb_memory_usage_alarm_ratio=0.8") + c.Assert(err.Error(), Equals, "Variable 'tidb_memory_usage_alarm_ratio' is a SESSION variable and can't be used with SET GLOBAL") +} + func (s *testSessionSuite2) TestSelectLockInShare(c *C) { tk1 := testkit.NewTestKitWithInit(c, s.store) tk1.MustExec("DROP TABLE IF EXISTS t_sel_in_share") diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 34e198237d5bc..271fd2b88b727 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1584,6 +1584,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { s.EnableAmendPessimisticTxn = TiDBOptOn(val) case TiDBTxnScope: s.TxnScope = val + case TiDBMemoryUsageAlarmRatio: + MemoryUsageAlarmRatio.Store(tidbOptFloat64(val, 0.8)) } s.systems[name] = val return nil diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 84daed5f8dada..ec73d5e6c6473 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1052,6 +1052,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeSession, Name: TiDBWaitSplitRegionTimeout, Value: strconv.Itoa(DefWaitSplitRegionTimeout), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt64}, {Scope: ScopeSession, Name: TiDBLowResolutionTSO, Value: BoolOff, Type: TypeBool}, {Scope: ScopeSession, Name: TiDBExpensiveQueryTimeThreshold, Value: strconv.Itoa(DefTiDBExpensiveQueryTimeThreshold), Type: TypeUnsigned, MinValue: int64(MinExpensiveQueryTimeThreshold), MaxValue: uint64(math.MaxInt64), AutoConvertOutOfRange: true}, + {Scope: ScopeSession, Name: TiDBMemoryUsageAlarmRatio, Value: strconv.FormatFloat(config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio, 'f', -1, 64), Type: TypeFloat, MinValue: 0.0, MaxValue: 1.0}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableNoopFuncs, Value: BoolToOnOff(DefTiDBEnableNoopFuncs), Type: TypeBool}, {Scope: ScopeSession, Name: TiDBReplicaRead, Value: "leader", Type: TypeEnum, PossibleValues: []string{"leader", "follower", "leader-and-follower"}}, {Scope: ScopeSession, Name: TiDBAllowRemoveAutoInc, Value: BoolToOnOff(DefTiDBAllowRemoveAutoInc), Type: TypeBool}, diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 352d009387735..cb64a34d6fb50 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -18,6 +18,7 @@ import ( "os" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/config" "github.com/uber-go/atomic" ) @@ -463,6 +464,9 @@ const ( // TiDBEnableAmendPessimisticTxn indicates if amend pessimistic transactions is enabled. TiDBEnableAmendPessimisticTxn = "tidb_enable_amend_pessimistic_txn" + + // TiDBMemoryUsageAlarmRatio indicates the alarm threshold when memory usage of the tidb-server exceeds. + TiDBMemoryUsageAlarmRatio = "tidb_memory_usage_alarm_ratio" ) // Default TiDB system variable values. @@ -596,4 +600,5 @@ var ( MinExpensiveQueryTimeThreshold uint64 = 10 //10s CapturePlanBaseline = serverGlobalVariable{globalVal: BoolOff} DefExecutorConcurrency = 5 + MemoryUsageAlarmRatio = atomic.NewFloat64(config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio) ) diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index 8c757ab81f848..d8642ea366fc0 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -126,6 +126,8 @@ func GetSessionOnlySysVars(s *SessionVars, key string) (string, bool, error) { return val, true, nil case TiDBExpensiveQueryTimeThreshold: return fmt.Sprintf("%d", atomic.LoadUint64(&ExpensiveQueryTimeThreshold)), true, nil + case TiDBMemoryUsageAlarmRatio: + return fmt.Sprintf("%g", MemoryUsageAlarmRatio.Load()), true, nil case TiDBConfig: conf := config.GetGlobalConfig() j, err := json.MarshalIndent(conf, "", "\t") diff --git a/util/expensivequery/expensivequery.go b/util/expensivequery/expensivequery.go index 8022cb564033a..56010728faa0e 100644 --- a/util/expensivequery/expensivequery.go +++ b/util/expensivequery/expensivequery.go @@ -55,7 +55,7 @@ func (eqh *Handle) Run() { ticker := time.NewTicker(tickInterval) defer ticker.Stop() sm := eqh.sm.Load().(util.SessionManager) - record := initMemoryUsageAlarmRecord() + record := &memoryUsageAlarm{} for { select { case <-ticker.C: @@ -75,6 +75,8 @@ func (eqh *Handle) Run() { } } threshold = atomic.LoadUint64(&variable.ExpensiveQueryTimeThreshold) + + record.memoryUsageAlarmRatio = variable.MemoryUsageAlarmRatio.Load() if record.err == nil { record.alarm4ExcessiveMemUsage(sm) } diff --git a/util/expensivequery/memory_usage_alarm.go b/util/expensivequery/memory_usage_alarm.go index d8f86b80eeee5..fbe9b6c5ff438 100644 --- a/util/expensivequery/memory_usage_alarm.go +++ b/util/expensivequery/memory_usage_alarm.go @@ -24,7 +24,6 @@ import ( "strings" "time" - "github.com/pingcap/errors" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/disk" @@ -36,8 +35,10 @@ import ( type memoryUsageAlarm struct { err error + initialized bool isServerMemoryQuotaSet bool serverMemoryQuota uint64 + memoryUsageAlarmRatio float64 lastCheckTime time.Time tmpDir string @@ -45,17 +46,11 @@ type memoryUsageAlarm struct { lastProfileFileName [][]string // heap, goroutine } -func initMemoryUsageAlarmRecord() (record *memoryUsageAlarm) { - record = &memoryUsageAlarm{} - if alert := config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio; alert == 0 || alert == 1 { - record.err = errors.New("close memory usage alarm recorder") - return - } +func (record *memoryUsageAlarm) initMemoryUsageAlarmRecord() { if quota := config.GetGlobalConfig().Performance.ServerMemoryQuota; quota != 0 { record.serverMemoryQuota = quota record.isServerMemoryQuotaSet = true } else { - // TODO: Get the memory info in container directly. record.serverMemoryQuota, record.err = memory.MemTotal() if record.err != nil { logutil.BgLogger().Error("get system total memory fail", zap.Error(record.err)) @@ -72,7 +67,8 @@ func initMemoryUsageAlarmRecord() (record *memoryUsageAlarm) { // Read last records files, err := ioutil.ReadDir(record.tmpDir) if err != nil { - return record + record.err = err + return } for _, f := range files { name := filepath.Join(record.tmpDir, f.Name()) @@ -86,13 +82,23 @@ func initMemoryUsageAlarmRecord() (record *memoryUsageAlarm) { record.lastProfileFileName[1] = append(record.lastProfileFileName[1], name) } } - - return record + record.initialized = true + return } // If Performance.ServerMemoryQuota is set, use `ServerMemoryQuota * MemoryUsageAlarmRatio` to check oom risk. // If Performance.ServerMemoryQuota is not set, use `system total memory size * MemoryUsageAlarmRatio` to check oom risk. func (record *memoryUsageAlarm) alarm4ExcessiveMemUsage(sm util.SessionManager) { + if record.memoryUsageAlarmRatio <= 0.0 || record.memoryUsageAlarmRatio >= 1.0 { + return + } + if !record.initialized { + record.initMemoryUsageAlarmRecord() + if record.err != nil { + return + } + } + var memoryUsage uint64 instanceStats := &runtime.MemStats{} runtime.ReadMemStats(instanceStats) @@ -107,7 +113,7 @@ func (record *memoryUsageAlarm) alarm4ExcessiveMemUsage(sm util.SessionManager) } // TODO: Consider NextGC to record SQLs. - if float64(memoryUsage) > float64(record.serverMemoryQuota)*config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio { + if float64(memoryUsage) > float64(record.serverMemoryQuota)*record.memoryUsageAlarmRatio { // At least ten seconds between two recordings that memory usage is less than threshold (default 80% system memory). // If the memory is still exceeded, only records once. interval := time.Since(record.lastCheckTime) @@ -129,7 +135,7 @@ func (record *memoryUsageAlarm) doRecord(memUsage uint64, instanceMemoryUsage ui fields = append(fields, zap.Any("system memory usage", memUsage)) fields = append(fields, zap.Any("tidb-server memory usage", instanceMemoryUsage)) } - fields = append(fields, zap.Any("memory-usage-alarm-ratio", config.GetGlobalConfig().Performance.MemoryUsageAlarmRatio)) + fields = append(fields, zap.Any("memory-usage-alarm-ratio", record.memoryUsageAlarmRatio)) fields = append(fields, zap.Any("record path", record.tmpDir)) logutil.BgLogger().Warn("tidb-server has the risk of OOM. Running SQLs and heap profile will be recorded in record path", fields...) From 2e918a2e8fb92777318bb5b4949202bb0d9090e4 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Thu, 19 Nov 2020 13:54:23 +0800 Subject: [PATCH 29/37] executor: specially handle empty input for apply's outer child aggregate (#20544) --- executor/join.go | 40 ++++++++++++++++++++++++++++++++++++++++ executor/join_test.go | 21 +++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/executor/join.go b/executor/join.go index dc94e24111a35..eb43d70093bb7 100644 --- a/executor/join.go +++ b/executor/join.go @@ -869,6 +869,39 @@ func (e *NestedLoopApplyExec) Open(ctx context.Context) error { return nil } +// aggExecutorTreeInputEmpty checks whether the executor tree returns empty if without aggregate operators. +// Note that, the prerequisite is that this executor tree has been executed already and it returns one row. +func aggExecutorTreeInputEmpty(e Executor) bool { + children := e.base().children + if len(children) == 0 { + return false + } + if len(children) > 1 { + _, ok := e.(*UnionExec) + if !ok { + // It is a Join executor. + return false + } + for _, child := range children { + if !aggExecutorTreeInputEmpty(child) { + return false + } + } + return true + } + // Single child executors. + if aggExecutorTreeInputEmpty(children[0]) { + return true + } + if hashAgg, ok := e.(*HashAggExec); ok { + return hashAgg.isChildReturnEmpty + } + if streamAgg, ok := e.(*StreamAggExec); ok { + return streamAgg.isChildReturnEmpty + } + return false +} + func (e *NestedLoopApplyExec) fetchSelectedOuterRow(ctx context.Context, chk *chunk.Chunk) (*chunk.Row, error) { outerIter := chunk.NewIterator4Chunk(e.outerChunk) for { @@ -884,6 +917,13 @@ func (e *NestedLoopApplyExec) fetchSelectedOuterRow(ctx context.Context, chk *ch if err != nil { return nil, err } + // For cases like `select count(1), (select count(1) from s where s.a > t.a) as sub from t where t.a = 1`, + // if outer child has no row satisfying `t.a = 1`, `sub` should be `null` instead of `0` theoretically; however, the + // outer `count(1)` produces one row <0, null> over the empty input, we should specially mark this outer row + // as not selected, to trigger the mismatch join procedure. + if e.outerChunkCursor == 0 && e.outerChunk.NumRows() == 1 && e.outerSelected[0] && aggExecutorTreeInputEmpty(e.outerExec) { + e.outerSelected[0] = false + } e.outerChunkCursor = 0 } outerRow := e.outerChunk.GetRow(e.outerChunkCursor) diff --git a/executor/join_test.go b/executor/join_test.go index c7b023b58d4b2..2e8bbe6a8a20e 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -2299,6 +2299,27 @@ func (s *testSuite9) TestIssue18572_3(c *C) { c.Assert(strings.Contains(err.Error(), "mockIndexHashJoinBuildErr"), IsTrue) } +func (s *testSuite9) TestApplyOuterAggEmptyInput(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(a int)") + tk.MustExec("create table t2(a int)") + tk.MustExec("insert into t1 values(1)") + tk.MustExec("insert into t2 values(1)") + tk.MustQuery("select count(1), (select count(1) from t2 where t2.a > t1.a) as field from t1 where t1.a = 100").Check(testkit.Rows( + "0 ", + )) + tk.MustQuery("select /*+ agg_to_cop() */ count(1), (select count(1) from t2 where t2.a > t1.a) as field from t1 where t1.a = 100").Check(testkit.Rows( + "0 ", + )) + tk.MustQuery("select count(1), (select count(1) from t2 where t2.a > t1.a) as field from t1 where t1.a = 1").Check(testkit.Rows( + "1 0", + )) + tk.MustQuery("select /*+ agg_to_cop() */ count(1), (select count(1) from t2 where t2.a > t1.a) as field from t1 where t1.a = 1").Check(testkit.Rows( + "1 0", + )) +} + func (s *testSuite9) TestIssue19112(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("drop table if exists t1, t2") From 91cd12f83f90e85adaf747cd1ebdd06e76b92479 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Thu, 19 Nov 2020 15:30:43 +0800 Subject: [PATCH 30/37] store: update the PD client to support the Global/Local TSO (#20980) --- go.mod | 22 +- go.sum | 454 ++++++++++++++++++++-- session/pessimistic_test.go | 5 +- session/session_test.go | 5 +- session/txn.go | 4 +- store/mockoracle/oracle.go | 18 +- store/mockstore/mocktikv/executor_test.go | 5 +- store/mockstore/mocktikv/pd.go | 20 +- store/mockstore/unistore/pd.go | 22 +- store/tikv/1pc_test.go | 3 +- store/tikv/2pc.go | 6 +- store/tikv/2pc_test.go | 6 +- store/tikv/async_commit_test.go | 13 +- store/tikv/gcworker/gc_worker_test.go | 2 +- store/tikv/kv.go | 2 +- store/tikv/lock_resolver.go | 13 +- store/tikv/lock_test.go | 21 +- store/tikv/oracle/oracle.go | 23 +- store/tikv/oracle/oracles/export_test.go | 7 +- store/tikv/oracle/oracles/local.go | 18 +- store/tikv/oracle/oracles/local_test.go | 13 +- store/tikv/oracle/oracles/pd.go | 140 +++++-- store/tikv/oracle/oracles/pd_test.go | 2 +- store/tikv/split_region.go | 4 +- store/tikv/store_test.go | 35 +- 25 files changed, 684 insertions(+), 179 deletions(-) diff --git a/go.mod b/go.mod index 0bc5aa87e1d8e..f1cfacedb197b 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,11 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect github.com/Jeffail/gabs/v2 v2.5.1 - github.com/aws/aws-sdk-go v1.30.24 // indirect github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d github.com/carlmjohnson/flagext v0.20.2 // indirect github.com/cheggaaa/pb/v3 v3.0.4 // indirect github.com/cockroachdb/pebble v0.0.0-20200617141519-3b241b76ed3b // indirect github.com/codahale/hdrhistogram v0.9.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 @@ -28,21 +25,15 @@ require ( github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 - github.com/gorilla/websocket v1.4.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 - github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 // indirect github.com/klauspost/cpuid v1.2.1 github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-colorable v0.1.6 // indirect - github.com/montanaflynn/stats v0.5.0 // indirect github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef github.com/ngaut/unistore v0.0.0-20201113064408-907e3fcf8e7d github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect - github.com/onsi/ginkgo v1.9.0 // indirect - github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/basictracer-go v1.0.0 github.com/opentracing/opentracing-go v1.1.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 @@ -54,8 +45,8 @@ require ( github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 - github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9 - github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 + github.com/pingcap/kvproto v0.0.0-20201113092725-08f2872278eb + github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 github.com/pingcap/parser v0.0.0-20201112105242-773b8b74f44e github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible @@ -67,27 +58,21 @@ require ( github.com/shirou/gopsutil v2.20.3+incompatible github.com/sirupsen/logrus v1.6.0 github.com/soheilhy/cmux v0.1.4 - github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/cobra v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.6.1 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/pd v1.1.0-beta.0.20200925080742-6d0aa719fa17 + github.com/tikv/pd v1.1.0-beta.0.20201119053953-51eff4c1fa05 github.com/twmb/murmur3 v1.1.3 github.com/uber-go/atomic v1.4.0 github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/xitongsys/parquet-go v1.5.4 // indirect github.com/zhangjinpeng1987/raft v0.0.0-20200819064223-df31bb68a018 // indirect - go.etcd.io/bbolt v1.3.4 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b go.uber.org/atomic v1.7.0 go.uber.org/automaxprocs v1.2.0 go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect - golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200904194848-62affa334b73 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20200819171115-d785dc25833f @@ -99,7 +84,6 @@ require ( gopkg.in/yaml.v2 v2.3.0 // indirect honnef.co/go/tools v0.0.1-2020.1.6 // indirect modernc.org/mathutil v1.1.1 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) diff --git a/go.sum b/go.sum index 10e8e7b78372d..64518f6aaa5f1 100644 --- a/go.sum +++ b/go.sum @@ -1,94 +1,158 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0 h1:0E3eE8MX426vUOs7aHfI7aN1BrIzzzf4ccKCSfSjGmc= +cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0 h1:9/vpR43S4aJaROxqQHQ3nH9lfyKKV0dC3vOmnw8ebQQ= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/spanner v1.1.0/go.mod h1:TzTaF9l2ZY2CIetNvVpUu6ZQy8YEOtzB6ICa5EwYjL0= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AstroProfundis/sysinfo v0.0.0-20200423033635-f6f7687215fd/go.mod h1:4m15VhW6ZffaqJNAK/HtN3Qvf97aCe1T39u3UXaA2lA= +github.com/AstroProfundis/tabby v1.1.0-color/go.mod h1:Wcm+uinH1saEOFGLK2LdY37lAOts8HLevz64Y3y3M3Q= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClickHouse/clickhouse-go v1.3.13/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= +github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg= github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= github.com/Jeffail/gabs/v2 v2.5.1 h1:ANfZYjpMlfTTKebycu4X1AgkVWumFVDYQl7JwOr4mDk= github.com/Jeffail/gabs/v2 v2.5.1/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/MichaelS11/go-cql-driver v0.0.0-20190914174813-cf3b3196aa43/go.mod h1:nW8K1gl1mu8o29Ns1Sv/EvYe9BBrh1T/GqucnYcO9PI= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RoaringBitmap/roaring v0.4.21/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo= +github.com/ScaleFT/sshkeys v0.0.0-20181112160850-82451a803681/go.mod h1:WfDateMPQ/55dPbZRp5Zxrux5WiEaHsjk9puUhz0KgY= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= +github.com/VoltDB/voltdb-client-go v1.0.1/go.mod h1:FSuyUyPbMimaMJ9DItBzhVzjFj5S4XHcEISlKEnFZ5Q= +github.com/XiaoMi/pegasus-go-client v0.0.0-20181029071519-9400942c5d1c/go.mod h1:KcL6D/4RZ8RAYzQ5gKI0odcdWUmCVlbQTOlWrhP71CY= +github.com/aerospike/aerospike-client-go v1.35.2/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= +github.com/alecthomas/chroma v0.6.8/go.mod h1:o9ohftueRi7H5be3+Q2cQCNa/YnLBFUNx40ZJfGVFKA= +github.com/alecthomas/chroma v0.7.1/go.mod h1:gHw09mkX1Qp80JlYbmN9L3+4R5o6DJJ3GRShh+AICNc= +github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= +github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= +github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= +github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA= +github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexbrainman/odbc v0.0.0-20190102080306-cf37ce290779/go.mod h1:WEQLoRNIjBhywJqaxe0olilzzBDABc5EVeETiprzR00= +github.com/amsokol/ignite-go-client v0.12.2/go.mod h1:K3tKJGcLQORFD+ds7f0f9fl88tv0KZcpfuNhzRyuLVE= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/apache/calcite-avatica-go/v4 v4.0.0/go.mod h1:fg6MgnbY4Ta6JI0KuNaL9o/LrOMZdprQSMHWflcNs1c= +github.com/apache/thrift v0.0.0-20171203172758-327ebb6c2b6d/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apple/foundationdb/bindings/go v0.0.0-20200112054404-407dc0907f4f/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U= +github.com/appleboy/easyssh-proxy v1.3.2/go.mod h1:Kk57I3w7OCafOjp5kgZFvxk2fO8Tca5CriBTOsbSbjY= github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/EventBus v0.0.0-20180315140547-d46933a94f05/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.24 h1:y3JPD51VuEmVqN3BEDVm4amGpDma2cKJcDPuAU1OR58= -github.com/aws/aws-sdk-go v1.30.24/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= +github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= +github.com/blevesearch/bleve v1.0.7/go.mod h1:3xvmBtaw12Y4C9iA1RTzwWCof5j5HjydjCTiDE2TeE0= +github.com/blevesearch/bleve v1.0.8-0.20200520165604-f0ded112bb1b/go.mod h1:3xvmBtaw12Y4C9iA1RTzwWCof5j5HjydjCTiDE2TeE0= +github.com/blevesearch/blevex v0.0.0-20190916190636-152f0fe5c040/go.mod h1:WH+MU2F4T0VmSdaPX+Wu5GYoZBrYWdOZWSjzvYcDmqQ= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/zap/v11 v11.0.7/go.mod h1:bJoY56fdU2m/IP4LLz/1h4jY2thBoREvoqbuJ8zhm9k= +github.com/blevesearch/zap/v12 v12.0.7/go.mod h1:70DNK4ZN4tb42LubeDbfpp6xnm8g3ROYVvvZ6pEoXD8= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/carlmjohnson/flagext v0.20.2 h1:qvpMM+TytSrlh3+EIVn/pzOwwq9y13hXZab6Y4Gvqpo= github.com/carlmjohnson/flagext v0.20.2/go.mod h1:Eenv0epIUAr4NuedNmkzI8WmBmjIxZC239XcKxYS2ac= +github.com/cavaliercoder/grab v2.0.1-0.20200331080741-9f014744ee41+incompatible/go.mod h1:tTBkfNqSBfuMmMBFaO2phgyhdYhiZQ/+iXCZDzcDsMI= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb v2.0.7+incompatible h1:gLKifR1UkZ/kLkda5gC0K6c8g+jU2sINPtBeOiNlMhU= +github.com/cheggaaa/pb v2.0.7+incompatible/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM= github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw= +github.com/cheynewallace/tabby v1.1.0/go.mod h1:Pba/6cUL8uYqvOc9RkyvFbHGrQ9wShyrn6/S/1OYVys= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= @@ -107,11 +171,14 @@ github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2 github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2/go.mod h1:7qG7YFnOALvsx6tKTNmQot8d7cGFXM9TidzvRFLWYwM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -119,11 +186,20 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= +github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= +github.com/couchbase/go-couchbase v0.0.0-20191217190632-b2754d72cc98/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= +github.com/couchbase/go_n1ql v0.0.0-20160215142504-6cf4e348b127/go.mod h1:Rn19fO9CVfhJkqyIED9ixL5Kh5XuH7hXgDTxyfGY7hM= +github.com/couchbase/gomemcached v0.0.0-20191205220529-2b26ed9d054e/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= +github.com/couchbase/goutils v0.0.0-20191018232750-b49639060d85/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= +github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= +github.com/couchbase/vellum v1.0.1/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.3.0/go.mod h1:CIEEvs7oIVZm30R8VxtFJs+4k201gReYyuYHJxZc68I= +github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= @@ -131,36 +207,59 @@ github.com/cznic/parser v0.0.0-20160622100904-31edd927e5b1/go.mod h1:2B43mz36vGZ github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGda1NeOBQwqlv7nUXpm+rIVHGxZZ4= github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= +github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= +github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 h1:X6mKGhCFOxrKeeHAjv/3UvT6e5RRxW6wRdlqlV6/H4w= github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs= github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= +github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= github.com/frankban/quicktest v1.11.1 h1:stwUsXhUGliQs9t0ZS39BWCltFdOHgABiIlihop8AD4= github.com/frankban/quicktest v1.11.1/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -171,6 +270,8 @@ github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JY github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -178,11 +279,17 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= +github.com/gizak/termui/v3 v3.1.0/go.mod h1:bXQEBkJpzxUAKf0+xq9MSWAvWZlE7c+aidmyFlkYTrY= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.49.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -199,36 +306,50 @@ github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwoh github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-redis/redis v6.15.1+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= +github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= +github.com/gocql/gocql v0.0.0-20200103014340-68f928edb90a/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godror/godror v0.10.3/go.mod h1:9MVLtu25FBJBMHkPs0m3Ngf/VmwGcLpM2HS8PlNGw9U= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gohxs/readline v0.0.0-20171011095936-a780388e6e7c/go.mod h1:9S/fKAutQ6wVHqm1jnp9D9sc5hu689s9AaTWFS92LaU= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= +github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= @@ -241,14 +362,15 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -260,24 +382,34 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 h1:K+lX49/3eURCE1IjlaZN//u6c+9nfDAMnyQ9E2dsJbY= github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= +github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -287,13 +419,17 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdR github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -305,13 +441,31 @@ github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73t github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/flux v0.65.0/go.mod h1:BwN2XG2lMszOoquQaFdPET8FRQfrXiZsWmcMO9rkaVY= +github.com/influxdata/influxdb v1.8.0/go.mod h1:SIzcnsjaHRFpmlxpJ4S3NT64qtEKYweNTUMb/vh0OMQ= +github.com/influxdata/influxql v1.1.0/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= +github.com/jackc/pgx v3.6.1+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jeremywohl/flatten v1.0.1/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 h1:3Jr6Mtili6DsXSF0RwRlAqpOUWXcSVUxdOm5kFPb3xY= github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= @@ -326,20 +480,36 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= +github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kevinburke/go-bindata v3.18.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc= github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -348,16 +518,23 @@ github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/markbates/pkger v0.16.0/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= +github.com/mattn/go-adodb v0.0.1/go.mod h1:jaSTRde4bohMuQgYQPxW3xRTPtX/cZKyxPrFVseJULo= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -372,15 +549,25 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo= +github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -389,9 +576,13 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nakagami/firebirdsql v0.0.0-20191128141720-8048f7ca3088/go.mod h1:yUx1EqfvqHBg0QcpcloWfeqH/yoLoI2j7sllgVhHHSQ= github.com/ncw/directio v1.0.4 h1:CojwI07mCEmRkajgx42Pf8jyCwTs1ji9/Ij9/PJG12k= github.com/ncw/directio v1.0.4/go.mod h1:CKGdcN7StAaqjT7Qack3lAXeX4pjnyc46YeqZH1yWVY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= @@ -404,6 +595,8 @@ github.com/ngaut/unistore v0.0.0-20201113064408-907e3fcf8e7d/go.mod h1:ZR3NH+Hzq github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= @@ -418,20 +611,32 @@ github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.3.0 h1:e5+lF2E4Y2WCIxBefVowBuB0iHrUH4HZ8q+6mGF7fJc= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200921100341-0e148dfc0029/go.mod h1:X3r7/4Wr9fSC5KlsfezBh/5noeWGEJNQuSvjgS2rvdI= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200407064406-b2b8ad403d01/go.mod h1:77fCh8d3oKzC5ceOJWeZXAS/mLzVgdZ7rKniwmOyFuo= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200807020752-01f0abe88e93/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20201022065613-94d8dc38a204/go.mod h1:X3r7/4Wr9fSC5KlsfezBh/5noeWGEJNQuSvjgS2rvdI= github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19 h1:IXpGy7y9HyoShAFmzW2OPF0xCA5EOoSTyZHwsgYk9Ro= github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= github.com/pingcap/br v4.0.0-beta.2.0.20201014031603-5676c8fdad1a+incompatible h1:RMx7D+dQtUTUZjelJyV0WwYr3GFxrjsSarFXhV2SWzI= @@ -441,75 +646,122 @@ github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JH github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 h1:R8gStypOBmpnHEx1qi//SaqxJVI4inOqljg/Aj5/390= github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/dm v1.1.0-alpha.0.20200521025928-83063141c5fd/go.mod h1:I5AAhwb0JPfLZINukML5VU9rB6mCcVA/Jq5OFFtMuEk= +github.com/pingcap/dumpling v0.0.0-20200423082233-887d037b5b5c/go.mod h1:VJTcnA0MLL9tzDceTDoRh3k5UnOq9Hk6wh/ATo+B8I8= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= +github.com/pingcap/errors v0.11.5-0.20200820035142-66eb5bf1d1cd/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 h1:wQKuKP2HUtej2gSvx1cZmY4DENUH6tlOxRkfvPT8EBU= github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= +github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMtVcOkjUcuQKh+YrluSo7+7YMCQSzy30= github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= +github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZLmhahmvHm7n9DUxGRQT00208= github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= +github.com/pingcap/go-tpc v1.0.4-0.20200525052430-dc963cdeef62/go.mod h1:YToE6BW+r+aWksQm1kuFnzKgEzaTKsVIHD36rxVYaWc= +github.com/pingcap/go-ycsb v0.0.0-20200226103513-00ca633a87d8/go.mod h1:B9UJ3Lbpk4r+qFNDAeS2l6ORGkVaVwMPO1mSqDXiNQc= +github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= +github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= +github.com/pingcap/kvproto v0.0.0-20190305055742-ab7debc182d9/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/kvproto v0.0.0-20190506024016-26344dff8f48/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200417092353-efbe03bcffbd/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200420075417-e0c6e8842f22/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200810113304-6157337686b1/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200916031750-f9473f2c5379/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9 h1:o/MpH2TJ2qJk0EwSI1O8Fw/dzx/ImSSKp2gXEUDvH6w= -github.com/pingcap/kvproto v0.0.0-20201023092649-e6d6090277c9/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20201113092725-08f2872278eb h1:K3r4KjVQeD4nLnfj44ibdLIXnUh58aQpkgVNWuBO9z0= +github.com/pingcap/kvproto v0.0.0-20201113092725-08f2872278eb/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 h1:Jboj+s4jSCp5E1WDgmRUv5rIFKFHaaSWuSZ4wMwXIcc= -github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 h1:M+DNpOu/I3uDmwee6vcnoPd6GgSMqND4gxvDQ/W584U= +github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20200422082501-7329d80eaf2c/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= github.com/pingcap/parser v0.0.0-20201112105242-773b8b74f44e h1:TCSZlqvNP2Ipt0xUwZ5Z6ehmKo1dlk5KCU1SOLXUYL4= github.com/pingcap/parser v0.0.0-20201112105242-773b8b74f44e/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE= +github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= +github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2/go.mod h1:s+utZtXDznOiL24VK0qGmtoHjjXNsscJx3m1n8cC56s= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= +github.com/pingcap/sysutil v0.0.0-20200408114249-ed3bd6f7fdb1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20200715082929-4c47bcac246a/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2 h1:b2G/eqDeywtdJF3w9nIUdqMmXChsmpLvf4FzUxJ9Vmk= github.com/pingcap/sysutil v0.0.0-20201021075216-f93ced2829e2/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= +github.com/pingcap/tidb v1.1.0-beta.0.20200423094549-0ad7ce6cdae6/go.mod h1:KJXj2xHYfl1x4zcusC2JEANzVci+ietFOMh/CAmrYdw= +github.com/pingcap/tidb-insight v0.3.1/go.mod h1:HZ6cGBYxViftTKLbl/V4moRol79bifZzI9ImGavJms0= github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible h1:y6cwuJJo8tLeHjgjODpwCSqNAv1g+9WUtcsFOHu/7ag= github.com/pingcap/tidb-lightning v4.0.9-0.20201106041742-a1ac97827a27+incompatible/go.mod h1:UrN2l06neiEfkass87Tjvv5JFIMsOMb2DT7P7s+fwMc= +github.com/pingcap/tidb-tools v3.0.13+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible h1:qPppnsXVh3KswqRZdSAShGLLPd7dB+5w4lXDnpYn0SQ= github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f h1:J+0TAI+7Hvebz4bM4GnRCRT4MpjYnUxbyi9ky5ZQUsU= github.com/pingcap/tipb v0.0.0-20201026044621-45e60c77588f/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tiup v1.2.3/go.mod h1:q8WzflNHjE1U49k2qstTL0clx2pKh8pkOzUFV4RTvQo= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prestodb/presto-go-client v0.0.0-20191203220353-4980913e2459/go.mod h1:psTrT5qzpQmYNpaWiyurHCiVTLMYYpAd5o0YQAJWpQg= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/r3labs/diff v0.0.0-20200627101315-aecd9dd05dd2/go.mod h1:7WjXasNzi0vJetRcB/RqNl5dlIsmXcTTLmF5IoH6Xig= +github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/relex/aini v1.2.0/go.mod h1:oFQyhvkzwi8GChiLukpBHkV2v142ls2L1MTeOSD2vic= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -517,25 +769,42 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.20.3+incompatible h1:0JVooMPsT7A7HqEYdydp/OfjSOYSjhXV7w1hkKj/NPQ= github.com/shirou/gopsutil v2.20.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20191125035519-b054a8dfd10d/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= +github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= +github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= +github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/snowflakedb/gosnowflake v1.3.4/go.mod h1:NsRq2QeiMUuoNUJhp5Q6xGC4uBrsS9g6LwZVEkTWgsE= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -546,6 +815,7 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -555,84 +825,129 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= +github.com/swaggo/http-swagger v0.0.0-20200103000832-0e9263c4b516/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= +github.com/swaggo/swag v1.6.5/go.mod h1:Y7ZLSS0d0DdxhWGVhQdu+Bu1QhaF5k0RD7FKdiAykeY= +github.com/swaggo/swag v1.6.6-0.20200323071853-8e21f4cefeea/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= +github.com/thda/tds v0.1.7/go.mod h1:isLIF1oZdXfkqVMJM8RyNrsjlHPlTKnPlnsBs7ngZcM= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tikv/pd v1.1.0-beta.0.20200925080742-6d0aa719fa17 h1:xI/CKdl55/WeoIvMS3040I9vJ52bIPkFK2Z3HbbOgO0= -github.com/tikv/pd v1.1.0-beta.0.20200925080742-6d0aa719fa17/go.mod h1:U5MWRAfLdLeeUQxs6/+3HdiOT2g/EXxHRs1+i6BDHbU= +github.com/tikv/client-go v0.0.0-20190421092910-44b82dcc9f4a/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc= +github.com/tikv/pd v1.1.0-beta.0.20200824114021-f8c45ae287fd/go.mod h1:quwjWtCmawAvS+YdxtSKG08sEexLzkhQgAno59wW+lI= +github.com/tikv/pd v1.1.0-beta.0.20201119053953-51eff4c1fa05 h1:XkZi+XIOUSZPm9i0xUEmWFOH3G8BIt7BGgkfW7vkvKo= +github.com/tikv/pd v1.1.0-beta.0.20201119053953-51eff4c1fa05/go.mod h1:qqRJlVDCqe6r+Fs4yWPORwxVOa3VDinww8fg7zGiKfk= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/go-css v0.0.0-20191108133013-220a796d1705/go.mod h1:e+JPLQ9wyQCgRnPenX2bo7MJoLphBHz5c1WUqaANSeA= +github.com/tj/go-termd v0.0.2-0.20200115111609-7f6aeb166380/go.mod h1:7JlPhw1+Bkn5PLz+kqAfzL8ij69OlQ1a4O5bbY98axo= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchtv/retool v1.3.8-0.20180918173430-41330f8b4e07/go.mod h1:88ZJJaNbdr4rvnbFzU5l5d17iJ69JoQX2+JJ5B0Wteo= github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/vertica/vertica-sql-go v0.1.6/go.mod h1:2LGtkNSdFF5CTJYeUA5qWfREuvYaql+51fNzmoD5W7c= github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xinsnake/go-http-digest-auth-client v0.4.0/go.mod h1:QK1t1v7ylyGb363vGWu+6Irh7gyFj+N7+UZzM0L6g8I= github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= github.com/xitongsys/parquet-go v1.5.4 h1:zsdMNZcCv9t3YnlOfysMI78vBw+cN65jQznQlizVtqE= github.com/xitongsys/parquet-go v1.5.4/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xo/dburl v0.0.0-20191219122722-3cca8608d645/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4= +github.com/xo/tblfmt v0.0.0-20190609041254-28c54ec42ce8/go.mod h1:3U5kKQdIhwACye7ml3acccHmjGExY9WmUGU7rnDWgv0= +github.com/xo/tblfmt v0.0.0-20191214134155-4c686ae34009/go.mod h1:+aPrd+upGAjb50ADyldbFNzNTjF3NvYl3FTr0gWNA5g= +github.com/xo/terminfo v0.0.0-20190125114736-1a4775eeeb62/go.mod h1:6Yhx5ZJl5942QrNRWLwITArVT9okUXc5c3brgWJMoDc= +github.com/xo/usql v0.7.8/go.mod h1:lmjb2xMb8yY1le5mlSvOSat4EoMM2HseDhVIPKAhAps= +github.com/xo/xoutil v0.0.0-20171112033149-46189f4026a5/go.mod h1:GngMELAA694UVFs172352HAA2KQEf4XuETgWmL4XSoY= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= +github.com/zaf/temp v0.0.0-20170209143821-94e385923345/go.mod h1:sXsZgXwh6DB0qlskmZVB4HE93e5YrktMrgUDPy9iYmY= github.com/zhangjinpeng1987/raft v0.0.0-20200819064223-df31bb68a018 h1:T3OrqVdcH6z6SakR7WkECvGpdkfB0MAur/6zf66GPxQ= github.com/zhangjinpeng1987/raft v0.0.0-20200819064223-df31bb68a018/go.mod h1:rTSjwgeYU2on64W50csWDlhyy0x9UYVYJUovHlYdt5s= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b h1:3kC4J3eQF6p1UEfQTkC67eEeb3rTk+shQqdX6tFyq9Q= go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -660,6 +975,8 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= @@ -668,29 +985,37 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -719,6 +1044,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -741,6 +1067,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= @@ -757,16 +1085,18 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181011152604-fa43e7bc11ba/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -781,14 +1111,19 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -796,6 +1131,8 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f h1:KJuwZVtZBVzDmEDtB2zro9CXkD9O0dpCv4o2LHbQIAw= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -812,13 +1149,16 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -828,12 +1168,14 @@ golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190802003818-e9bb7d36c060/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191105231337-689d0f08e67a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -841,8 +1183,10 @@ golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191210221141-98df12377212/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -851,7 +1195,10 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200422205258-72e4a01eba43/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200820010801-b793a1359eac h1:DugppSxw0LSF8lcjaODPJZoDzq0ElTGskTst3ZaBkHI= golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -859,6 +1206,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -873,6 +1226,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -881,14 +1235,19 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= @@ -904,37 +1263,49 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +gopkg.in/VividCortex/ewma.v1 v1.1.1/go.mod h1:TekXuFipeiHWiAlO1+wSS23vTcyFau5u3rxXUSXj710= gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v2 v2.0.7/go.mod h1:0CiZ1p8pvtxBlQpLXkHuUTpdJ1shm3OqCF1QugkjHL4= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fatih/color.v1 v1.7.0/go.mod h1:P7yosIhqIl/sX8J8UypY5M+dDpD2KmyfP5IRs5v/fo0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v6 v6.1.1/go.mod h1:NFjHNLrHQiruory+EmqDXCGv6CrjkeYeA+bR9mIfNFk= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/mattn/go-colorable.v0 v0.1.0/go.mod h1:BVJlBXzARQxdi3nZo6f6bnl5yR20/tOL6p+V0KejgSY= +gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc= +gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -942,7 +1313,6 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -954,17 +1324,33 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc= honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= +modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= +modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= +modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/mathutil v1.1.1 h1:FeylZSVX8S+58VsyJlkEj2bcpdytmp9MmDKZkKx8OIE= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY= +modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +software.sslmate.com/src/go-pkcs12 v0.0.0-20200619203921-c9ed90bd32dc/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= +sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI= diff --git a/session/pessimistic_test.go b/session/pessimistic_test.go index ed0419031c8fb..0de76c9adbec0 100644 --- a/session/pessimistic_test.go +++ b/session/pessimistic_test.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/testkit" @@ -1950,14 +1951,14 @@ func (s *testPessimisticSuite) TestSelectForUpdateConflictRetry(c *C) { tsCh := make(chan uint64) go func() { tk3.MustExec("update tk set c2 = c2 + 1 where c1 = 1") - lastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background()) + lastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) tsCh <- lastTS tk3.MustExec("commit") tsCh <- lastTS }() // tk2LastTS should be its forUpdateTS - tk2LastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background()) + tk2LastTS, err := s.store.GetOracle().GetLowResolutionTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) tk2.MustExec("commit") diff --git a/session/session_test.go b/session/session_test.go index fc3e99cd6ddaa..e8dd43126a67f 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -47,6 +47,7 @@ import ( "github.com/pingcap/tidb/store/mockstore/cluster" "github.com/pingcap/tidb/store/mockstore/mocktikv" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/sqlexec" @@ -3241,7 +3242,7 @@ func (s *testSessionSuite2) TestSetTxnScope(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) // assert default value result := tk.MustQuery("select @@txn_scope;") - result.Check(testkit.Rows(config.DefTxnScope)) + result.Check(testkit.Rows(oracle.GlobalTxnScope)) // assert set sys variable tk.MustExec("set @@session.txn_scope = 'dc-1';") @@ -3253,7 +3254,7 @@ func (s *testSessionSuite2) TestSetTxnScope(c *C) { c.Check(err, IsNil) tk.Se = se result = tk.MustQuery("select @@txn_scope;") - result.Check(testkit.Rows(config.DefTxnScope)) + result.Check(testkit.Rows(oracle.GlobalTxnScope)) } func (s *testSessionSuite3) TestSetVarHint(c *C) { diff --git a/session/txn.go b/session/txn.go index 3b756c022efc6..ff9c41df9d1b9 100644 --- a/session/txn.go +++ b/session/txn.go @@ -358,9 +358,9 @@ func (s *session) getTxnFuture(ctx context.Context) *txnFuture { oracleStore := s.store.GetOracle() var tsFuture oracle.Future if s.sessionVars.LowResolutionTSO { - tsFuture = oracleStore.GetLowResolutionTimestampAsync(ctx) + tsFuture = oracleStore.GetLowResolutionTimestampAsync(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) } else { - tsFuture = oracleStore.GetTimestampAsync(ctx) + tsFuture = oracleStore.GetTimestampAsync(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) } ret := &txnFuture{future: tsFuture, store: s.store} failpoint.InjectContext(ctx, "mockGetTSFail", func() { diff --git a/store/mockoracle/oracle.go b/store/mockoracle/oracle.go index c822696657d77..3e242f548a340 100644 --- a/store/mockoracle/oracle.go +++ b/store/mockoracle/oracle.go @@ -55,7 +55,7 @@ func (o *MockOracle) AddOffset(d time.Duration) { } // GetTimestamp implements oracle.Oracle interface. -func (o *MockOracle) GetTimestamp(context.Context) (uint64, error) { +func (o *MockOracle) GetTimestamp(ctx context.Context, _ *oracle.Option) (uint64, error) { o.Lock() defer o.Unlock() @@ -77,26 +77,26 @@ type mockOracleFuture struct { } func (m *mockOracleFuture) Wait() (uint64, error) { - return m.o.GetTimestamp(m.ctx) + return m.o.GetTimestamp(m.ctx, &oracle.Option{}) } // GetTimestampAsync implements oracle.Oracle interface. -func (o *MockOracle) GetTimestampAsync(ctx context.Context) oracle.Future { +func (o *MockOracle) GetTimestampAsync(ctx context.Context, _ *oracle.Option) oracle.Future { return &mockOracleFuture{o, ctx} } // GetLowResolutionTimestamp implements oracle.Oracle interface. -func (o *MockOracle) GetLowResolutionTimestamp(ctx context.Context) (uint64, error) { - return o.GetTimestamp(ctx) +func (o *MockOracle) GetLowResolutionTimestamp(ctx context.Context, opt *oracle.Option) (uint64, error) { + return o.GetTimestamp(ctx, opt) } // GetLowResolutionTimestampAsync implements oracle.Oracle interface. -func (o *MockOracle) GetLowResolutionTimestampAsync(ctx context.Context) oracle.Future { - return o.GetTimestampAsync(ctx) +func (o *MockOracle) GetLowResolutionTimestampAsync(ctx context.Context, opt *oracle.Option) oracle.Future { + return o.GetTimestampAsync(ctx, opt) } // IsExpired implements oracle.Oracle interface. -func (o *MockOracle) IsExpired(lockTimestamp uint64, TTL uint64) bool { +func (o *MockOracle) IsExpired(lockTimestamp, TTL uint64, _ *oracle.Option) bool { o.RLock() defer o.RUnlock() @@ -104,7 +104,7 @@ func (o *MockOracle) IsExpired(lockTimestamp uint64, TTL uint64) bool { } // UntilExpired implement oracle.Oracle interface. -func (o *MockOracle) UntilExpired(lockTimeStamp uint64, TTL uint64) int64 { +func (o *MockOracle) UntilExpired(lockTimeStamp, TTL uint64, _ *oracle.Option) int64 { o.RLock() defer o.RUnlock() return oracle.ExtractPhysical(lockTimeStamp) + int64(TTL) - oracle.GetPhysical(time.Now().Add(o.offset)) diff --git a/store/mockstore/mocktikv/executor_test.go b/store/mockstore/mocktikv/executor_test.go index a9c964a4591ba..b98981828287e 100644 --- a/store/mockstore/mocktikv/executor_test.go +++ b/store/mockstore/mocktikv/executor_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/mockstore/mocktikv" "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/testkit" ) @@ -72,8 +73,8 @@ func (s *testExecutorSuite) TestResolvedLargeTxnLocks(c *C) { tk.MustExec("insert into t values (1, 1)") - oracle := s.store.GetOracle() - tso, err := oracle.GetTimestamp(context.Background()) + o := s.store.GetOracle() + tso, err := o.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) key := tablecodec.EncodeRowKeyWithHandle(tbl.Meta().ID, kv.IntHandle(1)) diff --git a/store/mockstore/mocktikv/pd.go b/store/mockstore/mocktikv/pd.go index cf653978085b2..b2db0de8fb343 100644 --- a/store/mockstore/mocktikv/pd.go +++ b/store/mockstore/mocktikv/pd.go @@ -69,10 +69,18 @@ func (c *pdClient) GetTS(context.Context) (int64, int64, error) { return tsMu.physicalTS, tsMu.logicalTS, nil } +func (c *pdClient) GetLocalTS(ctx context.Context, dcLocation string) (int64, int64, error) { + return c.GetTS(ctx) +} + func (c *pdClient) GetTSAsync(ctx context.Context) pd.TSFuture { return &mockTSFuture{c, ctx, false} } +func (c *pdClient) GetLocalTSAsync(ctx context.Context, dcLocation string) pd.TSFuture { + return c.GetTSAsync(ctx) +} + type mockTSFuture struct { pdc *pdClient ctx context.Context @@ -92,6 +100,10 @@ func (c *pdClient) GetRegion(ctx context.Context, key []byte) (*pd.Region, error return &pd.Region{Meta: region, Leader: peer}, nil } +func (c *pdClient) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string) (*pd.Region, error) { + return &pd.Region{}, nil +} + func (c *pdClient) GetPrevRegion(ctx context.Context, key []byte) (*pd.Region, error) { region, peer := c.cluster.GetPrevRegionByKey(key) return &pd.Region{Meta: region, Leader: peer}, nil @@ -167,8 +179,12 @@ func (c *pdClient) ScatterRegion(ctx context.Context, regionID uint64) error { return nil } -func (c *pdClient) ScatterRegionWithOption(ctx context.Context, regionID uint64, opts ...pd.ScatterRegionOption) error { - return nil +func (c *pdClient) ScatterRegions(ctx context.Context, regionsID []uint64, opts ...pd.RegionsOption) (*pdpb.ScatterRegionResponse, error) { + return nil, nil +} + +func (c *pdClient) SplitRegions(ctx context.Context, splitKeys [][]byte, opts ...pd.RegionsOption) (*pdpb.SplitRegionsResponse, error) { + return nil, nil } func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { diff --git a/store/mockstore/unistore/pd.go b/store/mockstore/unistore/pd.go index 11f5e1f32ea78..ea27bb50c7771 100644 --- a/store/mockstore/unistore/pd.go +++ b/store/mockstore/unistore/pd.go @@ -40,10 +40,18 @@ func newPDClient(pd *us.MockPD) *pdClient { } } +func (c *pdClient) GetLocalTS(ctx context.Context, dcLocation string) (int64, int64, error) { + return c.GetTS(ctx) +} + func (c *pdClient) GetTSAsync(ctx context.Context) pd.TSFuture { return &mockTSFuture{c, ctx, false} } +func (c *pdClient) GetLocalTSAsync(ctx context.Context, dcLocation string) pd.TSFuture { + return &mockTSFuture{c, ctx, false} +} + type mockTSFuture struct { pdc *pdClient ctx context.Context @@ -93,10 +101,18 @@ func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetO return &pdpb.GetOperatorResponse{Status: pdpb.OperatorStatus_SUCCESS}, nil } -func (c *pdClient) ScatterRegionWithOption(ctx context.Context, regionID uint64, opts ...pd.ScatterRegionOption) error { - return nil +func (c *pdClient) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { + return nil, nil +} + +func (c *pdClient) ScatterRegions(ctx context.Context, regionsID []uint64, opts ...pd.RegionsOption) (*pdpb.ScatterRegionResponse, error) { + return nil, nil } -func (c *pdClient) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { +func (c *pdClient) SplitRegions(ctx context.Context, splitKeys [][]byte, opts ...pd.RegionsOption) (*pdpb.SplitRegionsResponse, error) { + return nil, nil +} + +func (c *pdClient) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string) (*pd.Region, error) { return nil, nil } diff --git a/store/tikv/1pc_test.go b/store/tikv/1pc_test.go index abbd330a3e90f..09d07aad3ec99 100644 --- a/store/tikv/1pc_test.go +++ b/store/tikv/1pc_test.go @@ -19,6 +19,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/store/tikv/oracle" ) type testOnePCSuite struct { @@ -166,7 +167,7 @@ func (s *testOnePCSuite) Test1PCIsolation(c *C) { // Make `txn`'s commitTs more likely to be less than `txn2`'s startTs if there's bug in commitTs // calculation. for i := 0; i < 10; i++ { - _, err := s.store.oracle.GetTimestamp(ctx) + _, err := s.store.oracle.GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) } diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 47c77816898c4..34275d727397f 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -799,7 +799,7 @@ func (tm *ttlManager) keepAlive(c *twoPhaseCommitter) { return } bo := NewBackofferWithVars(context.Background(), pessimisticLockMaxBackoff, c.txn.vars) - now, err := c.store.GetOracle().GetTimestamp(bo.ctx) + now, err := c.store.GetOracle().GetTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if err != nil { err1 := bo.Backoff(BoPDRPC, err) if err1 != nil { @@ -1015,7 +1015,7 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { // than the snapshot TS of all existent readers. So we get a new timestamp // from PD as our MinCommitTS. if commitTSMayBeCalculated && config.GetGlobalConfig().TiKVClient.ExternalConsistency { - minCommitTS, err := c.store.oracle.GetTimestamp(ctx) + minCommitTS, err := c.store.oracle.GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) // If we fail to get a timestamp from PD, we just propagate the failure // instead of falling back to the normal 2PC because a normal 2PC will // also be likely to fail due to the same timestamp issue. @@ -1158,7 +1158,7 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) (err error) { } c.commitTS = commitTS - if c.store.oracle.IsExpired(c.startTS, kv.MaxTxnTimeUse) { + if c.store.oracle.IsExpired(c.startTS, kv.MaxTxnTimeUse, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) { err = errors.Errorf("conn %d txn takes too much time, txnStartTS: %d, comm: %d", c.connID, c.startTS, c.commitTS) return err diff --git a/store/tikv/2pc_test.go b/store/tikv/2pc_test.go index 534e339c4ee41..c0b4782ca4990 100644 --- a/store/tikv/2pc_test.go +++ b/store/tikv/2pc_test.go @@ -226,7 +226,7 @@ func (s *testCommitterSuite) TestPrewriteRollback(c *C) { err = committer.prewriteMutations(NewBackofferWithVars(ctx, PrewriteMaxBackoff, nil), committer.mutations) c.Assert(err, IsNil) } - committer.commitTS, err = s.store.oracle.GetTimestamp(ctx) + committer.commitTS, err = s.store.oracle.GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) err = committer.commitMutations(NewBackofferWithVars(ctx, int(atomic.LoadUint64(&CommitMaxBackoff)), nil), &PlainMutations{keys: [][]byte{[]byte("a")}}) c.Assert(err, IsNil) @@ -677,7 +677,7 @@ func (s *testCommitterSuite) TestPessimisticTTL(c *C) { err = txn.LockKeys(context.Background(), lockCtx, key2) c.Assert(err, IsNil) lockInfo := s.getLockInfo(c, key) - msBeforeLockExpired := s.store.GetOracle().UntilExpired(txn.StartTS(), lockInfo.LockTtl) + msBeforeLockExpired := s.store.GetOracle().UntilExpired(txn.StartTS(), lockInfo.LockTtl, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(msBeforeLockExpired, GreaterEqual, int64(100)) lr := newLockResolver(s.store) @@ -690,7 +690,7 @@ func (s *testCommitterSuite) TestPessimisticTTL(c *C) { for i := 0; i < 50; i++ { lockInfoNew := s.getLockInfo(c, key) if lockInfoNew.LockTtl > lockInfo.LockTtl { - currentTS, err := lr.store.GetOracle().GetTimestamp(bo.ctx) + currentTS, err := lr.store.GetOracle().GetTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) // Check that the TTL is update to a reasonable range. expire := oracle.ExtractPhysical(txn.startTS) + int64(lockInfoNew.LockTtl) diff --git a/store/tikv/async_commit_test.go b/store/tikv/async_commit_test.go index d33dc686b13e8..7b5ce75fd0616 100644 --- a/store/tikv/async_commit_test.go +++ b/store/tikv/async_commit_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/mockstore/cluster" "github.com/pingcap/tidb/store/mockstore/unistore" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" ) @@ -161,7 +162,7 @@ func (s *testAsyncCommitSuite) lockKeys(c *C, keys, values [][]byte, primaryKey, c.Assert(err, IsNil) if commitPrimary { - tpc.commitTS, err = s.store.oracle.GetTimestamp(ctx) + tpc.commitTS, err = s.store.oracle.GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) err = tpc.commitMutations(NewBackofferWithVars(ctx, int(atomic.LoadUint64(&CommitMaxBackoff)), nil), tpc.mutationsOfKeys([][]byte{primaryKey})) c.Assert(err, IsNil) @@ -192,13 +193,13 @@ func (s *testAsyncCommitSuite) TestCheckSecondaries(c *C) { s.lockKeys(c, [][]byte{}, [][]byte{}, []byte("z"), []byte("z"), false) lock := s.mustGetLock(c, []byte("z")) lock.UseAsyncCommit = true - ts, err := s.store.oracle.GetTimestamp(context.Background()) + ts, err := s.store.oracle.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) status := TxnStatus{primaryLock: &kvrpcpb.LockInfo{Secondaries: [][]byte{}, UseAsyncCommit: true, MinCommitTs: ts}} err = s.store.lockResolver.resolveLockAsync(s.bo, lock, status) c.Assert(err, IsNil) - currentTS, err := s.store.oracle.GetTimestamp(context.Background()) + currentTS, err := s.store.oracle.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) status, err = s.store.lockResolver.getTxnStatus(s.bo, lock.TxnID, []byte("z"), currentTS, currentTS, true) c.Assert(err, IsNil) @@ -206,7 +207,7 @@ func (s *testAsyncCommitSuite) TestCheckSecondaries(c *C) { c.Assert(status.CommitTS(), Equals, ts) // One key is committed (i), one key is locked (a). Should get committed. - ts, err = s.store.oracle.GetTimestamp(context.Background()) + ts, err = s.store.oracle.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) commitTs := ts + 10 @@ -284,7 +285,7 @@ func (s *testAsyncCommitSuite) TestCheckSecondaries(c *C) { c.Assert(gotResolve, Equals, int64(1)) // One key has been rolled back (b), one is locked (a). Should be rolled back. - ts, err = s.store.oracle.GetTimestamp(context.Background()) + ts, err = s.store.oracle.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) commitTs = ts + 10 @@ -340,7 +341,7 @@ func (s *testAsyncCommitSuite) TestRepeatableRead(c *C) { txn1.Set([]byte("k1"), []byte("v2")) for i := 0; i < 20; i++ { - _, err := s.store.GetOracle().GetTimestamp(ctx) + _, err := s.store.GetOracle().GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) } diff --git a/store/tikv/gcworker/gc_worker_test.go b/store/tikv/gcworker/gc_worker_test.go index 4d010570839b1..11b981aaa1758 100644 --- a/store/tikv/gcworker/gc_worker_test.go +++ b/store/tikv/gcworker/gc_worker_test.go @@ -137,7 +137,7 @@ func (s *testGCWorkerSuite) mustGetNone(c *C, key string, ts uint64) { } func (s *testGCWorkerSuite) mustAllocTs(c *C) uint64 { - ts, err := s.oracle.GetTimestamp(context.Background()) + ts, err := s.oracle.GetTimestamp(context.Background(), &oracle.Option{}) c.Assert(err, IsNil) return ts } diff --git a/store/tikv/kv.go b/store/tikv/kv.go index 53dae50d13c04..dfe8159bb27e5 100644 --- a/store/tikv/kv.go +++ b/store/tikv/kv.go @@ -397,7 +397,7 @@ func (s *tikvStore) getTimestampWithRetry(bo *Backoffer) (uint64, error) { } for { - startTS, err := s.oracle.GetTimestamp(bo.ctx) + startTS, err := s.oracle.GetTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) // mockGetTSErrorInRetry should wait MockCommitErrorOnce first, then will run into retry() logic. // Then mockGetTSErrorInRetry will return retryable error when first retry. // Before PR #8743, we don't cleanup txn after meet error such as error like: PD server timeout diff --git a/store/tikv/lock_resolver.go b/store/tikv/lock_resolver.go index d39e94a0ab8d3..c3584afa9c484 100644 --- a/store/tikv/lock_resolver.go +++ b/store/tikv/lock_resolver.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" @@ -218,7 +219,7 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi // locks have been cleaned before GC. expiredLocks := locks - callerStartTS, err := lr.store.GetOracle().GetTimestamp(bo.ctx) + callerStartTS, err := lr.store.GetOracle().GetTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if err != nil { return false, errors.Trace(err) } @@ -376,7 +377,7 @@ func (lr *LockResolver) resolveLocks(bo *Backoffer, callerStartTS uint64, locks tikvLockResolverCountWithNotExpired.Inc() // If the lock is valid, the txn may be a pessimistic transaction. // Update the txn expire time. - msBeforeLockExpired := lr.store.GetOracle().UntilExpired(l.TxnID, status.ttl) + msBeforeLockExpired := lr.store.GetOracle().UntilExpired(l.TxnID, status.ttl, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) msBeforeTxnExpired.update(msBeforeLockExpired) if forWrite { // Write conflict detected! @@ -446,7 +447,7 @@ func (t *txnExpireTime) value() int64 { func (lr *LockResolver) GetTxnStatus(txnID uint64, callerStartTS uint64, primary []byte) (TxnStatus, error) { var status TxnStatus bo := NewBackoffer(context.Background(), cleanupMaxBackoff) - currentTS, err := lr.store.GetOracle().GetLowResolutionTimestamp(bo.ctx) + currentTS, err := lr.store.GetOracle().GetLowResolutionTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if err != nil { return status, err } @@ -469,7 +470,7 @@ func (lr *LockResolver) getTxnStatusFromLock(bo *Backoffer, l *Lock, callerStart // Set currentTS to max uint64 to make the lock expired. currentTS = math.MaxUint64 } else { - currentTS, err = lr.store.GetOracle().GetLowResolutionTimestamp(bo.ctx) + currentTS, err = lr.store.GetOracle().GetLowResolutionTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if err != nil { return TxnStatus{}, err } @@ -502,7 +503,7 @@ func (lr *LockResolver) getTxnStatusFromLock(bo *Backoffer, l *Lock, callerStart logutil.Logger(bo.ctx).Warn("getTxnStatusFromLock backoff fail", zap.Error(err)) } - if lr.store.GetOracle().UntilExpired(l.TxnID, l.TTL) <= 0 { + if lr.store.GetOracle().UntilExpired(l.TxnID, l.TTL, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) <= 0 { logutil.Logger(bo.ctx).Warn("lock txn not found, lock has expired", zap.Uint64("CallerStartTs", callerStartTS), zap.Stringer("lock str", l)) @@ -594,7 +595,7 @@ func (lr *LockResolver) getTxnStatus(bo *Backoffer, txnID uint64, primary []byte status.primaryLock = cmdResp.LockInfo if status.primaryLock != nil && status.primaryLock.UseAsyncCommit { - if !lr.store.GetOracle().IsExpired(txnID, cmdResp.LockTtl) { + if !lr.store.GetOracle().IsExpired(txnID, cmdResp.LockTtl, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) { status.ttl = cmdResp.LockTtl } } else if cmdResp.LockTtl != 0 { diff --git a/store/tikv/lock_test.go b/store/tikv/lock_test.go index c992b2d12df8a..11a29b3fc478f 100644 --- a/store/tikv/lock_test.go +++ b/store/tikv/lock_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" ) @@ -72,7 +73,7 @@ func (s *testLockSuite) lockKey(c *C, key, value, primaryKey, primaryValue []byt c.Assert(err, IsNil) if commitPrimary { - tpc.commitTS, err = s.store.oracle.GetTimestamp(ctx) + tpc.commitTS, err = s.store.oracle.GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) err = tpc.commitMutations(NewBackofferWithVars(ctx, int(atomic.LoadUint64(&CommitMaxBackoff)), nil), tpc.mutationsOfKeys([][]byte{primaryKey})) c.Assert(err, IsNil) @@ -214,7 +215,7 @@ func (s *testLockSuite) TestCheckTxnStatusTTL(c *C) { bo := NewBackofferWithVars(context.Background(), PrewriteMaxBackoff, nil) lr := newLockResolver(s.store) - callerStartTS, err := lr.store.GetOracle().GetTimestamp(bo.ctx) + callerStartTS, err := lr.store.GetOracle().GetTimestamp(bo.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) // Check the lock TTL of a transaction. @@ -279,8 +280,8 @@ func (s *testLockSuite) TestCheckTxnStatus(c *C) { txn.Set(kv.Key("second"), []byte("xxx")) s.prewriteTxnWithTTL(c, txn.(*tikvTxn), 1000) - oracle := s.store.GetOracle() - currentTS, err := oracle.GetTimestamp(context.Background()) + o := s.store.GetOracle() + currentTS, err := o.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) c.Assert(currentTS, Greater, txn.StartTS()) @@ -307,7 +308,7 @@ func (s *testLockSuite) TestCheckTxnStatus(c *C) { c.Assert(timeBeforeExpire, Equals, int64(0)) // Then call getTxnStatus again and check the lock status. - currentTS, err = oracle.GetTimestamp(context.Background()) + currentTS, err = o.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) status, err = newLockResolver(s.store).getTxnStatus(bo, txn.StartTS(), []byte("key"), currentTS, 0, true) c.Assert(err, IsNil) @@ -338,8 +339,8 @@ func (s *testLockSuite) TestCheckTxnStatusNoWait(c *C) { err = committer.prewriteMutations(NewBackofferWithVars(context.Background(), PrewriteMaxBackoff, nil), committer.mutationsOfKeys([][]byte{[]byte("second")})) c.Assert(err, IsNil) - oracle := s.store.GetOracle() - currentTS, err := oracle.GetTimestamp(context.Background()) + o := s.store.GetOracle() + currentTS, err := o.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) bo := NewBackofferWithVars(context.Background(), PrewriteMaxBackoff, nil) resolver := newLockResolver(s.store) @@ -369,7 +370,7 @@ func (s *testLockSuite) TestCheckTxnStatusNoWait(c *C) { c.Assert(committer.cleanupMutations(bo, committer.mutations), IsNil) // Call getTxnStatusFromLock to cover TxnNotFound and retry timeout. - startTS, err := oracle.GetTimestamp(context.Background()) + startTS, err := o.GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(err, IsNil) lock = &Lock{ Key: []byte("second"), @@ -491,9 +492,9 @@ func (s *testLockSuite) TestBatchResolveLocks(c *C) { } // Locks may not expired - msBeforeLockExpired := s.store.GetOracle().UntilExpired(locks[0].TxnID, locks[1].TTL) + msBeforeLockExpired := s.store.GetOracle().UntilExpired(locks[0].TxnID, locks[1].TTL, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(msBeforeLockExpired, Greater, int64(0)) - msBeforeLockExpired = s.store.GetOracle().UntilExpired(locks[3].TxnID, locks[3].TTL) + msBeforeLockExpired = s.store.GetOracle().UntilExpired(locks[3].TxnID, locks[3].TTL, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) c.Assert(msBeforeLockExpired, Greater, int64(0)) lr := newLockResolver(s.store) diff --git a/store/tikv/oracle/oracle.go b/store/tikv/oracle/oracle.go index 9381622e3b3a9..c49412c3110d5 100644 --- a/store/tikv/oracle/oracle.go +++ b/store/tikv/oracle/oracle.go @@ -22,14 +22,19 @@ import ( "go.uber.org/zap" ) +// Option represents available options for the oracle.Oracle. +type Option struct { + TxnScope string +} + // Oracle is the interface that provides strictly ascending timestamps. type Oracle interface { - GetTimestamp(ctx context.Context) (uint64, error) - GetTimestampAsync(ctx context.Context) Future - GetLowResolutionTimestamp(ctx context.Context) (uint64, error) - GetLowResolutionTimestampAsync(ctx context.Context) Future - IsExpired(lockTimestamp uint64, TTL uint64) bool - UntilExpired(lockTimeStamp uint64, TTL uint64) int64 + GetTimestamp(ctx context.Context, opt *Option) (uint64, error) + GetTimestampAsync(ctx context.Context, opt *Option) Future + GetLowResolutionTimestamp(ctx context.Context, opt *Option) (uint64, error) + GetLowResolutionTimestampAsync(ctx context.Context, opt *Option) Future + IsExpired(lockTimestamp, TTL uint64, opt *Option) bool + UntilExpired(lockTimeStamp, TTL uint64, opt *Option) int64 Close() } @@ -38,7 +43,11 @@ type Future interface { Wait() (uint64, error) } -const physicalShiftBits = 18 +const ( + physicalShiftBits = 18 + // GlobalTxnScope is the default transaction scope for a Oracle service. + GlobalTxnScope = "global" +) // ComposeTS creates a ts from physical and logical parts. func ComposeTS(physical, logical int64) uint64 { diff --git a/store/tikv/oracle/oracles/export_test.go b/store/tikv/oracle/oracles/export_test.go index 999c8cf5f6e39..3a6ceb965420e 100644 --- a/store/tikv/oracle/oracles/export_test.go +++ b/store/tikv/oracle/oracles/export_test.go @@ -14,6 +14,7 @@ package oracles import ( + "sync/atomic" "time" "github.com/pingcap/tidb/store/tikv/oracle" @@ -45,10 +46,12 @@ func NewEmptyPDOracle() oracle.Oracle { return &pdOracle{} } -// SetEmptyPDOracleLastTs exports PD oracle's last ts to test. +// SetEmptyPDOracleLastTs exports PD oracle's global last ts to test. func SetEmptyPDOracleLastTs(oc oracle.Oracle, ts uint64) { switch o := oc.(type) { case *pdOracle: - o.lastTS = ts + lastTSInterface, _ := o.lastTSMap.LoadOrStore(oracle.GlobalTxnScope, new(uint64)) + lastTSPointer := lastTSInterface.(*uint64) + atomic.StoreUint64(lastTSPointer, ts) } } diff --git a/store/tikv/oracle/oracles/local.go b/store/tikv/oracle/oracles/local.go index 8c4f069f1c0fa..aaaba70f2a11b 100644 --- a/store/tikv/oracle/oracles/local.go +++ b/store/tikv/oracle/oracles/local.go @@ -37,7 +37,7 @@ func NewLocalOracle() oracle.Oracle { return &localOracle{} } -func (l *localOracle) IsExpired(lockTS uint64, TTL uint64) bool { +func (l *localOracle) IsExpired(lockTS, TTL uint64, _ *oracle.Option) bool { now := time.Now() if l.hook != nil { now = l.hook.currentTime @@ -45,7 +45,7 @@ func (l *localOracle) IsExpired(lockTS uint64, TTL uint64) bool { return oracle.GetPhysical(now) >= oracle.ExtractPhysical(lockTS)+int64(TTL) } -func (l *localOracle) GetTimestamp(context.Context) (uint64, error) { +func (l *localOracle) GetTimestamp(ctx context.Context, _ *oracle.Option) (uint64, error) { l.Lock() defer l.Unlock() now := time.Now() @@ -63,19 +63,19 @@ func (l *localOracle) GetTimestamp(context.Context) (uint64, error) { return ts, nil } -func (l *localOracle) GetTimestampAsync(ctx context.Context) oracle.Future { +func (l *localOracle) GetTimestampAsync(ctx context.Context, _ *oracle.Option) oracle.Future { return &future{ ctx: ctx, l: l, } } -func (l *localOracle) GetLowResolutionTimestamp(ctx context.Context) (uint64, error) { - return l.GetTimestamp(ctx) +func (l *localOracle) GetLowResolutionTimestamp(ctx context.Context, opt *oracle.Option) (uint64, error) { + return l.GetTimestamp(ctx, opt) } -func (l *localOracle) GetLowResolutionTimestampAsync(ctx context.Context) oracle.Future { - return l.GetTimestampAsync(ctx) +func (l *localOracle) GetLowResolutionTimestampAsync(ctx context.Context, opt *oracle.Option) oracle.Future { + return l.GetTimestampAsync(ctx, opt) } type future struct { @@ -84,11 +84,11 @@ type future struct { } func (f *future) Wait() (uint64, error) { - return f.l.GetTimestamp(f.ctx) + return f.l.GetTimestamp(f.ctx, &oracle.Option{}) } // UntilExpired implement oracle.Oracle interface. -func (l *localOracle) UntilExpired(lockTimeStamp uint64, TTL uint64) int64 { +func (l *localOracle) UntilExpired(lockTimeStamp, TTL uint64, opt *oracle.Option) int64 { now := time.Now() if l.hook != nil { now = l.hook.currentTime diff --git a/store/tikv/oracle/oracles/local_test.go b/store/tikv/oracle/oracles/local_test.go index d2dd31bfe5ee5..becc89725f9b5 100644 --- a/store/tikv/oracle/oracles/local_test.go +++ b/store/tikv/oracle/oracles/local_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/oracle/oracles" ) @@ -26,7 +27,7 @@ func TestLocalOracle(t *testing.T) { defer l.Close() m := map[uint64]struct{}{} for i := 0; i < 100000; i++ { - ts, err := l.GetTimestamp(context.Background()) + ts, err := l.GetTimestamp(context.Background(), &oracle.Option{}) if err != nil { t.Error(err) } @@ -43,13 +44,13 @@ func TestIsExpired(t *testing.T) { defer o.Close() start := time.Now() oracles.SetOracleHookCurrentTime(o, start) - ts, _ := o.GetTimestamp(context.Background()) + ts, _ := o.GetTimestamp(context.Background(), &oracle.Option{}) oracles.SetOracleHookCurrentTime(o, start.Add(10*time.Millisecond)) - expire := o.IsExpired(ts, 5) + expire := o.IsExpired(ts, 5, &oracle.Option{}) if !expire { t.Error("should expired") } - expire = o.IsExpired(ts, 200) + expire = o.IsExpired(ts, 200, &oracle.Option{}) if expire { t.Error("should not expired") } @@ -60,9 +61,9 @@ func TestLocalOracle_UntilExpired(t *testing.T) { defer o.Close() start := time.Now() oracles.SetOracleHookCurrentTime(o, start) - ts, _ := o.GetTimestamp(context.Background()) + ts, _ := o.GetTimestamp(context.Background(), &oracle.Option{}) oracles.SetOracleHookCurrentTime(o, start.Add(10*time.Millisecond)) - if o.UntilExpired(ts, 5) != -5 || o.UntilExpired(ts, 15) != 5 { + if o.UntilExpired(ts, 5, &oracle.Option{}) != -5 || o.UntilExpired(ts, 15, &oracle.Option{}) != 5 { t.Error("until expired should be +-5") } } diff --git a/store/tikv/oracle/oracles/pd.go b/store/tikv/oracle/oracles/pd.go index a088ab5106271..116f6d200d14b 100644 --- a/store/tikv/oracle/oracles/pd.go +++ b/store/tikv/oracle/oracles/pd.go @@ -15,6 +15,7 @@ package oracles import ( "context" + "sync" "sync/atomic" "time" @@ -22,7 +23,7 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/util/logutil" - "github.com/tikv/pd/client" + pd "github.com/tikv/pd/client" "go.uber.org/zap" ) @@ -32,9 +33,10 @@ const slowDist = 30 * time.Millisecond // pdOracle is an Oracle that uses a placement driver client as source. type pdOracle struct { - c pd.Client - lastTS uint64 - quit chan struct{} + c pd.Client + // txn_scope (string) -> lastTSPointer (*uint64) + lastTSMap sync.Map + quit chan struct{} } // NewPdOracle create an Oracle that uses a pd client source. @@ -49,8 +51,8 @@ func NewPdOracle(pdClient pd.Client, updateInterval time.Duration) (oracle.Oracl } ctx := context.TODO() go o.updateTS(ctx, updateInterval) - // Initialize lastTS by Get. - _, err := o.GetTimestamp(ctx) + // Initialize the timestamp of the global txnScope by Get. + _, err := o.GetTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if err != nil { o.Close() return nil, errors.Trace(err) @@ -60,24 +62,28 @@ func NewPdOracle(pdClient pd.Client, updateInterval time.Duration) (oracle.Oracl // IsExpired returns whether lockTS+TTL is expired, both are ms. It uses `lastTS` // to compare, may return false negative result temporarily. -func (o *pdOracle) IsExpired(lockTS, TTL uint64) bool { - lastTS := atomic.LoadUint64(&o.lastTS) +func (o *pdOracle) IsExpired(lockTS, TTL uint64, opt *oracle.Option) bool { + lastTS, exist := o.getLastTS(opt.TxnScope) + if !exist { + return true + } return oracle.ExtractPhysical(lastTS) >= oracle.ExtractPhysical(lockTS)+int64(TTL) } // GetTimestamp gets a new increasing time. -func (o *pdOracle) GetTimestamp(ctx context.Context) (uint64, error) { - ts, err := o.getTimestamp(ctx) +func (o *pdOracle) GetTimestamp(ctx context.Context, opt *oracle.Option) (uint64, error) { + ts, err := o.getTimestamp(ctx, opt.TxnScope) if err != nil { return 0, errors.Trace(err) } - o.setLastTS(ts) + o.setLastTS(ts, opt.TxnScope) return ts, nil } type tsFuture struct { pd.TSFuture - o *pdOracle + o *pdOracle + txnScope string } // Wait implements the oracle.Future interface. @@ -89,18 +95,31 @@ func (f *tsFuture) Wait() (uint64, error) { return 0, errors.Trace(err) } ts := oracle.ComposeTS(physical, logical) - f.o.setLastTS(ts) + f.o.setLastTS(ts, f.txnScope) return ts, nil } -func (o *pdOracle) GetTimestampAsync(ctx context.Context) oracle.Future { - ts := o.c.GetTSAsync(ctx) - return &tsFuture{ts, o} +func (o *pdOracle) GetTimestampAsync(ctx context.Context, opt *oracle.Option) oracle.Future { + var ts pd.TSFuture + if opt.TxnScope == oracle.GlobalTxnScope || opt.TxnScope == "" { + ts = o.c.GetTSAsync(ctx) + } else { + ts = o.c.GetLocalTSAsync(ctx, opt.TxnScope) + } + return &tsFuture{ts, o, opt.TxnScope} } -func (o *pdOracle) getTimestamp(ctx context.Context) (uint64, error) { +func (o *pdOracle) getTimestamp(ctx context.Context, txnScope string) (uint64, error) { now := time.Now() - physical, logical, err := o.c.GetTS(ctx) + var ( + physical, logical int64 + err error + ) + if txnScope == oracle.GlobalTxnScope || txnScope == "" { + physical, logical, err = o.c.GetTS(ctx) + } else { + physical, logical, err = o.c.GetLocalTS(ctx, txnScope) + } if err != nil { return 0, errors.Trace(err) } @@ -112,11 +131,35 @@ func (o *pdOracle) getTimestamp(ctx context.Context) (uint64, error) { return oracle.ComposeTS(physical, logical), nil } -func (o *pdOracle) setLastTS(ts uint64) { - lastTS := atomic.LoadUint64(&o.lastTS) - if ts > lastTS { - atomic.CompareAndSwapUint64(&o.lastTS, lastTS, ts) +func (o *pdOracle) setLastTS(ts uint64, txnScope string) { + if txnScope == "" { + txnScope = oracle.GlobalTxnScope + } + lastTSInterface, ok := o.lastTSMap.Load(txnScope) + if !ok { + lastTSInterface, _ = o.lastTSMap.LoadOrStore(txnScope, new(uint64)) + } + lastTSPointer := lastTSInterface.(*uint64) + for { + lastTS := atomic.LoadUint64(lastTSPointer) + if ts <= lastTS { + return + } + if atomic.CompareAndSwapUint64(lastTSPointer, lastTS, ts) { + return + } + } +} + +func (o *pdOracle) getLastTS(txnScope string) (uint64, bool) { + if txnScope == "" { + txnScope = oracle.GlobalTxnScope } + lastTSInterface, ok := o.lastTSMap.Load(txnScope) + if !ok { + return 0, false + } + return atomic.LoadUint64(lastTSInterface.(*uint64)), true } func (o *pdOracle) updateTS(ctx context.Context, interval time.Duration) { @@ -125,12 +168,17 @@ func (o *pdOracle) updateTS(ctx context.Context, interval time.Duration) { for { select { case <-ticker.C: - ts, err := o.getTimestamp(ctx) - if err != nil { - logutil.Logger(ctx).Error("updateTS error", zap.Error(err)) - break - } - o.setLastTS(ts) + // Update the timestamp for each txnScope + o.lastTSMap.Range(func(key, _ interface{}) bool { + txnScope := key.(string) + ts, err := o.getTimestamp(ctx, txnScope) + if err != nil { + logutil.Logger(ctx).Error("updateTS error", zap.String("txnScope", txnScope), zap.Error(err)) + return true + } + o.setLastTS(ts, txnScope) + return true + }) case <-o.quit: return } @@ -138,8 +186,11 @@ func (o *pdOracle) updateTS(ctx context.Context, interval time.Duration) { } // UntilExpired implement oracle.Oracle interface. -func (o *pdOracle) UntilExpired(lockTS uint64, TTL uint64) int64 { - lastTS := atomic.LoadUint64(&o.lastTS) +func (o *pdOracle) UntilExpired(lockTS uint64, TTL uint64, opt *oracle.Option) int64 { + lastTS, ok := o.getLastTS(opt.TxnScope) + if !ok { + return 0 + } return oracle.ExtractPhysical(lockTS) + int64(TTL) - oracle.ExtractPhysical(lastTS) } @@ -148,20 +199,35 @@ func (o *pdOracle) Close() { } // A future that resolves immediately to a low resolution timestamp. -type lowResolutionTsFuture uint64 +type lowResolutionTsFuture struct { + ts uint64 + err error +} // Wait implements the oracle.Future interface. func (f lowResolutionTsFuture) Wait() (uint64, error) { - return uint64(f), nil + return f.ts, f.err } // GetLowResolutionTimestamp gets a new increasing time. -func (o *pdOracle) GetLowResolutionTimestamp(ctx context.Context) (uint64, error) { - lastTS := atomic.LoadUint64(&o.lastTS) +func (o *pdOracle) GetLowResolutionTimestamp(ctx context.Context, opt *oracle.Option) (uint64, error) { + lastTS, ok := o.getLastTS(opt.TxnScope) + if !ok { + return 0, errors.Errorf("get low resolution timestamp fail, invalid txnScope = %s", opt.TxnScope) + } return lastTS, nil } -func (o *pdOracle) GetLowResolutionTimestampAsync(ctx context.Context) oracle.Future { - lastTS := atomic.LoadUint64(&o.lastTS) - return lowResolutionTsFuture(lastTS) +func (o *pdOracle) GetLowResolutionTimestampAsync(ctx context.Context, opt *oracle.Option) oracle.Future { + lastTS, ok := o.getLastTS(opt.TxnScope) + if !ok { + return lowResolutionTsFuture{ + ts: 0, + err: errors.Errorf("get low resolution timestamp async fail, invalid txnScope = %s", opt.TxnScope), + } + } + return lowResolutionTsFuture{ + ts: lastTS, + err: nil, + } } diff --git a/store/tikv/oracle/oracles/pd_test.go b/store/tikv/oracle/oracles/pd_test.go index e5199fbf2ad7f..1cd6aa8646075 100644 --- a/store/tikv/oracle/oracles/pd_test.go +++ b/store/tikv/oracle/oracles/pd_test.go @@ -32,7 +32,7 @@ func TestPDOracle_UntilExpired(t *testing.T) { start := time.Now() oracles.SetEmptyPDOracleLastTs(o, oracle.ComposeTS(oracle.GetPhysical(start), 0)) lockTs := oracle.ComposeTS(oracle.GetPhysical(start.Add(time.Duration(lockAfter)*time.Millisecond)), 1) - waitTs := o.UntilExpired(lockTs, uint64(lockExp)) + waitTs := o.UntilExpired(lockTs, uint64(lockExp), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) if waitTs != int64(lockAfter+lockExp) { t.Errorf("waitTs shoulb be %d but got %d", int64(lockAfter+lockExp), waitTs) } diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 353b9f5a9e6e0..1c46146384333 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -213,11 +213,11 @@ func (s *tikvStore) scatterRegion(bo *Backoffer, regionID uint64, tableID *int64 logutil.BgLogger().Info("start scatter region", zap.Uint64("regionID", regionID)) for { - opts := make([]pd.ScatterRegionOption, 0, 1) + opts := make([]pd.RegionsOption, 0, 1) if tableID != nil { opts = append(opts, pd.WithGroup(fmt.Sprintf("%v", *tableID))) } - err := s.pdClient.ScatterRegionWithOption(bo.ctx, regionID, opts...) + _, err := s.pdClient.ScatterRegions(bo.ctx, []uint64{regionID}, opts...) failpoint.Inject("MockScatterRegionTimeout", func(val failpoint.Value) { if val.(bool) { diff --git a/store/tikv/store_test.go b/store/tikv/store_test.go index 2f07d28db6120..3dcdd7a9b1e6b 100644 --- a/store/tikv/store_test.go +++ b/store/tikv/store_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/mockoracle" + "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/store/tikv/tikvrpc" pd "github.com/tikv/pd/client" ) @@ -67,14 +68,14 @@ func (s *testStoreSuite) TestOracle(c *C) { c.Assert(err, IsNil) c.Assert(t1, Less, t2) - t1, err = o.GetLowResolutionTimestamp(ctx) + t1, err = o.GetLowResolutionTimestamp(ctx, &oracle.Option{}) c.Assert(err, IsNil) - t2, err = o.GetLowResolutionTimestamp(ctx) + t2, err = o.GetLowResolutionTimestamp(ctx, &oracle.Option{}) c.Assert(err, IsNil) c.Assert(t1, Less, t2) - f := o.GetLowResolutionTimestampAsync(ctx) + f := o.GetLowResolutionTimestampAsync(ctx, &oracle.Option{}) c.Assert(f, NotNil) - _ = o.UntilExpired(0, 0) + _ = o.UntilExpired(0, 0, &oracle.Option{}) // Check retry. var wg sync.WaitGroup @@ -92,7 +93,7 @@ func (s *testStoreSuite) TestOracle(c *C) { t3, err := s.store.getTimestampWithRetry(NewBackofferWithVars(ctx, tsoMaxBackoff, nil)) c.Assert(err, IsNil) c.Assert(t2, Less, t3) - expired := s.store.oracle.IsExpired(t2, 50) + expired := s.store.oracle.IsExpired(t2, 50, &oracle.Option{}) c.Assert(expired, IsTrue) }() @@ -135,10 +136,18 @@ func (c *mockPDClient) GetTS(ctx context.Context) (int64, int64, error) { return c.client.GetTS(ctx) } +func (c *mockPDClient) GetLocalTS(ctx context.Context, dcLocation string) (int64, int64, error) { + return c.GetTS(ctx) +} + func (c *mockPDClient) GetTSAsync(ctx context.Context) pd.TSFuture { return nil } +func (c *mockPDClient) GetLocalTSAsync(ctx context.Context, dcLocation string) pd.TSFuture { + return nil +} + func (c *mockPDClient) GetRegion(ctx context.Context, key []byte) (*pd.Region, error) { c.RLock() defer c.RUnlock() @@ -149,6 +158,10 @@ func (c *mockPDClient) GetRegion(ctx context.Context, key []byte) (*pd.Region, e return c.client.GetRegion(ctx, key) } +func (c *mockPDClient) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string) (*pd.Region, error) { + return nil, nil +} + func (c *mockPDClient) GetPrevRegion(ctx context.Context, key []byte) (*pd.Region, error) { c.RLock() defer c.RUnlock() @@ -213,16 +226,20 @@ func (c *mockPDClient) ScatterRegion(ctx context.Context, regionID uint64) error return nil } +func (c *mockPDClient) ScatterRegions(ctx context.Context, regionsID []uint64, opts ...pd.RegionsOption) (*pdpb.ScatterRegionResponse, error) { + return nil, nil +} + +func (c *mockPDClient) SplitRegions(ctx context.Context, splitKeys [][]byte, opts ...pd.RegionsOption) (*pdpb.SplitRegionsResponse, error) { + return nil, nil +} + func (c *mockPDClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { return &pdpb.GetOperatorResponse{Status: pdpb.OperatorStatus_SUCCESS}, nil } func (c *mockPDClient) GetLeaderAddr() string { return "mockpd" } -func (c *mockPDClient) ScatterRegionWithOption(ctx context.Context, regionID uint64, opts ...pd.ScatterRegionOption) error { - return nil -} - type checkRequestClient struct { Client priority pb.CommandPri From aa9f7584f8fbcdb1dec35e3fa520224825000112 Mon Sep 17 00:00:00 2001 From: wjHuang Date: Thu, 19 Nov 2020 16:20:33 +0800 Subject: [PATCH 31/37] server: remove prepare stmt if no retry in stmt close() (#20867) Signed-off-by: wjhuang2016 --- server/driver_tidb.go | 20 +++++++++++++++++--- server/tidb_test.go | 22 ++++++++++++++++++++++ sessionctx/variable/session.go | 11 ++++++----- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/server/driver_tidb.go b/server/driver_tidb.go index 206d73dc735fb..8292029590b8a 100644 --- a/server/driver_tidb.go +++ b/server/driver_tidb.go @@ -18,6 +18,7 @@ import ( "crypto/tls" "sync/atomic" + "github.com/pingcap/errors" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/mysql" @@ -149,9 +150,22 @@ func (ts *TiDBStatement) Reset() { // Close implements PreparedStatement Close method. func (ts *TiDBStatement) Close() error { //TODO close at tidb level - err := ts.ctx.DropPreparedStmt(ts.id) - if err != nil { - return err + if ts.ctx.GetSessionVars().TxnCtx != nil && ts.ctx.GetSessionVars().TxnCtx.CouldRetry { + err := ts.ctx.DropPreparedStmt(ts.id) + if err != nil { + return err + } + } else { + if core.PreparedPlanCacheEnabled() { + preparedPointer := ts.ctx.GetSessionVars().PreparedStmts[ts.id] + preparedObj, ok := preparedPointer.(*core.CachedPrepareStmt) + if !ok { + return errors.Errorf("invalid CachedPrepareStmt type") + } + ts.ctx.PreparedPlanCache().Delete(core.NewPSTMTPlanCacheKey( + ts.ctx.GetSessionVars(), ts.id, preparedObj.PreparedAst.SchemaVersion)) + } + ts.ctx.GetSessionVars().RemovePreparedStmt(ts.id) } delete(ts.ctx.stmts, int(ts.id)) diff --git a/server/tidb_test.go b/server/tidb_test.go index 96a698c3f9b9c..eeac9534499ba 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -27,6 +27,7 @@ import ( "net/http" "os" "path/filepath" + "sync/atomic" "time" "github.com/go-sql-driver/mysql" @@ -38,6 +39,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/logutil" @@ -994,3 +996,23 @@ func (ts *tidbTestSuite) TestPessimisticInsertSelectForUpdate(c *C) { c.Assert(err, IsNil) c.Assert(rs, IsNil) // should be no delay } + +func (ts *tidbTestSerialSuite) TestPrepareCount(c *C) { + qctx, err := ts.tidbdrv.OpenCtx(uint64(0), 0, uint8(tmysql.DefaultCollationID), "test", nil) + c.Assert(err, IsNil) + prepareCnt := atomic.LoadInt64(&variable.PreparedStmtCount) + ctx := context.Background() + _, err = Execute(ctx, qctx, "use test;") + c.Assert(err, IsNil) + _, err = Execute(ctx, qctx, "drop table if exists t1") + c.Assert(err, IsNil) + _, err = Execute(ctx, qctx, "create table t1 (id int)") + c.Assert(err, IsNil) + stmt, _, _, err := qctx.Prepare("insert into t1 values (?)") + c.Assert(err, IsNil) + c.Assert(atomic.LoadInt64(&variable.PreparedStmtCount), Equals, prepareCnt+1) + c.Assert(err, IsNil) + err = qctx.GetStatement(stmt.ID()).Close() + c.Assert(err, IsNil) + c.Assert(atomic.LoadInt64(&variable.PreparedStmtCount), Equals, prepareCnt) +} diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 271fd2b88b727..8f87e01ccc439 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -56,7 +56,8 @@ import ( atomic2 "go.uber.org/atomic" ) -var preparedStmtCount int64 +// PreparedStmtCount is exported for test. +var PreparedStmtCount int64 // RetryInfo saves retry information. type RetryInfo struct { @@ -1186,9 +1187,9 @@ func (s *SessionVars) AddPreparedStmt(stmtID uint32, stmt interface{}) error { if err != nil { maxPreparedStmtCount = DefMaxPreparedStmtCount } - newPreparedStmtCount := atomic.AddInt64(&preparedStmtCount, 1) + newPreparedStmtCount := atomic.AddInt64(&PreparedStmtCount, 1) if maxPreparedStmtCount >= 0 && newPreparedStmtCount > maxPreparedStmtCount { - atomic.AddInt64(&preparedStmtCount, -1) + atomic.AddInt64(&PreparedStmtCount, -1) return ErrMaxPreparedStmtCountReached.GenWithStackByArgs(maxPreparedStmtCount) } metrics.PreparedStmtGauge.Set(float64(newPreparedStmtCount)) @@ -1204,7 +1205,7 @@ func (s *SessionVars) RemovePreparedStmt(stmtID uint32) { return } delete(s.PreparedStmts, stmtID) - afterMinus := atomic.AddInt64(&preparedStmtCount, -1) + afterMinus := atomic.AddInt64(&PreparedStmtCount, -1) metrics.PreparedStmtGauge.Set(float64(afterMinus)) } @@ -1214,7 +1215,7 @@ func (s *SessionVars) WithdrawAllPreparedStmt() { if psCount == 0 { return } - afterMinus := atomic.AddInt64(&preparedStmtCount, -int64(psCount)) + afterMinus := atomic.AddInt64(&PreparedStmtCount, -int64(psCount)) metrics.PreparedStmtGauge.Set(float64(afterMinus)) } From 42d0208032a97c9fb44dbcec4767b17d7d815e99 Mon Sep 17 00:00:00 2001 From: Null not nil <67764674+nullnotnil@users.noreply.github.com> Date: Thu, 19 Nov 2020 01:47:33 -0700 Subject: [PATCH 32/37] expression: fix builtin IF truncation of type len (#20743) --- expression/builtin_control.go | 13 +++++++++++-- expression/expr_to_pb_test.go | 4 ++-- expression/integration_test.go | 10 ++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/expression/builtin_control.go b/expression/builtin_control.go index cfba5f3bb32fd..a0b4eaa53e52f 100644 --- a/expression/builtin_control.go +++ b/expression/builtin_control.go @@ -54,6 +54,14 @@ var ( _ builtinFunc = &builtinIfJSONSig{} ) +func maxlen(lhsFlen, rhsFlen int) int { + // -1 indicates that the length is unknown, such as the case for expressions. + if lhsFlen < 0 || rhsFlen < 0 { + return mysql.MaxRealWidth + } + return mathutil.Max(lhsFlen, rhsFlen) +} + // InferType4ControlFuncs infer result type for builtin IF, IFNULL, NULLIF, LEAD and LAG. func InferType4ControlFuncs(lexp, rexp Expression) *types.FieldType { lhs, rhs := lexp.GetType(), rexp.GetType() @@ -114,9 +122,10 @@ func InferType4ControlFuncs(lexp, rexp Expression) *types.FieldType { if lhs.Decimal != types.UnspecifiedLength { rhsFlen -= rhs.Decimal } - resultFieldType.Flen = mathutil.Max(lhsFlen, rhsFlen) + resultFieldType.Decimal + 1 + flen := maxlen(lhsFlen, rhsFlen) + resultFieldType.Decimal + 1 // account for -1 len fields + resultFieldType.Flen = mathutil.Min(flen, mysql.MaxDecimalWidth) // make sure it doesn't overflow } else { - resultFieldType.Flen = mathutil.Max(lhs.Flen, rhs.Flen) + resultFieldType.Flen = maxlen(lhs.Flen, rhs.Flen) } } // Fix decimal for int and string. diff --git a/expression/expr_to_pb_test.go b/expression/expr_to_pb_test.go index 7c9bdd12c1d0b..5ae93fe87359c 100644 --- a/expression/expr_to_pb_test.go +++ b/expression/expr_to_pb_test.go @@ -558,8 +558,8 @@ func (s *testEvaluatorSuite) TestControlFunc2Pb(c *C) { c.Assert(err, IsNil) jsons := []string{ "{\"tp\":10000,\"children\":[{\"tp\":201,\"val\":\"gAAAAAAAAAE=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAI=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAM=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}}],\"sig\":4208,\"field_type\":{\"tp\":3,\"flag\":128,\"flen\":-1,\"decimal\":0,\"collate\":63,\"charset\":\"\"}}", - "{\"tp\":10000,\"children\":[{\"tp\":201,\"val\":\"gAAAAAAAAAE=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAI=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAM=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}}],\"sig\":4107,\"field_type\":{\"tp\":3,\"flag\":128,\"flen\":-1,\"decimal\":0,\"collate\":63,\"charset\":\"binary\"}}", - "{\"tp\":10000,\"children\":[{\"tp\":201,\"val\":\"gAAAAAAAAAE=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAI=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}}],\"sig\":4101,\"field_type\":{\"tp\":3,\"flag\":128,\"flen\":-1,\"decimal\":0,\"collate\":63,\"charset\":\"binary\"}}", + "{\"tp\":10000,\"children\":[{\"tp\":201,\"val\":\"gAAAAAAAAAE=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAI=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAM=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}}],\"sig\":4107,\"field_type\":{\"tp\":3,\"flag\":128,\"flen\":24,\"decimal\":0,\"collate\":63,\"charset\":\"binary\"}}", + "{\"tp\":10000,\"children\":[{\"tp\":201,\"val\":\"gAAAAAAAAAE=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}},{\"tp\":201,\"val\":\"gAAAAAAAAAI=\",\"sig\":0,\"field_type\":{\"tp\":3,\"flag\":0,\"flen\":-1,\"decimal\":-1,\"collate\":63,\"charset\":\"\"}}],\"sig\":4101,\"field_type\":{\"tp\":3,\"flag\":128,\"flen\":24,\"decimal\":0,\"collate\":63,\"charset\":\"binary\"}}", "null", } for i, pbExpr := range pbExprs { diff --git a/expression/integration_test.go b/expression/integration_test.go index b9c15e8691b1f..617aab9ff23ba 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -7681,6 +7681,16 @@ func (s *testIntegrationSuite) TestIssue20180(c *C) { tk.MustQuery("select * from t where a > 1 and a = \"b\";").Check(testkit.Rows("b")) } +func (s *testIntegrationSuite) TestIssue20730(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("DROP TABLE IF EXISTS tmp;") + tk.MustExec("CREATE TABLE tmp (id int(11) NOT NULL,value int(1) NOT NULL,PRIMARY KEY (id))") + tk.MustExec("INSERT INTO tmp VALUES (1, 1),(2,2),(3,3),(4,4),(5,5)") + tk.MustExec("SET @sum := 10") + tk.MustQuery("SELECT @sum := IF(@sum=20,4,@sum + tmp.value) sum FROM tmp ORDER BY tmp.id").Check(testkit.Rows("11", "13", "16", "20", "4")) +} + func (s *testIntegrationSerialSuite) TestClusteredIndexAndNewCollation(c *C) { collate.SetNewCollationEnabledForTest(true) defer collate.SetNewCollationEnabledForTest(false) From 61b9930c5c7c7c4f382cb53b1032dca2ac75c637 Mon Sep 17 00:00:00 2001 From: Yilin Chen Date: Thu, 19 Nov 2020 17:02:53 +0800 Subject: [PATCH 33/37] tikv: don't set undetermined error for cancelled prewrites (#20983) Signed-off-by: Yilin Chen --- store/mockstore/unistore/rpc.go | 16 ++++++++---- store/tikv/2pc.go | 14 +++++----- store/tikv/async_commit_fail_test.go | 38 ++++++++++++++++++++++++++++ store/tikv/prewrite.go | 4 ++- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/store/mockstore/unistore/rpc.go b/store/mockstore/unistore/rpc.go index bf8c65f183dc6..1cd93f9e0d289 100644 --- a/store/mockstore/unistore/rpc.go +++ b/store/mockstore/unistore/rpc.go @@ -91,11 +91,17 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R resp.Resp, err = c.usSvr.KvScan(ctx, req.Scan()) case tikvrpc.CmdPrewrite: failpoint.Inject("rpcPrewriteResult", func(val failpoint.Value) { - switch val.(string) { - case "notLeader": - failpoint.Return(&tikvrpc.Response{ - Resp: &kvrpcpb.PrewriteResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, - }, nil) + if val != nil { + switch val.(string) { + case "notLeader": + failpoint.Return(&tikvrpc.Response{ + Resp: &kvrpcpb.PrewriteResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + }, nil) + case "writeConflict": + failpoint.Return(&tikvrpc.Response{ + Resp: &kvrpcpb.PrewriteResponse{Errors: []*kvrpcpb.KeyError{{Conflict: &kvrpcpb.WriteConflict{}}}}, + }, nil) + } } }) diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 34275d727397f..4647c48dd023b 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -114,12 +114,13 @@ type twoPhaseCommitter struct { noFallBack bool } - useAsyncCommit uint32 - minCommitTS uint64 - maxCommitTS uint64 - prewriteStarted bool - useOnePC uint32 - onePCCommitTS uint64 + useAsyncCommit uint32 + minCommitTS uint64 + maxCommitTS uint64 + prewriteStarted bool + prewriteCancelled uint32 + useOnePC uint32 + onePCCommitTS uint64 } type memBufferMutations struct { @@ -1614,6 +1615,7 @@ func (batchExe *batchExecutor) process(batches []batchMutations) error { zap.Uint64("conn", batchExe.committer.connID), zap.Stringer("action type", batchExe.action), zap.Uint64("txnStartTS", batchExe.committer.startTS)) + atomic.StoreUint32(&batchExe.committer.prewriteCancelled, 1) cancel() } if err == nil { diff --git a/store/tikv/async_commit_fail_test.go b/store/tikv/async_commit_fail_test.go index 3bc71ea1bbdbb..178bba5369b0b 100644 --- a/store/tikv/async_commit_fail_test.go +++ b/store/tikv/async_commit_fail_test.go @@ -79,6 +79,44 @@ func (s *testAsyncCommitFailSuite) TestFailAsyncCommitPrewriteRpcErrors(c *C) { c.Assert(bytes.Equal(res, []byte("a1")), IsTrue) } +func (s *testAsyncCommitFailSuite) TestAsyncCommitPrewriteCancelled(c *C) { + // This test doesn't support tikv mode because it needs setting failpoint in unistore. + if *WithTiKV { + return + } + + defer config.RestoreFunc()() + config.UpdateGlobal(func(conf *config.Config) { + conf.TiKVClient.AsyncCommit.Enable = true + }) + + // Split into two regions. + splitKey := "s" + bo := NewBackofferWithVars(context.Background(), 5000, nil) + loc, err := s.store.GetRegionCache().LocateKey(bo, []byte(splitKey)) + c.Assert(err, IsNil) + newRegionID := s.cluster.AllocID() + newPeerID := s.cluster.AllocID() + s.cluster.Split(loc.Region.GetID(), newRegionID, []byte(splitKey), []uint64{newPeerID}, newPeerID) + s.store.GetRegionCache().InvalidateCachedRegion(loc.Region) + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/mockstore/unistore/rpcPrewriteResult", `1*return("writeConflict")->sleep(50)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/mockstore/unistore/rpcPrewriteResult"), IsNil) + }() + + t1, err := s.store.Begin() + c.Assert(err, IsNil) + err = t1.Set([]byte("a"), []byte("a")) + c.Assert(err, IsNil) + err = t1.Set([]byte("z"), []byte("z")) + c.Assert(err, IsNil) + ctx := context.WithValue(context.Background(), sessionctx.ConnID, uint64(1)) + err = t1.Commit(ctx) + c.Assert(err, NotNil) + c.Assert(kv.ErrWriteConflict.Equal(err), IsTrue, Commentf("%s", errors.ErrorStack(err))) +} + func (s *testAsyncCommitFailSuite) TestPointGetWithAsyncCommit(c *C) { defer config.RestoreFunc()() config.UpdateGlobal(func(conf *config.Config) { diff --git a/store/tikv/prewrite.go b/store/tikv/prewrite.go index 9703a43882f4f..f20413118038e 100644 --- a/store/tikv/prewrite.go +++ b/store/tikv/prewrite.go @@ -119,7 +119,9 @@ func (action actionPrewrite) handleSingleBatch(c *twoPhaseCommitter, bo *Backoff // If we fail to receive response for async commit prewrite, it will be undetermined whether this // transaction has been successfully committed. - if (c.isAsyncCommit() || c.isOnePC()) && sender.rpcError != nil { + // If prewrite has been cancelled, all ongoing prewrite RPCs will become errors, we needn't set undetermined + // errors. + if (c.isAsyncCommit() || c.isOnePC()) && sender.rpcError != nil && atomic.LoadUint32(&c.prewriteCancelled) == 0 { c.setUndeterminedErr(errors.Trace(sender.rpcError)) } From 38bbb0dd21976206229f5cdcfdfed58e67abe71d Mon Sep 17 00:00:00 2001 From: Zhuomin Liu Date: Thu, 19 Nov 2020 17:17:53 +0800 Subject: [PATCH 34/37] planner: ban (index) merge join heuristically when convert eq cond to other cond (#21138) --- cmd/explaintest/r/explain_complex.result | 36 +++---- .../r/explain_complex_stats.result | 30 +++--- cmd/explaintest/r/tpch.result | 102 +++++++++--------- executor/testdata/prepare_suite_out.json | 40 ++++--- planner/core/exhaust_physical_plans.go | 64 +++++------ planner/core/explain.go | 18 +++- planner/core/testdata/analyze_suite_out.json | 6 +- .../integration_serial_suite_out.json | 36 +++---- .../core/testdata/integration_suite_out.json | 32 +++--- planner/core/testdata/plan_suite_out.json | 8 +- 10 files changed, 191 insertions(+), 181 deletions(-) diff --git a/cmd/explaintest/r/explain_complex.result b/cmd/explaintest/r/explain_complex.result index 8d66939ba7c01..a24054cbae871 100644 --- a/cmd/explaintest/r/explain_complex.result +++ b/cmd/explaintest/r/explain_complex.result @@ -118,28 +118,28 @@ id estRows task access object operator info Projection_13 1.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext, test.st.t └─Limit_16 1.00 root offset:0, count:2500 └─HashAgg_19 1.00 root group by:test.dd.dic, test.st.aid, funcs:firstrow(test.st.id)->test.st.id, funcs:firstrow(test.st.aid)->test.st.aid, funcs:firstrow(test.st.cm)->test.st.cm, funcs:firstrow(test.st.p1)->test.st.p1, funcs:firstrow(test.st.p2)->test.st.p2, funcs:firstrow(test.st.p3)->test.st.p3, funcs:firstrow(test.st.p4)->test.st.p4, funcs:firstrow(test.st.p5)->test.st.p5, funcs:firstrow(test.st.p6_md5)->test.st.p6_md5, funcs:firstrow(test.st.p7_md5)->test.st.p7_md5, funcs:firstrow(test.st.ext)->test.st.ext, funcs:firstrow(test.st.t)->test.st.t, funcs:firstrow(test.dd.id)->test.dd.id, funcs:firstrow(test.dd.dic)->test.dd.dic, funcs:firstrow(test.dd.ip)->test.dd.ip, funcs:firstrow(test.dd.t)->test.dd.t - └─HashJoin_34 0.00 root inner join, equal:[eq(test.dd.aid, test.st.aid) eq(test.dd.ip, test.st.ip)], other cond:gt(test.dd.t, test.st.t) - ├─IndexLookUp_52(Build) 0.00 root - │ ├─IndexRangeScan_49(Build) 3333.33 cop[tikv] table:dd, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo - │ └─Selection_51(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), not(isnull(test.dd.ip)) - │ └─TableRowIDScan_50 3333.33 cop[tikv] table:dd keep order:false, stats:pseudo - └─IndexLookUp_41(Probe) 3.33 root - ├─IndexRangeScan_38(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo - └─Selection_40(Probe) 3.33 cop[tikv] eq(test.st.pt, "android"), not(isnull(test.st.ip)) - └─TableRowIDScan_39 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo + └─HashJoin_33 0.00 root inner join, equal:[eq(test.dd.aid, test.st.aid) eq(test.dd.ip, test.st.ip)], other cond:gt(test.dd.t, test.st.t) + ├─IndexLookUp_51(Build) 0.00 root + │ ├─IndexRangeScan_48(Build) 3333.33 cop[tikv] table:dd, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo + │ └─Selection_50(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), not(isnull(test.dd.ip)) + │ └─TableRowIDScan_49 3333.33 cop[tikv] table:dd keep order:false, stats:pseudo + └─IndexLookUp_40(Probe) 3.33 root + ├─IndexRangeScan_37(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1478143908,+inf], keep order:false, stats:pseudo + └─Selection_39(Probe) 3.33 cop[tikv] eq(test.st.pt, "android"), not(isnull(test.st.ip)) + └─TableRowIDScan_38 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext from st gad join dd sdk on gad.aid = sdk.aid and gad.dic = sdk.mac and gad.t < sdk.t where gad.t > 1477971479 and gad.bm = 0 and gad.pt = 'ios' and gad.dit = 'mac' and sdk.t > 1477971479 and sdk.bm = 0 and sdk.pt = 'ios' limit 3000; id estRows task access object operator info Projection_10 0.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext └─Limit_13 0.00 root offset:0, count:3000 - └─IndexMergeJoin_26 0.00 root inner join, inner:IndexLookUp_24, outer key:test.st.aid, inner key:test.dd.aid, other cond:eq(test.st.dic, test.dd.mac), lt(test.st.t, test.dd.t) - ├─IndexLookUp_35(Build) 0.00 root - │ ├─IndexRangeScan_32(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1477971479,+inf], keep order:false, stats:pseudo - │ └─Selection_34(Probe) 0.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), not(isnull(test.st.dic)) - │ └─TableRowIDScan_33 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo - └─IndexLookUp_24(Probe) 0.00 root - ├─IndexRangeScan_21(Build) 10000.00 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true, stats:pseudo - └─Selection_23(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t)) - └─TableRowIDScan_22 10000.00 cop[tikv] table:sdk keep order:false, stats:pseudo + └─IndexJoin_18 0.00 root inner join, inner:IndexLookUp_17, outer key:test.st.aid, inner key:test.dd.aid, equal cond:eq(test.st.aid, test.dd.aid), eq(test.st.dic, test.dd.mac), other cond:lt(test.st.t, test.dd.t) + ├─IndexLookUp_34(Build) 0.00 root + │ ├─IndexRangeScan_31(Build) 3333.33 cop[tikv] table:gad, index:t(t) range:(1477971479,+inf], keep order:false, stats:pseudo + │ └─Selection_33(Probe) 0.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), not(isnull(test.st.dic)) + │ └─TableRowIDScan_32 3333.33 cop[tikv] table:gad keep order:false, stats:pseudo + └─IndexLookUp_17(Probe) 0.00 root + ├─IndexRangeScan_14(Build) 10000.00 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:false, stats:pseudo + └─Selection_16(Probe) 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t)) + └─TableRowIDScan_15 10000.00 cop[tikv] table:sdk keep order:false, stats:pseudo explain SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5; id estRows task access object operator info Projection_5 1.00 root test.st.cm, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, Column#20, Column#21 diff --git a/cmd/explaintest/r/explain_complex_stats.result b/cmd/explaintest/r/explain_complex_stats.result index ed307ab8dcb1b..7d015e0b5170a 100644 --- a/cmd/explaintest/r/explain_complex_stats.result +++ b/cmd/explaintest/r/explain_complex_stats.result @@ -128,25 +128,25 @@ id estRows task access object operator info Projection_13 424.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext, test.st.t └─Limit_16 424.00 root offset:0, count:2500 └─HashAgg_19 424.00 root group by:test.dd.dic, test.st.aid, funcs:firstrow(test.st.id)->test.st.id, funcs:firstrow(test.st.aid)->test.st.aid, funcs:firstrow(test.st.cm)->test.st.cm, funcs:firstrow(test.st.p1)->test.st.p1, funcs:firstrow(test.st.p2)->test.st.p2, funcs:firstrow(test.st.p3)->test.st.p3, funcs:firstrow(test.st.p4)->test.st.p4, funcs:firstrow(test.st.p5)->test.st.p5, funcs:firstrow(test.st.p6_md5)->test.st.p6_md5, funcs:firstrow(test.st.p7_md5)->test.st.p7_md5, funcs:firstrow(test.st.ext)->test.st.ext, funcs:firstrow(test.st.t)->test.st.t, funcs:firstrow(test.dd.id)->test.dd.id, funcs:firstrow(test.dd.dic)->test.dd.dic, funcs:firstrow(test.dd.ip)->test.dd.ip, funcs:firstrow(test.dd.t)->test.dd.t - └─HashJoin_34 424.00 root inner join, equal:[eq(test.st.aid, test.dd.aid) eq(test.st.ip, test.dd.ip)], other cond:gt(test.dd.t, test.st.t) - ├─TableReader_37(Build) 424.00 root data:Selection_36 - │ └─Selection_36 424.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.pt, "android"), gt(test.st.t, 1478143908), not(isnull(test.st.ip)) - │ └─TableRangeScan_35 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false - └─TableReader_44(Probe) 455.80 root data:Selection_43 - └─Selection_43 455.80 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), gt(test.dd.t, 1478143908), not(isnull(test.dd.ip)), not(isnull(test.dd.t)) - └─TableRangeScan_42 2000.00 cop[tikv] table:dd range:[0,+inf], keep order:false + └─HashJoin_33 424.00 root inner join, equal:[eq(test.st.aid, test.dd.aid) eq(test.st.ip, test.dd.ip)], other cond:gt(test.dd.t, test.st.t) + ├─TableReader_36(Build) 424.00 root data:Selection_35 + │ └─Selection_35 424.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.pt, "android"), gt(test.st.t, 1478143908), not(isnull(test.st.ip)) + │ └─TableRangeScan_34 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false + └─TableReader_43(Probe) 455.80 root data:Selection_42 + └─Selection_42 455.80 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), gt(test.dd.t, 1478143908), not(isnull(test.dd.ip)), not(isnull(test.dd.t)) + └─TableRangeScan_41 2000.00 cop[tikv] table:dd range:[0,+inf], keep order:false explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext from st gad join dd sdk on gad.aid = sdk.aid and gad.dic = sdk.mac and gad.t < sdk.t where gad.t > 1477971479 and gad.bm = 0 and gad.pt = 'ios' and gad.dit = 'mac' and sdk.t > 1477971479 and sdk.bm = 0 and sdk.pt = 'ios' limit 3000; id estRows task access object operator info Projection_10 170.34 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd.dic, test.dd.ip, test.dd.t, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, test.st.ext └─Limit_13 170.34 root offset:0, count:3000 - └─IndexMergeJoin_26 170.34 root inner join, inner:IndexLookUp_24, outer key:test.st.aid, inner key:test.dd.aid, other cond:eq(test.st.dic, test.dd.mac), lt(test.st.t, test.dd.t) - ├─TableReader_31(Build) 170.34 root data:Selection_30 - │ └─Selection_30 170.34 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), gt(test.st.t, 1477971479), not(isnull(test.st.dic)) - │ └─TableRangeScan_29 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false - └─IndexLookUp_24(Probe) 1.00 root - ├─IndexRangeScan_21(Build) 3.93 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true - └─Selection_23(Probe) 1.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t)) - └─TableRowIDScan_22 3.93 cop[tikv] table:sdk keep order:false + └─IndexJoin_18 170.34 root inner join, inner:IndexLookUp_17, outer key:test.st.aid, inner key:test.dd.aid, equal cond:eq(test.st.aid, test.dd.aid), eq(test.st.dic, test.dd.mac), other cond:lt(test.st.t, test.dd.t) + ├─TableReader_30(Build) 170.34 root data:Selection_29 + │ └─Selection_29 170.34 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), gt(test.st.t, 1477971479), not(isnull(test.st.dic)) + │ └─TableRangeScan_28 1999.00 cop[tikv] table:gad range:[0,+inf], keep order:false + └─IndexLookUp_17(Probe) 1.00 root + ├─IndexRangeScan_14(Build) 3.93 cop[tikv] table:sdk, index:aid(aid, dic) range: decided by [eq(test.dd.aid, test.st.aid)], keep order:false + └─Selection_16(Probe) 1.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t)) + └─TableRowIDScan_15 3.93 cop[tikv] table:sdk keep order:false explain SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5; id estRows task access object operator info Projection_5 39.28 root test.st.cm, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, Column#20, Column#21 diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index 0282fc5e9be67..4b8a8a827c95e 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -184,37 +184,37 @@ limit 100; id estRows task access object operator info Projection_37 100.00 root tpch.supplier.s_acctbal, tpch.supplier.s_name, tpch.nation.n_name, tpch.part.p_partkey, tpch.part.p_mfgr, tpch.supplier.s_address, tpch.supplier.s_phone, tpch.supplier.s_comment └─TopN_40 100.00 root tpch.supplier.s_acctbal:desc, tpch.nation.n_name, tpch.supplier.s_name, tpch.part.p_partkey, offset:0, count:100 - └─HashJoin_46 155496.00 root inner join, equal:[eq(tpch.part.p_partkey, tpch.partsupp.ps_partkey) eq(tpch.partsupp.ps_supplycost, Column#50)] - ├─HashJoin_60(Build) 155496.00 root inner join, equal:[eq(tpch.partsupp.ps_partkey, tpch.part.p_partkey)] - │ ├─TableReader_90(Build) 155496.00 root data:Selection_89 - │ │ └─Selection_89 155496.00 cop[tikv] eq(tpch.part.p_size, 30), like(tpch.part.p_type, "%STEEL", 92) - │ │ └─TableFullScan_88 10000000.00 cop[tikv] table:part keep order:false - │ └─HashJoin_63(Probe) 8155010.44 root inner join, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] - │ ├─HashJoin_65(Build) 100000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] - │ │ ├─HashJoin_78(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] - │ │ │ ├─TableReader_83(Build) 1.00 root data:Selection_82 - │ │ │ │ └─Selection_82 1.00 cop[tikv] eq(tpch.region.r_name, "ASIA") - │ │ │ │ └─TableFullScan_81 5.00 cop[tikv] table:region keep order:false - │ │ │ └─TableReader_80(Probe) 25.00 root data:TableFullScan_79 - │ │ │ └─TableFullScan_79 25.00 cop[tikv] table:nation keep order:false - │ │ └─TableReader_85(Probe) 500000.00 root data:TableFullScan_84 - │ │ └─TableFullScan_84 500000.00 cop[tikv] table:supplier keep order:false - │ └─TableReader_87(Probe) 40000000.00 root data:TableFullScan_86 - │ └─TableFullScan_86 40000000.00 cop[tikv] table:partsupp keep order:false - └─Selection_91(Probe) 6524008.35 root not(isnull(Column#50)) - └─HashAgg_94 8155010.44 root group by:tpch.partsupp.ps_partkey, funcs:min(tpch.partsupp.ps_supplycost)->Column#50, funcs:firstrow(tpch.partsupp.ps_partkey)->tpch.partsupp.ps_partkey - └─HashJoin_98 8155010.44 root inner join, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] - ├─HashJoin_100(Build) 100000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] - │ ├─HashJoin_113(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] - │ │ ├─TableReader_118(Build) 1.00 root data:Selection_117 - │ │ │ └─Selection_117 1.00 cop[tikv] eq(tpch.region.r_name, "ASIA") - │ │ │ └─TableFullScan_116 5.00 cop[tikv] table:region keep order:false - │ │ └─TableReader_115(Probe) 25.00 root data:TableFullScan_114 - │ │ └─TableFullScan_114 25.00 cop[tikv] table:nation keep order:false - │ └─TableReader_120(Probe) 500000.00 root data:TableFullScan_119 - │ └─TableFullScan_119 500000.00 cop[tikv] table:supplier keep order:false - └─TableReader_122(Probe) 40000000.00 root data:TableFullScan_121 - └─TableFullScan_121 40000000.00 cop[tikv] table:partsupp keep order:false + └─HashJoin_45 155496.00 root inner join, equal:[eq(tpch.part.p_partkey, tpch.partsupp.ps_partkey) eq(tpch.partsupp.ps_supplycost, Column#50)] + ├─HashJoin_58(Build) 155496.00 root inner join, equal:[eq(tpch.partsupp.ps_partkey, tpch.part.p_partkey)] + │ ├─TableReader_88(Build) 155496.00 root data:Selection_87 + │ │ └─Selection_87 155496.00 cop[tikv] eq(tpch.part.p_size, 30), like(tpch.part.p_type, "%STEEL", 92) + │ │ └─TableFullScan_86 10000000.00 cop[tikv] table:part keep order:false + │ └─HashJoin_61(Probe) 8155010.44 root inner join, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] + │ ├─HashJoin_63(Build) 100000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] + │ │ ├─HashJoin_76(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] + │ │ │ ├─TableReader_81(Build) 1.00 root data:Selection_80 + │ │ │ │ └─Selection_80 1.00 cop[tikv] eq(tpch.region.r_name, "ASIA") + │ │ │ │ └─TableFullScan_79 5.00 cop[tikv] table:region keep order:false + │ │ │ └─TableReader_78(Probe) 25.00 root data:TableFullScan_77 + │ │ │ └─TableFullScan_77 25.00 cop[tikv] table:nation keep order:false + │ │ └─TableReader_83(Probe) 500000.00 root data:TableFullScan_82 + │ │ └─TableFullScan_82 500000.00 cop[tikv] table:supplier keep order:false + │ └─TableReader_85(Probe) 40000000.00 root data:TableFullScan_84 + │ └─TableFullScan_84 40000000.00 cop[tikv] table:partsupp keep order:false + └─Selection_89(Probe) 6524008.35 root not(isnull(Column#50)) + └─HashAgg_92 8155010.44 root group by:tpch.partsupp.ps_partkey, funcs:min(tpch.partsupp.ps_supplycost)->Column#50, funcs:firstrow(tpch.partsupp.ps_partkey)->tpch.partsupp.ps_partkey + └─HashJoin_96 8155010.44 root inner join, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] + ├─HashJoin_98(Build) 100000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] + │ ├─HashJoin_111(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] + │ │ ├─TableReader_116(Build) 1.00 root data:Selection_115 + │ │ │ └─Selection_115 1.00 cop[tikv] eq(tpch.region.r_name, "ASIA") + │ │ │ └─TableFullScan_114 5.00 cop[tikv] table:region keep order:false + │ │ └─TableReader_113(Probe) 25.00 root data:TableFullScan_112 + │ │ └─TableFullScan_112 25.00 cop[tikv] table:nation keep order:false + │ └─TableReader_118(Probe) 500000.00 root data:TableFullScan_117 + │ └─TableFullScan_117 500000.00 cop[tikv] table:supplier keep order:false + └─TableReader_120(Probe) 40000000.00 root data:TableFullScan_119 + └─TableFullScan_119 40000000.00 cop[tikv] table:partsupp keep order:false /* Q3 Shipping Priority Query This query retrieves the 10 unshipped orders with the highest value. @@ -345,26 +345,26 @@ id estRows task access object operator info Sort_23 5.00 root Column#49:desc └─Projection_25 5.00 root tpch.nation.n_name, Column#49 └─HashAgg_28 5.00 root group by:Column#52, funcs:sum(Column#50)->Column#49, funcs:firstrow(Column#51)->tpch.nation.n_name - └─Projection_86 11822812.50 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#50, tpch.nation.n_name, tpch.nation.n_name - └─HashJoin_38 11822812.50 root inner join, equal:[eq(tpch.supplier.s_nationkey, tpch.customer.c_nationkey) eq(tpch.orders.o_custkey, tpch.customer.c_custkey)] - ├─TableReader_84(Build) 7500000.00 root data:TableFullScan_83 - │ └─TableFullScan_83 7500000.00 cop[tikv] table:customer keep order:false - └─HashJoin_52(Probe) 11822812.50 root inner join, equal:[eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)] - ├─TableReader_82(Build) 11822812.50 root data:Selection_81 - │ └─Selection_81 11822812.50 cop[tikv] ge(tpch.orders.o_orderdate, 1994-01-01 00:00:00.000000), lt(tpch.orders.o_orderdate, 1995-01-01) - │ └─TableFullScan_80 75000000.00 cop[tikv] table:orders keep order:false - └─HashJoin_55(Probe) 61163763.01 root inner join, equal:[eq(tpch.supplier.s_suppkey, tpch.lineitem.l_suppkey)] - ├─HashJoin_57(Build) 100000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] - │ ├─HashJoin_70(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] - │ │ ├─TableReader_75(Build) 1.00 root data:Selection_74 - │ │ │ └─Selection_74 1.00 cop[tikv] eq(tpch.region.r_name, "MIDDLE EAST") - │ │ │ └─TableFullScan_73 5.00 cop[tikv] table:region keep order:false - │ │ └─TableReader_72(Probe) 25.00 root data:TableFullScan_71 - │ │ └─TableFullScan_71 25.00 cop[tikv] table:nation keep order:false - │ └─TableReader_77(Probe) 500000.00 root data:TableFullScan_76 - │ └─TableFullScan_76 500000.00 cop[tikv] table:supplier keep order:false - └─TableReader_79(Probe) 300005811.00 root data:TableFullScan_78 - └─TableFullScan_78 300005811.00 cop[tikv] table:lineitem keep order:false + └─Projection_85 11822812.50 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))->Column#50, tpch.nation.n_name, tpch.nation.n_name + └─HashJoin_37 11822812.50 root inner join, equal:[eq(tpch.supplier.s_nationkey, tpch.customer.c_nationkey) eq(tpch.orders.o_custkey, tpch.customer.c_custkey)] + ├─TableReader_83(Build) 7500000.00 root data:TableFullScan_82 + │ └─TableFullScan_82 7500000.00 cop[tikv] table:customer keep order:false + └─HashJoin_51(Probe) 11822812.50 root inner join, equal:[eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)] + ├─TableReader_81(Build) 11822812.50 root data:Selection_80 + │ └─Selection_80 11822812.50 cop[tikv] ge(tpch.orders.o_orderdate, 1994-01-01 00:00:00.000000), lt(tpch.orders.o_orderdate, 1995-01-01) + │ └─TableFullScan_79 75000000.00 cop[tikv] table:orders keep order:false + └─HashJoin_54(Probe) 61163763.01 root inner join, equal:[eq(tpch.supplier.s_suppkey, tpch.lineitem.l_suppkey)] + ├─HashJoin_56(Build) 100000.00 root inner join, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] + │ ├─HashJoin_69(Build) 5.00 root inner join, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] + │ │ ├─TableReader_74(Build) 1.00 root data:Selection_73 + │ │ │ └─Selection_73 1.00 cop[tikv] eq(tpch.region.r_name, "MIDDLE EAST") + │ │ │ └─TableFullScan_72 5.00 cop[tikv] table:region keep order:false + │ │ └─TableReader_71(Probe) 25.00 root data:TableFullScan_70 + │ │ └─TableFullScan_70 25.00 cop[tikv] table:nation keep order:false + │ └─TableReader_76(Probe) 500000.00 root data:TableFullScan_75 + │ └─TableFullScan_75 500000.00 cop[tikv] table:supplier keep order:false + └─TableReader_78(Probe) 300005811.00 root data:TableFullScan_77 + └─TableFullScan_77 300005811.00 cop[tikv] table:lineitem keep order:false /* Q6 Forecasting Revenue Change Query This query quantifies the amount of revenue increase that would have resulted from eliminating certain companywide diff --git a/executor/testdata/prepare_suite_out.json b/executor/testdata/prepare_suite_out.json index 5caa86bded46b..8e289819f6fe7 100644 --- a/executor/testdata/prepare_suite_out.json +++ b/executor/testdata/prepare_suite_out.json @@ -140,17 +140,15 @@ } ], "Plan": [ - "MergeJoin_9 6387.21 root inner join, left key:test.t1.a, right key:test.t2.a, other cond:eq(test.t1.b, test.t2.b)", - "├─Projection_59(Build) 99.80 root test.t2.a, test.t2.b, test.t2.c", - "│ └─IndexLookUp_58 99.80 root ", - "│ ├─Selection_57(Build) 99.80 cop[tikv] not(isnull(test.t2.b))", - "│ │ └─IndexRangeScan_55 99.90 cop[tikv] table:t2, index:b(b, a) range:[1 -inf,1 +inf], keep order:true, stats:pseudo", - "│ └─TableRowIDScan_56(Probe) 99.80 cop[tikv] table:t2 keep order:false, stats:pseudo", - "└─Projection_54(Probe) 99.80 root test.t1.a, test.t1.b, test.t1.c", - " └─IndexLookUp_53 99.80 root ", - " ├─Selection_52(Build) 99.80 cop[tikv] not(isnull(test.t1.b))", - " │ └─IndexRangeScan_50 99.90 cop[tikv] table:t1, index:b(b, a) range:[1 -inf,1 +inf], keep order:true, stats:pseudo", - " └─TableRowIDScan_51(Probe) 99.80 cop[tikv] table:t1 keep order:false, stats:pseudo" + "HashJoin_37 6387.21 root inner join, equal:[eq(test.t1.b, test.t2.b) eq(test.t1.a, test.t2.a)]", + "├─IndexLookUp_62(Build) 99.80 root ", + "│ ├─Selection_61(Build) 99.80 cop[tikv] not(isnull(test.t2.b))", + "│ │ └─IndexRangeScan_59 99.90 cop[tikv] table:t2, index:b(b, a) range:[1 -inf,1 +inf], keep order:false, stats:pseudo", + "│ └─TableRowIDScan_60(Probe) 99.80 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp_55(Probe) 99.80 root ", + " ├─Selection_54(Build) 99.80 cop[tikv] not(isnull(test.t1.b))", + " │ └─IndexRangeScan_52 99.90 cop[tikv] table:t1, index:b(b, a) range:[1 -inf,1 +inf], keep order:false, stats:pseudo", + " └─TableRowIDScan_53(Probe) 99.80 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "LastPlanUseCache": "0", "Result": null @@ -164,17 +162,15 @@ } ], "Plan": [ - "MergeJoin_9 6387.21 root inner join, left key:test.t1.a, right key:test.t2.a, other cond:eq(test.t1.b, test.t2.b)", - "├─Projection_59(Build) 99.80 root test.t2.a, test.t2.b, test.t2.c", - "│ └─IndexLookUp_58 99.80 root ", - "│ ├─Selection_57(Build) 99.80 cop[tikv] not(isnull(test.t2.b))", - "│ │ └─IndexRangeScan_55 99.90 cop[tikv] table:t2, index:b(b, a) range:[2 -inf,2 +inf], keep order:true, stats:pseudo", - "│ └─TableRowIDScan_56(Probe) 99.80 cop[tikv] table:t2 keep order:false, stats:pseudo", - "└─Projection_54(Probe) 99.80 root test.t1.a, test.t1.b, test.t1.c", - " └─IndexLookUp_53 99.80 root ", - " ├─Selection_52(Build) 99.80 cop[tikv] not(isnull(test.t1.b))", - " │ └─IndexRangeScan_50 99.90 cop[tikv] table:t1, index:b(b, a) range:[2 -inf,2 +inf], keep order:true, stats:pseudo", - " └─TableRowIDScan_51(Probe) 99.80 cop[tikv] table:t1 keep order:false, stats:pseudo" + "HashJoin_37 6387.21 root inner join, equal:[eq(test.t1.b, test.t2.b) eq(test.t1.a, test.t2.a)]", + "├─IndexLookUp_62(Build) 99.80 root ", + "│ ├─Selection_61(Build) 99.80 cop[tikv] not(isnull(test.t2.b))", + "│ │ └─IndexRangeScan_59 99.90 cop[tikv] table:t2, index:b(b, a) range:[2 -inf,2 +inf], keep order:false, stats:pseudo", + "│ └─TableRowIDScan_60(Probe) 99.80 cop[tikv] table:t2 keep order:false, stats:pseudo", + "└─IndexLookUp_55(Probe) 99.80 root ", + " ├─Selection_54(Build) 99.80 cop[tikv] not(isnull(test.t1.b))", + " │ └─IndexRangeScan_52 99.90 cop[tikv] table:t1, index:b(b, a) range:[2 -inf,2 +inf], keep order:false, stats:pseudo", + " └─TableRowIDScan_53(Probe) 99.80 cop[tikv] table:t1 keep order:false, stats:pseudo" ], "LastPlanUseCache": "1", "Result": [ diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index b3ca49f4b0820..5af99616a8aec 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -153,7 +153,9 @@ func (p *LogicalJoin) GetMergeJoin(prop *property.PhysicalProperty, schema *expr } for _, lhsChildProperty := range p.leftProperties { offsets := getMaxSortPrefix(lhsChildProperty, leftJoinKeys) - if len(offsets) == 0 { + // If not all equal conditions hit properties. We ban merge join heuristically. Because in this case, merge join + // may get a very low performance. In executor, executes join results before other conditions filter it. + if len(offsets) < len(leftJoinKeys) { continue } @@ -391,7 +393,6 @@ func (p *LogicalJoin) getHashJoin(prop *property.PhysicalProperty, innerIdx int, // When inner plan is TableReader, the parameter `ranges` will be nil. Because pk only have one column. So all of its range // is generated during execution time. func (p *LogicalJoin) constructIndexJoin( - joinTP string, prop *property.PhysicalProperty, outerIdx int, innerTask task, @@ -440,36 +441,33 @@ func (p *LogicalJoin) constructIndexJoin( } var outerHashKeys, innerHashKeys []*expression.Column - // HashKey is only used for IndexJoin and IndexHashJoin since they need to - // build hash tables. - if joinTP != plancodec.TypeIndexMergeJoin { - outerHashKeys, innerHashKeys = make([]*expression.Column, len(newOuterKeys)), make([]*expression.Column, len(newInnerKeys)) - copy(outerHashKeys, newOuterKeys) - copy(innerHashKeys, newInnerKeys) - // we can use the `col col` in `OtherCondition` to build the hashtable to avoid the unnecessary calculating. - for i := len(newOtherConds) - 1; i >= 0; i = i - 1 { - switch c := newOtherConds[i].(type) { - case *expression.ScalarFunction: - if c.FuncName.L == ast.EQ { - lhs, ok1 := c.GetArgs()[0].(*expression.Column) - rhs, ok2 := c.GetArgs()[1].(*expression.Column) - if ok1 && ok2 { - outerSchema, innerSchema := p.Children()[outerIdx].Schema(), p.Children()[1-outerIdx].Schema() - if outerSchema.Contains(lhs) && innerSchema.Contains(rhs) { - outerHashKeys = append(outerHashKeys, lhs) - innerHashKeys = append(innerHashKeys, rhs) - } else if innerSchema.Contains(lhs) && outerSchema.Contains(rhs) { - outerHashKeys = append(outerHashKeys, rhs) - innerHashKeys = append(innerHashKeys, lhs) - } - newOtherConds = append(newOtherConds[:i], newOtherConds[i+1:]...) + outerHashKeys, innerHashKeys = make([]*expression.Column, len(newOuterKeys)), make([]*expression.Column, len(newInnerKeys)) + copy(outerHashKeys, newOuterKeys) + copy(innerHashKeys, newInnerKeys) + // we can use the `col col` in `OtherCondition` to build the hashtable to avoid the unnecessary calculating. + for i := len(newOtherConds) - 1; i >= 0; i = i - 1 { + switch c := newOtherConds[i].(type) { + case *expression.ScalarFunction: + if c.FuncName.L == ast.EQ { + lhs, ok1 := c.GetArgs()[0].(*expression.Column) + rhs, ok2 := c.GetArgs()[1].(*expression.Column) + if ok1 && ok2 { + outerSchema, innerSchema := p.Children()[outerIdx].Schema(), p.Children()[1-outerIdx].Schema() + if outerSchema.Contains(lhs) && innerSchema.Contains(rhs) { + outerHashKeys = append(outerHashKeys, lhs) + innerHashKeys = append(innerHashKeys, rhs) + } else if innerSchema.Contains(lhs) && outerSchema.Contains(rhs) { + outerHashKeys = append(outerHashKeys, rhs) + innerHashKeys = append(innerHashKeys, lhs) } + newOtherConds = append(newOtherConds[:i], newOtherConds[i+1:]...) } - default: - continue } + default: + continue } } + baseJoin := basePhysicalJoin{ InnerChildIdx: 1 - outerIdx, LeftConditions: p.LeftConditions, @@ -506,10 +504,14 @@ func (p *LogicalJoin) constructIndexMergeJoin( path *util.AccessPath, compareFilters *ColWithCmpFuncManager, ) []PhysicalPlan { - indexJoins := p.constructIndexJoin(plancodec.TypeIndexMergeJoin, prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) + indexJoins := p.constructIndexJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) indexMergeJoins := make([]PhysicalPlan, 0, len(indexJoins)) for _, plan := range indexJoins { join := plan.(*PhysicalIndexJoin) + // Index merge join can't handle hash keys. So we ban it heuristically. + if len(join.InnerHashKeys) > len(join.InnerJoinKeys) { + return nil + } hasPrefixCol := false for _, l := range join.IdxColLens { if l != types.UnspecifiedLength { @@ -591,7 +593,7 @@ func (p *LogicalJoin) constructIndexHashJoin( path *util.AccessPath, compareFilters *ColWithCmpFuncManager, ) []PhysicalPlan { - indexJoins := p.constructIndexJoin(plancodec.TypeIndexHashJoin, prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) + indexJoins := p.constructIndexJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, path, compareFilters) indexHashJoins := make([]PhysicalPlan, 0, len(indexJoins)) for _, plan := range indexJoins { join := plan.(*PhysicalIndexJoin) @@ -759,7 +761,7 @@ func (p *LogicalJoin) buildIndexJoinInner2TableScan( failpoint.Return(p.constructIndexHashJoin(prop, outerIdx, innerTask, nil, keyOff2IdxOff, nil, nil)) } }) - joins = append(joins, p.constructIndexJoin(plancodec.TypeIndexJoin, prop, outerIdx, innerTask, ranges, keyOff2IdxOff, nil, nil)...) + joins = append(joins, p.constructIndexJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, nil, nil)...) // We can reuse the `innerTask` here since index nested loop hash join // do not need the inner child to promise the order. joins = append(joins, p.constructIndexHashJoin(prop, outerIdx, innerTask, ranges, keyOff2IdxOff, nil, nil)...) @@ -794,7 +796,7 @@ func (p *LogicalJoin) buildIndexJoinInner2IndexScan( failpoint.Return(p.constructIndexHashJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)) } }) - joins = append(joins, p.constructIndexJoin(plancodec.TypeIndexJoin, prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)...) + joins = append(joins, p.constructIndexJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)...) // We can reuse the `innerTask` here since index nested loop hash join // do not need the inner child to promise the order. joins = append(joins, p.constructIndexHashJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, helper.chosenPath, helper.lastColManager)...) diff --git a/planner/core/explain.go b/planner/core/explain.go index 79a5569b4f6b1..0df1abd644452 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -489,10 +489,15 @@ func (p *basePhysicalAgg) ExplainNormalizedInfo() string { // ExplainInfo implements Plan interface. func (p *PhysicalIndexJoin) ExplainInfo() string { - return p.explainInfo(false) + return p.explainInfo(false, false) +} + +// ExplainInfo implements Plan interface. +func (p *PhysicalIndexMergeJoin) ExplainInfo() string { + return p.explainInfo(false, true) } -func (p *PhysicalIndexJoin) explainInfo(normalized bool) string { +func (p *PhysicalIndexJoin) explainInfo(normalized bool, isIndexMergeJoin bool) string { sortedExplainExpressionList := expression.SortedExplainExpressionList if normalized { sortedExplainExpressionList = expression.SortedExplainNormalizedExpressionList @@ -513,7 +518,7 @@ func (p *PhysicalIndexJoin) explainInfo(normalized bool) string { expression.ExplainColumnList(p.InnerJoinKeys)) } - if len(p.OuterHashKeys) > 0 { + if len(p.OuterHashKeys) > 0 && !isIndexMergeJoin { exprs := make([]expression.Expression, 0, len(p.OuterHashKeys)) for i := range p.OuterHashKeys { expr, err := expression.NewFunctionBase(MockContext(), ast.EQ, types.NewFieldType(mysql.TypeLonglong), p.OuterHashKeys[i], p.InnerHashKeys[i]) @@ -541,7 +546,12 @@ func (p *PhysicalIndexJoin) explainInfo(normalized bool) string { // ExplainNormalizedInfo implements Plan interface. func (p *PhysicalIndexJoin) ExplainNormalizedInfo() string { - return p.explainInfo(true) + return p.explainInfo(true, false) +} + +// ExplainNormalizedInfo implements Plan interface. +func (p *PhysicalIndexMergeJoin) ExplainNormalizedInfo() string { + return p.explainInfo(true, true) } // ExplainInfo implements Plan interface. diff --git a/planner/core/testdata/analyze_suite_out.json b/planner/core/testdata/analyze_suite_out.json index ef7a4b7303c05..994ad6b67e179 100644 --- a/planner/core/testdata/analyze_suite_out.json +++ b/planner/core/testdata/analyze_suite_out.json @@ -143,9 +143,9 @@ ], "Plan": [ "IndexJoin_9 12475.01 root inner join, inner:IndexReader_8, outer key:test.t1.a, inner key:test.t2.a, equal cond:eq(test.t1.a, test.t2.a), eq(test.t1.c, test.t2.c), other cond:gt(test.t2.b, minus(test.t1.b, 1)), lt(test.t2.b, plus(test.t1.b, 1))", - "├─TableReader_19(Build) 9980.01 root data:Selection_18", - "│ └─Selection_18 9980.01 cop[tikv] not(isnull(test.t1.a)), not(isnull(test.t1.c))", - "│ └─TableFullScan_17 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", + "├─TableReader_18(Build) 9980.01 root data:Selection_17", + "│ └─Selection_17 9980.01 cop[tikv] not(isnull(test.t1.a)), not(isnull(test.t1.c))", + "│ └─TableFullScan_16 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", "└─IndexReader_8(Probe) 1.25 root index:Selection_7", " └─Selection_7 1.25 cop[tikv] not(isnull(test.t2.a)), not(isnull(test.t2.c))", " └─IndexRangeScan_6 1.25 cop[tikv] table:t2, index:idx(a, b, c) range: decided by [eq(test.t2.a, test.t1.a) gt(test.t2.b, minus(test.t1.b, 1)) lt(test.t2.b, plus(test.t1.b, 1))], keep order:false, stats:pseudo" diff --git a/planner/core/testdata/integration_serial_suite_out.json b/planner/core/testdata/integration_serial_suite_out.json index 918d08fa7dcfc..937afcb4cc61e 100644 --- a/planner/core/testdata/integration_serial_suite_out.json +++ b/planner/core/testdata/integration_serial_suite_out.json @@ -514,9 +514,9 @@ "SQL": "explain select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.b = s.b", "Plan": [ "IndexJoin_11 12475.01 root inner join, inner:IndexLookUp_10, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), eq(test.t.b, test.s.b)", - "├─TableReader_24(Build) 9980.01 root data:Selection_23", - "│ └─Selection_23 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", - "│ └─TableFullScan_22 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "├─TableReader_23(Build) 9980.01 root data:Selection_22", + "│ └─Selection_22 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", + "│ └─TableFullScan_21 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─IndexLookUp_10(Probe) 1.25 root ", " ├─Selection_8(Build) 1.25 cop[tikv] not(isnull(test.s.a))", " │ └─IndexRangeScan_6 1.25 cop[tikv] table:s, index:a(a) range: decided by [eq(test.s.a, test.t.a)], keep order:false, stats:pseudo", @@ -528,9 +528,9 @@ "SQL": "explain select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.b = s.a", "Plan": [ "IndexJoin_10 12475.01 root inner join, inner:IndexLookUp_9, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), eq(test.t.b, test.s.a)", - "├─TableReader_22(Build) 9980.01 root data:Selection_21", - "│ └─Selection_21 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", - "│ └─TableFullScan_20 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "├─TableReader_21(Build) 9980.01 root data:Selection_20", + "│ └─Selection_20 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", + "│ └─TableFullScan_19 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─IndexLookUp_9(Probe) 1.25 root ", " ├─Selection_8(Build) 1.25 cop[tikv] not(isnull(test.s.a))", " │ └─IndexRangeScan_6 1.25 cop[tikv] table:s, index:a(a) range: decided by [eq(test.s.a, test.t.a)], keep order:false, stats:pseudo", @@ -541,9 +541,9 @@ "SQL": "explain select /*+ inl_join(s) */ * from t join s on t.a=s.a and t.a = s.b", "Plan": [ "IndexJoin_11 12475.01 root inner join, inner:IndexLookUp_10, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), eq(test.t.a, test.s.b)", - "├─TableReader_24(Build) 9990.00 root data:Selection_23", - "│ └─Selection_23 9990.00 cop[tikv] not(isnull(test.t.a))", - "│ └─TableFullScan_22 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "├─TableReader_23(Build) 9990.00 root data:Selection_22", + "│ └─Selection_22 9990.00 cop[tikv] not(isnull(test.t.a))", + "│ └─TableFullScan_21 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─IndexLookUp_10(Probe) 1.25 root ", " ├─Selection_8(Build) 1.25 cop[tikv] not(isnull(test.s.a))", " │ └─IndexRangeScan_6 1.25 cop[tikv] table:s, index:a(a) range: decided by [eq(test.s.a, test.t.a)], keep order:false, stats:pseudo", @@ -555,9 +555,9 @@ "SQL": "explain select /*+ inl_hash_join(s) */ * from t join s on t.a=s.a and t.b = s.b", "Plan": [ "IndexHashJoin_13 12475.01 root inner join, inner:IndexLookUp_10, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), eq(test.t.b, test.s.b)", - "├─TableReader_24(Build) 9980.01 root data:Selection_23", - "│ └─Selection_23 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", - "│ └─TableFullScan_22 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "├─TableReader_23(Build) 9980.01 root data:Selection_22", + "│ └─Selection_22 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", + "│ └─TableFullScan_21 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─IndexLookUp_10(Probe) 1.25 root ", " ├─Selection_8(Build) 1.25 cop[tikv] not(isnull(test.s.a))", " │ └─IndexRangeScan_6 1.25 cop[tikv] table:s, index:a(a) range: decided by [eq(test.s.a, test.t.a)], keep order:false, stats:pseudo", @@ -569,9 +569,9 @@ "SQL": "explain select /*+ inl_hash_join(s) */ * from t join s on t.a=s.a and t.b = s.a", "Plan": [ "IndexHashJoin_12 12475.01 root inner join, inner:IndexLookUp_9, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), eq(test.t.b, test.s.a)", - "├─TableReader_22(Build) 9980.01 root data:Selection_21", - "│ └─Selection_21 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", - "│ └─TableFullScan_20 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "├─TableReader_21(Build) 9980.01 root data:Selection_20", + "│ └─Selection_20 9980.01 cop[tikv] not(isnull(test.t.a)), not(isnull(test.t.b))", + "│ └─TableFullScan_19 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─IndexLookUp_9(Probe) 1.25 root ", " ├─Selection_8(Build) 1.25 cop[tikv] not(isnull(test.s.a))", " │ └─IndexRangeScan_6 1.25 cop[tikv] table:s, index:a(a) range: decided by [eq(test.s.a, test.t.a)], keep order:false, stats:pseudo", @@ -582,9 +582,9 @@ "SQL": "explain select /*+ inl_hash_join(s) */ * from t join s on t.a=s.a and t.a = s.b", "Plan": [ "IndexHashJoin_13 12475.01 root inner join, inner:IndexLookUp_10, outer key:test.t.a, inner key:test.s.a, equal cond:eq(test.t.a, test.s.a), eq(test.t.a, test.s.b)", - "├─TableReader_24(Build) 9990.00 root data:Selection_23", - "│ └─Selection_23 9990.00 cop[tikv] not(isnull(test.t.a))", - "│ └─TableFullScan_22 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "├─TableReader_23(Build) 9990.00 root data:Selection_22", + "│ └─Selection_22 9990.00 cop[tikv] not(isnull(test.t.a))", + "│ └─TableFullScan_21 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", "└─IndexLookUp_10(Probe) 1.25 root ", " ├─Selection_8(Build) 1.25 cop[tikv] not(isnull(test.s.a))", " │ └─IndexRangeScan_6 1.25 cop[tikv] table:s, index:a(a) range: decided by [eq(test.s.a, test.t.a)], keep order:false, stats:pseudo", diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 38d70b5c6ba3d..0ad109e42dcb2 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -69,8 +69,8 @@ "SQL": "explain select /*+ inl_join(t2) */ * from t1, t2 where t1.a = t2.a and t1.b = t2.b and t1.c = t2.c", "Plan": [ "IndexJoin_10 3.00 root inner join, inner:IndexLookUp_9, outer key:test.t1.c, inner key:test.t2.c, equal cond:eq(test.t1.a, test.t2.a), eq(test.t1.b, test.t2.b), eq(test.t1.c, test.t2.c)", - "├─TableReader_20(Build) 3.00 root data:TableFullScan_19", - "│ └─TableFullScan_19 3.00 cop[tikv] table:t1 keep order:false", + "├─TableReader_19(Build) 3.00 root data:TableFullScan_18", + "│ └─TableFullScan_18 3.00 cop[tikv] table:t1 keep order:false", "└─IndexLookUp_9(Probe) 1.00 root ", " ├─IndexRangeScan_7(Build) 1.00 cop[tikv] table:t2, index:idx2(c) range: decided by [eq(test.t2.c, test.t1.c)], keep order:false", " └─TableRowIDScan_8(Probe) 1.00 cop[tikv] table:t2 keep order:false" @@ -175,8 +175,8 @@ "SQL": "explain select /*+ TIDB_INLJ(t2) */ * from t1 join t2 on t1.a = t2.a and t1.c = t2.c", "Plan": [ "IndexJoin_9 2.00 root inner join, inner:IndexLookUp_8, outer key:test.t1.a, inner key:test.t2.a, equal cond:eq(test.t1.a, test.t2.a), eq(test.t1.c, test.t2.c)", - "├─TableReader_19(Build) 1.00 root data:TableFullScan_18", - "│ └─TableFullScan_18 1.00 cop[tikv] table:t1 keep order:false", + "├─TableReader_18(Build) 1.00 root data:TableFullScan_17", + "│ └─TableFullScan_17 1.00 cop[tikv] table:t1 keep order:false", "└─IndexLookUp_8(Probe) 2.00 root ", " ├─IndexRangeScan_6(Build) 2.00 cop[tikv] table:t2, index:PRIMARY(a, b) range: decided by [eq(test.t2.a, test.t1.a)], keep order:false", " └─TableRowIDScan_7(Probe) 2.00 cop[tikv] table:t2 keep order:false" @@ -348,23 +348,23 @@ { "SQL": "desc select /*+ TIDB_INLJ(t2)*/ * from t1, t2 where t1.a = t2.a and t1.b = t2.b", "Plan": [ - "IndexJoin_15 12487.50 root inner join, inner:TableReader_11, outer key:test.t1.a, inner key:test.t2.a, equal cond:eq(test.t1.a, test.t2.a), eq(test.t1.b, test.t2.b)", - "├─IndexReader_21(Build) 9990.00 root index:IndexFullScan_20", - "│ └─IndexFullScan_20 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", - "└─TableReader_11(Probe) 1.00 root data:Selection_10", - " └─Selection_10 1.00 cop[tikv] not(isnull(test.t2.b))", - " └─TableRangeScan_9 1.00 cop[tikv] table:t2 range: decided by [test.t1.a], keep order:false, stats:pseudo" + "IndexJoin_14 12487.50 root inner join, inner:TableReader_10, outer key:test.t1.a, inner key:test.t2.a, equal cond:eq(test.t1.a, test.t2.a), eq(test.t1.b, test.t2.b)", + "├─IndexReader_19(Build) 9990.00 root index:IndexFullScan_18", + "│ └─IndexFullScan_18 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", + "└─TableReader_10(Probe) 1.00 root data:Selection_9", + " └─Selection_9 1.00 cop[tikv] not(isnull(test.t2.b))", + " └─TableRangeScan_8 1.00 cop[tikv] table:t2 range: decided by [test.t1.a], keep order:false, stats:pseudo" ] }, { "SQL": "desc select /*+ TIDB_INLJ(t2)*/ * from t1, t2 where t1.a = t2.a and t1.b = t2.a and t1.b = t2.b", "Plan": [ - "IndexJoin_15 12487.50 root inner join, inner:TableReader_11, outer key:test.t1.a, test.t1.b, inner key:test.t2.a, test.t2.a, equal cond:eq(test.t1.a, test.t2.a), eq(test.t1.b, test.t2.a), eq(test.t1.b, test.t2.b)", - "├─IndexReader_20(Build) 9990.00 root index:IndexFullScan_19", - "│ └─IndexFullScan_19 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", - "└─TableReader_11(Probe) 1.00 root data:Selection_10", - " └─Selection_10 1.00 cop[tikv] not(isnull(test.t2.b))", - " └─TableRangeScan_9 1.00 cop[tikv] table:t2 range: decided by [test.t1.a test.t1.b], keep order:false, stats:pseudo" + "IndexJoin_13 12487.50 root inner join, inner:TableReader_9, outer key:test.t1.a, test.t1.b, inner key:test.t2.a, test.t2.a, equal cond:eq(test.t1.a, test.t2.a), eq(test.t1.b, test.t2.a), eq(test.t1.b, test.t2.b)", + "├─IndexReader_18(Build) 9990.00 root index:IndexFullScan_17", + "│ └─IndexFullScan_17 9990.00 cop[tikv] table:t1, index:idx_t1_b(b) keep order:false, stats:pseudo", + "└─TableReader_9(Probe) 1.00 root data:Selection_8", + " └─Selection_8 1.00 cop[tikv] not(isnull(test.t2.b))", + " └─TableRangeScan_7 1.00 cop[tikv] table:t2 range: decided by [test.t1.a test.t1.b], keep order:false, stats:pseudo" ] } ] diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 75ffa492dc74a..d2c771a448cf5 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -446,7 +446,7 @@ }, { "SQL": "select * from t where t.c in (select b from t s where s.a = t.a)", - "Best": "MergeSemiJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t.a,test.t.a)" + "Best": "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t.a,test.t.a)(test.t.c,test.t.b)" }, { "SQL": "select t.c in (select b from t s where s.a = t.a) from t", @@ -2232,7 +2232,8 @@ "TableReader_7 2048.00 root data:Selection_6", "└─Selection_6 2048.00 cop[tikv] eq(test.test.age, 5)", " └─TableFullScan_5 2048.00 cop[tikv] table:test keep order:false" - ] + ], + "Result": null } ] }, @@ -2245,7 +2246,8 @@ "IndexLookUp_7 2048.00 root ", "├─IndexRangeScan_5(Build) 2048.00 cop[tikv] table:test, index:idx_age(age) range:[5,5], keep order:false", "└─TableRowIDScan_6(Probe) 2048.00 cop[tikv] table:test keep order:false" - ] + ], + "Result": null } ] } From 49a15278933eb0d182861d0f42bcfc5611581085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=99=8E?= Date: Thu, 19 Nov 2020 19:04:53 +0800 Subject: [PATCH 35/37] expression: adjust int constant when compare with year type (#21099) --- executor/executor_test.go | 13 +++++++++++++ expression/builtin_compare.go | 19 ++++++++++++++++++- types/time.go | 4 ++-- util/ranger/ranger_test.go | 18 +++++++++++++----- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index f36cd4a021246..e9f45aacccba3 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -7013,5 +7013,18 @@ func (s *testSuite) TestIssue20975SelectForUpdateBatchPointGetWithPartitionTable tk1.MustExec("select * from t1 where id in (1, 11) for update") tk2.MustExec("drop table t2") tk1.MustExec("commit") +} +func (s *testSuite) TestIssue20305(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t2 (a year(4))") + tk.MustExec("insert into t2 values(69)") + tk.MustQuery("select * from t2 where a <= 69").Check(testkit.Rows("2069")) + // the following test is a regression test that matches MySQL's behavior. + tk.MustExec("drop table if exists t3") + tk.MustExec("CREATE TABLE `t3` (`y` year DEFAULT NULL, `a` int DEFAULT NULL)") + tk.MustExec("INSERT INTO `t3` VALUES (2069, 70), (2010, 11), (2155, 2156), (2069, 69)") + tk.MustQuery("SELECT * FROM `t3` where y <= a").Check(testkit.Rows("2155 2156")) } diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index 5564b1f00d11c..24618a046cf15 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -1282,7 +1282,8 @@ func RefineComparedConstant(ctx sessionctx.Context, targetFieldType types.FieldT } // refineArgs will rewrite the arguments if the compare expression is `int column non-int constant` or -// `non-int constant int column`. E.g., `a < 1.1` will be rewritten to `a < 2`. +// `non-int constant int column`. E.g., `a < 1.1` will be rewritten to `a < 2`. It also handles comparing year type +// with int constant if the int constant falls into a sensible year representation. func (c *compareFunctionClass) refineArgs(ctx sessionctx.Context, args []Expression) []Expression { if ContainMutableConst(ctx, args) { return args @@ -1325,6 +1326,22 @@ func (c *compareFunctionClass) refineArgs(ctx sessionctx.Context, args []Express } } } + // int constant [cmp] year type + if arg0IsCon && arg0IsInt && arg1Type.Tp == mysql.TypeYear { + adjusted, failed := types.AdjustYear(arg0.Value.GetInt64(), false) + if failed == nil { + arg0.Value.SetInt64(adjusted) + finalArg0 = arg0 + } + } + // year type [cmp] int constant + if arg1IsCon && arg1IsInt && arg0Type.Tp == mysql.TypeYear { + adjusted, failed := types.AdjustYear(arg1.Value.GetInt64(), false) + if failed == nil { + arg1.Value.SetInt64(adjusted) + finalArg1 = arg1 + } + } if isExceptional && (c.op == opcode.EQ || c.op == opcode.NullEQ) { // This will always be false. return []Expression{NewZero(), NewOne()} diff --git a/types/time.go b/types/time.go index 90bfe7aa82985..cd353153a5057 100644 --- a/types/time.go +++ b/types/time.go @@ -1224,8 +1224,8 @@ func adjustYear(y int) int { } // AdjustYear is used for adjusting year and checking its validation. -func AdjustYear(y int64, shouldAdjust bool) (int64, error) { - if y == 0 && !shouldAdjust { +func AdjustYear(y int64, adjustZero bool) (int64, error) { + if y == 0 && !adjustZero { return y, nil } y = int64(adjustYear(int(y))) diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 3e9d7b8542210..f6c0e48f82f0b 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1343,6 +1343,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { testKit.MustExec("INSERT INTO t VALUES (1), (70), (99), (0), ('0')") testKit.MustQuery("SELECT * FROM t WHERE a < 15698").Check(testkit.Rows("0", "1970", "1999", "2000", "2001")) testKit.MustQuery("SELECT * FROM t WHERE a <= 0").Check(testkit.Rows("0")) + testKit.MustQuery("SELECT * FROM t WHERE a <= 1").Check(testkit.Rows("0", "1970", "1999", "2000", "2001")) testKit.MustQuery("SELECT * FROM t WHERE a < 2000").Check(testkit.Rows("0", "1970", "1999")) testKit.MustQuery("SELECT * FROM t WHERE a > -1").Check(testkit.Rows("0", "1970", "1999", "2000", "2001")) @@ -1370,14 +1371,21 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { { indexPos: 0, exprStr: `a not in (1, 2, 70)`, - accessConds: "[not(in(test.t.a, 1, 2, 70))]", + accessConds: "[not(in(test.t.a, 1, 2, 70))]", // this is in accordance with MySQL, MySQL won't interpret 70 here as 1970 filterConds: "[]", resultStr: `[(NULL,1970) (1970,2001) (2002,+inf]]`, }, + { + indexPos: 0, + exprStr: `a = 1 or a = 2 or a = 70`, + accessConds: "[or(eq(test.t.a, 2001), or(eq(test.t.a, 2002), eq(test.t.a, 1970)))]", // this is in accordance with MySQL, MySQL won't interpret 70 here as 1970 + filterConds: "[]", + resultStr: `[[1970,1970] [2001,2002]]`, + }, { indexPos: 0, exprStr: `a not in (99)`, - accessConds: "[ne(test.t.a, 99)]", + accessConds: "[ne(test.t.a, 1999)]", // this is in accordance with MySQL filterConds: "[]", resultStr: `[[-inf,1999) (1999,+inf]]`, }, @@ -1405,7 +1413,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { { indexPos: 0, exprStr: `a != 1`, - accessConds: "[ne(test.t.a, 1)]", + accessConds: "[ne(test.t.a, 2001)]", filterConds: "[]", resultStr: `[[-inf,2001) (2001,+inf]]`, }, @@ -1418,13 +1426,13 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) { }, { exprStr: "a < 99 or a > 01", - accessConds: "[or(lt(test.t.a, 99), gt(test.t.a, 1))]", + accessConds: "[or(lt(test.t.a, 1999), gt(test.t.a, 2001))]", filterConds: "[]", resultStr: "[[-inf,1999) (2001,+inf]]", }, { exprStr: "a >= 70 and a <= 69", - accessConds: "[ge(test.t.a, 70) le(test.t.a, 69)]", + accessConds: "[ge(test.t.a, 1970) le(test.t.a, 2069)]", filterConds: "[]", resultStr: "[[1970,2069]]", }, From 22feeb4aef750724f9f1321267a6bd5a661eb5e7 Mon Sep 17 00:00:00 2001 From: Alkaid Jiang <38248129+jyz0309@users.noreply.github.com> Date: Thu, 19 Nov 2020 19:36:54 +0800 Subject: [PATCH 36/37] executor:Add runtime stat for IndexMergeReaderExecutor (#20653) Signed-off-by: jyz0309 <45495947@qq.com> --- executor/executor_test.go | 25 ++++++ executor/index_merge_reader.go | 139 ++++++++++++++++++++++++++++++-- util/execdetails/execdetails.go | 2 + 3 files changed, 160 insertions(+), 6 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index e9f45aacccba3..a6036bfb07952 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -6527,6 +6527,31 @@ func (s *testSerialSuite1) TestCollectCopRuntimeStats(c *C) { c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/tikvStoreRespResult"), IsNil) } +func (s *testSerialSuite1) TestIndexMergeRuntimeStats(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test;") + tk.MustExec("drop table if exists t1") + tk.MustExec("set @@tidb_enable_index_merge = 1") + tk.MustExec("create table t1(id int primary key, a int, b int, c int, d int)") + tk.MustExec("create index t1a on t1(a)") + tk.MustExec("create index t1b on t1(b)") + tk.MustExec("insert into t1 values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5)") + sql := "explain analyze select /*+ use_index_merge(t1, primary, t1a) */ * from t1 where id < 2 or a > 4;" + rows := tk.MustQuery(sql).Rows() + c.Assert(len(rows), Equals, 4) + explain := fmt.Sprintf("%v", rows[0]) + c.Assert(explain, Matches, ".*time:.*loops:.*index_task:{fetch_handle:.*, merge:.*}.*table_task:{num.*concurrency.*fetch_row.*wait_time.*}.*") + tableRangeExplain := fmt.Sprintf("%v", rows[1]) + indexExplain := fmt.Sprintf("%v", rows[2]) + tableExplain := fmt.Sprintf("%v", rows[3]) + c.Assert(tableRangeExplain, Matches, ".*time:.*loops:.*cop_task:.*") + c.Assert(indexExplain, Matches, ".*time:.*loops:.*cop_task:.*") + c.Assert(tableExplain, Matches, ".*time:.*loops:.*cop_task:.*") + tk.MustExec("set @@tidb_enable_collect_execution_info=0;") + sql = "select /*+ use_index_merge(t1, primary, t1a) */ * from t1 where id < 2 or a > 4 order by a" + tk.MustQuery(sql).Check(testkit.Rows("1 1 1 1 1", "5 5 5 5 5")) +} + func (s *testSerialSuite1) TestIndexlookupRuntimeStats(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test;") diff --git a/executor/index_merge_reader.go b/executor/index_merge_reader.go index 689e466e3659c..22de3f61dbfbe 100644 --- a/executor/index_merge_reader.go +++ b/executor/index_merge_reader.go @@ -14,10 +14,13 @@ package executor import ( + "bytes" "context" + "fmt" "runtime/trace" "sync" "sync/atomic" + "time" "unsafe" "github.com/pingcap/errors" @@ -33,6 +36,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/ranger" @@ -102,11 +106,13 @@ type IndexMergeReaderExecutor struct { colLens [][]int handleCols plannercore.HandleCols + stats *IndexMergeRuntimeStat } // Open implements the Executor Open interface func (e *IndexMergeReaderExecutor) Open(ctx context.Context) error { e.keyRanges = make([][]kv.KeyRange, 0, len(e.partialPlans)) + e.initRuntimeStats() for i, plan := range e.partialPlans { _, ok := plan[0].(*plannercore.PhysicalIndexScan) if !ok { @@ -169,7 +175,9 @@ func (e *IndexMergeReaderExecutor) waitPartialWorkersAndCloseFetchChan(partialWo } func (e *IndexMergeReaderExecutor) startIndexMergeProcessWorker(ctx context.Context, workCh chan<- *lookupTableTask, fetch <-chan *lookupTableTask) { - idxMergeProcessWorker := &indexMergeProcessWorker{} + idxMergeProcessWorker := &indexMergeProcessWorker{ + stats: e.stats, + } e.processWokerWg.Add(1) go func() { defer trace.StartRegion(ctx, "IndexMergeProcessWorker").End() @@ -202,14 +210,15 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context, if err != nil { return err } - - result, err := distsql.SelectWithRuntimeStats(ctx, e.ctx, kvReq, e.handleCols.GetFieldsTypes(), e.feedbacks[workID], getPhysicalPlanIDs(e.partialPlans[workID]), e.id) + result, err := distsql.SelectWithRuntimeStats(ctx, e.ctx, kvReq, e.handleCols.GetFieldsTypes(), e.feedbacks[workID], getPhysicalPlanIDs(e.partialPlans[workID]), e.getPartitalPlanID(workID)) if err != nil { return err } result.Fetch(ctx) worker := &partialIndexWorker{ + stats: e.stats, + idxID: e.getPartitalPlanID(workID), sc: e.ctx, batchSize: e.maxChunkSize, maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize, @@ -250,7 +259,7 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context, func (e *IndexMergeReaderExecutor) buildPartialTableReader(ctx context.Context, workID int) Executor { tableReaderExec := &TableReaderExecutor{ - baseExecutor: newBaseExecutor(e.ctx, e.schema, 0), + baseExecutor: newBaseExecutor(e.ctx, e.schema, e.getPartitalPlanID(workID)), table: e.table, dagPB: e.dagPBs[workID], startTS: e.startTS, @@ -272,6 +281,7 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context, } tableInfo := e.partialPlans[workID][0].(*plannercore.PhysicalTableScan).Table worker := &partialTableWorker{ + stats: e.stats, sc: e.ctx, batchSize: e.maxChunkSize, maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize, @@ -306,7 +316,31 @@ func (e *IndexMergeReaderExecutor) startPartialTableWorker(ctx context.Context, return nil } +func (e *IndexMergeReaderExecutor) initRuntimeStats() { + if e.runtimeStats != nil && e.stats == nil { + e.stats = &IndexMergeRuntimeStat{ + Concurrency: e.ctx.GetSessionVars().IndexLookupConcurrency(), + } + e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(e.id, e.stats) + } +} + +func (e *IndexMergeReaderExecutor) getPartitalPlanID(workID int) int { + if len(e.partialPlans[workID]) > 0 { + return e.partialPlans[workID][len(e.partialPlans[workID])-1].ID() + } + return 0 +} + +func (e *IndexMergeReaderExecutor) getTablePlanRootID() int { + if len(e.tblPlans) > 0 { + return e.tblPlans[len(e.tblPlans)-1].ID() + } + return e.id +} + type partialTableWorker struct { + stats *IndexMergeRuntimeStat sc sessionctx.Context batchSize int maxBatchSize int @@ -318,7 +352,12 @@ type partialTableWorker struct { func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *lookupTableTask, resultCh chan<- *lookupTableTask, finished <-chan struct{}, handleCols plannercore.HandleCols) (count int64, err error) { chk := chunk.NewChunkWithCapacity(retTypes(w.tableReader), w.maxChunkSize) + var basic *execdetails.BasicRuntimeStats + if be := w.tableReader.base(); be != nil && be.runtimeStats != nil { + basic = be.runtimeStats + } for { + start := time.Now() handles, retChunk, err := w.extractTaskHandles(ctx, chk, handleCols) if err != nil { doneCh := make(chan error, 1) @@ -333,6 +372,9 @@ func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan str } count += int64(len(handles)) task := w.buildTableTask(handles, retChunk) + if w.stats != nil { + atomic.AddInt64(&w.stats.FetchIdxTime, int64(time.Since(start))) + } select { case <-ctx.Done(): return count, ctx.Err() @@ -342,6 +384,9 @@ func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan str return count, nil case fetchCh <- task: } + if basic != nil { + basic.Record(time.Since(start), chk.NumRows()) + } } } @@ -387,6 +432,7 @@ func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Co e.tblWorkerWg.Add(lookupConcurrencyLimit) for i := 0; i < lookupConcurrencyLimit; i++ { worker := &indexMergeTableScanWorker{ + stats: e.stats, workCh: workCh, finished: e.finished, buildTblReader: e.buildFinalTableReader, @@ -409,7 +455,7 @@ func (e *IndexMergeReaderExecutor) startIndexMergeTableScanWorker(ctx context.Co func (e *IndexMergeReaderExecutor) buildFinalTableReader(ctx context.Context, handles []kv.Handle) (Executor, error) { tableReaderExec := &TableReaderExecutor{ - baseExecutor: newBaseExecutor(e.ctx, e.schema, 0), + baseExecutor: newBaseExecutor(e.ctx, e.schema, e.getTablePlanRootID()), table: e.table, dagPB: e.tableRequest, startTS: e.startTS, @@ -505,6 +551,7 @@ func (e *IndexMergeReaderExecutor) Close() error { } type indexMergeProcessWorker struct { + stats *IndexMergeRuntimeStat } func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan *lookupTableTask, @@ -515,8 +562,8 @@ func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan }() distinctHandles := kv.NewHandleMap() - for task := range fetchCh { + start := time.Now() handles := task.handles fhs := make([]kv.Handle, 0, 8) for _, h := range handles { @@ -532,6 +579,9 @@ func (w *indexMergeProcessWorker) fetchLoop(ctx context.Context, fetchCh <-chan handles: fhs, doneCh: make(chan error, 1), } + if w.stats != nil { + w.stats.IndexMergeProcess += time.Since(start) + } select { case <-ctx.Done(): return @@ -560,7 +610,9 @@ func (w *indexMergeProcessWorker) handleLoopFetcherPanic(ctx context.Context, re } type partialIndexWorker struct { + stats *IndexMergeRuntimeStat sc sessionctx.Context + idxID int batchSize int maxBatchSize int maxChunkSize int @@ -575,7 +627,15 @@ func (w *partialIndexWorker) fetchHandles( finished <-chan struct{}, handleCols plannercore.HandleCols) (count int64, err error) { chk := chunk.NewChunkWithCapacity(handleCols.GetFieldsTypes(), w.maxChunkSize) + var basicStats *execdetails.BasicRuntimeStats + if w.stats != nil { + if w.idxID != 0 { + basicStats = &execdetails.BasicRuntimeStats{} + w.sc.GetSessionVars().StmtCtx.RuntimeStatsColl.RegisterStats(w.idxID, basicStats) + } + } for { + start := time.Now() handles, retChunk, err := w.extractTaskHandles(ctx, chk, result, handleCols) if err != nil { doneCh := make(chan error, 1) @@ -590,6 +650,9 @@ func (w *partialIndexWorker) fetchHandles( } count += int64(len(handles)) task := w.buildTableTask(handles, retChunk) + if w.stats != nil { + atomic.AddInt64(&w.stats.FetchIdxTime, int64(time.Since(start))) + } select { case <-ctx.Done(): return count, ctx.Err() @@ -599,6 +662,9 @@ func (w *partialIndexWorker) fetchHandles( return count, nil case fetchCh <- task: } + if basicStats != nil { + basicStats.Record(time.Since(start), chk.NumRows()) + } } } @@ -640,6 +706,7 @@ func (w *partialIndexWorker) buildTableTask(handles []kv.Handle, retChk *chunk.C } type indexMergeTableScanWorker struct { + stats *IndexMergeRuntimeStat workCh <-chan *lookupTableTask finished <-chan struct{} buildTblReader func(ctx context.Context, handles []kv.Handle) (Executor, error) @@ -652,6 +719,7 @@ type indexMergeTableScanWorker struct { func (w *indexMergeTableScanWorker) pickAndExecTask(ctx context.Context) (task *lookupTableTask) { var ok bool for { + waitStart := time.Now() select { case task, ok = <-w.workCh: if !ok { @@ -660,7 +728,13 @@ func (w *indexMergeTableScanWorker) pickAndExecTask(ctx context.Context) (task * case <-w.finished: return } + execStart := time.Now() err := w.executeTask(ctx, task) + if w.stats != nil { + atomic.AddInt64(&w.stats.WaitTime, int64(execStart.Sub(waitStart))) + atomic.AddInt64(&w.stats.FetchRow, int64(time.Since(execStart))) + atomic.AddInt64(&w.stats.TableTaskNum, 1) + } task.doneCh <- err } } @@ -717,3 +791,56 @@ func (w *indexMergeTableScanWorker) executeTask(ctx context.Context, task *looku } return nil } + +// IndexMergeRuntimeStat record the indexMerge runtime stat +type IndexMergeRuntimeStat struct { + IndexMergeProcess time.Duration + FetchIdxTime int64 + WaitTime int64 + FetchRow int64 + TableTaskNum int64 + Concurrency int +} + +func (e *IndexMergeRuntimeStat) String() string { + var buf bytes.Buffer + if e.FetchIdxTime != 0 { + buf.WriteString(fmt.Sprintf("index_task:{fetch_handle:%s", time.Duration(e.FetchIdxTime))) + if e.IndexMergeProcess != 0 { + buf.WriteString(fmt.Sprintf(", merge:%s", e.IndexMergeProcess)) + } + buf.WriteByte('}') + } + if e.FetchRow != 0 { + if buf.Len() > 0 { + buf.WriteByte(',') + } + buf.WriteString(fmt.Sprintf(" table_task:{num:%d, concurrency:%d, fetch_row:%s, wait_time:%s}", e.TableTaskNum, e.Concurrency, time.Duration(e.FetchRow), time.Duration(e.WaitTime))) + } + return buf.String() +} + +// Clone implements the RuntimeStats interface. +func (e *IndexMergeRuntimeStat) Clone() execdetails.RuntimeStats { + newRs := *e + return &newRs +} + +// Merge implements the RuntimeStats interface. +func (e *IndexMergeRuntimeStat) Merge(other execdetails.RuntimeStats) { + tmp, ok := other.(*IndexMergeRuntimeStat) + if !ok { + return + } + e.IndexMergeProcess += tmp.IndexMergeProcess + e.FetchIdxTime += tmp.FetchIdxTime + e.FetchRow += tmp.FetchRow + e.WaitTime += e.WaitTime + e.TableTaskNum += tmp.TableTaskNum + e.Concurrency += tmp.Concurrency +} + +// Tp implements the RuntimeStats interface. +func (e *IndexMergeRuntimeStat) Tp() int { + return execdetails.TpIndexMergeRunTimeStats +} diff --git a/util/execdetails/execdetails.go b/util/execdetails/execdetails.go index 7cb1b567ec89e..0cf37c3c66867 100644 --- a/util/execdetails/execdetails.go +++ b/util/execdetails/execdetails.go @@ -495,6 +495,8 @@ const ( TpIndexLookUpRunTimeStats // TpSlowQueryRuntimeStat is the tp for TpSlowQueryRuntimeStat TpSlowQueryRuntimeStat + // TpIndexMergeRunTimeStats is the tp for TpIndexMergeRunTimeStats + TpIndexMergeRunTimeStats ) // RuntimeStats is used to express the executor runtime information. From eca1ce192b0a5aae1626c9eb005cd83ee670520f Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Fri, 20 Nov 2020 11:14:23 +0800 Subject: [PATCH 37/37] util: fix wrong calculation to get memory usage. (#21158) --- util/memory/meminfo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/memory/meminfo.go b/util/memory/meminfo.go index b18b2500e4002..d08ccec9d4c50 100644 --- a/util/memory/meminfo.go +++ b/util/memory/meminfo.go @@ -36,7 +36,7 @@ func MemTotalNormal() (uint64, error) { // MemUsedNormal returns the total used amount of RAM on this system in non-container environment. func MemUsedNormal() (uint64, error) { v, err := mem.VirtualMemory() - return v.Total - (v.Free + v.Buffers + v.Cached), err + return v.Used, err } const (