diff --git a/pkg/bindinfo/BUILD.bazel b/pkg/bindinfo/BUILD.bazel index c62004f988526..488c4980439f3 100644 --- a/pkg/bindinfo/BUILD.bazel +++ b/pkg/bindinfo/BUILD.bazel @@ -41,6 +41,8 @@ go_library( "//pkg/util/table-filter", "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", + "@org_golang_x_sync//singleflight", "@org_uber_go_zap//:zap", ], ) diff --git a/pkg/bindinfo/binding_cache.go b/pkg/bindinfo/binding_cache.go index 341d1c0e22aee..abd345ca2b273 100644 --- a/pkg/bindinfo/binding_cache.go +++ b/pkg/bindinfo/binding_cache.go @@ -18,6 +18,7 @@ import ( "errors" "sync" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/bindinfo/norm" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" @@ -25,8 +26,10 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util/hack" "github.com/pingcap/tidb/pkg/util/kvcache" + "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/memory" + "go.uber.org/zap" ) // FuzzyBindingCache is based on BindingCache, and provide some more advanced features, like @@ -52,24 +55,40 @@ type fuzzyBindingCache struct { fuzzy2SQLDigests map[string][]string // fuzzyDigest --> sqlDigests sql2FuzzyDigest map[string]string // sqlDigest --> fuzzyDigest + + // loadBindingFromStorageFunc is used to load binding from storage if cache miss. + loadBindingFromStorageFunc func(sqlDigest string) (Bindings, error) } -func newFuzzyBindingCache() FuzzyBindingCache { +func newFuzzyBindingCache(loadBindingFromStorageFunc func(string) (Bindings, error)) FuzzyBindingCache { return &fuzzyBindingCache{ - BindingCache: newBindCache(), - fuzzy2SQLDigests: make(map[string][]string), - sql2FuzzyDigest: make(map[string]string), + BindingCache: newBindCache(), + fuzzy2SQLDigests: make(map[string][]string), + sql2FuzzyDigest: make(map[string]string), + loadBindingFromStorageFunc: loadBindingFromStorageFunc, } } func (fbc *fuzzyBindingCache) FuzzyMatchingBinding(sctx sessionctx.Context, fuzzyDigest string, tableNames []*ast.TableName) (matchedBinding Binding, isMatched bool) { + matchedBinding, isMatched, missingSQLDigest := fbc.getFromMemory(sctx, fuzzyDigest, tableNames) + if len(missingSQLDigest) == 0 { + return + } + if fbc.loadBindingFromStorageFunc == nil { + return + } + fbc.loadFromStore(missingSQLDigest) // loadFromStore's SetBinding has a Mutex inside, so it's safe to call it without lock + matchedBinding, isMatched, _ = fbc.getFromMemory(sctx, fuzzyDigest, tableNames) + return +} + +func (fbc *fuzzyBindingCache) getFromMemory(sctx sessionctx.Context, fuzzyDigest string, tableNames []*ast.TableName) (matchedBinding Binding, isMatched bool, missingSQLDigest []string) { fbc.mu.RLock() defer fbc.mu.RUnlock() bindingCache := fbc.BindingCache if bindingCache.Size() == 0 { return } - leastWildcards := len(tableNames) + 1 enableFuzzyBinding := sctx.GetSessionVars().EnableFuzzyBinding for _, sqlDigest := range fbc.fuzzy2SQLDigests[fuzzyDigest] { @@ -86,9 +105,33 @@ func (fbc *fuzzyBindingCache) FuzzyMatchingBinding(sctx sessionctx.Context, fuzz break } } + } else { + missingSQLDigest = append(missingSQLDigest, sqlDigest) + } + } + return matchedBinding, isMatched, missingSQLDigest +} + +func (fbc *fuzzyBindingCache) loadFromStore(missingSQLDigest []string) { + for _, sqlDigest := range missingSQLDigest { + bindings, err := fbc.loadBindingFromStorageFunc(sqlDigest) + if err != nil { + logutil.BgLogger().Warn("loadBindingFromStorageFunc binding failed", zap.String("sqlDigest", sqlDigest), zap.Error(err)) + continue + } + // put binding into the cache + oldBinding := fbc.GetBinding(sqlDigest) + newBindings := removeDeletedBindings(merge(oldBinding, bindings)) + if len(newBindings) > 0 { + err = fbc.SetBinding(sqlDigest, newBindings) + if err != nil { + // When the memory capacity of bing_cache is not enough, + // there will be some memory-related errors in multiple places. + // Only needs to be handled once. + logutil.BgLogger().Warn("BindHandle.Update", zap.String("category", "sql-bind"), zap.Error(err)) + } } } - return } func (fbc *fuzzyBindingCache) SetBinding(sqlDigest string, bindings Bindings) (err error) { @@ -153,9 +196,10 @@ func (fbc *fuzzyBindingCache) Copy() (c FuzzyBindingCache, err error) { fuzzy2SQLDigests[k] = newList } return &fuzzyBindingCache{ - BindingCache: bc, - fuzzy2SQLDigests: fuzzy2SQLDigests, - sql2FuzzyDigest: sql2FuzzyDigest, + BindingCache: bc, + fuzzy2SQLDigests: fuzzy2SQLDigests, + sql2FuzzyDigest: sql2FuzzyDigest, + loadBindingFromStorageFunc: fbc.loadBindingFromStorageFunc, }, nil } @@ -273,6 +317,9 @@ func (c *bindingCache) delete(key bindingCacheKey) bool { // The return value is not read-only, but it shouldn't be changed in the caller functions. // The function is thread-safe. func (c *bindingCache) GetBinding(sqlDigest string) Bindings { + failpoint.Inject("get_binding_return_nil", func(_ failpoint.Value) { + failpoint.Return(nil) + }) c.lock.Lock() defer c.lock.Unlock() return c.get(bindingCacheKey(sqlDigest)) diff --git a/pkg/bindinfo/binding_cache_test.go b/pkg/bindinfo/binding_cache_test.go index 28b733bba5dce..12cf1a94bb32c 100644 --- a/pkg/bindinfo/binding_cache_test.go +++ b/pkg/bindinfo/binding_cache_test.go @@ -35,7 +35,7 @@ func bindingFuzzyDigest(t *testing.T, b Binding) string { } func TestFuzzyBindingCache(t *testing.T) { - fbc := newFuzzyBindingCache().(*fuzzyBindingCache) + fbc := newFuzzyBindingCache(nil).(*fuzzyBindingCache) b1 := Binding{BindSQL: "SELECT * FROM db1.t1", SQLDigest: "b1"} fDigest1 := bindingFuzzyDigest(t, b1) b2 := Binding{BindSQL: "SELECT * FROM db2.t1", SQLDigest: "b2"} diff --git a/pkg/bindinfo/fuzzy_binding_test.go b/pkg/bindinfo/fuzzy_binding_test.go index d9f17661ffc68..75366744757af 100644 --- a/pkg/bindinfo/fuzzy_binding_test.go +++ b/pkg/bindinfo/fuzzy_binding_test.go @@ -39,21 +39,6 @@ func showBinding(tk *testkit.TestKit, showStmt string) [][]any { return result } -func removeAllBindings(tk *testkit.TestKit, global bool) { - scope := "session" - if global { - scope = "global" - } - res := showBinding(tk, fmt.Sprintf("show %v bindings", scope)) - for _, r := range res { - if r[4] == "builtin" { - continue - } - tk.MustExec(fmt.Sprintf("drop %v binding for sql digest '%v'", scope, r[5])) - } - tk.MustQuery(fmt.Sprintf("show %v bindings", scope)).Check(testkit.Rows()) // empty -} - func TestFuzzyBindingBasic(t *testing.T) { store := testkit.CreateMockStore(t) tk1 := testkit.NewTestKit(t, store) @@ -355,156 +340,3 @@ func TestFuzzyBindingPlanCache(t *testing.T) { tk.MustExec(`execute stmt using @v`) hasPlan("IndexFullScan", "index:c(c)") } - -func TestFuzzyBindingHints(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec(`use test`) - - for _, db := range []string{"db1", "db2", "db3"} { - tk.MustExec(`create database ` + db) - tk.MustExec(`use ` + db) - tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) - tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) - tk.MustExec(`create table t3 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) - } - tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) - - for _, c := range []struct { - binding string - qTemplate string - }{ - // use index - {`create global binding using select /*+ use_index(t1, c) */ * from *.t1 where a=1`, - `select * from %st1 where a=1000`}, - {`create global binding using select /*+ use_index(t1, c) */ * from *.t1 where d<1`, - `select * from %st1 where d<10000`}, - {`create global binding using select /*+ use_index(t1, c) */ * from *.t1, *.t2 where t1.d<1`, - `select * from %st1, t2 where t1.d<100`}, - {`create global binding using select /*+ use_index(t1, c) */ * from *.t1, *.t2 where t1.d<1`, - `select * from t1, %st2 where t1.d<100`}, - {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2 where t1.d<1`, - `select * from %st1, t2 where t1.d<100`}, - {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2 where t1.d<1`, - `select * from t1, %st2 where t1.d<100`}, - {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2, *.t3 where t1.d<1`, - `select * from %st1, t2, t3 where t1.d<100`}, - {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2, *.t3 where t1.d<1`, - `select * from t1, t2, %st3 where t1.d<100`}, - - // ignore index - {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b=1`, - `select * from %st1 where b=1000`}, - {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b>1`, - `select * from %st1 where b>1000`}, - {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b in (1,2)`, - `select * from %st1 where b in (1)`}, - {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b in (1,2)`, - `select * from %st1 where b in (1,2,3,4,5)`}, - - // order index hint - {`create global binding using select /*+ order_index(t1, a) */ a from *.t1 where a<10 order by a limit 10`, - `select a from %st1 where a<10000 order by a limit 10`}, - {`create global binding using select /*+ order_index(t1, b) */ b from *.t1 where b>10 order by b limit 1111`, - `select b from %st1 where b>2 order by b limit 10`}, - - // no order index hint - {`create global binding using select /*+ no_order_index(t1, c) */ c from *.t1 where c<10 order by c limit 10`, - `select c from %st1 where c<10000 order by c limit 10`}, - {`create global binding using select /*+ no_order_index(t1, d) */ d from *.t1 where d>10 order by d limit 1111`, - `select d from %st1 where d>2 order by d limit 10`}, - - // agg hint - {`create global binding using select /*+ hash_agg() */ count(*) from *.t1 group by a`, - `select count(*) from %st1 group by a`}, - {`create global binding using select /*+ stream_agg() */ count(*) from *.t1 group by b`, - `select count(*) from %st1 group by b`}, - - // to_cop hint - {`create global binding using select /*+ agg_to_cop() */ sum(a) from *.t1`, - `select sum(a) from %st1`}, - {`create global binding using select /*+ limit_to_cop() */ a from *.t1 limit 10`, - `select a from %st1 limit 101`}, - - // index merge hint - {`create global binding using select /*+ use_index_merge(t1, c, d) */ * from *.t1 where c=1 or d=1`, - `select * from %st1 where c=1000 or d=1000`}, - {`create global binding using select /*+ no_index_merge() */ * from *.t1 where a=1 or b=1`, - `select * from %st1 where a=1000 or b=1000`}, - - // join type hint - {`create global binding using select /*+ hash_join(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from %st1, t2 where t1.a=t2.a`}, - {`create global binding using select /*+ hash_join(t2) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from t1, %st2 where t1.a=t2.a`}, - {`create global binding using select /*+ hash_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - {`create global binding using select /*+ hash_join_build(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from t1, %st2 where t1.a=t2.a`}, - {`create global binding using select /*+ hash_join_probe(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from t1, %st2 where t1.a=t2.a`}, - {`create global binding using select /*+ merge_join(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from %st1, t2 where t1.a=t2.a`}, - {`create global binding using select /*+ merge_join(t2) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from t1, %st2 where t1.a=t2.a`}, - {`create global binding using select /*+ merge_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - {`create global binding using select /*+ inl_join(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from %st1, t2 where t1.a=t2.a`}, - {`create global binding using select /*+ inl_join(t2) */ * from *.t1, *.t2 where t1.a=t2.a`, - `select * from t1, %st2 where t1.a=t2.a`}, - {`create global binding using select /*+ inl_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - - // no join type hint - {`create global binding using select /*+ no_hash_join(t1) */ * from *.t1, *.t2 where t1.b=t2.b`, - `select * from %st1, t2 where t1.b=t2.b`}, - {`create global binding using select /*+ no_hash_join(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, - `select * from t1, %st2 where t1.c=t2.c`}, - {`create global binding using select /*+ no_hash_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - {`create global binding using select /*+ no_merge_join(t1) */ * from *.t1, *.t2 where t1.b=t2.b`, - `select * from %st1, t2 where t1.b=t2.b`}, - {`create global binding using select /*+ no_merge_join(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, - `select * from t1, %st2 where t1.c=t2.c`}, - {`create global binding using select /*+ no_merge_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - {`create global binding using select /*+ no_index_join(t1) */ * from *.t1, *.t2 where t1.b=t2.b`, - `select * from %st1, t2 where t1.b=t2.b`}, - {`create global binding using select /*+ no_index_join(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, - `select * from t1, %st2 where t1.c=t2.c`}, - {`create global binding using select /*+ no_index_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - - // join order hint - {`create global binding using select /*+ leading(t2) */ * from *.t1, *.t2 where t1.b=t2.b`, - `select * from %st1, t2 where t1.b=t2.b`}, - {`create global binding using select /*+ leading(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, - `select * from t1, %st2 where t1.c=t2.c`}, - {`create global binding using select /*+ leading(t2, t1) */ * from *.t1, *.t2 where t1.c=t2.c`, - `select * from t1, %st2 where t1.c=t2.c`}, - {`create global binding using select /*+ leading(t1, t2) */ * from *.t1, *.t2 where t1.c=t2.c`, - `select * from t1, %st2 where t1.c=t2.c`}, - {`create global binding using select /*+ leading(t1) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - {`create global binding using select /*+ leading(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - {`create global binding using select /*+ leading(t2,t3) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - {`create global binding using select /*+ leading(t2,t3,t1) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, - `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, - } { - removeAllBindings(tk, true) - tk.MustExec(c.binding) - for _, currentDB := range []string{"db1", "db2", "db3"} { - tk.MustExec(`use ` + currentDB) - for _, db := range []string{"db1.", "db2.", "db3.", ""} { - query := fmt.Sprintf(c.qTemplate, db) - tk.MustExec(query) - tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning - tk.MustExec(query) - tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1")) - } - } - } -} diff --git a/pkg/bindinfo/global_handle.go b/pkg/bindinfo/global_handle.go index e033d69089032..9166711e6b7ba 100644 --- a/pkg/bindinfo/global_handle.go +++ b/pkg/bindinfo/global_handle.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/tidb/pkg/util/logutil" utilparser "github.com/pingcap/tidb/pkg/util/parser" "go.uber.org/zap" + "golang.org/x/sync/singleflight" ) // GlobalBindingHandle is used to handle all global sql bind operations. @@ -118,6 +119,9 @@ type globalBindingHandle struct { // invalidBindings indicates the invalid bindings found during querying. // A binding will be deleted from this map, after 2 bind-lease, after it is dropped from the kv. invalidBindings *invalidBindingCache + + // syncBindingSingleflight is used to synchronize the execution of `LoadFromStorageToCache` method. + syncBindingSingleflight singleflight.Group } // Lease influences the duration of loading bind info and handling invalid bind. @@ -163,7 +167,7 @@ func (h *globalBindingHandle) setCache(c FuzzyBindingCache) { func (h *globalBindingHandle) Reset() { h.lastUpdateTime.Store(types.ZeroTimestamp) h.invalidBindings = newInvalidBindingCache() - h.setCache(newFuzzyBindingCache()) + h.setCache(newFuzzyBindingCache(h.LoadBindingsFromStorage)) variable.RegisterStatistics(h) } @@ -183,7 +187,7 @@ func (h *globalBindingHandle) LoadFromStorageToCache(fullLoad bool) (err error) if fullLoad { lastUpdateTime = types.ZeroTimestamp timeCondition = "" - newCache = newFuzzyBindingCache() + newCache = newFuzzyBindingCache(h.LoadBindingsFromStorage) } else { lastUpdateTime = h.getLastUpdateTime() timeCondition = fmt.Sprintf("WHERE update_time>'%s'", lastUpdateTime.String()) @@ -626,7 +630,7 @@ func (*paramMarkerChecker) Leave(in ast.Node) (ast.Node, bool) { // Clear resets the bind handle. It is only used for test. func (h *globalBindingHandle) Clear() { - h.setCache(newFuzzyBindingCache()) + h.setCache(newFuzzyBindingCache(h.LoadBindingsFromStorage)) h.setLastUpdateTime(types.ZeroTimestamp) h.invalidBindings.reset() } @@ -647,7 +651,6 @@ func (h *globalBindingHandle) callWithSCtx(wrapTxn bool, f func(sctx sessionctx. h.sPool.Put(resource) } }() - sctx := resource.(sessionctx.Context) if wrapTxn { if _, err = exec(sctx, "BEGIN PESSIMISTIC"); err != nil { @@ -682,3 +685,46 @@ func (h *globalBindingHandle) Stats(_ *variable.SessionVars) (map[string]any, er m[lastPlanBindingUpdateTime] = h.getLastUpdateTime().String() return m, nil } + +// LoadBindingsFromStorageToCache loads global bindings from storage to the memory cache. +func (h *globalBindingHandle) LoadBindingsFromStorage(sqlDigest string) (Bindings, error) { + if sqlDigest == "" { + return nil, nil + } + bindings, err, _ := h.syncBindingSingleflight.Do(sqlDigest, func() (any, error) { + return h.loadBindingsFromStorageInternal(sqlDigest) + }) + if err != nil { + logutil.BgLogger().Warn("fail to LoadBindingsFromStorageToCache", zap.Error(err)) + return nil, err + } + if bindings == nil { + return nil, nil + } + return bindings.(Bindings), err +} + +func (h *globalBindingHandle) loadBindingsFromStorageInternal(sqlDigest string) (any, error) { + var bindings Bindings + err := h.callWithSCtx(false, func(sctx sessionctx.Context) error { + rows, _, err := execRows(sctx, "SELECT original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source, sql_digest, plan_digest FROM mysql.bind_info where sql_digest = ?", sqlDigest) + if err != nil { + return err + } + bindings = make([]Binding, 0, len(rows)) + for _, row := range rows { + // Skip the builtin record which is designed for binding synchronization. + if row.GetString(0) == BuiltinPseudoSQL4BindLock { + continue + } + _, binding, err := newBinding(sctx, row) + if err != nil { + logutil.BgLogger().Warn("failed to generate bind record from data row", zap.String("category", "sql-bind"), zap.Error(err)) + continue + } + bindings = append(bindings, binding) + } + return nil + }) + return bindings, err +} diff --git a/pkg/bindinfo/session_handle.go b/pkg/bindinfo/session_handle.go index afa4736d8971b..5b9f933a628b1 100644 --- a/pkg/bindinfo/session_handle.go +++ b/pkg/bindinfo/session_handle.go @@ -61,7 +61,7 @@ type sessionBindingHandle struct { // NewSessionBindingHandle creates a new SessionBindingHandle. func NewSessionBindingHandle() SessionBindingHandle { sessionHandle := &sessionBindingHandle{} - sessionHandle.ch = newFuzzyBindingCache() + sessionHandle.ch = newFuzzyBindingCache(nil) return sessionHandle } diff --git a/pkg/bindinfo/tests/BUILD.bazel b/pkg/bindinfo/tests/BUILD.bazel index 033d3b6ca0440..d64cd29e36d0e 100644 --- a/pkg/bindinfo/tests/BUILD.bazel +++ b/pkg/bindinfo/tests/BUILD.bazel @@ -2,14 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test") go_test( name = "tests_test", - timeout = "short", + timeout = "moderate", srcs = [ "bind_test.go", "main_test.go", ], flaky = True, race = "on", - shard_count = 16, + shard_count = 18, deps = [ "//pkg/bindinfo", "//pkg/bindinfo/internal", @@ -23,6 +23,7 @@ go_test( "//pkg/util", "//pkg/util/parser", "//pkg/util/stmtsummary", + "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", "@org_uber_go_goleak//:goleak", ], diff --git a/pkg/bindinfo/tests/bind_test.go b/pkg/bindinfo/tests/bind_test.go index 17329931f041b..d3b9c4d2bc1f5 100644 --- a/pkg/bindinfo/tests/bind_test.go +++ b/pkg/bindinfo/tests/bind_test.go @@ -20,6 +20,7 @@ import ( "strconv" "testing" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/bindinfo/internal" "github.com/pingcap/tidb/pkg/bindinfo/norm" @@ -880,3 +881,232 @@ func TestNormalizeStmtForBinding(t *testing.T) { require.Equal(t, test.digest, digest) } } + +func showBinding(tk *testkit.TestKit, showStmt string) [][]any { + rows := tk.MustQuery(showStmt).Sort().Rows() + result := make([][]any, len(rows)) + for i, r := range rows { + result[i] = append(result[i], r[:4]...) + result[i] = append(result[i], r[8:10]...) + } + return result +} + +func removeAllBindings(tk *testkit.TestKit, global bool) { + scope := "session" + if global { + scope = "global" + } + res := showBinding(tk, fmt.Sprintf("show %v bindings", scope)) + for _, r := range res { + if r[4] == "builtin" { + continue + } + tk.MustExec(fmt.Sprintf("drop %v binding for sql digest '%v'", scope, r[5])) + } + tk.MustQuery(fmt.Sprintf("show %v bindings", scope)).Check(testkit.Rows()) // empty +} + +func testFuzzyBindingHints(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`use test`) + + for _, db := range []string{"db1", "db2", "db3"} { + tk.MustExec(`create database ` + db) + tk.MustExec(`use ` + db) + tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + tk.MustExec(`create table t3 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + } + tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) + + for _, c := range []struct { + binding string + qTemplate string + }{ + // use index + {`create global binding using select /*+ use_index(t1, c) */ * from *.t1 where a=1`, + `select * from %st1 where a=1000`}, + {`create global binding using select /*+ use_index(t1, c) */ * from *.t1 where d<1`, + `select * from %st1 where d<10000`}, + {`create global binding using select /*+ use_index(t1, c) */ * from *.t1, *.t2 where t1.d<1`, + `select * from %st1, t2 where t1.d<100`}, + {`create global binding using select /*+ use_index(t1, c) */ * from *.t1, *.t2 where t1.d<1`, + `select * from t1, %st2 where t1.d<100`}, + {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2 where t1.d<1`, + `select * from %st1, t2 where t1.d<100`}, + {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2 where t1.d<1`, + `select * from t1, %st2 where t1.d<100`}, + {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2, *.t3 where t1.d<1`, + `select * from %st1, t2, t3 where t1.d<100`}, + {`create global binding using select /*+ use_index(t1, c), use_index(t2, a) */ * from *.t1, *.t2, *.t3 where t1.d<1`, + `select * from t1, t2, %st3 where t1.d<100`}, + + // ignore index + {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b=1`, + `select * from %st1 where b=1000`}, + {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b>1`, + `select * from %st1 where b>1000`}, + {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b in (1,2)`, + `select * from %st1 where b in (1)`}, + {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b in (1,2)`, + `select * from %st1 where b in (1,2,3,4,5)`}, + + // order index hint + {`create global binding using select /*+ order_index(t1, a) */ a from *.t1 where a<10 order by a limit 10`, + `select a from %st1 where a<10000 order by a limit 10`}, + {`create global binding using select /*+ order_index(t1, b) */ b from *.t1 where b>10 order by b limit 1111`, + `select b from %st1 where b>2 order by b limit 10`}, + + // no order index hint + {`create global binding using select /*+ no_order_index(t1, c) */ c from *.t1 where c<10 order by c limit 10`, + `select c from %st1 where c<10000 order by c limit 10`}, + {`create global binding using select /*+ no_order_index(t1, d) */ d from *.t1 where d>10 order by d limit 1111`, + `select d from %st1 where d>2 order by d limit 10`}, + + // agg hint + {`create global binding using select /*+ hash_agg() */ count(*) from *.t1 group by a`, + `select count(*) from %st1 group by a`}, + {`create global binding using select /*+ stream_agg() */ count(*) from *.t1 group by b`, + `select count(*) from %st1 group by b`}, + + // to_cop hint + {`create global binding using select /*+ agg_to_cop() */ sum(a) from *.t1`, + `select sum(a) from %st1`}, + {`create global binding using select /*+ limit_to_cop() */ a from *.t1 limit 10`, + `select a from %st1 limit 101`}, + + // index merge hint + {`create global binding using select /*+ use_index_merge(t1, c, d) */ * from *.t1 where c=1 or d=1`, + `select * from %st1 where c=1000 or d=1000`}, + {`create global binding using select /*+ no_index_merge() */ * from *.t1 where a=1 or b=1`, + `select * from %st1 where a=1000 or b=1000`}, + + // join type hint + {`create global binding using select /*+ hash_join(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from %st1, t2 where t1.a=t2.a`}, + {`create global binding using select /*+ hash_join(t2) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from t1, %st2 where t1.a=t2.a`}, + {`create global binding using select /*+ hash_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + {`create global binding using select /*+ hash_join_build(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from t1, %st2 where t1.a=t2.a`}, + {`create global binding using select /*+ hash_join_probe(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from t1, %st2 where t1.a=t2.a`}, + {`create global binding using select /*+ merge_join(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from %st1, t2 where t1.a=t2.a`}, + {`create global binding using select /*+ merge_join(t2) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from t1, %st2 where t1.a=t2.a`}, + {`create global binding using select /*+ merge_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + {`create global binding using select /*+ inl_join(t1) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from %st1, t2 where t1.a=t2.a`}, + {`create global binding using select /*+ inl_join(t2) */ * from *.t1, *.t2 where t1.a=t2.a`, + `select * from t1, %st2 where t1.a=t2.a`}, + {`create global binding using select /*+ inl_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + + // no join type hint + {`create global binding using select /*+ no_hash_join(t1) */ * from *.t1, *.t2 where t1.b=t2.b`, + `select * from %st1, t2 where t1.b=t2.b`}, + {`create global binding using select /*+ no_hash_join(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, + `select * from t1, %st2 where t1.c=t2.c`}, + {`create global binding using select /*+ no_hash_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + {`create global binding using select /*+ no_merge_join(t1) */ * from *.t1, *.t2 where t1.b=t2.b`, + `select * from %st1, t2 where t1.b=t2.b`}, + {`create global binding using select /*+ no_merge_join(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, + `select * from t1, %st2 where t1.c=t2.c`}, + {`create global binding using select /*+ no_merge_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + {`create global binding using select /*+ no_index_join(t1) */ * from *.t1, *.t2 where t1.b=t2.b`, + `select * from %st1, t2 where t1.b=t2.b`}, + {`create global binding using select /*+ no_index_join(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, + `select * from t1, %st2 where t1.c=t2.c`}, + {`create global binding using select /*+ no_index_join(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + + // join order hint + {`create global binding using select /*+ leading(t2) */ * from *.t1, *.t2 where t1.b=t2.b`, + `select * from %st1, t2 where t1.b=t2.b`}, + {`create global binding using select /*+ leading(t2) */ * from *.t1, *.t2 where t1.c=t2.c`, + `select * from t1, %st2 where t1.c=t2.c`}, + {`create global binding using select /*+ leading(t2, t1) */ * from *.t1, *.t2 where t1.c=t2.c`, + `select * from t1, %st2 where t1.c=t2.c`}, + {`create global binding using select /*+ leading(t1, t2) */ * from *.t1, *.t2 where t1.c=t2.c`, + `select * from t1, %st2 where t1.c=t2.c`}, + {`create global binding using select /*+ leading(t1) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + {`create global binding using select /*+ leading(t2) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + {`create global binding using select /*+ leading(t2,t3) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + {`create global binding using select /*+ leading(t2,t3,t1) */ * from *.t1, *.t2, *.t3 where t1.a=t2.a and t3.b=t2.b`, + `select * from t1, %st2, t3 where t1.a=t2.a and t3.b=t2.b`}, + } { + removeAllBindings(tk, true) + tk.MustExec(c.binding) + for _, currentDB := range []string{"db1", "db2", "db3"} { + tk.MustExec(`use ` + currentDB) + for _, db := range []string{"db1.", "db2.", "db3.", ""} { + query := fmt.Sprintf(c.qTemplate, db) + tk.MustExec(query) + tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + tk.MustExec(query) + tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1")) + } + } + } +} + +func TestFuzzyBindingHints(t *testing.T) { + testFuzzyBindingHints(t) +} + +func TestFuzzyBindingHintsWithSourceReturning(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec(`use test`) + + for _, db := range []string{"db1", "db2", "db3"} { + tk.MustExec(`create database ` + db) + tk.MustExec(`use ` + db) + tk.MustExec(`create table t1 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + tk.MustExec(`create table t2 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + tk.MustExec(`create table t3 (a int, b int, c int, d int, key(a), key(b), key(c), key(d))`) + } + tk.MustExec(`set @@tidb_opt_enable_fuzzy_binding=1`) + + for _, c := range []struct { + binding string + qTemplate string + }{ + // use index + {`create global binding using select /*+ use_index(t1, c) */ * from *.t1 where a=1`, + `select * from %st1 where a=1000`}, + + // ignore index + {`create global binding using select /*+ ignore_index(t1, b) */ * from *.t1 where b=1`, + `select * from %st1 where b=1000`}, + + // order index hint + {`create global binding using select /*+ order_index(t1, a) */ a from *.t1 where a<10 order by a limit 10`, + `select a from %st1 where a<10000 order by a limit 10`}, + } { + removeAllBindings(tk, true) + tk.MustExec(c.binding) + for _, currentDB := range []string{"db1", "db2", "db3"} { + tk.MustExec(`use ` + currentDB) + for _, db := range []string{"db1.", "db2.", "db3.", ""} { + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/bindinfo/get_binding_return_nil", `return()`)) + query := fmt.Sprintf(c.qTemplate, db) + tk.MustExec(query) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/bindinfo/get_binding_return_nil")) + tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + tk.MustExec(query) + tk.MustQuery(`select @@last_plan_from_binding`).Check(testkit.Rows("1")) + } + } + } +}