diff --git a/build/BUILD.bazel b/build/BUILD.bazel new file mode 100644 index 0000000000000..5ee4c194e4fbd --- /dev/null +++ b/build/BUILD.bazel @@ -0,0 +1,154 @@ +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//go:def.bzl", "nogo") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("//build/linter/staticcheck:def.bzl", "staticcheck_analyzers") + +bool_flag( + name = "with_nogo_flag", + build_setting_default = False, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_nogo", + flag_values = { + ":with_nogo_flag": "true", + }, + visibility = ["//visibility:public"], +) + +STATICHECK_ANALYZERS = [ + "S1000", + "S1001", + "S1002", + "S1003", + "S1004", + "S1005", + "S1006", + "S1007", + "S1008", + "S1009", + "S1010", + "S1011", + "S1012", + "S1016", + "S1017", + "S1018", + "S1019", + "S1020", + "S1021", + "S1023", + "S1024", + "S1025", + "S1028", + "S1029", + "S1030", + "S1031", + "S1032", + "S1033", + "S1034", + "S1035", + "S1036", + "S1037", + "S1038", + "S1039", + "S1040", + "SA1019", + "SA1029", + "SA2000", + "SA2001", + "SA2003", + "SA3000", + "SA3001", + "SA4009", + "SA5000", + "SA5001", + "SA5002", + "SA5003", + "SA5004", + "SA5005", + "SA5007", + "SA5008", + "SA5009", + "SA5010", + #"SA5011", + "SA5012", + "SA6000", + "SA6001", + "SA6005", + "U1000", +] + +nogo( + name = "tidb_nogo", + config = ":nogo_config.json", + visibility = ["//visibility:public"], # must have public visibility + deps = [ + "@org_golang_x_tools//go/analysis/passes/asmdecl:go_default_library", + "@org_golang_x_tools//go/analysis/passes/assign:go_default_library", + "@org_golang_x_tools//go/analysis/passes/atomic:go_default_library", + "@org_golang_x_tools//go/analysis/passes/atomicalign:go_default_library", + "@org_golang_x_tools//go/analysis/passes/bools:go_default_library", + "@org_golang_x_tools//go/analysis/passes/buildssa:go_default_library", + "@org_golang_x_tools//go/analysis/passes/buildtag:go_default_library", + # https://github.com/bazelbuild/rules_go/issues/2396 + # "@org_golang_x_tools//go/analysis/passes/cgocall:go_default_library", + "@org_golang_x_tools//go/analysis/passes/composite:go_default_library", + "@org_golang_x_tools//go/analysis/passes/copylock:go_default_library", + "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_default_library", + "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_default_library", + "@org_golang_x_tools//go/analysis/passes/errorsas:go_default_library", + "@org_golang_x_tools//go/analysis/passes/fieldalignment:go_default_library", + "@org_golang_x_tools//go/analysis/passes/findcall:go_default_library", + "@org_golang_x_tools//go/analysis/passes/httpresponse:go_default_library", + "@org_golang_x_tools//go/analysis/passes/ifaceassert:go_default_library", + "@org_golang_x_tools//go/analysis/passes/inspect:go_default_library", + "@org_golang_x_tools//go/analysis/passes/loopclosure:go_default_library", + "@org_golang_x_tools//go/analysis/passes/lostcancel:go_default_library", + "@org_golang_x_tools//go/analysis/passes/nilfunc:go_default_library", + "@org_golang_x_tools//go/analysis/passes/nilness:go_default_library", + "@org_golang_x_tools//go/analysis/passes/pkgfact:go_default_library", + "@org_golang_x_tools//go/analysis/passes/printf:go_default_library", + "@org_golang_x_tools//go/analysis/passes/shift:go_default_library", + "@org_golang_x_tools//go/analysis/passes/sortslice:go_default_library", + "@org_golang_x_tools//go/analysis/passes/stdmethods:go_default_library", + "@org_golang_x_tools//go/analysis/passes/stringintconv:go_default_library", + "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library", + "@org_golang_x_tools//go/analysis/passes/testinggoroutine:go_default_library", + "@org_golang_x_tools//go/analysis/passes/tests:go_default_library", + "@org_golang_x_tools//go/analysis/passes/timeformat:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library", + "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library", + "//build/linter/asciicheck", + "//build/linter/bodyclose", + "//build/linter/durationcheck", + "//build/linter/etcdconfig", + "//build/linter/exportloopref", + "//build/linter/forcetypeassert", + "//build/linter/gofmt", + "//build/linter/gci", + "//build/linter/gosec", + "//build/linter/ineffassign", + "//build/linter/makezero", + "//build/linter/mirror", + "//build/linter/misspell", + "//build/linter/noloopclosure", + "//build/linter/prealloc", + "//build/linter/predeclared", + "//build/linter/unconvert", + "//build/linter/rowserrcheck", + ] + staticcheck_analyzers(STATICHECK_ANALYZERS) + + select({ + "//build:with_nogo": [ + "//build/linter/allrevive", + "//build/linter/errcheck", + "//build/linter/filepermission", + "//build/linter/lll", + "//build/linter/revive", + ], + "//conditions:default": [], + }), +) diff --git a/build/linter/etcdconfig/BUILD.bazel b/build/linter/etcdconfig/BUILD.bazel new file mode 100644 index 0000000000000..beff516fd4837 --- /dev/null +++ b/build/linter/etcdconfig/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "etcdconfig", + srcs = ["analyzer.go"], + importpath = "github.com/pingcap/tidb/build/linter/etcdconfig", + visibility = ["//visibility:public"], + deps = [ + "@org_golang_x_tools//go/analysis", + "@org_golang_x_tools//go/analysis/passes/inspect", + ], +) diff --git a/build/linter/etcdconfig/analyzer.go b/build/linter/etcdconfig/analyzer.go new file mode 100644 index 0000000000000..566a920a2f10e --- /dev/null +++ b/build/linter/etcdconfig/analyzer.go @@ -0,0 +1,100 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdconfig + +import ( + "go/ast" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" +) + +// Analyzer is the analyzer struct of unconvert. +var Analyzer = &analysis.Analyzer{ + Name: "etcdconfig", + Doc: "Check necessary fields of etcd config", + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +const ( + configPackagePath = "go.etcd.io/etcd/client/v3" + configPackageName = "clientv3" + configStructName = "Config" +) + +// Adapted from https://github.com/mdempsky/unconvert/blob/beb68d938016d2dec1d1b078054f4d3db25f97be/unconvert.go#L371-L414. +func run(pass *analysis.Pass) (interface{}, error) { + for _, file := range pass.Files { + packageName := "" + for _, spec := range file.Imports { + if spec.Path.Value != "\""+configPackagePath+"\"" { + continue + } + if spec.Name != nil { + packageName = spec.Name.Name + } else { + packageName = configPackageName + } + } + if packageName == "" { + continue + } + + for _, decl := range file.Decls { + ast.Inspect(decl, func(n ast.Node) bool { + lit, ok := n.(*ast.CompositeLit) + if !ok { + return true + } + tp, ok := lit.Type.(*ast.SelectorExpr) + if !ok { + return true + } + litPackage, ok := tp.X.(*ast.Ident) + if !ok { + return true + } + if litPackage.Name != packageName { + return true + } + if tp.Sel.Name != configStructName { + return true + } + + found := false + for _, field := range lit.Elts { + kv, ok := field.(*ast.KeyValueExpr) + if !ok { + continue + } + key, ok := kv.Key.(*ast.Ident) + if !ok { + continue + } + if key.Name == "AutoSyncInterval" { + found = true + break + } + } + if !found { + pass.Reportf(lit.Pos(), "missing field AutoSyncInterval") + } + return true + }) + } + } + return nil, nil +} diff --git a/build/nogo_config.json b/build/nogo_config.json new file mode 100644 index 0000000000000..8e67e7f621666 --- /dev/null +++ b/build/nogo_config.json @@ -0,0 +1,1172 @@ +{ + "all_revive": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "/rules_go_work-*": "ignore generated code", + ".*_/testmain\\.go$": "ignore code" + } + }, + "asciicheck": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "br/pkg/lightning/web/res_vfsdata.go": "ignore code" + } + }, + "asmdecl": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "assign": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "atomic": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "atomicalign": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "bodyclose": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "bools": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "buildtag": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "printf": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "unreachable": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "composites": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "br/pkg/glue/console_glue_test.go": "ignore code", + "br/pkg/restore/db_test.go": "ignore code", + ".*_/testmain\\.go$": "ignore code" + } + }, + "copylocks": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "/cgo/": "ignore cgo code" + } + }, + "ctrlflow": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "deadcode": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "deepequalerrors": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "durationcheck": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + "/rules_go_work-*": "ignore generated code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "errorsas": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "errcheck": { + "exclude_files": { + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + ".*_test\\.go$": "ignore generated code", + "util/logutil": "ignore util/logutil code", + "tools/": "ignore tools code", + "/src/net/conf.go": "ignore code", + "/rules_go_work-*": "ignore generated code", + "GOROOT/": "ignore code", + "/parser/": "ignore code", + ".*_/testmain\\.go$": "ignore code" + } + }, + "exportloopref": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "filepermission": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + ".*_/testmain\\.go$": "ignore code" + } + }, + "fieldalignment": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + ".*_/testmain\\.go$": "ignore code", + ".*_test\\.go$": "ignore test code" + }, + "only_files": { + "util/checksum": "util/checksum code", + "util/processinfo.go": "util/processinfo.go code", + "util/cpuprofile/": "util/cpuprofile/ code", + "util/cteutil/": "util/cteutil/ code", + "util/dbutil/": "util/dbutil/ code", + "util/deadlockhistory/": "util/deadlockhistory/ code", + "util/domainutil/": "util/domainutil/ code", + "util/encrypt/": "util/encrypt/ code", + "util/etcd/": "util/etcd/ code", + "util/expensivequery/": "util/expensivequery/ code", + "util/filter/": "util/filter/ code", + "util/importer/": "util/importer/ code", + "util/keydecoder/": "util/keydecoder/ code", + "util/kvcache/": "util/kvcache/ code", + "util/localpool/": "util/localpool/ code", + "util/mathutil/": "util/mathutil/ code", + "util/memory/": "util/memory/ code", + "util/mock/": "util/mock/ code", + "util/mvmap/": "util/mvmap/ code", + "util/profile/": "util/profile/ code", + "util/ranger/": "util/ranger/ code", + "util/regexpr-router/": "util/regexpr-router/ code", + "util/schemacmp/": "util/schemacmp/ code", + "util/sqlexec/": "util/sqlexec/ code", + "util/stringutil/": "util/stringutil/ code", + "util/table-router/": "util/table-router/ code", + "util/timeutil/": "util/timeutil/ code", + "util/topsql/": "util/topsql/ code", + "util/tracing/": "util/tracing/ code", + "util/trxevents/": "util/trxevents/ code", + "util/watcher/": "util/watcher/ code", + "util/gctuner": "util/gctuner", + "store/mockstore/unistore/util": "store/mockstore/unistore/util code", + "ddl/util/": "ddl/util code" + } + }, + "findcall": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "forcetypeassert": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + }, + "only_files": { + "util/gctuner": "only for util/gctuner", + "br/pkg/lightning/mydump/": "only for br/pkg/lightning/mydump/", + "br/pkg/lightning/importer/opts": "only for br/pkg/lightning/importer/opts", + "executor/aggregate.go": "only for executor/aggregate.go", + "types/json_binary_functions.go": "only for types/json_binary_functions.go", + "types/json_binary_test.go": "only for types/json_binary_test.go", + "ddl/backfilling.go": "only for ddl/backfilling.go", + "ddl/column.go": "only for ddl/column.go", + "ddl/index.go": "only for ddl/index.go", + "ddl/ingest/": "only for ddl/ingest/", + "util/cgroup": "only for util/cgroup code", + "server/conn.go": "only for server/conn.go", + "server/conn_stmt.go": "only for server/conn_stmt.go", + "server/conn_test.go": "only for server/conn_test.go", + "planner/core/plan.go": "only for planner/core/plan.go", + "errno/": "only for errno/", + "extension/": "extension code" + } + }, + "gofmt": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "/cgo/": "ignore cgo code", + "/rules_go_work-*": "ignore generated code", + ".*test_/testmain\\.go$": "ignore generated code", + ".*failpoint_binding__.go$": "ignore generated code" + } + }, + "gci": { + "exclude_files": { + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "/cgo/": "ignore cgo code", + ".*\\.pb\\.go$": "generated code", + "/rules_go_work-*": "ignore generated code", + ".*test_/testmain\\.go$": "ignore generated code", + ".*failpoint_binding__.go$": "ignore generated code", + "util/printer/printer.go": "ignore util/printer code", + "parser/parser.go": "ignore parser code", + "parser/hintparser.go": "ignore parser/hintparser code" + } + }, + "gosec": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + "parser/goyacc/": "ignore goyacc code", + ".*_test\\.go$": "ignore generated code", + "/cgo/": "ignore cgo code", + "/rules_go_work-*": "ignore generated code", + "tools/check/ut.go": "ignore tools/check code", + "tools/check/xprog.go": "ignore tools/check code", + "cmd/pluginpkg/pluginpkg.go": "ignore cmd/pluginpkg code", + "tools/check/xprog.go:": "ignore tools/check code", + "cmd/explaintest/main.go": "ignore cmd/explaintest code", + "GOROOT/": "ignore code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "httpresponse": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "ifaceassert": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "ineffassign": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "/cgo/": "no need to vet cgo code" + } + }, + "inspect": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "loopclosure": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "lostcancel": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "lll": { + "exclude_files": { + "/cgo/": "ignore cgo code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + ".*pb\\.go$": "ignore generated code", + "br/pkg/streamhelper/.*_test\\.go$": "ignore test code", + "br/pkg/errors/errors.go": "ignore error", + "br/pkg/rtree/logging_test.go": "ignore rtree code", + "br/pkg/logutil/logging_test.go": "ignore logutil code", + "br/pkg/storage/": "ignore storage code", + "br/pkg/mock/storage/storage.go": "ignore storage code", + "br/pkg/utils/": "ignore utils code", + "br/pkg/version/version.go": "ignore version code", + "br/pkg/lightning/common/errors.go": "ignore error", + "br/pkg/lightning/config/config.go": "ignore config code", + "br/pkg/lightning/config/config_test.go": "ignore config code", + "br/pkg/lightning/config/global.go": "ignore global code", + "br/pkg/lightning/mydump/": "ignore mydump code", + "br/pkg/lightning/checkpoints/checkpoints.go": "ignore checkpoints code", + "br/pkg/lightning/checkpoints/glue_checkpoint.go": "ignore glue_checkpoint code", + "br/pkg/lightning/backend/local/": "ignore local code", + "br/pkg/lightning/backend/tidb/tidb.go": "ignore tidb code", + "br/pkg/lightning/backend/tidb/tidb_test.go": "ignore tidb code", + "br/pkg/lightning/checkpoints/checkpoints_test.go": "ignore checkpoints code", + "br/pkg/lightning/checkpoints/checkpoints_sql_test.go": "ignore checkpoints code", + "br/pkg/lightning/errormanager/errormanager_test.go": "ignore errormanager code", + "br/pkg/lightning/backend/kv/sql2kv_test.go": "ignore sql2kv code", + "br/pkg/stream/rewrite_meta_rawkv_test.go": "ignore rewrite_meta_rawkv code", + "br/pkg/lightning/lightning.go": "ignore lightning code", + "br/pkg/lightning/importer": "ignore importer code", + "br/cmd/tidb-lightning-ctl/main.go": "ignore main code", + "br/pkg/stream/rewrite_meta_rawkv.go": "ignore rewrite_meta_rawkv code", + "br/pkg/lightning/web/": "ignore web code", + "br/pkg/mock/": "ignore mock code", + "br/pkg/aws/ebs.go": "ignore ebs code", + "br/pkg/backup/": "ignore backup code", + "br/pkg/restore/": "ignore restore code", + "br/pkg/task/": "ignore task code", + "br/pkg/version/": "ignore version code", + "ddl/util/util.go": "ignore util code", + "ddl/syncer/syncer.go": "ignore syncer code", + "ddl/ingest/backend.go": "ignore backend code", + "ddl/ingest/backend_mgr.go": "ignore backend_mgr code", + "ddl/ingest/engine_mgr.go": "ignore engine_mgr code", + "util/column-mapping/column.go": "ignore error", + "parser/mysql/": "ignore mysql code", + "parser/auth": "ignore auth code", + "parser/charset/charset.go": "ignore charset code", + "parser/types/field_type.go": "ignore field_type code", + "parser/model/ddl.go": "ignore ddl code", + "parser/model/model.go": "ignore model code", + "parser/model/model_test.go": "ignore tables code", + "parser/ast": "ignore ast code", + "parser/parser.go": "ignore parser code", + "parser/hintparser.go": "ignore hintparser code", + "parser/yy_parser.go": "ignore yy_parser code", + "parser/digester.go": "ignore digester code", + "parser/consistent_test.go": "ignore consistent code", + "parser/bench_test.go": "ignore bench code", + "parser/parser_test.go": "ignore parser_test code", + "parser/digester_test.go": "ignore digester_test code", + "parser/hintparser_test.go": "ignore hintparser_test code", + "errno/errname.go": "ignore errname code", + "planner/funcdep/fd_graph.go": "ignore fd_graph code", + "planner/funcdep/doc.go": "ignore funcdep code", + "planner/util/path.go": "ignore path code", + "planner/memo/": "ignore memo code", + "ttl/cache/ttlstatus.go": "ignore ttlstatus code", + "ttl/ttlworker/": "ignore ttlworker code", + "ttl/sqlbuilder/sql_test.go": "ignore sql_test code", + "planner/core/": "ignore core code", + "planner/optimize.go": "ignore optimize code", + "planner/cascades/": "ignore cascades code", + "planner/funcdep/extract_fd_test.go": "ignore extract_fd code", + "planner/funcdep/only_full_group_by_test.go": "ignore only_full_group_by code", + "dumpling/export": "ignore export code", + "ddl/placement/": "ignore placement code" + }, + "only_files": { + "br/": "only for br code", + "ttl/": "only for ttl code", + "planner/": "only for planner code", + "parser/": "only for parser code", + "dumpling/": "only for dumpling code" + } + }, + "makezero": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "/cgo/": "ignore cgo code", + ".*_test\\.go$": "ignore generated code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "mirror": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "/cgo/": "ignore cgo code", + "external/": "no need to vet third party code", + "tools/": "ignore tools code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "misspell": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "/cgo/": "ignore cgo code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "nilfunc": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "nilness": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "/cgo/": "ignore cgo" + } + }, + "noloopclosure": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + }, + "only_files": { + "br/pkg/lightning/mydump/": "br/pkg/lightning/mydump/", + "br/pkg/lightning/importer/opts": "br/pkg/lightning/importer/opts", + "kv/": "kv code", + "util/memory": "util/memory", + "ddl/": "ddl", + "planner/": "planner", + "extension/": "extension code" + } + }, + "pkgfact": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "revive": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + "GOROOT/": "ignore code", + "/cgo/": "ignore cgo", + "tools/": "ignore tool code", + "parser/goyacc/": "ignore goyacc code", + "parser/ast/": "ignore parser/ast code", + "parser/test_driver/": "ignore parser/test_driver code", + "dumpling/export/sql_type.go": "please fix it", + ".*_test\\.go$": "ignore generated code", + ".*_generated\\.go$": "ignore generated code", + "planner/core/flat_plan.go": "planner/core/flat_plan.go", + "planner/core/resolve_indices.go": "planner/core/resolve_indices.go", + "planner/core/rule_result_reorder.go": "planner/core/rule_result_reorder.go", + "planner/core/rule_join_reorder_dp.go": "planner/core/rule_join_reorder_dp.go", + "planner/core/plan_cache_param.go": "planner/core/plan_cache_param.go", + "planner/core/foreign_key.go": "planner/core/foreign_key.go", + "planner/core/rule_predicate_push_down.go": "planner/core/rule_predicate_push_down.go", + "planner/core/rule_aggregation_push_down.go": "planner/core/rule_aggregation_push_down.go", + "planner/core/rule_max_min_eliminate.go": "planner/core/rule_max_min_eliminate.go", + "planner/core/rule_predicate_simplification.go": "planner/core/rule_predicate_simplification.go", + "planner/core/indexmerge_path.go": "planner/core/indexmerge_path.go", + "planner/core/rule_join_reorder.go": "planner/core/rule_join_reorder.go", + "planner/core/plan_cacheable_checker.go": "planner/core/plan_cacheable_checker.go", + "planner/core/rule_decorrelate.go": "planner/core/rule_decorrelate.go", + "planner/core/plan_cache_utils.go": "planner/core/plan_cache_utils.go", + "planner/core/rule_aggregation_skew_rewrite.go": "planner/core/rule_aggregation_skew_rewrite.go", + "planner/core/rule_topn_push_down.go": "planner/core/rule_topn_push_down.go", + "planner/core/plan_cost_ver1.go": "planner/core/plan_cost_ver1.go", + "planner/core/expression_rewriter.go": "planner/core/expression_rewriter.go", + "planner/core/find_best_task.go": "planner/core/find_best_task.go", + "planner/core/task.go": "planner/core/task.go", + "planner/core/preprocess.go": "planner/core/preprocess.go", + "planner/core/rule_partition_processor.go": "planner/core/rule_partition_processor.go", + "planner/core/exhaust_physical_plans.go": "planner/core/exhaust_physical_plans.go", + "planner/core/plan_cache_lru.go": "planner/core/plan_cache_lru.go", + "planner/core/optimizer.go": "planner/core/optimizer.go", + "planner/core/common_plans.go": "planner/core/common_plans.go", + "planner/core/plan_cost_ver2.go": "planner/core/plan_cost_ver2.go", + "planner/core/logical_plans.go": "planner/core/logical_plans.go", + "plugin/conn_ip_example/": "plugin/conn_ip_example/" + }, + "only_files": { + "autoid_service/": "autoid_service", + "bindinfo/": "bindinfo", + "br/pkg/lightning/": "br/pkg/lightning/", + "executor/aggregate.go": "executor/aggregate.go", + "executor/analyze_utils.go": "executor/analyze_utils.go", + "types/json_binary_functions.go": "types/json_binary_functions.go", + "types/json_binary_test.go": "types/json_binary_test.go", + "ddl/backfilling.go": "ddl/backfilling.go", + "ddl/column.go": "ddl/column.go", + "ddl/index.go": "ddl/index.go", + "ddl/ttl.go": "ddl/ttl.go", + "ddl/ttl_test.go": "ddl/ttl_test.go", + "ddl/job_table.go": "ddl/job_table.go", + "ddl/ingest/": "ddl/ingest/", + "ddl/internal/": "ddl/internal/", + "ddl/syncer/": "ddl/syncer/", + "expression/builtin_cast.go": "expression/builtin_cast code", + "server/conn.go": "server/conn.go", + "server/conn_stmt.go": "server/conn_stmt.go", + "server/conn_test.go": "server/conn_test.go", + "planner/core/rule_partition_eliminate.go": "planner/core/rule_partition_eliminate code", + "distsql/": "ignore distsql code", + "disttask": "disttask code", + "dumpling/export": "dumpling/export code", + "lock/": "lock file", + "errno/": "errno code", + "session/": "session code", + "structure/": "structure code", + "telemetry/": "telemetry code", + "parser/": "parser code", + "planner/": "planner code", + "plugin/": "plugin code", + "util/": "util code", + "meta/": "parser code", + "extension/": "extension code", + "resourcemanager/": "resourcemanager code", + "keyspace/": "keyspace code", + "owner/": "owner code", + "timer/": "timer code" + } + }, + "shift": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "sortslice": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "stdmethods": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "stringintconv": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "structtag": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "testinggoroutine": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "tests": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "unconvert": { + "exclude_files": { + "external/": "no need to vet third party code", + ".*\\.pb\\.go$": "generated code", + "parser/parser.go": "generated code", + "/cgo/": "no need to vet third party code for cgo", + "br/pkg/lightning/common/conn.go": "ignore: to fix it", + "br/pkg/lightning/common/storage_unix.go": "ignore: to fix it", + "br/pkg/task/common.go": "ignore: to fix it", + ".*_generated\\.go$": "ignore generated code" + } + }, + "unmarshal": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "unsafeptr": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "parser/digester.go": "ignore code" + } + }, + "unusedresult": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "parser/digester_test.go": "ignore code" + } + }, + "rowserrcheck": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "server/tidb_test.go": "ignore test code", + "server/tidb_serial_test.go": "ignore test code", + "server/statistics_handler_test.go": "ignore test code", + "server/server_test.go": "ignore test code", + "server/optimize_trace_test.go": "ignore test code", + "server/plan_replayer_test.go": "ignore test code" + } + }, + "S1000": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1001": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1002": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1003": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1004": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1005": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1006": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1007": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1008": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1009": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1010": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1011": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1012": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1013": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1014": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1015": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1016": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1017": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1018": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1019": { + "exclude_files": { + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "parser/parser.go": "ignore code" + } + }, + "S1020": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1021": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "tools/check/ut.go": "ignore code" + } + }, + "S1022": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1023": { + "exclude_files": { + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "parser/parser.go": "ignore code" + } + }, + "S1024": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1025": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1026": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1027": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1028": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1029": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1030": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1031": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1032": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1033": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1034": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1035": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1036": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1037": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1038": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1039": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "S1040": { + "exclude_files": { + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "parser/parser.go": "ignore generated code" + } + }, + "SA1019": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "/build/": "no need to linter code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + ".*_test\\.go$": "ignore test code", + "br/pkg/restore/split/client.go": "github.com/golang/protobuf deprecated", + "br/pkg/streamhelper/advancer_cliext.go": "github.com/golang/protobuf deprecated", + "br/pkg/lightning/checkpoints/checkpoints.go": "cfg.TikvImporter.Addr is deprecated", + "br/pkg/lightning/checkpoints/glue_checkpoint.go": "cfg.TikvImporter.Addr is deprecated", + "br/pkg/lightning/backend/local/local.go": "grpc Compressor/Decompressor is deprecated", + "br/pkg/lightning/backend/local/compress.go": "grpc Compressor/Decompressor is deprecated" + }, + "only_files": { + "expression/bench_test.go": "expression/bench_test.go", + "domain/": "domain code", + "util/gctuner": "util/gctuner", + "util/cgroup": "util/cgroup code", + "util/watcher": "util/watcher", + "br/pkg/": "br/pkg", + "executor/aggregate.go": "executor/aggregate.go", + "types/": "types", + "ddl/": "enable to ddl", + "expression/builtin_cast.go": "enable expression/builtin_cast.go", + "planner/core/plan.go": "planner/core/plan.go", + "extension/": "extension code", + "resourcemanager/": "resourcemanager code", + "keyspace/": "keyspace code", + "server/": "server code", + "owner/": "owner code", + "meta": "meta code", + "timer/": "timer code" + } + }, + "SA1029": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "/external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + ".*_test\\.go$": "ignore test code" + } + }, + "SA2000": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA2001": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA2003": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA3000": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA3001": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA4009": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5000": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5001": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5002": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5003": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5004": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5005": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5007": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5008": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5009": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5010": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5011": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA5012": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA6000": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA6001": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "SA6005": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "prealloc": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "parser/yy_parser.go": "ignore generated code", + "/cgo/": "no need to vet third party code for cgo" + } + }, + "predeclared": { + "exclude_files": { + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code", + "parser/yy_parser.go": "ignore generated code", + "parser/parser.go": "ignore generated code", + "/cgo/": "no need to vet third party code for cgo" + } + }, + "U1000": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + "external/": "no need to vet third party code", + ".*_generated\\.go$": "ignore generated code" + } + }, + "etcdconfig": { + "exclude_files": { + "parser/parser.go": "parser/parser.go code", + ".*_test.go": "ignore test code", + "external/": "no need to vet third party code" + } + } +} diff --git a/dumpling/export/util.go b/dumpling/export/util.go new file mode 100644 index 0000000000000..9f75d487254e6 --- /dev/null +++ b/dumpling/export/util.go @@ -0,0 +1,122 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package export + +import ( + "context" + "database/sql" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/br/pkg/version" + tcontext "github.com/pingcap/tidb/dumpling/context" + clientv3 "go.etcd.io/etcd/client/v3" + "golang.org/x/exp/slices" +) + +const tidbServerInformationPath = "/tidb/server/info" + +func getPdDDLIDs(pCtx context.Context, cli *clientv3.Client) ([]string, error) { + ctx, cancel := context.WithTimeout(pCtx, 10*time.Second) + defer cancel() + + resp, err := cli.Get(ctx, tidbServerInformationPath, clientv3.WithPrefix()) + if err != nil { + return nil, errors.Trace(err) + } + pdDDLIds := make([]string, len(resp.Kvs)) + for i, kv := range resp.Kvs { + items := strings.Split(string(kv.Key), "/") + pdDDLIds[i] = items[len(items)-1] + } + return pdDDLIds, nil +} + +func checkSameCluster(tctx *tcontext.Context, db *sql.DB, pdAddrs []string) (bool, error) { + cli, err := clientv3.New(clientv3.Config{ + Endpoints: pdAddrs, + DialTimeout: defaultEtcdDialTimeOut, + AutoSyncInterval: 30 * time.Second, + }) + if err != nil { + return false, errors.Trace(err) + } + tidbDDLIDs, err := GetTiDBDDLIDs(tctx, db) + if err != nil { + return false, err + } + pdDDLIDs, err := getPdDDLIDs(tctx, cli) + if err != nil { + return false, err + } + slices.Sort(tidbDDLIDs) + slices.Sort(pdDDLIDs) + + return sameStringArray(tidbDDLIDs, pdDDLIDs), nil +} + +func sameStringArray(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func string2Map(a, b []string) map[string]string { + a2b := make(map[string]string, len(a)) + for i, str := range a { + a2b[str] = b[i] + } + return a2b +} + +func needRepeatableRead(serverType version.ServerType, consistency string) bool { + return consistency != ConsistencyTypeSnapshot || serverType != version.ServerTypeTiDB +} + +func infiniteChan[T any]() (chan<- T, <-chan T) { + in, out := make(chan T), make(chan T) + + go func() { + var ( + q []T + e T + ok bool + ) + handleRead := func() bool { + if !ok { + for _, e = range q { + out <- e + } + close(out) + return true + } + q = append(q, e) + return false + } + for { + if len(q) > 0 { + select { + case e, ok = <-in: + if handleRead() { + return + } + case out <- q[0]: + q = q[1:] + } + } else { + e, ok = <-in + if handleRead() { + return + } + } + } + }() + return in, out +} diff --git a/util/etcd/etcd.go b/util/etcd/etcd.go new file mode 100644 index 0000000000000..bb4dac73f009e --- /dev/null +++ b/util/etcd/etcd.go @@ -0,0 +1,344 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcd + +import ( + "context" + "crypto/tls" + "fmt" + "path" + "strings" + "time" + + "github.com/pingcap/errors" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/namespace" +) + +// Node organizes the ectd query result as a Trie tree +type Node struct { + Childs map[string]*Node + Value []byte +} + +// OpType is operation's type in etcd +type OpType string + +var ( + // CreateOp is create operation type + CreateOp OpType = "create" + + // UpdateOp is update operation type + UpdateOp OpType = "update" + + // DeleteOp is delete operation type + DeleteOp OpType = "delete" +) + +// Operation represents an operation in etcd, include create, update and delete. +type Operation struct { + Tp OpType + Key string + Value string + Opts []clientv3.OpOption + TTL int64 + WithPrefix bool +} + +// String implements Stringer interface. +func (o *Operation) String() string { + return fmt.Sprintf("{Tp: %s, Key: %s, Value: %s, TTL: %d, WithPrefix: %v, Opts: %v}", o.Tp, o.Key, o.Value, o.TTL, o.WithPrefix, o.Opts) +} + +// Client is a wrapped etcd client that support some simple method +type Client struct { + client *clientv3.Client + rootPath string +} + +// NewClient returns a wrapped etcd client +func NewClient(cli *clientv3.Client, root string) *Client { + return &Client{ + client: cli, + rootPath: root, + } +} + +// NewClientFromCfg returns a wrapped etcd client +func NewClientFromCfg(endpoints []string, dialTimeout time.Duration, root string, security *tls.Config) (*Client, error) { + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: dialTimeout, + TLS: security, + AutoSyncInterval: 30 * time.Second, + }) + if err != nil { + return nil, errors.Trace(err) + } + + return &Client{ + client: cli, + rootPath: root, + }, nil +} + +// Close shutdowns the connection to etcd +func (e *Client) Close() error { + if err := e.client.Close(); err != nil { + return errors.Trace(err) + } + return nil +} + +// GetClient returns client +func (e *Client) GetClient() *clientv3.Client { + return e.client +} + +// Create guarantees to set a key = value with some options(like ttl) +func (e *Client) Create(ctx context.Context, key string, val string, opts []clientv3.OpOption) (int64, error) { + key = keyWithPrefix(e.rootPath, key) + txnResp, err := e.client.KV.Txn(ctx).If( + clientv3.Compare(clientv3.ModRevision(key), "=", 0), + ).Then( + clientv3.OpPut(key, val, opts...), + ).Commit() + if err != nil { + return 0, errors.Trace(err) + } + + if !txnResp.Succeeded { + return 0, errors.AlreadyExistsf("key %s in etcd", key) + } + + if txnResp.Header != nil { + return txnResp.Header.Revision, nil + } + + // impossible to happen + return 0, errors.New("revision is unknown") +} + +// Get returns a key/value matchs the given key +func (e *Client) Get(ctx context.Context, key string) (value []byte, revision int64, err error) { + key = keyWithPrefix(e.rootPath, key) + resp, err := e.client.KV.Get(ctx, key) + if err != nil { + return nil, -1, errors.Trace(err) + } + + if len(resp.Kvs) == 0 { + return nil, -1, errors.NotFoundf("key %s in etcd", key) + } + + return resp.Kvs[0].Value, resp.Header.Revision, nil +} + +// Update updates a key/value. +// set ttl 0 to disable the Lease ttl feature +func (e *Client) Update(ctx context.Context, key string, val string, ttl int64) error { + key = keyWithPrefix(e.rootPath, key) + + var opts []clientv3.OpOption + if ttl > 0 { + lcr, err := e.client.Lease.Grant(ctx, ttl) + if err != nil { + return errors.Trace(err) + } + + opts = []clientv3.OpOption{clientv3.WithLease(lcr.ID)} + } + + txnResp, err := e.client.KV.Txn(ctx).If( + clientv3.Compare(clientv3.ModRevision(key), ">", 0), + ).Then( + clientv3.OpPut(key, val, opts...), + ).Commit() + if err != nil { + return errors.Trace(err) + } + + if !txnResp.Succeeded { + return errors.NotFoundf("key %s in etcd", key) + } + + return nil +} + +// UpdateOrCreate updates a key/value, if the key does not exist then create, or update +func (e *Client) UpdateOrCreate(ctx context.Context, key string, val string, ttl int64) error { + key = keyWithPrefix(e.rootPath, key) + + var opts []clientv3.OpOption + if ttl > 0 { + lcr, err := e.client.Lease.Grant(ctx, ttl) + if err != nil { + return errors.Trace(err) + } + + opts = []clientv3.OpOption{clientv3.WithLease(lcr.ID)} + } + + _, err := e.client.KV.Do(ctx, clientv3.OpPut(key, val, opts...)) + if err != nil { + return errors.Trace(err) + } + return nil +} + +// List returns the trie struct that constructed by the key/value with same prefix +func (e *Client) List(ctx context.Context, key string) (node *Node, revision int64, err error) { + key = keyWithPrefix(e.rootPath, key) + if !strings.HasSuffix(key, "/") { + key += "/" + } + + resp, err := e.client.KV.Get(ctx, key, clientv3.WithPrefix()) + if err != nil { + return nil, -1, errors.Trace(err) + } + + root := new(Node) + length := len(key) + for _, kv := range resp.Kvs { + key := string(kv.Key) + if len(key) <= length { + continue + } + + keyTail := key[length:] + tailNode := parseToDirTree(root, keyTail) + tailNode.Value = kv.Value + } + + return root, resp.Header.Revision, nil +} + +// Delete deletes the key/values with matching prefix or key +func (e *Client) Delete(ctx context.Context, key string, withPrefix bool) error { + key = keyWithPrefix(e.rootPath, key) + var opts []clientv3.OpOption + if withPrefix { + opts = []clientv3.OpOption{clientv3.WithPrefix()} + } + + _, err := e.client.KV.Delete(ctx, key, opts...) + if err != nil { + return errors.Trace(err) + } + + return nil +} + +// Watch watchs the events of key with prefix. +func (e *Client) Watch(ctx context.Context, prefix string, revision int64) clientv3.WatchChan { + if revision > 0 { + return e.client.Watch(ctx, prefix, clientv3.WithPrefix(), clientv3.WithRev(revision)) + } + return e.client.Watch(ctx, prefix, clientv3.WithPrefix()) +} + +// DoTxn does some operation in one transaction. +// Note: should only have one opereration for one key, otherwise will get duplicate key error. +func (e *Client) DoTxn(ctx context.Context, operations []*Operation) (int64, error) { + cmps := make([]clientv3.Cmp, 0, len(operations)) + ops := make([]clientv3.Op, 0, len(operations)) + + for _, operation := range operations { + operation.Key = keyWithPrefix(e.rootPath, operation.Key) + + if operation.TTL > 0 { + if operation.Tp == DeleteOp { + return 0, errors.Errorf("unexpected TTL in delete operation") + } + + lcr, err := e.client.Lease.Grant(ctx, operation.TTL) + if err != nil { + return 0, errors.Trace(err) + } + operation.Opts = append(operation.Opts, clientv3.WithLease(lcr.ID)) + } + + if operation.WithPrefix { + operation.Opts = append(operation.Opts, clientv3.WithPrefix()) + } + + switch operation.Tp { + case CreateOp: + cmps = append(cmps, clientv3.Compare(clientv3.ModRevision(operation.Key), "=", 0)) + ops = append(ops, clientv3.OpPut(operation.Key, operation.Value, operation.Opts...)) + case UpdateOp: + cmps = append(cmps, clientv3.Compare(clientv3.ModRevision(operation.Key), ">", 0)) + ops = append(ops, clientv3.OpPut(operation.Key, operation.Value, operation.Opts...)) + case DeleteOp: + ops = append(ops, clientv3.OpDelete(operation.Key, operation.Opts...)) + default: + return 0, errors.Errorf("unknown operation type %s", operation.Tp) + } + } + + txnResp, err := e.client.KV.Txn(ctx).If( + cmps..., + ).Then( + ops..., + ).Commit() + if err != nil { + return 0, errors.Trace(err) + } + + if !txnResp.Succeeded { + return 0, errors.Errorf("do transaction failed, operations: %+v", operations) + } + + return txnResp.Header.Revision, nil +} + +func parseToDirTree(root *Node, p string) *Node { + pathDirs := strings.Split(p, "/") + current := root + var next *Node + var ok bool + + for _, dir := range pathDirs { + if current.Childs == nil { + current.Childs = make(map[string]*Node) + } + + next, ok = current.Childs[dir] + if !ok { + current.Childs[dir] = new(Node) + next = current.Childs[dir] + } + + current = next + } + + return current +} + +func keyWithPrefix(prefix, key string) string { + if strings.HasPrefix(key, prefix) { + return key + } + + return path.Join(prefix, key) +} + +// SetEtcdCliByNamespace is used to add an etcd namespace prefix before etcd path. +func SetEtcdCliByNamespace(cli *clientv3.Client, namespacePrefix string) { + cli.KV = namespace.NewKV(cli.KV, namespacePrefix) + cli.Watcher = namespace.NewWatcher(cli.Watcher, namespacePrefix) + cli.Lease = namespace.NewLease(cli.Lease, namespacePrefix) +}