From cc190c19dd25195de43e684141fab34817223f8c Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 11 Jun 2020 00:37:44 +0800 Subject: [PATCH] backend: add local kv storage backend to get rid of importer (#326) * backend: add local backend, try to move importer's sort kv and split & scatter into lightning * address comment * use sync.Map * write to every peer * add retry on write and ingest * fix split region * udpate leveldb config * use badger * use pebble * update pebble config * restrict concurrency * use workerPool to restrict concurrency for all engines * make range concurrency as config * flush db at close engine * wip: use sstable to split range * fast split ranges by encode key and file size * fix split bug * use bigEndian to split range * fix split region bug & ingest lost write meta bug * add send-kv-pairs config * fix duplicate write error * fix checksum in small table * use go routine to write tikv and ingest * fix * update sort * update sort * fix * fix * fix retry * update * fix concurrency bug * fix retry bug * fix iter.Next * try fix checksum mismatch * fix deadlock * update * fix * optimize memory usage * add checkpoint for local mode * fix checkpoint * do not write chunk checkpoint in local mode * fix remote checkpoint * fix checkpoint * manually destroy checkpoint * only flush index if checkpoint is on * remove some useless code * format code * fix unit test * fix test * fix * fix tls * fix review comment * add c comment for local.Close * add some comment * fix local backend checkpoint * fix unit test * refine some test with local backend * address comment * fix close engine * checkpoint integrateion_test for local backend * try fix * return nil if engine not exist in CloseEngine and ImportEngine * address comment * test localbackend checkpoint * adjust config to save coverage * fix review comments * change ParseIndexKey method * remove saveCpChan channel buf * add test to save coverage * test ingest failed * inject failpoint before dataengine importer * fix review comments * fix test format Co-authored-by: luancheng --- .gitignore | 1 + Makefile | 2 +- cmd/tidb-lightning-ctl/main.go | 51 +- cmd/tidb-lightning/main.go | 14 +- go.mod | 5 + go.sum | 173 +-- lightning/backend/backend.go | 29 +- lightning/backend/importer.go | 8 +- lightning/backend/local.go | 1162 +++++++++++++++++ lightning/backend/localhelper.go | 286 ++++ lightning/backend/tidb.go | 2 +- lightning/common/security.go | 31 +- lightning/common/util.go | 9 + lightning/config/config.go | 38 +- lightning/config/const.go | 7 +- lightning/config/global.go | 11 +- lightning/manual/manual.go | 65 + lightning/manual/manual_nocgo.go | 19 + lightning/mydump/region.go | 4 + lightning/restore/restore.go | 120 +- lightning/restore/restore_test.go | 6 +- tests/_utils/run_lightning | 1 + tests/checkpoint/run.sh | 96 +- tests/checkpoint_engines/run.sh | 150 +-- tests/concurrent-restore/run.sh | 2 +- tests/csv/config.toml | 4 + tests/csv/run.sh | 52 +- tests/issue_282/run.sh | 2 +- tests/local_backend/config.toml | 9 + .../data/cpeng-schema-create.sql | 1 + tests/local_backend/data/cpeng.a-schema.sql | 1 + tests/local_backend/data/cpeng.a.1.sql | 1 + tests/local_backend/data/cpeng.a.2.sql | 1 + tests/local_backend/data/cpeng.a.3.sql | 1 + tests/local_backend/data/cpeng.b-schema.sql | 1 + tests/local_backend/data/cpeng.b.1.sql | 4 + tests/local_backend/data/cpeng.b.2.sql | 1 + tests/local_backend/run.sh | 77 ++ tests/run.sh | 2 - tests/various_types/run.sh | 156 +-- 40 files changed, 2164 insertions(+), 441 deletions(-) create mode 100644 lightning/backend/local.go create mode 100644 lightning/backend/localhelper.go create mode 100644 lightning/manual/manual.go create mode 100644 lightning/manual/manual_nocgo.go create mode 100644 tests/local_backend/config.toml create mode 100644 tests/local_backend/data/cpeng-schema-create.sql create mode 100644 tests/local_backend/data/cpeng.a-schema.sql create mode 100644 tests/local_backend/data/cpeng.a.1.sql create mode 100644 tests/local_backend/data/cpeng.a.2.sql create mode 100644 tests/local_backend/data/cpeng.a.3.sql create mode 100644 tests/local_backend/data/cpeng.b-schema.sql create mode 100644 tests/local_backend/data/cpeng.b.1.sql create mode 100644 tests/local_backend/data/cpeng.b.2.sql create mode 100755 tests/local_backend/run.sh diff --git a/.gitignore b/.gitignore index 574f4e556..dab4e360d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store bin/ test_*/ *.log diff --git a/Makefile b/Makefile index 00d1d8192..bd35b55e5 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH))) export PATH := $(path_to_add):$(PATH) GO := go -GOBUILD := GO111MODULE=on CGO_ENABLED=0 $(GO) build +GOBUILD := GO111MODULE=on CGO_ENABLED=1 $(GO) build GOTEST := GO111MODULE=on CGO_ENABLED=1 $(GO) test -p 3 ARCH := "`uname -s`" diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index 3bdbcd370..ca2fb2168 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -197,12 +197,6 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common } defer target.Close() - importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) - if err != nil { - return errors.Trace(err) - } - defer importer.Close() - targetTables, err := cpdb.DestroyErrorCheckpoint(ctx, tableName) if err != nil { return errors.Trace(err) @@ -219,15 +213,42 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common } } - for _, table := range targetTables { - for engineID := table.MinEngineID; engineID <= table.MaxEngineID; engineID++ { - fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID) - closedEngine, err := importer.UnsafeCloseEngine(ctx, table.TableName, engineID) - if err != nil { - fmt.Fprintln(os.Stderr, "* Encountered error while closing engine:", err) - lastErr = err - } else { - closedEngine.Cleanup(ctx) + if cfg.TikvImporter.Backend == "importer" { + importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) + if err != nil { + return errors.Trace(err) + } + defer importer.Close() + + for _, table := range targetTables { + for engineID := table.MinEngineID; engineID <= table.MaxEngineID; engineID++ { + fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID) + closedEngine, err := importer.UnsafeCloseEngine(ctx, table.TableName, engineID) + if err != nil { + fmt.Fprintln(os.Stderr, "* Encountered error while closing engine:", err) + lastErr = err + } else { + closedEngine.Cleanup(ctx) + } + } + } + } + // For importer backend, engine was stored in importer's memory, we can retrieve it from alive importer process. + // But in local backend, if we want to use common API `UnsafeCloseEngine` and `Cleanup`, + // we need either lightning process alive or engine map persistent. + // both of them seems unnecessary if we only need to do is cleanup specify engine directory. + // so we didn't choose to use common API. + if cfg.TikvImporter.Backend == "local" { + for _, table := range targetTables { + for engineID := table.MinEngineID; engineID <= table.MaxEngineID; engineID++ { + fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID) + _, eID := kv.MakeUUID(table.TableName, engineID) + file := kv.LocalFile{Uuid: eID} + err := file.Cleanup(cfg.TikvImporter.SortedKVDir) + if err != nil { + fmt.Fprintln(os.Stderr, "* Encountered error while cleanup engine:", err) + lastErr = err + } } } } diff --git a/cmd/tidb-lightning/main.go b/cmd/tidb-lightning/main.go index 146d0bbe1..219b454a2 100644 --- a/cmd/tidb-lightning/main.go +++ b/cmd/tidb-lightning/main.go @@ -54,12 +54,16 @@ func main() { // The time need of `encode kv data and write` step reduce from 52m4s to 37m30s when change // GOGC from 100 to 500, the total time needed reduce near 15m too. // The cost of this is the memory of lightnin at runtime grow from about 200M to 700M, but it's acceptable. - // // So we set the gc percentage as 500 default to reduce the GC frequency instead of 100. - gogc := os.Getenv("GOGC") - if gogc == "" { - old := debug.SetGCPercent(500) - log.L().Debug("set gc percentage", zap.Int("old", old), zap.Int("new", 500)) + // + // Local mode need much more memory than importer/tidb mode, if the gc percentage is too high, + // lightning memory usage will also be high. + if cfg.TikvImporter.Backend != config.BackendLocal { + gogc := os.Getenv("GOGC") + if gogc == "" { + old := debug.SetGCPercent(500) + log.L().Debug("set gc percentage", zap.Int("old", old), zap.Int("new", 500)) + } } err := app.GoServe() diff --git a/go.mod b/go.mod index 95321f489..2491f2544 100644 --- a/go.mod +++ b/go.mod @@ -5,25 +5,30 @@ go 1.13 require ( github.com/BurntSushi/toml v0.3.1 github.com/DATA-DOG/go-sqlmock v1.4.1 + github.com/cockroachdb/pebble v0.0.0-20200601233547-7956a7440a70 github.com/coreos/go-semver v0.3.0 + github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 // indirect github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 github.com/gogo/protobuf v1.3.1 github.com/golang/mock v1.4.3 github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf // indirect github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 + github.com/pingcap/br v0.0.0-20200521085655-53201addd4ad github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 github.com/pingcap/failpoint v0.0.0-20200506114213-c17f16071c53 github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad github.com/pingcap/parser v0.0.0-20200522094936-3b720a0512a6 + github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181 github.com/pingcap/tidb v1.1.0-beta.0.20200527030457-572bba0499e1 github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/satori/go.uuid v1.2.0 github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 + github.com/spaolacci/murmur3 v1.1.0 // indirect go.uber.org/zap v1.15.0 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/text v0.3.2 diff --git a/go.sum b/go.sum index 43154600d..bf3988240 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,7 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0 h1:0E3eE8MX426vUOs7aHfI7aN1BrIzzzf4ccKCSfSjGmc= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -13,21 +14,20 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.4.0/go.mod h1:ZusYJWlOshgSBGbt6K3GnB3MT3H1xs2id9+TCl4fDBA= +cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -38,30 +38,37 @@ github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpT github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.26.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.24 h1:y3JPD51VuEmVqN3BEDVm4amGpDma2cKJcDPuAU1OR58= github.com/aws/aws-sdk-go v1.30.24/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= +github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb/v3 v3.0.1/go.mod h1:SqqeMF/pMOIu3xgGoxtPYhMNQP258xE4x/XRTYua+KU= +github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM= github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20200601233547-7956a7440a70 h1:oinoYvbVDh+9Kh+/00vNKBS7px1xPz0k6wHLfpepOMg= +github.com/cockroachdb/pebble v0.0.0-20200601233547-7956a7440a70/go.mod h1:crLnbSFbwAcQNs9FPfI1avHb5BqVgqZcr4r+IzpJ5FM= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coocood/badger v1.5.1-0.20200515070411-e02af0688441/go.mod h1:klY8SfH2lNZ/23/SIxwHoJw+T6wYGB12YPCF9MUoiu0= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= @@ -73,7 +80,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= @@ -96,41 +102,40 @@ github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0 github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs= github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.15.0/go.mod h1:HNxAJ/+FY/XSsxuwz8iIYdp2GtMmPbJ8WQjjGMxd6Qk= github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -144,12 +149,12 @@ github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7N github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -182,46 +187,40 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= +github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3 h1:3CYI9xg87xNAD+es02gZxbX/ky4KQeoFBsNOzuoAQZg= github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9 h1:K+lX49/3eURCE1IjlaZN//u6c+9nfDAMnyQ9E2dsJbY= github.com/google/pprof v0.0.0-20200407044318-7d83b28da2e9/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= @@ -229,23 +228,19 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.14.3 h1:OCJlWkOUoTnl0neNGlf4fUm3TmbEtguw7vR+nGtnDjY= github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= @@ -253,18 +248,17 @@ github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZO github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hypnoglow/gormzap v0.3.0/go.mod h1:5Wom8B7Jl2oK0Im9hs6KQ+Kl92w4Y7gKCrj66rhyvw0= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf h1:Ut4tTtPNmInWiEWJRernsWm688R0RN6PFO8sZhwI0sk= github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/sqltocsv v0.0.0-20190824231449-5650f27fd5b6 h1:3Jr6Mtili6DsXSF0RwRlAqpOUWXcSVUxdOm5kFPb3xY= @@ -295,22 +289,17 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -322,15 +311,18 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -347,7 +339,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 h1:pmpDGKLw4n82EtrNiLqB+xSz/JQwFOaZuMALYUHwX5s= github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -362,60 +353,49 @@ github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= -github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= -github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200302022638-35a6e979dca9/go.mod h1:YUceA4BHY/MTtp63yZLTYP22waFSwMNo9lXq2FDtzVw= github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200407064406-b2b8ad403d01/go.mod h1:77fCh8d3oKzC5ceOJWeZXAS/mLzVgdZ7rKniwmOyFuo= github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200514075710-eecc9a4525b5/go.mod h1:8q+yDx0STBPri8xS4A2duS1dAf+xO0cMtjwe0t6MWJk= +github.com/pingcap/br v0.0.0-20200413082646-b37ae8f0b494/go.mod h1:RVNBJsnSdGiDJwoageXIFaCeeBhXD/19sHI1w3rAOLE= github.com/pingcap/br v0.0.0-20200426093517-dd11ae28b885/go.mod h1:4w3meMnk7HDNpNgjuRAxavruTeKJvUiXxoEWTjzXPnA= +github.com/pingcap/br v0.0.0-20200521085655-53201addd4ad h1:nptiQT0kWdIUghh49OyaTBYb4DtdxJmsLHOxbU25kW4= github.com/pingcap/br v0.0.0-20200521085655-53201addd4ad/go.mod h1:SlSUHWY7QUoooiYxOKuJ8kUh2KjI29ogBh89YXz2dLA= -github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= -github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 h1:R8gStypOBmpnHEx1qi//SaqxJVI4inOqljg/Aj5/390= github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik= github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 h1:58naV4XMEqm0hl9LcYo6cZoGBGiLtefMQMF/vo3XLgQ= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+v8Jlc98uMBvKIzr1a+UhnLyVYn8Q5Q= github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= -github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798 h1:6DMbRqPI1qzQ8N1xc3+nKY8IxSACd9VqQKkRVvbyoIg= github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/failpoint v0.0.0-20200506114213-c17f16071c53 h1:8sC8OLinmaw24xLeeJlYBFvUBsOiOYBtNqTuVOTnynQ= github.com/pingcap/failpoint v0.0.0-20200506114213-c17f16071c53/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= -github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d h1:rCmRK0lCRrHMUbS99BKFYhK9YxJDNw0xB033cQbYo0s= github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= -github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20190227013052-e71ca0165a5f/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5 h1:knEvP4R5v5b2T107/Q6VzB0C8/6T7NXB/V7Vl1FtQsg= -github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200330093347-98f910b71904/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200417092353-efbe03bcffbd/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200420075417-e0c6e8842f22/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= @@ -424,52 +404,41 @@ github.com/pingcap/kvproto v0.0.0-20200424032552-6650270c39c3/go.mod h1:IOdRDPLy github.com/pingcap/kvproto v0.0.0-20200428135407-0f5ffe459677/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29 h1:NpW1OuYrIl+IQrSsVbtyHpHpazmSCHy+ysrOixY0xY4= github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad h1:SveG82rmu/GFxYanffxsSF503SiQV+2JLnWEiGiF+Tc= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87 h1:533jEUp3mtfWjk0el+awLbyGVxiHcUIGWcR1Y7gB+fg= -github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= github.com/pingcap/parser v0.0.0-20200424075042-8222d8b724a4/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= github.com/pingcap/parser v0.0.0-20200507022230-f3bf29096657/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= github.com/pingcap/parser v0.0.0-20200522094936-3b720a0512a6 h1:sa3NjjIEU9JOY/P8bhn+SJupcKZ497b3sfPVfvyCm2A= github.com/pingcap/parser v0.0.0-20200522094936-3b720a0512a6/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= -github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs= github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2/go.mod h1:s+utZtXDznOiL24VK0qGmtoHjjXNsscJx3m1n8cC56s= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181 h1:FM+PzdoR3fmWAJx3ug+p5aOgs5aZYwFkoDL7Potdsz0= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181/go.mod h1:q4HTx/bA8aKBa4S7L+SQKHvjRPXCRV0tA0yRw0qkZSA= -github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd h1:k7CIHMFVKjHsda3PKkiN4zv++NEnexlUwiJEhryWpG0= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= -github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb h1:bDbgLaNTRNK6Qw7KjvEqqfCQstY8WMEcXyXTU7yzYKg= github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20200408114249-ed3bd6f7fdb1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= -github.com/pingcap/tidb v0.0.0-20200317142013-5268094afe05 h1:J8K5s6rPrMc23vz3gruEGCBCWx3ESi6XtwQLFfALT/U= -github.com/pingcap/tidb v0.0.0-20200317142013-5268094afe05/go.mod h1:4CGOiKZSaOU/Da3QYMtp0c3uBE2SxpcLOpESXmeQhcs= +github.com/pingcap/tidb v0.0.0-20200401141416-959eca8f3a39/go.mod h1:btnHsqUQvJnY18+OP2Z6MCRq1tX4B8JUCrmqctSKxOg= github.com/pingcap/tidb v1.1.0-beta.0.20200424154252-5ede18f10eed/go.mod h1:m2VDlJDbUeHPCXAfKPajqLmB1uLvWpkKk3zALNqDYdw= github.com/pingcap/tidb v1.1.0-beta.0.20200509133407-a9dc72cf2558/go.mod h1:cXNbVSQAkwwmjFQmEnEPI00Z2/Y/KOhouttUPERiInE= github.com/pingcap/tidb v1.1.0-beta.0.20200513065557-5a0787dfa915/go.mod h1:khS9Z9YlbtxsaZsSsSahelgh5L16EtP30QADFmPiI/I= github.com/pingcap/tidb v1.1.0-beta.0.20200527030457-572bba0499e1 h1:4ihqSPKpupQiBP+aeAEH3WED+tiVhM3y+JKe6l+quI4= github.com/pingcap/tidb v1.1.0-beta.0.20200527030457-572bba0499e1/go.mod h1:2W+Vb1nUJL7Ai/UqdK7ftaZbsNJBjawLGolvCCbl7Fk= -github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible h1:84F7MFMfdAYObrznvRslmVu43aoihrlL+7mMyMlOi0o= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible h1:/JKsYjsa5Ug8v5CN4zIbJGIqsvgBUkGwaP/rEScVvWM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= -github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60 h1:aJPXrT1u4VfUSGFA2oQVwl4pOXzqe+YI6wed01cjDH4= github.com/pingcap/tipb v0.0.0-20200212061130-c4d518eb1d60/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee h1:XJQ6/LGzOSc/jo33AD8t7jtc4GohxcyODsYnb+kZXJM= github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -478,7 +447,6 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= @@ -509,36 +477,31 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI9CLrl2ZvDLiahto= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49VXJ6UZja7gU9QLHwRpIPBN89SKEo= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByijiI8yYiuwLwQ2ZxXobW7ua/8GRB3pI= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= @@ -546,9 +509,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= @@ -559,25 +520,21 @@ github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+t github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= github.com/swaggo/swag v1.6.5/go.mod h1:Y7ZLSS0d0DdxhWGVhQdu+Bu1QhaF5k0RD7FKdiAykeY= github.com/swaggo/swag v1.6.6-0.20200323071853-8e21f4cefeea/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= -github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 h1:uSDYjYejelKyceA6DiCsngFof9jAyeaSyX9XC5a1a7Q= github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= -github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= -github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= @@ -592,7 +549,6 @@ github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -608,43 +564,34 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.10.0/go.mod h1:vLRicqpG/qQEzno4SYU86iCwfT95EZza+Eba0ItuxqY= -go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -652,8 +599,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -662,6 +609,8 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -671,18 +620,15 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -700,31 +646,28 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0 h1:2mqDk8w/o6UmeUCu5Qiq2y7iMf6anbx+YA8d1JFoFrs= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -750,22 +693,22 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= @@ -778,7 +721,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -794,7 +736,6 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191107010934-f79515f33823 h1:akkRBeitX2EZP59KdtKw310CI4WGPCNPyrLbE7WZA8Y= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -805,16 +746,14 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200301222351-066e0c02454c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200313205530-4303120df7d8 h1:gkI/wGGwpcG5W4hLCzZNGxA4wzWBGGDStRI1MrjDl2Q= -golang.org/x/tools v0.0.0-20200313205530-4303120df7d8/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1 h1:odiryKYJy7CjdrZxhrcE1Z8L9+kGyGZOnfpuauvdCeU= golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -823,8 +762,8 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.1 h1:5mMS6mYvK5LVB8+ujVBC33Y8gltBo/kT6HBm6kU80G4= google.golang.org/api v0.15.1/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -839,7 +778,6 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -854,7 +792,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -863,13 +800,10 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= @@ -879,27 +813,21 @@ gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3M gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= @@ -908,9 +836,6 @@ modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4 h1:VO9oZbbkvTwqLimlQt15QNdOOBArT2dw/bvzsMZBiqQ= sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/lightning/backend/backend.go b/lightning/backend/backend.go index 24b82b1a6..6c9be5062 100644 --- a/lightning/backend/backend.go +++ b/lightning/backend/backend.go @@ -73,6 +73,12 @@ func makeLogger(tag string, engineUUID uuid.UUID) log.Logger { ) } +func MakeUUID(tableName string, engineID int32) (string, uuid.UUID) { + tag := makeTag(tableName, engineID) + engineUUID := uuid.NewV5(engineNamespace, tag) + return tag, engineUUID +} + var engineNamespace = uuid.Must(uuid.FromString("d68d6abe-c59e-45d6-ade8-e2b0ceb7bedf")) // AbstractBackend is the abstract interface behind Backend. @@ -136,6 +142,15 @@ type AbstractBackend interface { FetchRemoteTableModels(schemaName string) ([]*model.TableInfo, error) } +func fetchRemoteTableModelsFromTLS(tls *common.TLS, schema string) ([]*model.TableInfo, error) { + var tables []*model.TableInfo + err := tls.GetJSON("/schema/"+schema, &tables) + if err != nil { + return nil, errors.Annotatef(err, "cannot read schema '%s' from remote", schema) + } + return tables, nil +} + // Backend is the delivery target for Lightning type Backend struct { abstract AbstractBackend @@ -199,8 +214,7 @@ func (be Backend) FetchRemoteTableModels(schemaName string) ([]*model.TableInfo, // OpenEngine opens an engine with the given table name and engine ID. func (be Backend) OpenEngine(ctx context.Context, tableName string, engineID int32) (*OpenedEngine, error) { - tag := makeTag(tableName, engineID) - engineUUID := uuid.NewV5(engineNamespace, tag) + tag, engineUUID := MakeUUID(tableName, engineID) logger := makeLogger(tag, engineUUID) if err := be.abstract.OpenEngine(ctx, engineUUID); err != nil { @@ -241,6 +255,14 @@ func (engine *OpenedEngine) Close(ctx context.Context) (*ClosedEngine, error) { return closedEngine, err } +// Flush current written data for local backend +func (engine *OpenedEngine) Flush() error { + if l, ok := engine.backend.(*local); ok { + return l.Flush(engine.uuid) + } + return nil +} + // WriteRows writes a collection of encoded rows into the engine. func (engine *OpenedEngine) WriteRows(ctx context.Context, columnNames []string, rows Rows) error { var err error @@ -270,8 +292,7 @@ outside: // knows via other ways that the engine has already been opened, e.g. when // resuming from a checkpoint. func (be Backend) UnsafeCloseEngine(ctx context.Context, tableName string, engineID int32) (*ClosedEngine, error) { - tag := makeTag(tableName, engineID) - engineUUID := uuid.NewV5(engineNamespace, tag) + tag, engineUUID := MakeUUID(tableName, engineID) return be.UnsafeCloseEngineWithUUID(ctx, tag, engineUUID) } diff --git a/lightning/backend/importer.go b/lightning/backend/importer.go index 51c20c29f..174d06ef2 100644 --- a/lightning/backend/importer.go +++ b/lightning/backend/importer.go @@ -41,7 +41,6 @@ var ( requiredTiDBVersion = *semver.New("2.1.0") requiredPDVersion = *semver.New("2.1.0") requiredTiKVVersion = *semver.New("2.1.0") - // ^ TODO: remember to use 4.0.0 for "local" backend. ) // importer represents a gRPC connection to tikv-importer. This type is @@ -336,10 +335,5 @@ func checkVersion(component string, expected, actual semver.Version) error { } func (importer *importer) FetchRemoteTableModels(schema string) ([]*model.TableInfo, error) { - var tables []*model.TableInfo - err := importer.tls.GetJSON("/schema/"+schema, &tables) - if err != nil { - return nil, errors.Annotatef(err, "cannot read schema '%s' from remote", schema) - } - return tables, nil + return fetchRemoteTableModelsFromTLS(importer.tls, schema) } diff --git a/lightning/backend/local.go b/lightning/backend/local.go new file mode 100644 index 000000000..c5442381c --- /dev/null +++ b/lightning/backend/local.go @@ -0,0 +1,1162 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package backend + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "io/ioutil" + "math" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/cockroachdb/pebble" + "github.com/coreos/go-semver/semver" + split "github.com/pingcap/br/pkg/restore" + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/errorpb" + sst "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/parser/model" + pd "github.com/pingcap/pd/v4/client" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/codec" + uuid "github.com/satori/go.uuid" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + + "github.com/pingcap/tidb-lightning/lightning/common" + "github.com/pingcap/tidb-lightning/lightning/log" + "github.com/pingcap/tidb-lightning/lightning/manual" + "github.com/pingcap/tidb-lightning/lightning/worker" +) + +const ( + dialTimeout = 5 * time.Second + bigValueSize = 1 << 16 // 64K + engineMetaFileSuffix = ".meta" + + gRPCKeepAliveTime = 10 * time.Second + gRPCKeepAliveTimeout = 3 * time.Second + gRPCBackOffMaxDelay = 3 * time.Second +) + +var ( + localMinTiDBVersion = *semver.New("4.0.0") + localMinTiKVVersion = *semver.New("4.0.0") + localMinPDVersion = *semver.New("4.0.0") +) + +// Range record start and end key for localStoreDir.DB +// so we can write it to tikv in streaming +type Range struct { + start []byte + end []byte + length int +} + +// localFileMeta contains some field that is necessary to continue the engine restore/import process. +// These field should be written to disk when we update chunk checkpoint +type localFileMeta struct { + Ts uint64 `json:"ts"` + Length int64 `json:"length"` + TotalSize int64 `json:"total_size"` +} + +type LocalFile struct { + localFileMeta + db *pebble.DB + Uuid uuid.UUID +} + +func (e *LocalFile) Close() error { + return e.db.Close() +} + +// Cleanup remove meta and db files +func (e *LocalFile) Cleanup(dataDir string) error { + metaPath := filepath.Join(dataDir, e.Uuid.String()+engineMetaFileSuffix) + if err := os.Remove(metaPath); err != nil && !os.IsNotExist(err) { + return errors.Trace(err) + } + + dbPath := filepath.Join(dataDir, e.Uuid.String()) + return os.RemoveAll(dbPath) +} + +type grpcClis struct { + mu sync.Mutex + clis map[uint64]*grpc.ClientConn +} + +type local struct { + engines sync.Map + grpcClis grpcClis + splitCli split.SplitClient + tls *common.TLS + pdAddr string + + localStoreDir string + regionSplitSize int64 + + rangeConcurrency *worker.Pool + batchWriteKVPairs int + checkpointEnabled bool +} + +// NewLocalBackend creates new connections to tikv. +func NewLocalBackend( + ctx context.Context, + tls *common.TLS, + pdAddr string, + regionSplitSize int64, + localFile string, + rangeConcurrency int, + sendKVPairs int, + enableCheckpoint bool, +) (Backend, error) { + pdCli, err := pd.NewClient([]string{pdAddr}, tls.ToPDSecurityOption()) + if err != nil { + return MakeBackend(nil), errors.Annotate(err, "construct pd client failed") + } + splitCli := split.NewSplitClient(pdCli, tls.TLSConfig()) + + shouldCreate := true + if enableCheckpoint { + if info, err := os.Stat(localFile); err != nil { + if !os.IsNotExist(err) { + return MakeBackend(nil), err + } + } else if info.IsDir() { + shouldCreate = false + } + } + + if shouldCreate { + err = os.Mkdir(localFile, 0700) + if err != nil { + return MakeBackend(nil), err + } + } + + local := &local{ + engines: sync.Map{}, + splitCli: splitCli, + tls: tls, + pdAddr: pdAddr, + + localStoreDir: localFile, + regionSplitSize: regionSplitSize, + + rangeConcurrency: worker.NewPool(ctx, rangeConcurrency, "range"), + batchWriteKVPairs: sendKVPairs, + checkpointEnabled: enableCheckpoint, + } + local.grpcClis.clis = make(map[uint64]*grpc.ClientConn) + return MakeBackend(local), nil +} + +func (local *local) getGrpcConnLocked(ctx context.Context, storeID uint64) (*grpc.ClientConn, error) { + + store, err := local.splitCli.GetStore(ctx, storeID) + if err != nil { + return nil, errors.Trace(err) + } + opt := grpc.WithInsecure() + if local.tls.TLSConfig() != nil { + opt = grpc.WithTransportCredentials(credentials.NewTLS(local.tls.TLSConfig())) + } + ctx, cancel := context.WithTimeout(ctx, dialTimeout) + + bfConf := backoff.DefaultConfig + bfConf.MaxDelay = gRPCBackOffMaxDelay + conn, err := grpc.DialContext( + ctx, + store.GetAddress(), + opt, + grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: gRPCKeepAliveTime, + Timeout: gRPCKeepAliveTimeout, + PermitWithoutStream: true, + }), + ) + cancel() + if err != nil { + return nil, errors.WithStack(err) + } + // Cache the conn. + local.grpcClis.clis[storeID] = conn + return conn, nil +} + +// Close the importer connection. +func (local *local) Close() { + local.engines.Range(func(k, v interface{}) bool { + v.(*LocalFile).Close() + return true + }) + + // if checkpoint is disable or we finish load all data successfully, then files in this + // dir will be useless, so we clean up this dir and all files in it. + if !local.checkpointEnabled || common.IsEmptyDir(local.localStoreDir) { + err := os.RemoveAll(local.localStoreDir) + if err != nil { + log.L().Warn("remove local db file failed", zap.Error(err)) + } + } +} + +// Flush ensure the written data is saved successfully, to make sure no data lose after restart +func (local *local) Flush(engineId uuid.UUID) error { + if engine, ok := local.engines.Load(engineId); ok { + engineFile := engine.(*LocalFile) + if err := engineFile.db.Flush(); err != nil { + return err + } + return local.saveEngineMeta(engineFile) + } + return errors.Errorf("engine '%s' not found", engineId) +} + +func (local *local) RetryImportDelay() time.Duration { + return defaultRetryBackoffTime +} + +func (local *local) MaxChunkSize() int { + // a batch size write to leveldb + return int(local.regionSplitSize) +} + +func (local *local) ShouldPostProcess() bool { + return true +} + +func (local *local) openEngineDB(engineUUID uuid.UUID, readOnly bool) (*pebble.DB, error) { + opt := &pebble.Options{ + MemTableSize: 128 << 20, + MaxConcurrentCompactions: 16, + MinCompactionRate: 1 << 30, + L0CompactionThreshold: math.MaxInt32, // set to max try to disable compaction + L0StopWritesThreshold: math.MaxInt32, // set to max try to disable compaction + MaxOpenFiles: 10000, + DisableWAL: true, + ReadOnly: readOnly, + } + dbPath := filepath.Join(local.localStoreDir, engineUUID.String()) + return pebble.Open(dbPath, opt) +} + +func (local *local) saveEngineMeta(engine *LocalFile) error { + jsonBytes, err := json.Marshal(&engine.localFileMeta) + if err != nil { + return errors.Trace(err) + } + metaPath := filepath.Join(local.localStoreDir, engine.Uuid.String()+engineMetaFileSuffix) + return errors.Trace(ioutil.WriteFile(metaPath, jsonBytes, 0644)) +} + +func (local *local) LoadEngineMeta(engineUUID uuid.UUID) (localFileMeta, error) { + var meta localFileMeta + + mataPath := filepath.Join(local.localStoreDir, engineUUID.String()+engineMetaFileSuffix) + f, err := os.Open(mataPath) + if err != nil { + return meta, err + } + err = json.NewDecoder(f).Decode(&meta) + return meta, err +} + +func (local *local) OpenEngine(ctx context.Context, engineUUID uuid.UUID) error { + meta, err := local.LoadEngineMeta(engineUUID) + if err != nil { + meta = localFileMeta{} + } + db, err := local.openEngineDB(engineUUID, false) + if err != nil { + return err + } + local.engines.Store(engineUUID, &LocalFile{localFileMeta: meta, db: db, Uuid: engineUUID}) + return nil +} + +// Close backend engine by uuid +// NOTE: we will return nil if engine is not exist. This will happen if engine import&cleanup successfully +// but exit before update checkpoint. Thus after restart, we will try to import this engine again. +func (local *local) CloseEngine(ctx context.Context, engineUUID uuid.UUID) error { + // flush mem table to storage, to free memory, + // ask others' advise, looks like unnecessary, but with this we can control memory precisely. + engine, ok := local.engines.Load(engineUUID) + if !ok { + // recovery mode, we should reopen this engine file + meta, err := local.LoadEngineMeta(engineUUID) + if err != nil { + // if engine meta not exist, just skip + if os.IsNotExist(err) { + return nil + } + return err + } + db, err := local.openEngineDB(engineUUID, true) + if err != nil { + return err + } + engineFile := &LocalFile{ + localFileMeta: meta, + Uuid: engineUUID, + db: db, + } + local.engines.Store(engineUUID, engineFile) + return nil + } + engineFile := engine.(*LocalFile) + err := engineFile.db.Flush() + if err != nil { + return err + } + return local.saveEngineMeta(engineFile) +} + +func (local *local) getImportClient(ctx context.Context, peer *metapb.Peer) (sst.ImportSSTClient, error) { + local.grpcClis.mu.Lock() + defer local.grpcClis.mu.Unlock() + var err error + + conn, ok := local.grpcClis.clis[peer.GetStoreId()] + if !ok { + conn, err = local.getGrpcConnLocked(ctx, peer.GetStoreId()) + if err != nil { + log.L().Error("could not get grpc connect ", zap.Uint64("storeId", peer.GetStoreId())) + return nil, err + } + } + return sst.NewImportSSTClient(conn), nil +} + +// WriteToTiKV writer engine key-value pairs to tikv and return the sst meta generated by tikv. +// we don't need to do cleanup for the pairs written to tikv if encounters an error, +// tikv will takes the responsibility to do so. +func (local *local) WriteToTiKV( + ctx context.Context, + engineFile *LocalFile, + region *split.RegionInfo) ([]*sst.SSTMeta, error) { + var startKey, endKey []byte + if len(region.Region.StartKey) > 0 { + _, startKey, _ = codec.DecodeBytes(region.Region.StartKey, []byte{}) + } + if len(region.Region.EndKey) > 0 { + _, endKey, _ = codec.DecodeBytes(region.Region.EndKey, []byte{}) + } + opt := &pebble.IterOptions{LowerBound: startKey, UpperBound: endKey} + iter := engineFile.db.NewIter(opt) + defer iter.Close() + + iter.First() + firstKey := codec.EncodeBytes([]byte{}, iter.Key()) + iter.Last() + lastKey := codec.EncodeBytes([]byte{}, iter.Key()) + + meta := &sst.SSTMeta{ + Uuid: uuid.NewV4().Bytes(), + RegionId: region.Region.GetId(), + RegionEpoch: region.Region.GetRegionEpoch(), + Range: &sst.Range{ + Start: firstKey, + End: lastKey, + }, + } + + leaderID := region.Leader.GetId() + clients := make([]sst.ImportSST_WriteClient, 0, len(region.Region.GetPeers())) + requests := make([]*sst.WriteRequest, 0, len(region.Region.GetPeers())) + for _, peer := range region.Region.GetPeers() { + cli, err := local.getImportClient(ctx, peer) + if err != nil { + return nil, err + } + + wstream, err := cli.Write(ctx) + if err != nil { + return nil, err + } + + // Bind uuid for this write request + req := &sst.WriteRequest{ + Chunk: &sst.WriteRequest_Meta{ + Meta: meta, + }, + } + if err = wstream.Send(req); err != nil { + return nil, err + } + clients = append(clients, wstream) + requests = append(requests, req) + } + + bytesBuf := newBytesBuffer() + defer bytesBuf.destroy() + pairs := make([]*sst.Pair, 0, local.batchWriteKVPairs) + count := 0 + size := int64(0) + totalCount := 0 + for iter.First(); iter.Valid(); iter.Next() { + size += int64(len(iter.Key()) + len(iter.Value())) + pair := &sst.Pair{ + Key: bytesBuf.addBytes(iter.Key()), + Value: bytesBuf.addBytes(iter.Value()), + } + + pairs = append(pairs, pair) + count++ + totalCount++ + + if count >= local.batchWriteKVPairs { + for i := range clients { + requests[i].Reset() + requests[i].Chunk = &sst.WriteRequest_Batch{ + Batch: &sst.WriteBatch{ + CommitTs: engineFile.Ts, + Pairs: pairs, + }, + } + if err := clients[i].Send(requests[i]); err != nil { + return nil, err + } + } + count = 0 + pairs = pairs[:0] + bytesBuf.reset() + } + } + + if count > 0 { + for i := range clients { + requests[i].Reset() + requests[i].Chunk = &sst.WriteRequest_Batch{ + Batch: &sst.WriteBatch{ + CommitTs: engineFile.Ts, + Pairs: pairs, + }, + } + if err := clients[i].Send(requests[i]); err != nil { + return nil, err + } + } + } + + if iter.Error() != nil { + return nil, errors.Trace(iter.Error()) + } + + var leaderPeerMetas []*sst.SSTMeta + for i, wStream := range clients { + if resp, closeErr := wStream.CloseAndRecv(); closeErr != nil { + return nil, closeErr + } else { + if leaderID == region.Region.Peers[i].GetId() { + leaderPeerMetas = resp.Metas + log.L().Debug("get metas after write kv stream to tikv", zap.Reflect("metas", leaderPeerMetas)) + } + } + } + + log.L().Debug("write to kv", zap.Reflect("region", region), zap.Uint64("leader", leaderID), + zap.Reflect("meta", meta), zap.Reflect("return metas", leaderPeerMetas), + zap.Int("kv_pairs", totalCount), zap.Int64("total_bytes", size), + zap.Int64("buf_size", bytesBuf.totalSize())) + + return leaderPeerMetas, nil +} + +func (local *local) Ingest(ctx context.Context, meta *sst.SSTMeta, region *split.RegionInfo) (*sst.IngestResponse, error) { + leader := region.Leader + if leader == nil { + leader = region.Region.GetPeers()[0] + } + + cli, err := local.getImportClient(ctx, leader) + if err != nil { + return nil, err + } + reqCtx := &kvrpcpb.Context{ + RegionId: region.Region.GetId(), + RegionEpoch: region.Region.GetRegionEpoch(), + Peer: leader, + } + + req := &sst.IngestRequest{ + Context: reqCtx, + Sst: meta, + } + resp, err := cli.Ingest(ctx, req) + if err != nil { + return nil, err + } + return resp, nil +} + +func (local *local) ReadAndSplitIntoRange(engineFile *LocalFile, engineUUID uuid.UUID) ([]Range, error) { + if engineFile.Length == 0 { + return nil, nil + } + + iter := engineFile.db.NewIter(nil) + defer iter.Close() + + var startKey, endKey []byte + if iter.First() { + startKey = append([]byte{}, iter.Key()...) + } else { + return nil, errors.New("could not find first pair, this shouldn't happen") + } + if iter.Last() { + endKey = append([]byte{}, iter.Key()...) + } else { + return nil, errors.New("could not find last pair, this shouldn't happen") + } + + // <= 96MB no need to split into range + if engineFile.TotalSize <= local.regionSplitSize { + ranges := []Range{{start: startKey, end: nextKey(endKey), length: int(engineFile.Length)}} + return ranges, nil + } + + log.L().Info("ReadAndSplitIntoRange", zap.Binary("start", startKey), zap.Binary("end", endKey)) + + // split data into n ranges, then seek n times to get n + 1 ranges + n := engineFile.TotalSize / local.regionSplitSize + + ranges := make([]Range, 0, n+1) + if tablecodec.IsIndexKey(startKey) { + // index engine + tableID, startIndexID, _, err := tablecodec.DecodeIndexKeyPrefix(startKey) + if err != nil { + return nil, err + } + tableID, endIndexID, _, err := tablecodec.DecodeIndexKeyPrefix(endKey) + if err != nil { + return nil, err + } + indexCount := (endIndexID - startIndexID) + 1 + + // each index has to split into n / indexCount ranges + indexRangeCount := n / indexCount + + for i := startIndexID; i <= endIndexID; i++ { + k := tablecodec.EncodeTableIndexPrefix(tableID, i) + iter.SeekGE(k) + // get first key of index i + startKeyOfIndex := append([]byte{}, iter.Key()...) + + k = tablecodec.EncodeTableIndexPrefix(tableID, i+1) + // get last key of index i + iter.SeekLT(k) + + lastKeyOfIndex := append([]byte{}, iter.Key()...) + + _, startIndexID, startValues, err := tablecodec.DecodeIndexKeyPrefix(startKeyOfIndex) + if err != nil { + return nil, err + } + _, endIndexID, endValues, err := tablecodec.DecodeIndexKeyPrefix(lastKeyOfIndex) + if err != nil { + return nil, err + } + + if startIndexID != endIndexID { + // this shouldn't happen + log.L().DPanic("index ID not match", + zap.Int64("startID", startIndexID), zap.Int64("endID", endIndexID)) + return nil, errors.New("index ID not match") + } + + // if index is Unique or Primary, the key is encoded as + // tablePrefix{tableID}_indexPrefixSep{indexID}_indexedColumnsValue + // if index is non-Unique, key is encoded as + // tablePrefix{tableID}_indexPrefixSep{indexID}_indexedColumnsValue_rowID + + // we can split by indexColumnsValue to get indexRangeCount ranges from above Keys + + log.L().Info("split index to range", + zap.Int64("indexID", i), zap.Int64("rangeCount", indexRangeCount), + zap.Binary("start", startKeyOfIndex), zap.Binary("end", lastKeyOfIndex)) + + values := splitValuesToRange(startValues, endValues, indexRangeCount) + + keyPrefix := tablecodec.EncodeTableIndexPrefix(tableID, i) + for _, v := range values { + e := append(keyPrefix, v...) + rangeEnd := nextKey(e) + ranges = append(ranges, Range{start: startKeyOfIndex, end: rangeEnd}) + startKeyOfIndex = rangeEnd + } + } + } else { + // data engine + tableID, startHandleInterface, err := tablecodec.DecodeRecordKey(startKey) + if err != nil { + return nil, err + } + endHandleInterface, err := tablecodec.DecodeRowKey(endKey) + if err != nil { + return nil, err + } + endHandle := endHandleInterface.IntValue() + startHandle := startHandleInterface.IntValue() + step := (endHandle - startHandle) / n + index := int64(0) + var sKey, eKey []byte + + log.L().Info("split data ranges", zap.Int64("step", step), + zap.Int64("startHandle", startHandle), zap.Int64("endHandle", endHandle), + zap.String("engine", engineUUID.String())) + + for i := startHandle; i+step <= endHandle; i += step { + sKey = tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(i)) + eKey = tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(i+step-1)) + index = i + log.L().Debug("data engine append range", zap.Int64("start handle", i), + zap.Int64("end handle", i+step-1), zap.Binary("start key", sKey), + zap.Binary("end key", eKey), zap.Int64("step", step)) + + ranges = append(ranges, Range{start: sKey, end: nextKey(eKey)}) + } + log.L().Debug("data engine append range at final", + zap.Int64("start handle", index+step), zap.Int64("end handle", endHandle), + zap.Binary("start key", sKey), zap.Binary("end key", endKey), + zap.Int64("step", endHandle-index-step+1)) + + sKey = tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(index+step)) + ranges = append(ranges, Range{start: sKey, end: nextKey(endKey)}) + } + return ranges, nil +} + +type bytesRecycleChan struct { + ch chan []byte +} + +// recycleChan is used for reusing allocated []byte so we can use memory more efficiently +// +// NOTE: we don't used a `sync.Pool` because when will sync.Pool release is depending on the +// garbage collector which always release the memory so late. Use a fixed size chan to reuse +// can decrease the memory usage to 1/3 compare with sync.Pool. +var recycleChan *bytesRecycleChan + +func init() { + recycleChan = &bytesRecycleChan{ + ch: make(chan []byte, 1024), + } +} + +func (c *bytesRecycleChan) Acquire() []byte { + select { + case b := <-c.ch: + return b + default: + return manual.New(1 << 20) // 1M + } +} + +func (c *bytesRecycleChan) Release(w []byte) { + select { + case c.ch <- w: + return + default: + manual.Free(w) + } +} + +type bytesBuffer struct { + bufs [][]byte + curBuf []byte + curIdx int + curBufIdx int + curBufLen int +} + +func newBytesBuffer() *bytesBuffer { + return &bytesBuffer{bufs: make([][]byte, 0, 128), curBufIdx: -1} +} + +func (b *bytesBuffer) addBuf() { + if b.curBufIdx < len(b.bufs)-1 { + b.curBufIdx += 1 + b.curBuf = b.bufs[b.curBufIdx] + } else { + buf := recycleChan.Acquire() + b.bufs = append(b.bufs, buf) + b.curBuf = buf + b.curBufIdx = len(b.bufs) - 1 + } + + b.curBufLen = len(b.curBuf) + b.curIdx = 0 +} + +func (b *bytesBuffer) reset() { + if len(b.bufs) > 0 { + b.curBuf = b.bufs[0] + b.curBufLen = len(b.bufs[0]) + b.curBufIdx = 0 + b.curIdx = 0 + } +} + +func (b *bytesBuffer) destroy() { + for _, buf := range b.bufs { + recycleChan.Release(buf) + } + b.bufs = b.bufs[:0] +} + +func (b *bytesBuffer) totalSize() int64 { + return int64(len(b.bufs)) * int64(1<<20) +} + +func (b *bytesBuffer) addBytes(bytes []byte) []byte { + if len(bytes) > bigValueSize { + return append([]byte{}, bytes...) + } + + if b.curIdx+len(bytes) > b.curBufLen { + b.addBuf() + } + idx := b.curIdx + copy(b.curBuf[idx:], bytes) + b.curIdx += len(bytes) + return b.curBuf[idx:b.curIdx] +} + +func (local *local) writeAndIngestByRange( + ctx context.Context, + engineFile *LocalFile, + start, end []byte) error { + ito := &pebble.IterOptions{ + LowerBound: start, + UpperBound: end, + } + + iter := engineFile.db.NewIter(ito) + defer iter.Close() + // Needs seek to first because NewIter returns an iterator that is unpositioned + hasKey := iter.First() + if !hasKey { + log.L().Info("There is no pairs in iterator", + zap.Binary("start", start), + zap.Binary("end", end), + zap.Binary("next end", nextKey(end))) + return nil + } + pairStart := append([]byte{}, iter.Key()...) + iter.Last() + pairEnd := append([]byte{}, iter.Key()...) + + var regions []*split.RegionInfo + var err error + ctx, cancel := context.WithCancel(ctx) + defer cancel() +WriteAndIngest: + for retry := 0; retry < maxRetryTimes; retry++ { + if retry != 0 { + select { + case <-time.After(time.Second): + case <-ctx.Done(): + return ctx.Err() + } + } + startKey := codec.EncodeBytes([]byte{}, pairStart) + endKey := codec.EncodeBytes([]byte{}, nextKey(pairEnd)) + regions, err = paginateScanRegion(ctx, local.splitCli, startKey, endKey, 128) + if err != nil || len(regions) == 0 { + log.L().Warn("scan region failed", zap.Error(err), zap.Int("region_len", len(regions))) + continue WriteAndIngest + } + + shouldWait := false + errChan := make(chan error, len(regions)) + for _, region := range regions { + log.L().Debug("get region", zap.Int("retry", retry), zap.Binary("startKey", startKey), + zap.Binary("endKey", endKey), zap.Uint64("id", region.Region.GetId()), + zap.Stringer("epoch", region.Region.GetRegionEpoch()), zap.Binary("start", region.Region.GetStartKey()), + zap.Binary("end", region.Region.GetEndKey()), zap.Reflect("peers", region.Region.GetPeers())) + + // generate new uuid for concurrent write to tikv + + if len(regions) == 1 { + if err = local.WriteAndIngestPairs(ctx, engineFile, region); err != nil { + continue WriteAndIngest + } + } else { + shouldWait = true + go func(r *split.RegionInfo) { + errChan <- local.WriteAndIngestPairs(ctx, engineFile, r) + }(region) + } + } + if shouldWait { + shouldRetry := false + for i := 0; i < len(regions); i++ { + err1 := <-errChan + if err1 != nil { + err = err1 + log.L().Warn("should retry this range", zap.Int("retry", retry), zap.Error(err)) + shouldRetry = true + } + } + if !shouldRetry { + return nil + } else { + continue WriteAndIngest + } + } + return nil + } + if err == nil { + err = errors.New("all retry failed") + } + return err +} + +func (local *local) WriteAndIngestPairs( + ctx context.Context, + engineFile *LocalFile, + region *split.RegionInfo, +) error { + metas, err := local.WriteToTiKV(ctx, engineFile, region) + if err != nil { + log.L().Warn("write to tikv failed", zap.Error(err)) + return err + } + + for _, meta := range metas { + for i := 0; i < maxRetryTimes; i++ { + log.L().Debug("ingest meta", zap.Reflect("meta", meta)) + resp, err := local.Ingest(ctx, meta, region) + if err != nil { + log.L().Warn("ingest failed", zap.Error(err), zap.Reflect("meta", meta), + zap.Reflect("region", region)) + continue + } + failpoint.Inject("FailIngestMeta", func(val failpoint.Value) { + switch val.(string) { + case "notleader": + resp.Error.NotLeader = &errorpb.NotLeader{ + RegionId: region.Region.Id, Leader: region.Leader} + case "epochnotmatch": + resp.Error.EpochNotMatch = &errorpb.EpochNotMatch{ + CurrentRegions: []*metapb.Region{region.Region}} + } + }) + needRetry, newRegion, errIngest := isIngestRetryable(resp, region, meta) + if errIngest == nil { + // ingest next meta + break + } + if !needRetry { + log.L().Warn("ingest failed noretry", zap.Error(errIngest), zap.Reflect("meta", meta), + zap.Reflect("region", region)) + // met non-retryable error retry whole Write procedure + return errIngest + } + // retry with not leader and epoch not match error + if newRegion != nil && i < maxRetryTimes-1 { + region = newRegion + } else { + log.L().Warn("retry ingest due to", + zap.Reflect("meta", meta), zap.Reflect("region", region), + zap.Reflect("new region", newRegion), zap.Error(errIngest)) + return errIngest + } + } + } + return nil +} + +func (local *local) WriteAndIngestByRanges(ctx context.Context, engineFile *LocalFile, ranges []Range) error { + if engineFile.Length == 0 { + log.L().Error("the ranges is empty") + return nil + } + log.L().Debug("the ranges Length write to tikv", zap.Int("Length", len(ranges))) + + errCh := make(chan error, len(ranges)) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for _, r := range ranges { + startKey := r.start + endKey := r.end + w := local.rangeConcurrency.Apply() + go func(w *worker.Worker) { + defer func() { + local.rangeConcurrency.Recycle(w) + }() + var err error + for i := 0; i < maxRetryTimes; i++ { + if err = local.writeAndIngestByRange(ctx, engineFile, startKey, endKey); err != nil { + log.L().Warn("write and ingest by range failed", + zap.Int("retry time", i+1), zap.Error(err)) + } else { + break + } + } + errCh <- err + }(w) + } + + for i := 0; i < len(ranges); i++ { + e := <-errCh + if e != nil { + return e + } + } + return nil +} + +func (local *local) ImportEngine(ctx context.Context, engineUUID uuid.UUID) error { + engineFile, ok := local.engines.Load(engineUUID) + if !ok { + // skip if engine not exist. See the comment of `CloseEngine` for more detail. + return nil + } + // split sorted file into range by 96MB size per file + ranges, err := local.ReadAndSplitIntoRange(engineFile.(*LocalFile), engineUUID) + if err != nil { + return err + } + // split region by given ranges + err = local.SplitAndScatterRegionByRanges(ctx, ranges) + if err != nil { + log.L().Error("split & scatter ranges failed", zap.Error(err)) + return err + } + // start to write to kv and ingest + err = local.WriteAndIngestByRanges(ctx, engineFile.(*LocalFile), ranges) + if err != nil { + log.L().Error("write and ingest ranges failed", zap.Error(err)) + return err + } + log.L().Info("import engine success", zap.Stringer("uuid", engineUUID)) + return nil +} + +func (local *local) CleanupEngine(ctx context.Context, engineUUID uuid.UUID) error { + // release this engine after import success + engineFile, ok := local.engines.Load(engineUUID) + if ok { + localEngine := engineFile.(*LocalFile) + err := localEngine.Close() + if err != nil { + return err + } + err = localEngine.Cleanup(local.localStoreDir) + if err != nil { + return err + } + local.engines.Delete(engineUUID) + } else { + log.L().Error("could not find engine in cleanupEngine", zap.Stringer("uuid", engineUUID)) + } + return nil +} + +func (local *local) CheckRequirements() error { + if err := checkTiDBVersion(local.tls, localMinTiDBVersion); err != nil { + return err + } + if err := checkPDVersion(local.tls, local.pdAddr, localMinPDVersion); err != nil { + return err + } + if err := checkTiKVVersion(local.tls, local.pdAddr, localMinTiKVVersion); err != nil { + return err + } + return nil +} + +func (local *local) FetchRemoteTableModels(schemaName string) ([]*model.TableInfo, error) { + return fetchRemoteTableModelsFromTLS(local.tls, schemaName) +} + +func (local *local) WriteRows( + ctx context.Context, + engineUUID uuid.UUID, + tableName string, + columnNames []string, + ts uint64, + rows Rows, +) (finalErr error) { + kvs := rows.(kvPairs) + if len(kvs) == 0 { + return nil + } + + e, ok := local.engines.Load(engineUUID) + if !ok { + return errors.Errorf("could not find engine for %s", engineUUID.String()) + } + engineFile := e.(*LocalFile) + + // write to pebble to make them sorted + wb := engineFile.db.NewBatch() + defer wb.Close() + wo := &pebble.WriteOptions{Sync: false} + + size := int64(0) + for _, pair := range kvs { + wb.Set(pair.Key, pair.Val, wo) + size += int64(len(pair.Key) + len(pair.Val)) + } + if err := wb.Commit(wo); err != nil { + return err + } + atomic.AddInt64(&engineFile.Length, int64(len(kvs))) + atomic.AddInt64(&engineFile.TotalSize, size) + engineFile.Ts = ts + return +} + +func (local *local) MakeEmptyRows() Rows { + return kvPairs(nil) +} + +func (local *local) NewEncoder(tbl table.Table, options *SessionOptions) Encoder { + return NewTableKVEncoder(tbl, options) +} + +func isIngestRetryable(resp *sst.IngestResponse, region *split.RegionInfo, meta *sst.SSTMeta) (bool, *split.RegionInfo, error) { + if resp.GetError() == nil { + return false, nil, nil + } + + var newRegion *split.RegionInfo + switch errPb := resp.GetError(); { + case errPb.NotLeader != nil: + if newLeader := errPb.GetNotLeader().GetLeader(); newLeader != nil { + newRegion = &split.RegionInfo{ + Leader: newLeader, + Region: region.Region, + } + return true, newRegion, errors.Errorf("not leader: %s", errPb.GetMessage()) + } + case errPb.EpochNotMatch != nil: + if currentRegions := errPb.GetEpochNotMatch().GetCurrentRegions(); currentRegions != nil { + var currentRegion *metapb.Region + for _, r := range currentRegions { + if insideRegion(r, meta) { + currentRegion = r + break + } + } + if currentRegion != nil { + var newLeader *metapb.Peer + for _, p := range currentRegion.Peers { + if p.GetStoreId() == region.Leader.GetStoreId() { + newLeader = p + break + } + } + if newLeader != nil { + newRegion = &split.RegionInfo{ + Leader: newLeader, + Region: currentRegion, + } + } + } + } + return true, newRegion, errors.Errorf("epoch not match: %s", errPb.GetMessage()) + } + return false, nil, errors.Errorf("non retryable error: %s", resp.GetError().GetMessage()) +} + +func nextKey(key []byte) []byte { + if len(key) == 0 { + return []byte{} + } + res := make([]byte, 0, len(key)+1) + pos := 0 + for i := len(key) - 1; i >= 0; i-- { + if key[i] != '\xff' { + pos = i + break + } + } + s, e := key[:pos], key[pos]+1 + res = append(append(res, s...), e) + return res +} + +// splitValuesToRange try to cut [start, end] to count range approximately +// just like [start, v1], [v1, v2]... [vCount-1, end] +// return value []{v1, v2... vCount-1, End} +func splitValuesToRange(start []byte, end []byte, count int64) [][]byte { + if bytes.Equal(start, end) { + log.L().Info("couldn't split range due to start end are same", + zap.Binary("start", start), + zap.Binary("end", end), + zap.Int64("count", count)) + return [][]byte{end} + } + + startBytes := make([]byte, 8) + endBytes := make([]byte, 8) + + minLen := len(start) + if minLen > len(end) { + minLen = len(end) + } + + offset := 0 + for i := 0; i < minLen; i++ { + if start[i] != end[i] { + offset = i + break + } + } + + copy(startBytes, start[offset:]) + copy(endBytes, end[offset:]) + + sValue := binary.BigEndian.Uint64(startBytes) + eValue := binary.BigEndian.Uint64(endBytes) + + step := (eValue - sValue) / uint64(count) + if step == uint64(0) { + step = uint64(1) + } + + res := make([][]byte, 0, count) + for cur := sValue + step; cur <= eValue-step; cur += step { + curBytes := make([]byte, offset+8) + copy(curBytes, start[:offset]) + binary.BigEndian.PutUint64(curBytes[offset:], cur) + res = append(res, curBytes) + } + res = append(res, end) + + return res +} diff --git a/lightning/backend/localhelper.go b/lightning/backend/localhelper.go new file mode 100644 index 000000000..34debca5c --- /dev/null +++ b/lightning/backend/localhelper.go @@ -0,0 +1,286 @@ +package backend + +import ( + "bytes" + "context" + "encoding/hex" + "strings" + "time" + + split "github.com/pingcap/br/pkg/restore" + "github.com/pingcap/errors" + sst "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/codec" + "go.uber.org/zap" + + "github.com/pingcap/tidb-lightning/lightning/log" +) + +const SplitRetryTimes = 32 + +// TODO remove this file and use br internal functions +// This File include region split & scatter operation just like br. +// we can simply call br function, but we need to change some function signature of br + +func (local *local) SplitAndScatterRegionByRanges(ctx context.Context, ranges []Range) error { + if len(ranges) == 0 { + return nil + } + + minKey := codec.EncodeBytes([]byte{}, ranges[0].start) + maxKey := codec.EncodeBytes([]byte{}, ranges[len(ranges)-1].end) + + log.L().Info("split and scatter region", + zap.Binary("minKey", minKey), + zap.Binary("maxKey", maxKey), + ) + + regions, err := paginateScanRegion(ctx, local.splitCli, minKey, maxKey, 128) + if err != nil { + return err + } + + splitKeyMap := getSplitKeys(ranges, regions) + + regionMap := make(map[uint64]*split.RegionInfo) + for _, region := range regions { + regionMap[region.Region.GetId()] = region + } + + scatterRegions := make([]*split.RegionInfo, 0) +SplitRegions: + for i := 0; i < SplitRetryTimes; i++ { + for regionID, keys := range splitKeyMap { + var newRegions []*split.RegionInfo + region := regionMap[regionID] + newRegions, errSplit := local.BatchSplitRegions(ctx, region, keys) + if errSplit != nil { + if strings.Contains(errSplit.Error(), "no valid key") { + for _, key := range keys { + log.L().Error("no valid key", + zap.Binary("startKey", region.Region.StartKey), + zap.Binary("endKey", region.Region.EndKey), + zap.Binary("key", codec.EncodeBytes([]byte{}, key))) + } + return errors.Trace(errSplit) + } + log.L().Warn("split regions", zap.Error(errSplit)) + select { + case <-time.After(time.Second): + case <-ctx.Done(): + return ctx.Err() + } + continue SplitRegions + } + scatterRegions = append(scatterRegions, newRegions...) + } + break + } + + startTime := time.Now() + scatterCount := 0 + for _, region := range scatterRegions { + local.waitForScatterRegion(ctx, region) + if time.Since(startTime) > split.ScatterWaitUpperInterval { + break + } + scatterCount++ + } + if scatterCount == len(scatterRegions) { + log.L().Info("waiting for scattering regions done", + zap.Int("regions", len(scatterRegions)), zap.Duration("take", time.Since(startTime))) + } else { + log.L().Warn("waiting for scattering regions timeout", + zap.Int("scatterCount", scatterCount), + zap.Int("regions", len(scatterRegions)), + zap.Duration("take", time.Since(startTime))) + } + return nil +} + +func paginateScanRegion( + ctx context.Context, client split.SplitClient, startKey, endKey []byte, limit int, +) ([]*split.RegionInfo, error) { + if len(endKey) != 0 && bytes.Compare(startKey, endKey) >= 0 { + return nil, errors.Errorf("startKey > endKey, startKey %s, endkey %s", + hex.EncodeToString(startKey), hex.EncodeToString(endKey)) + } + + regions := []*split.RegionInfo{} + for { + batch, err := client.ScanRegions(ctx, startKey, endKey, limit) + if err != nil { + return nil, errors.Trace(err) + } + regions = append(regions, batch...) + if len(batch) < limit { + // No more region + break + } + startKey = batch[len(batch)-1].Region.GetEndKey() + if len(startKey) == 0 || + (len(endKey) > 0 && bytes.Compare(startKey, endKey) >= 0) { + // All key space have scanned + break + } + } + return regions, nil +} + +func (local *local) BatchSplitRegions(ctx context.Context, region *split.RegionInfo, keys [][]byte) ([]*split.RegionInfo, error) { + newRegions, err := local.splitCli.BatchSplitRegions(ctx, region, keys) + if err != nil { + return nil, err + } + for _, region := range newRegions { + // Wait for a while until the regions successfully splits. + local.waitForSplit(ctx, region.Region.Id) + if err = local.splitCli.ScatterRegion(ctx, region); err != nil { + log.L().Warn("scatter region failed", zap.Stringer("region", region.Region), zap.Error(err)) + } + } + return newRegions, nil +} + +func (local *local) hasRegion(ctx context.Context, regionID uint64) (bool, error) { + regionInfo, err := local.splitCli.GetRegionByID(ctx, regionID) + if err != nil { + return false, err + } + return regionInfo != nil, nil +} + +func (local *local) waitForSplit(ctx context.Context, regionID uint64) { + for i := 0; i < split.SplitCheckMaxRetryTimes; i++ { + ok, err := local.hasRegion(ctx, regionID) + if err != nil { + log.L().Warn("wait for split failed", zap.Error(err)) + return + } + if ok { + break + } + select { + case <-time.After(time.Second): + case <-ctx.Done(): + return + } + } +} + +func (local *local) waitForScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) { + regionID := regionInfo.Region.GetId() + for i := 0; i < split.ScatterWaitMaxRetryTimes; i++ { + ok, err := local.isScatterRegionFinished(ctx, regionID) + if err != nil { + log.L().Warn("scatter region failed: do not have the region", + zap.Stringer("region", regionInfo.Region)) + return + } + if ok { + break + } + select { + case <-time.After(time.Second): + case <-ctx.Done(): + return + } + } +} + +func (local *local) isScatterRegionFinished(ctx context.Context, regionID uint64) (bool, error) { + resp, err := local.splitCli.GetOperator(ctx, regionID) + if err != nil { + return false, err + } + // Heartbeat may not be sent to PD + if respErr := resp.GetHeader().GetError(); respErr != nil { + if respErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND { + return true, nil + } + return false, errors.Errorf("get operator error: %s", respErr.GetType()) + } + // If the current operator of the region is not 'scatter-region', we could assume + // that 'scatter-operator' has finished or timeout + ok := string(resp.GetDesc()) != "scatter-region" || resp.GetStatus() != pdpb.OperatorStatus_RUNNING + return ok, nil +} + +func getSplitKeys(ranges []Range, regions []*split.RegionInfo) map[uint64][][]byte { + splitKeyMap := make(map[uint64][][]byte) + checkKeys := make([][]byte, 0) + for _, rg := range ranges { + checkKeys = append(checkKeys, truncateRowKey(rg.end)) + } + for _, key := range checkKeys { + if region := needSplit(key, regions); region != nil { + splitKeys, ok := splitKeyMap[region.Region.GetId()] + if !ok { + splitKeys = make([][]byte, 0, 1) + } + splitKeyMap[region.Region.GetId()] = append(splitKeys, key) + log.L().Debug("get key for split region", + zap.Binary("key", key), + zap.Binary("startKey", region.Region.StartKey), + zap.Binary("endKey", region.Region.EndKey)) + } + } + return splitKeyMap +} + +// needSplit checks whether a key is necessary to split, if true returns the split region +func needSplit(splitKey []byte, regions []*split.RegionInfo) *split.RegionInfo { + // If splitKey is the max key. + if len(splitKey) == 0 { + return nil + } + splitKey = codec.EncodeBytes([]byte{}, splitKey) + + for _, region := range regions { + // If splitKey is the boundary of the region + log.L().Debug("need split", + zap.Binary("splitKey", splitKey), + zap.Binary("region start", region.Region.GetStartKey()), + zap.Binary("region end", region.Region.GetEndKey()), + ) + if bytes.Equal(splitKey, region.Region.GetStartKey()) { + return nil + } + // If splitKey is in a region + if bytes.Compare(splitKey, region.Region.GetStartKey()) > 0 && beforeEnd(splitKey, region.Region.GetEndKey()) { + return region + } + } + return nil +} + +var ( + tablePrefix = tablecodec.TablePrefix() + idLen = 8 + recordPrefix = []byte("_r") +) + +func truncateRowKey(key []byte) []byte { + if bytes.HasPrefix(key, tablePrefix) && + len(key) > tablecodec.RecordRowKeyLen && + bytes.HasPrefix(key[len(tablePrefix)+idLen:], recordPrefix) { + return key[:tablecodec.RecordRowKeyLen] + } + return key +} + +func beforeEnd(key []byte, end []byte) bool { + return bytes.Compare(key, end) < 0 || len(end) == 0 +} + +func insideRegion(region *metapb.Region, meta *sst.SSTMeta) bool { + rg := meta.GetRange() + return keyInsideRegion(region, rg.GetStart()) && keyInsideRegion(region, rg.GetEnd()) +} + +func keyInsideRegion(region *metapb.Region, key []byte) bool { + return bytes.Compare(key, region.GetStartKey()) >= 0 && (beforeEnd(key, region.GetEndKey())) +} diff --git a/lightning/backend/tidb.go b/lightning/backend/tidb.go index 7656d1315..f1643bba4 100644 --- a/lightning/backend/tidb.go +++ b/lightning/backend/tidb.go @@ -304,7 +304,7 @@ func (be *tidbBackend) WriteRows(ctx context.Context, _ uuid.UUID, tableName str insertStmt.WriteString(" VALUES") // Note: we are not going to do interpolation (prepared statements) to avoid - // complication arised from data length overflow of BIT and BINARY columns + // complication arise from data length overflow of BIT and BINARY columns for i, row := range rows { if i != 0 { diff --git a/lightning/common/security.go b/lightning/common/security.go index 7f90b61d3..a2fd22b22 100644 --- a/lightning/common/security.go +++ b/lightning/common/security.go @@ -22,15 +22,19 @@ import ( "net/http/httptest" "github.com/pingcap/errors" + pd "github.com/pingcap/pd/v4/client" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) // TLS type TLS struct { - inner *tls.Config - client *http.Client - url string + caPath string + certPath string + keyPath string + inner *tls.Config + client *http.Client + url string } // ToTLSConfig constructs a `*tls.Config` from the CA, certification and key @@ -90,9 +94,12 @@ func NewTLS(caPath, certPath, keyPath, host string) (*TLS, error) { transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = inner return &TLS{ - inner: inner, - client: &http.Client{Transport: transport}, - url: "https://" + host, + caPath: caPath, + certPath: certPath, + keyPath: keyPath, + inner: inner, + client: &http.Client{Transport: transport}, + url: "https://" + host, }, nil } @@ -141,3 +148,15 @@ func (tc *TLS) WrapListener(l net.Listener) net.Listener { func (tc *TLS) GetJSON(path string, v interface{}) error { return GetJSON(tc.client, tc.url+path, v) } + +func (tc *TLS) ToPDSecurityOption() pd.SecurityOption { + return pd.SecurityOption{ + CAPath: tc.caPath, + CertPath: tc.certPath, + KeyPath: tc.keyPath, + } +} + +func (tc *TLS) TLSConfig() *tls.Config { + return tc.inner +} diff --git a/lightning/common/util.go b/lightning/common/util.go index 5187ddf03..17570a72f 100644 --- a/lightning/common/util.go +++ b/lightning/common/util.go @@ -89,6 +89,15 @@ func IsDirExists(name string) bool { return f != nil && f.IsDir() } +// IsEmptyDir checks if dir is empty. +func IsEmptyDir(name string) bool { + entries, err := ioutil.ReadDir(name) + if err != nil { + return false + } + return len(entries) == 0 +} + // SQLWithRetry constructs a retryable transaction. type SQLWithRetry struct { DB *sql.DB diff --git a/lightning/config/config.go b/lightning/config/config.go index b834a4bce..7e03625cb 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -44,6 +44,9 @@ const ( BackendTiDB = "tidb" // BackendImporter is a constant for choosing the "Importer" backend in the configuration. BackendImporter = "importer" + // BackendLocal is a constant for choosing the "Local" backup in the configuration. + // In this mode, we write & sort kv pairs with local storage and directly write them to tikv. + BackendLocal = "local" // CheckpointDriverMySQL is a constant for choosing the "MySQL" checkpoint driver in the configuration. CheckpointDriverMySQL = "mysql" @@ -150,10 +153,14 @@ type MydumperRuntime struct { } type TikvImporter struct { - Addr string `toml:"addr" json:"addr"` - Backend string `toml:"backend" json:"backend"` - OnDuplicate string `toml:"on-duplicate" json:"on-duplicate"` - MaxKVPairs int `toml:"max-kv-pairs" json:"max-kv-pairs"` + Addr string `toml:"addr" json:"addr"` + Backend string `toml:"backend" json:"backend"` + OnDuplicate string `toml:"on-duplicate" json:"on-duplicate"` + MaxKVPairs int `toml:"max-kv-pairs" json:"max-kv-pairs"` + SendKVPairs int `toml:"send-kv-pairs" json:"send-kv-pairs"` + RegionSplitSize int64 `toml:"region-split-size" json:"region-split-size"` + SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"` + RangeConcurrency int `toml:"range-concurrency" json:"range-concurrency"` } type Checkpoint struct { @@ -252,9 +259,11 @@ func NewConfig() *Config { MaxRegionSize: MaxRegionSize, }, TikvImporter: TikvImporter{ - Backend: BackendImporter, - OnDuplicate: ReplaceOnDup, - MaxKVPairs: 32, + Backend: BackendImporter, + OnDuplicate: ReplaceOnDup, + MaxKVPairs: 32, + SendKVPairs: 100000, + RegionSplitSize: SplitRegionSize, }, PostRestore: PostRestore{ Checksum: true, @@ -280,6 +289,7 @@ func (cfg *Config) LoadFromGlobal(global *GlobalConfig) error { cfg.Mydumper.NoSchema = global.Mydumper.NoSchema cfg.TikvImporter.Addr = global.TikvImporter.Addr cfg.TikvImporter.Backend = global.TikvImporter.Backend + cfg.TikvImporter.SortedKVDir = global.TikvImporter.SortedKVDir cfg.Checkpoint.Enable = global.Checkpoint.Enable cfg.PostRestore.Checksum = global.PostRestore.Checksum cfg.PostRestore.Analyze = global.PostRestore.Analyze @@ -386,17 +396,29 @@ func (cfg *Config) Adjust() error { cfg.App.TableConcurrency = cfg.App.RegionConcurrency } mustHaveInternalConnections = false - case BackendImporter: + case BackendImporter, BackendLocal: if cfg.App.IndexConcurrency == 0 { cfg.App.IndexConcurrency = 2 } if cfg.App.TableConcurrency == 0 { cfg.App.TableConcurrency = 6 } + if cfg.TikvImporter.RangeConcurrency == 0 { + cfg.TikvImporter.RangeConcurrency = 32 + } + if cfg.TikvImporter.RegionSplitSize == 0 { + cfg.TikvImporter.RegionSplitSize = SplitRegionSize + } default: return errors.Errorf("invalid config: unsupported `tikv-importer.backend` (%s)", cfg.TikvImporter.Backend) } + if cfg.TikvImporter.Backend == BackendLocal { + if len(cfg.TikvImporter.SortedKVDir) == 0 { + return errors.Errorf("tikv-importer.sorted-kv-dir must not be empty!") + } + } + if cfg.TikvImporter.Backend == BackendTiDB { cfg.TikvImporter.OnDuplicate = strings.ToLower(cfg.TikvImporter.OnDuplicate) switch cfg.TikvImporter.OnDuplicate { diff --git a/lightning/config/const.go b/lightning/config/const.go index 8d24a13b4..ad7863708 100644 --- a/lightning/config/const.go +++ b/lightning/config/const.go @@ -19,9 +19,10 @@ const ( _G = _M << 10 // mydumper - ReadBlockSize int64 = 64 * _K - MinRegionSize int64 = 256 * _M - MaxRegionSize int64 = 256 * _M + ReadBlockSize int64 = 64 * _K + MinRegionSize int64 = 256 * _M + MaxRegionSize int64 = 256 * _M + SplitRegionSize int64 = 96 * _M BufferSizeScale = 5 diff --git a/lightning/config/global.go b/lightning/config/global.go index 03623cf49..888f51dd5 100644 --- a/lightning/config/global.go +++ b/lightning/config/global.go @@ -54,8 +54,9 @@ type GlobalMydumper struct { } type GlobalImporter struct { - Addr string `toml:"addr" json:"addr"` - Backend string `toml:"backend" json:"backend"` + Addr string `toml:"addr" json:"addr"` + Backend string `toml:"backend" json:"backend"` + SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"` } type GlobalConfig struct { @@ -145,7 +146,8 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon pdAddr := fs.String("pd-urls", "", "PD endpoint address") dataSrcPath := fs.String("d", "", "Directory of the dump to import") importerAddr := fs.String("importer", "", "address (host:port) to connect to tikv-importer") - backend := fs.String("backend", "", `delivery backend ("importer" or "tidb")`) + backend := fs.String("backend", "", `delivery backend ("importer" or "tidb" or "local")`) + sortedKVDir := fs.String("sorted-kv-dir", "", "path for KV pairs when local backend enabled") enableCheckpoint := fs.Bool("enable-checkpoint", true, "whether to enable checkpoints") noSchema := fs.Bool("no-schema", false, "ignore schema files, get schema directly from TiDB instead") checksum := fs.Bool("checksum", true, "compare checksum after importing") @@ -220,6 +222,9 @@ func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalCon if *backend != "" { cfg.TikvImporter.Backend = *backend } + if *sortedKVDir != "" { + cfg.TikvImporter.SortedKVDir = *sortedKVDir + } if !*enableCheckpoint { cfg.Checkpoint.Enable = false } diff --git a/lightning/manual/manual.go b/lightning/manual/manual.go new file mode 100644 index 000000000..fbc78814f --- /dev/null +++ b/lightning/manual/manual.go @@ -0,0 +1,65 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package manual + +// #include +import "C" +import "unsafe" + +// The go:linkname directives provides backdoor access to private functions in +// the runtime. Below we're accessing the throw function. + +//go:linkname throw runtime.throw +func throw(s string) + +// TODO(peter): Rather than relying an C malloc/free, we could fork the Go +// runtime page allocator and allocate large chunks of memory using mmap or +// similar. + +const ( + // MaxArrayLen is a safe maximum length for slices on this architecture. + MaxArrayLen = 1<<31 - 1 +) + +// New allocates a slice of size n. The returned slice is from manually managed +// memory and MUST be released by calling Free. Failure to do so will result in +// a memory leak. +func New(n int) []byte { + if n == 0 { + return make([]byte, 0) + } + // We need to be conscious of the Cgo pointer passing rules: + // + // https://golang.org/cmd/cgo/#hdr-Passing_pointers + // + // ... + // Note: the current implementation has a bug. While Go code is permitted + // to write nil or a C pointer (but not a Go pointer) to C memory, the + // current implementation may sometimes cause a runtime error if the + // contents of the C memory appear to be a Go pointer. Therefore, avoid + // passing uninitialized C memory to Go code if the Go code is going to + // store pointer values in it. Zero out the memory in C before passing it + // to Go. + ptr := C.calloc(C.size_t(n), 1) + if ptr == nil { + // NB: throw is like panic, except it guarantees the process will be + // terminated. The call below is exactly what the Go runtime invokes when + // it cannot allocate memory. + throw("out of memory") + } + // Interpret the C pointer as a pointer to a Go array, then slice. + return (*[MaxArrayLen]byte)(unsafe.Pointer(ptr))[:n:n] +} + +// Free frees the specified slice. +func Free(b []byte) { + if cap(b) != 0 { + if len(b) == 0 { + b = b[:cap(b)] + } + ptr := unsafe.Pointer(&b[0]) + C.free(ptr) + } +} diff --git a/lightning/manual/manual_nocgo.go b/lightning/manual/manual_nocgo.go new file mode 100644 index 000000000..466d1ee63 --- /dev/null +++ b/lightning/manual/manual_nocgo.go @@ -0,0 +1,19 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// +build !cgo + +package manual + +// Provides versions of New and Free when cgo is not available (e.g. cross +// compilation). + +// New allocates a slice of size n. +func New(n int) []byte { + return make([]byte, n) +} + +// Free frees the specified slice. +func Free(b []byte) { +} diff --git a/lightning/mydump/region.go b/lightning/mydump/region.go index 3d909d909..268094677 100644 --- a/lightning/mydump/region.go +++ b/lightning/mydump/region.go @@ -199,6 +199,10 @@ func MakeTableRegions( dataFileSizes = append(dataFileSizes, float64(dataFileSize)) } + log.L().Debug("in makeTableRegions", + zap.Int64("maxRegionSize", cfg.Mydumper.MaxRegionSize), + zap.Int("len fileRegions", len(filesRegions))) + AllocateEngineIDs(filesRegions, dataFileSizes, float64(cfg.Mydumper.BatchSize), cfg.Mydumper.BatchImportRatio, float64(cfg.App.TableConcurrency)) return filesRegions, nil } diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index ec2a9a27e..e7b578db0 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -180,6 +180,13 @@ func NewRestoreControllerWithPauser(ctx context.Context, dbMetas []*mydump.MDDat } case config.BackendTiDB: backend = kv.NewTiDBBackend(tidbMgr.db, cfg.TikvImporter.OnDuplicate) + case config.BackendLocal: + backend, err = kv.NewLocalBackend(ctx, tls, cfg.TiDB.PdAddr, cfg.TikvImporter.RegionSplitSize, + cfg.TikvImporter.SortedKVDir, cfg.TikvImporter.RangeConcurrency, cfg.TikvImporter.SendKVPairs, + cfg.Checkpoint.Enable) + if err != nil { + return nil, err + } default: return nil, errors.New("unknown backend: " + cfg.TikvImporter.Backend) } @@ -356,6 +363,9 @@ func (rc *RestoreController) estimateChunkCountIntoMetrics() { func (rc *RestoreController) saveStatusCheckpoint(tableName string, engineID int32, err error, statusIfSucceed CheckpointStatus) { merger := &StatusCheckpointMerger{Status: statusIfSucceed, EngineID: engineID} + log.L().Debug("update checkpoint", zap.String("table", tableName), zap.Int32("engine_id", engineID), + zap.Uint8("new_status", uint8(statusIfSucceed)), zap.Error(err)) + switch { case err == nil: break @@ -789,6 +799,7 @@ func (t *TableRestore) restoreEngines(ctx context.Context, rc *RestoreController if indexEngineCp.Status < CheckpointStatusImported && cp.Status < CheckpointStatusIndexImported { indexWorker := rc.indexWorkers.Apply() defer rc.indexWorkers.Recycle(indexWorker) + indexEngine, err := rc.backend.OpenEngine(ctx, t.tableName, indexEngineID) if err != nil { return errors.Trace(err) @@ -821,30 +832,36 @@ func (t *TableRestore) restoreEngines(ctx context.Context, rc *RestoreController continue } - wg.Add(1) + if engine.Status < CheckpointStatusImported { + wg.Add(1) - // Note: We still need tableWorkers to control the concurrency of tables. - // In the future, we will investigate more about - // the difference between restoring tables concurrently and restoring tables one by one. - restoreWorker := rc.tableWorkers.Apply() - - go func(w *worker.Worker, eid int32, ecp *EngineCheckpoint) { - defer wg.Done() - - engineLogTask := t.logger.With(zap.Int32("engineNumber", eid)).Begin(zap.InfoLevel, "restore engine") - dataClosedEngine, dataWorker, err := t.restoreEngine(ctx, rc, indexEngine, eid, ecp) - engineLogTask.End(zap.ErrorLevel, err) - rc.tableWorkers.Recycle(w) - if err != nil { - engineErr.Set(err) - return - } + // Note: We still need tableWorkers to control the concurrency of tables. + // In the future, we will investigate more about + // the difference between restoring tables concurrently and restoring tables one by one. + restoreWorker := rc.tableWorkers.Apply() - defer rc.closedEngineLimit.Recycle(dataWorker) - if err := t.importEngine(ctx, dataClosedEngine, rc, eid, ecp); err != nil { - engineErr.Set(err) - } - }(restoreWorker, engineID, engine) + go func(w *worker.Worker, eid int32, ecp *EngineCheckpoint) { + defer wg.Done() + + engineLogTask := t.logger.With(zap.Int32("engineNumber", eid)).Begin(zap.InfoLevel, "restore engine") + dataClosedEngine, dataWorker, err := t.restoreEngine(ctx, rc, indexEngine, eid, ecp) + engineLogTask.End(zap.ErrorLevel, err) + rc.tableWorkers.Recycle(w) + if err != nil { + engineErr.Set(err) + return + } + + failpoint.Inject("FailBeforeDataEngineImported", func() { + panic("forcing failure due to FailBeforeDataEngineImported") + }) + + defer rc.closedEngineLimit.Recycle(dataWorker) + if err := t.importEngine(ctx, dataClosedEngine, rc, eid, ecp); err != nil { + engineErr.Set(err) + } + }(restoreWorker, engineID, engine) + } } wg.Wait() @@ -872,9 +889,13 @@ func (t *TableRestore) restoreEngines(ctx context.Context, rc *RestoreController var err error if indexEngineCp.Status < CheckpointStatusImported { // the lock ensures the import() step will not be concurrent. - rc.postProcessLock.Lock() + if !rc.isLocalBackend() { + rc.postProcessLock.Lock() + } err = t.importKV(ctx, closedIndexEngine) - rc.postProcessLock.Unlock() + if !rc.isLocalBackend() { + rc.postProcessLock.Unlock() + } rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusImported) } @@ -981,13 +1002,32 @@ func (t *TableRestore) restoreEngine( zap.Int64("read", totalSQLSize), zap.Uint64("written", totalKVSize), ) - rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusAllWritten) + + // in local mode, this check-point make no sense, because we don't do flush now, + // so there may be data lose if exit at here. So we don't write this checkpoint + // here like other mode. + if !rc.isLocalBackend() { + rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusAllWritten) + } if err != nil { return nil, nil, errors.Trace(err) } dataWorker := rc.closedEngineLimit.Apply() closedDataEngine, err := dataEngine.Close(ctx) + // For local backend, if checkpoint is enabled, we must flush index engine to avoid data loss. + // this flush action impact up to 10% of the performance, so we only do it if necessary. + if err == nil && rc.cfg.Checkpoint.Enable && rc.isLocalBackend() { + if err = indexEngine.Flush(); err != nil { + // If any error occurred, recycle worker immediately + rc.closedEngineLimit.Recycle(dataWorker) + return nil, nil, errors.Trace(err) + } + // Currently we write all the checkpoints after data&index engine are flushed. + for _, chunk := range cp.Chunks { + saveCheckpoint(rc, t, engineID, chunk) + } + } rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusClosed) if err != nil { // If any error occurred, recycle worker immediately @@ -1012,9 +1052,13 @@ func (t *TableRestore) importEngine( // FIXME: flush is an asynchronous operation, what if flush failed? // the lock ensures the import() step will not be concurrent. - rc.postProcessLock.Lock() + if !rc.isLocalBackend() { + rc.postProcessLock.Lock() + } err := t.importKV(ctx, closedEngine) - rc.postProcessLock.Unlock() + if !rc.isLocalBackend() { + rc.postProcessLock.Unlock() + } rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusImported) if err != nil { return errors.Trace(err) @@ -1187,6 +1231,10 @@ func (rc *RestoreController) cleanCheckpoints(ctx context.Context) error { return errors.Annotate(err, "clean checkpoints") } +func (rc *RestoreController) isLocalBackend() bool { + return rc.cfg.TikvImporter.Backend == "local" +} + type chunkRestore struct { parser mydump.Parser index int @@ -1580,16 +1628,20 @@ func (cr *chunkRestore) deliverLoop( cr.chunk.Checksum.Add(&indexChecksum) cr.chunk.Chunk.Offset = offset cr.chunk.Chunk.PrevRowIDMax = rowID - if dataChecksum.SumKVS() != 0 || indexChecksum.SumKVS() != 0 { + // IN local mode, we should write these checkpoint after engine flushed + if !rc.isLocalBackend() && (dataChecksum.SumKVS() != 0 || indexChecksum.SumKVS() != 0) { // No need to save checkpoint if nothing was delivered. - cr.saveCheckpoint(t, engineID, rc) + saveCheckpoint(rc, t, engineID, cr.chunk) } + // TODO: for local backend, we may save checkpoint more frequently, e.g. after writen + //10GB kv pairs to data engine, we can do a flush for both data & index engine, then we + // can safely update current checkpoint. } return } -func (cr *chunkRestore) saveCheckpoint(t *TableRestore, engineID int32, rc *RestoreController) { +func saveCheckpoint(rc *RestoreController, t *TableRestore, engineID int32, chunk *ChunkCheckpoint) { // We need to update the AllocBase every time we've finished a file. // The AllocBase is determined by the maximum of the "handle" (_tidb_rowid // or integer primary key), which can only be obtained by reading all data. @@ -1610,10 +1662,10 @@ func (cr *chunkRestore) saveCheckpoint(t *TableRestore, engineID int32, rc *Rest tableName: t.tableName, merger: &ChunkCheckpointMerger{ EngineID: engineID, - Key: cr.chunk.Key, - Checksum: cr.chunk.Checksum, - Pos: cr.chunk.Chunk.Offset, - RowID: cr.chunk.Chunk.PrevRowIDMax, + Key: chunk.Key, + Checksum: chunk.Checksum, + Pos: chunk.Chunk.Offset, + RowID: chunk.Chunk.PrevRowIDMax, }, } } diff --git a/lightning/restore/restore_test.go b/lightning/restore/restore_test.go index 65c7f31be..e76600df1 100644 --- a/lightning/restore/restore_test.go +++ b/lightning/restore/restore_test.go @@ -620,7 +620,8 @@ func (s *chunkRestoreSuite) TestDeliverLoopEmptyData(c *C) { // Deliver nothing. - rc := &RestoreController{backend: importer} + cfg := &config.Config{} + rc := &RestoreController{cfg: cfg, backend: importer} kvsCh := make(chan []deliveredKVs, 1) kvsCh <- []deliveredKVs{} @@ -705,7 +706,8 @@ func (s *chunkRestoreSuite) TestDeliverLoop(c *C) { close(kvsCh) }() - rc := &RestoreController{saveCpCh: saveCpCh, backend: importer} + cfg := &config.Config{} + rc := &RestoreController{cfg: cfg, saveCpCh: saveCpCh, backend: importer} _, err = s.cr.deliverLoop(ctx, kvsCh, s.tr, 0, dataEngine, indexEngine, rc) c.Assert(err, IsNil) diff --git a/tests/_utils/run_lightning b/tests/_utils/run_lightning index 60f273848..22352f2b8 100755 --- a/tests/_utils/run_lightning +++ b/tests/_utils/run_lightning @@ -27,6 +27,7 @@ bin/tidb-lightning.test -test.coverprofile="$TEST_DIR/cov.$TEST_NAME.$$.out" DEV --config "tests/$TEST_NAME/config.toml" \ -d "tests/$TEST_NAME/data" \ --importer '127.0.0.1:8808' \ + --sorted-kv-dir "$TEST_DIR/sorted" \ --enable-checkpoint=0 \ --check-requirements=0 \ "$@" diff --git a/tests/checkpoint/run.sh b/tests/checkpoint/run.sh index 74a6a3fd3..354066740 100755 --- a/tests/checkpoint/run.sh +++ b/tests/checkpoint/run.sh @@ -55,57 +55,61 @@ for i in $(seq "$TABLE_COUNT"); do done PARTIAL_IMPORT_QUERY="$PARTIAL_IMPORT_QUERY AS s;" -# Set the failpoint to kill the lightning instance as soon as one table is imported -# If checkpoint does work, this should only kill 9 instances of lightnings. -SLOWDOWN_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500)' -export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeIndexEngineImported=return" +for BACKEND in importer tidb local; do -# Start importing the tables. -run_sql 'DROP DATABASE IF EXISTS cppk_tsr' -run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk' -run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680.bak`' + # Set the failpoint to kill the lightning instance as soon as one table is imported + # If checkpoint does work, this should only kill 9 instances of lightnings. + SLOWDOWN_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500)' + export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeIndexEngineImported=return" -# panic after saving index engine checkpoint status before saving table checkpoint status -set +e -for i in $(seq "$TABLE_COUNT"); do - echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" - run_lightning -d "$DBPATH" --enable-checkpoint=1 2> /dev/null - [ $? -ne 0 ] || exit 1 -done -set -e + # Start importing the tables. + run_sql 'DROP DATABASE IF EXISTS cppk_tsr' + run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk' + run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680.bak`' -export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS" -set +e -for i in $(seq "$TABLE_COUNT"); do - echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" - run_lightning -d "$DBPATH" --enable-checkpoint=1 2> /dev/null -done -set -e + # panic after saving index engine checkpoint status before saving table checkpoint status + set +e + for i in $(seq "$TABLE_COUNT"); do + echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" + run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1 2> /dev/null + [ $? -ne 0 ] || exit 1 + done + set -e -# Start importing the tables. -run_sql 'DROP DATABASE IF EXISTS cppk_tsr' -run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk' -run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680.bak`' + export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS" + set +e + for i in $(seq "$TABLE_COUNT"); do + echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" + run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1 2> /dev/null + done + set -e -export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb-lightning/lightning/SetTaskID=return(1357924680);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(1)" + # Start importing the tables. + run_sql 'DROP DATABASE IF EXISTS cppk_tsr' + run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk' + run_sql 'DROP DATABASE IF EXISTS `tidb_lightning_checkpoint_test_cppk.1357924680.bak`' -set +e -for i in $(seq "$TABLE_COUNT"); do - echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" - run_lightning -d "$DBPATH" --enable-checkpoint=1 2> /dev/null - [ $? -ne 0 ] || exit 1 -done -set -e + export GO_FAILPOINTS="$SLOWDOWN_FAILPOINTS;github.com/pingcap/tidb-lightning/lightning/SetTaskID=return(1357924680);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(1)" + + set +e + for i in $(seq "$TABLE_COUNT"); do + echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" + run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1 2> /dev/null + [ $? -ne 0 ] || exit 1 + done + set -e -# After everything is done, there should be no longer new calls to ImportEngine -# (and thus `kill_lightning_after_one_import` will spare this final check) -echo "******** Verify checkpoint no-op ********" -run_lightning -d "$DBPATH" --enable-checkpoint=1 -run_sql "$PARTIAL_IMPORT_QUERY" -check_contains "s: $(( (1000 * $CHUNK_COUNT + 1001) * $CHUNK_COUNT * $TABLE_COUNT ))" -run_sql 'SELECT count(*) FROM `tidb_lightning_checkpoint_test_cppk.1357924680.bak`.table_v6 WHERE status >= 200' -check_contains "count(*): $TABLE_COUNT" + # After everything is done, there should be no longer new calls to ImportEngine + # (and thus `kill_lightning_after_one_import` will spare this final check) + echo "******** Verify checkpoint no-op ********" + run_lightning -d "$DBPATH" --backend $BACKEND --enable-checkpoint=1 + run_sql "$PARTIAL_IMPORT_QUERY" + check_contains "s: $(( (1000 * $CHUNK_COUNT + 1001) * $CHUNK_COUNT * $TABLE_COUNT ))" + run_sql 'SELECT count(*) FROM `tidb_lightning_checkpoint_test_cppk.1357924680.bak`.table_v6 WHERE status >= 200' + check_contains "count(*): $TABLE_COUNT" -# Ensure there is no dangling open engines -ls -lA "$TEST_DIR"/importer/.temp/ -[ -z "$(ls -A "$TEST_DIR"/importer/.temp/)" ] + # Ensure there is no dangling open engines + ls -lA "$TEST_DIR"/importer/.temp/ + [ -z "$(ls -A "$TEST_DIR"/importer/.temp/)" ] + +done diff --git a/tests/checkpoint_engines/run.sh b/tests/checkpoint_engines/run.sh index 1fedf6b63..851034f7f 100755 --- a/tests/checkpoint_engines/run.sh +++ b/tests/checkpoint_engines/run.sh @@ -16,80 +16,82 @@ set -eux do_run_lightning() { - run_lightning --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-checkpoint-engines.log" --config "tests/$TEST_NAME/$1.toml" + run_lightning --backend $1 --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-checkpoint-engines.log" --config "tests/$TEST_NAME/$2.toml" } -# First, verify that a normal operation is fine. - -rm -f "$TEST_DIR/lightning-checkpoint-engines.log" -rm -f "/tmp/tidb_lightning_checkpoint.pb" -run_sql 'DROP DATABASE IF EXISTS cpeng;' - -do_run_lightning config - -# Check that we have indeed opened 6 engines (index + data engine) -DATA_ENGINE_COUNT=4 -INDEX_ENGINE_COUNT=2 -ENGINE_COUNT=6 -OPEN_ENGINES_COUNT=$(grep 'open engine' "$TEST_DIR/lightning-checkpoint-engines.log" | wc -l) -echo "Number of open engines: $OPEN_ENGINES_COUNT" -[ "$OPEN_ENGINES_COUNT" -eq $ENGINE_COUNT ] - -# Check that everything is correctly imported -run_sql 'SELECT count(*), sum(c) FROM cpeng.a' -check_contains 'count(*): 4' -check_contains 'sum(c): 10' - -run_sql 'SELECT count(*), sum(c) FROM cpeng.b' -check_contains 'count(*): 4' -check_contains 'sum(c): 46' - -# Now, verify it works with checkpoints as well. - -run_sql 'DROP DATABASE cpeng;' -rm -f "/tmp/tidb_lightning_checkpoint.pb" - -# Data engine part -export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(140)' -set +e -for i in $(seq "$ENGINE_COUNT"); do - echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" - do_run_lightning config 2> /dev/null - [ $? -ne 0 ] || exit 1 -done -set -e - -echo "******** Verify checkpoint no-op ********" -do_run_lightning config - -run_sql 'SELECT count(*), sum(c) FROM cpeng.a' -check_contains 'count(*): 4' -check_contains 'sum(c): 10' - -run_sql 'SELECT count(*), sum(c) FROM cpeng.b' -check_contains 'count(*): 4' -check_contains 'sum(c): 46' - -# Now, try again with MySQL checkpoints - -run_sql 'DROP DATABASE cpeng;' -run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint;' - -set +e -for i in $(seq "$ENGINE_COUNT"); do - echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" - do_run_lightning mysql 2> /dev/null - [ $? -ne 0 ] || exit 1 +for BACKEND in importer local; do + # First, verify that a normal operation is fine. + rm -f "$TEST_DIR/lightning-checkpoint-engines.log" + rm -f "/tmp/tidb_lightning_checkpoint.pb" + run_sql 'DROP DATABASE IF EXISTS cpeng;' + export GO_FAILPOINTS="" + + do_run_lightning $BACKEND config + + # Check that we have indeed opened 6 engines (index + data engine) + DATA_ENGINE_COUNT=4 + INDEX_ENGINE_COUNT=2 + ENGINE_COUNT=6 + OPEN_ENGINES_COUNT=$(grep 'open engine' "$TEST_DIR/lightning-checkpoint-engines.log" | wc -l) + echo "Number of open engines: $OPEN_ENGINES_COUNT" + [ "$OPEN_ENGINES_COUNT" -eq $ENGINE_COUNT ] + + # Check that everything is correctly imported + run_sql 'SELECT count(*), sum(c) FROM cpeng.a' + check_contains 'count(*): 4' + check_contains 'sum(c): 10' + + run_sql 'SELECT count(*), sum(c) FROM cpeng.b' + check_contains 'count(*): 4' + check_contains 'sum(c): 46' + + # Now, verify it works with checkpoints as well. + + run_sql 'DROP DATABASE cpeng;' + rm -f "/tmp/tidb_lightning_checkpoint.pb" + + # Data engine part + export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(140)' + set +e + for i in $(seq "$ENGINE_COUNT"); do + echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" + do_run_lightning $BACKEND config 2> /dev/null + [ $? -ne 0 ] || exit 1 + done + set -e + + echo "******** Verify checkpoint no-op ********" + do_run_lightning $BACKEND config + + run_sql 'SELECT count(*), sum(c) FROM cpeng.a' + check_contains 'count(*): 4' + check_contains 'sum(c): 10' + + run_sql 'SELECT count(*), sum(c) FROM cpeng.b' + check_contains 'count(*): 4' + check_contains 'sum(c): 46' + + # Now, try again with MySQL checkpoints + + run_sql 'DROP DATABASE cpeng;' + run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint;' + + set +e + for i in $(seq "$ENGINE_COUNT"); do + echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" + do_run_lightning $BACKEND mysql 2> /dev/null + [ $? -ne 0 ] || exit 1 + done + set -e + + echo "******** Verify checkpoint no-op ********" + do_run_lightning $BACKEND mysql + + run_sql 'SELECT count(*), sum(c) FROM cpeng.a' + check_contains 'count(*): 4' + check_contains 'sum(c): 10' + + run_sql 'SELECT count(*), sum(c) FROM cpeng.b' + check_contains 'count(*): 4' + check_contains 'sum(c): 46' done -set -e - -echo "******** Verify checkpoint no-op ********" -do_run_lightning mysql - -run_sql 'SELECT count(*), sum(c) FROM cpeng.a' -check_contains 'count(*): 4' -check_contains 'sum(c): 10' - -run_sql 'SELECT count(*), sum(c) FROM cpeng.b' -check_contains 'count(*): 4' -check_contains 'sum(c): 46' \ No newline at end of file diff --git a/tests/concurrent-restore/run.sh b/tests/concurrent-restore/run.sh index f0bc77fcd..b540da39a 100644 --- a/tests/concurrent-restore/run.sh +++ b/tests/concurrent-restore/run.sh @@ -45,4 +45,4 @@ done # check tikv_gc_life_time is recovered to the original value run_sql 'select VARIABLE_VALUE from mysql.tidb where VARIABLE_NAME = "tikv_gc_life_time"'; -check_contains "VARIABLE_VALUE: $ORIGINAL_TIKV_GC_LIFE_TIME" +check_contains "VARIABLE_VALUE: $ORIGINAL_TIKV_GC_LIFE_TIME" \ No newline at end of file diff --git a/tests/csv/config.toml b/tests/csv/config.toml index e61b4516b..95da85a3d 100644 --- a/tests/csv/config.toml +++ b/tests/csv/config.toml @@ -6,3 +6,7 @@ not-null = false null = '\N' backslash-escape = true trim-last-separator = false + +[tikv-importer] +send-kv-pairs=10 +region-split-size = 1024 diff --git a/tests/csv/run.sh b/tests/csv/run.sh index 08790ef26..f88248a50 100755 --- a/tests/csv/run.sh +++ b/tests/csv/run.sh @@ -2,39 +2,39 @@ set -eu -for BACKEND in importer tidb; do +for BACKEND in importer tidb local; do -run_sql 'DROP DATABASE IF EXISTS csv' + run_sql 'DROP DATABASE IF EXISTS csv' -run_lightning --backend $BACKEND + run_lightning --backend $BACKEND -run_sql 'SELECT count(*), sum(PROCESSLIST_TIME), sum(THREAD_OS_ID), count(PROCESSLIST_STATE) FROM csv.threads' -check_contains 'count(*): 43' -check_contains 'sum(PROCESSLIST_TIME): 322253' -check_contains 'sum(THREAD_OS_ID): 303775702' -check_contains 'count(PROCESSLIST_STATE): 3' + run_sql 'SELECT count(*), sum(PROCESSLIST_TIME), sum(THREAD_OS_ID), count(PROCESSLIST_STATE) FROM csv.threads' + check_contains 'count(*): 43' + check_contains 'sum(PROCESSLIST_TIME): 322253' + check_contains 'sum(THREAD_OS_ID): 303775702' + check_contains 'count(PROCESSLIST_STATE): 3' -run_sql 'SELECT count(*) FROM csv.threads WHERE PROCESSLIST_TIME IS NOT NULL' -check_contains 'count(*): 12' + run_sql 'SELECT count(*) FROM csv.threads WHERE PROCESSLIST_TIME IS NOT NULL' + check_contains 'count(*): 12' -run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 1' -check_contains 'hex(t): 5C' -check_contains 'j: {"?": []}' -check_contains 'hex(b): FFFFFFFF' + run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 1' + check_contains 'hex(t): 5C' + check_contains 'j: {"?": []}' + check_contains 'hex(b): FFFFFFFF' -run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 2' -check_contains 'hex(t): 22' -check_contains 'j: "\n\n\n"' -check_contains 'hex(b): 0D0A0D0A' + run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 2' + check_contains 'hex(t): 22' + check_contains 'j: "\n\n\n"' + check_contains 'hex(b): 0D0A0D0A' -run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 3' -check_contains 'hex(t): 0A' -check_contains 'j: [",,,"]' -check_contains 'hex(b): 5C2C5C2C' + run_sql 'SELECT hex(t), j, hex(b) FROM csv.escapes WHERE i = 3' + check_contains 'hex(t): 0A' + check_contains 'j: [",,,"]' + check_contains 'hex(b): 5C2C5C2C' -run_sql 'SELECT id FROM csv.empty_strings WHERE a = """"' -check_contains 'id: 3' -run_sql 'SELECT id FROM csv.empty_strings WHERE b <> ""' -check_not_contains 'id:' + run_sql 'SELECT id FROM csv.empty_strings WHERE a = """"' + check_contains 'id: 3' + run_sql 'SELECT id FROM csv.empty_strings WHERE b <> ""' + check_not_contains 'id:' done diff --git a/tests/issue_282/run.sh b/tests/issue_282/run.sh index ce2a0f9e6..f46889a39 100644 --- a/tests/issue_282/run.sh +++ b/tests/issue_282/run.sh @@ -15,7 +15,7 @@ set -eu -for backend in tidb importer; do +for backend in tidb importer local; do run_sql 'DROP DATABASE IF EXISTS issue282;' run_lightning --backend $backend diff --git a/tests/local_backend/config.toml b/tests/local_backend/config.toml new file mode 100644 index 000000000..412e5a01f --- /dev/null +++ b/tests/local_backend/config.toml @@ -0,0 +1,9 @@ +[lightning] +table-concurrency = 1 + +[checkpoint] +enable = true +driver = "file" + +[mydumper] +batch-size = 50 # force splitting the data into 4 batches diff --git a/tests/local_backend/data/cpeng-schema-create.sql b/tests/local_backend/data/cpeng-schema-create.sql new file mode 100644 index 000000000..1e23466ee --- /dev/null +++ b/tests/local_backend/data/cpeng-schema-create.sql @@ -0,0 +1 @@ +create database cpeng; diff --git a/tests/local_backend/data/cpeng.a-schema.sql b/tests/local_backend/data/cpeng.a-schema.sql new file mode 100644 index 000000000..fe3f493b6 --- /dev/null +++ b/tests/local_backend/data/cpeng.a-schema.sql @@ -0,0 +1 @@ +create table a (c int); diff --git a/tests/local_backend/data/cpeng.a.1.sql b/tests/local_backend/data/cpeng.a.1.sql new file mode 100644 index 000000000..58829b7d8 --- /dev/null +++ b/tests/local_backend/data/cpeng.a.1.sql @@ -0,0 +1 @@ +insert into a values (1); diff --git a/tests/local_backend/data/cpeng.a.2.sql b/tests/local_backend/data/cpeng.a.2.sql new file mode 100644 index 000000000..ccbcb5801 --- /dev/null +++ b/tests/local_backend/data/cpeng.a.2.sql @@ -0,0 +1 @@ +insert into a values (2); diff --git a/tests/local_backend/data/cpeng.a.3.sql b/tests/local_backend/data/cpeng.a.3.sql new file mode 100644 index 000000000..effdc8f3e --- /dev/null +++ b/tests/local_backend/data/cpeng.a.3.sql @@ -0,0 +1 @@ +insert into a values (3),(4); diff --git a/tests/local_backend/data/cpeng.b-schema.sql b/tests/local_backend/data/cpeng.b-schema.sql new file mode 100644 index 000000000..4a3c844ef --- /dev/null +++ b/tests/local_backend/data/cpeng.b-schema.sql @@ -0,0 +1 @@ +create table b (c int); diff --git a/tests/local_backend/data/cpeng.b.1.sql b/tests/local_backend/data/cpeng.b.1.sql new file mode 100644 index 000000000..cadf0227f --- /dev/null +++ b/tests/local_backend/data/cpeng.b.1.sql @@ -0,0 +1,4 @@ +insert into b values (10),(11),(12); +/* +padding to make the data file > 50 bytes +*/ diff --git a/tests/local_backend/data/cpeng.b.2.sql b/tests/local_backend/data/cpeng.b.2.sql new file mode 100644 index 000000000..83045aee9 --- /dev/null +++ b/tests/local_backend/data/cpeng.b.2.sql @@ -0,0 +1 @@ +insert into b values (13); diff --git a/tests/local_backend/run.sh b/tests/local_backend/run.sh new file mode 100755 index 000000000..346b3d71c --- /dev/null +++ b/tests/local_backend/run.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +ENGINE_COUNT=6 + +# First, verify that inject with not leader error is fine. +rm -f "$TEST_DIR/lightning-local.log" +rm -f "/tmp/tidb_lightning_checkpoint.pb" +run_sql 'DROP DATABASE IF EXISTS cpeng;' +export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/backend/local/FailIngestMeta=return("notleader")' + +run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml" + +# Check that everything is correctly imported +run_sql 'SELECT count(*), sum(c) FROM cpeng.a' +check_contains 'count(*): 4' +check_contains 'sum(c): 10' + +run_sql 'SELECT count(*), sum(c) FROM cpeng.b' +check_contains 'count(*): 4' +check_contains 'sum(c): 46' + +# Now, verify it works with epoch not match as well. +run_sql 'DROP DATABASE cpeng;' +rm -f "/tmp/tidb_lightning_checkpoint.pb" + +export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/backend/local/FailIngestMeta=return("epochnotmatch")' + +run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml" + +run_sql 'SELECT count(*), sum(c) FROM cpeng.a' +check_contains 'count(*): 4' +check_contains 'sum(c): 10' + +run_sql 'SELECT count(*), sum(c) FROM cpeng.b' +check_contains 'count(*): 4' +check_contains 'sum(c): 46' + + +# Now, verify it works with checkpoints as well. +run_sql 'DROP DATABASE cpeng;' +rm -f "/tmp/tidb_lightning_checkpoint.pb" + +set +e +export GO_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeDataEngineImported=return' +for i in $(seq "$ENGINE_COUNT"); do + echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" + run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml" + [ $? -ne 0 ] || exit 1 +done +set -e + +export GO_FAILPOINTS='' +echo "******** Verify checkpoint no-op ********" +run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml" + +run_sql 'SELECT count(*), sum(c) FROM cpeng.a' +check_contains 'count(*): 4' +check_contains 'sum(c): 10' + +run_sql 'SELECT count(*), sum(c) FROM cpeng.b' +check_contains 'count(*): 4' +check_contains 'sum(c): 46' diff --git a/tests/run.sh b/tests/run.sh index 8efe079b8..d71b86742 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -169,5 +169,3 @@ for script in tests/*/run.sh; do TEST_NAME="$(basename "$(dirname "$script")")" \ sh "$script" done - - diff --git a/tests/various_types/run.sh b/tests/various_types/run.sh index da2a0b789..93d250981 100755 --- a/tests/various_types/run.sh +++ b/tests/various_types/run.sh @@ -17,94 +17,94 @@ set -eu -for BACKEND in importer tidb; do +for BACKEND in importer tidb local; do -run_sql 'DROP DATABASE IF EXISTS vt;' -run_lightning --backend $BACKEND -echo Import using $BACKEND finished + run_sql 'DROP DATABASE IF EXISTS vt;' + run_lightning --backend $BACKEND + echo Import using $BACKEND finished -run_sql 'SELECT count(pk), bin(min(pk)), bin(max(pk)) FROM vt.bit' -check_contains 'count(pk): 16' -check_contains 'bin(min(pk)): 0' -check_contains 'bin(max(pk)): 11' -run_sql 'SELECT sum(ref) FROM vt.bit WHERE pk = 0b10' -check_contains 'sum(ref): 82' + run_sql 'SELECT count(pk), bin(min(pk)), bin(max(pk)) FROM vt.bit' + check_contains 'count(pk): 16' + check_contains 'bin(min(pk)): 0' + check_contains 'bin(max(pk)): 11' + run_sql 'SELECT sum(ref) FROM vt.bit WHERE pk = 0b10' + check_contains 'sum(ref): 82' -run_sql 'SELECT count(pk), min(pk), max(pk), sum(pk) FROM vt.decimal' -check_contains 'count(pk): 50' -check_contains 'min(pk): -99.9990' -check_contains 'max(pk): 99.9912' -check_contains 'sum(pk): -9.9123' -run_sql 'SELECT ref FROM vt.decimal WHERE pk BETWEEN -1.0 AND 0.0' -check_contains 'ref: 22' + run_sql 'SELECT count(pk), min(pk), max(pk), sum(pk) FROM vt.decimal' + check_contains 'count(pk): 50' + check_contains 'min(pk): -99.9990' + check_contains 'max(pk): 99.9912' + check_contains 'sum(pk): -9.9123' + run_sql 'SELECT ref FROM vt.decimal WHERE pk BETWEEN -1.0 AND 0.0' + check_contains 'ref: 22' -run_sql 'SELECT count(pk), min(pk), max(pk) FROM vt.double' -check_contains 'count(pk): 41' -check_contains 'min(pk): 9.85967654375977e-305' -check_contains 'max(pk): 1.0142320547350045e304' -run_sql 'SELECT ref FROM vt.double WHERE pk BETWEEN 1e100 AND 1e120' -check_contains 'ref: 245' + run_sql 'SELECT count(pk), min(pk), max(pk) FROM vt.double' + check_contains 'count(pk): 41' + check_contains 'min(pk): 9.85967654375977e-305' + check_contains 'max(pk): 1.0142320547350045e304' + run_sql 'SELECT ref FROM vt.double WHERE pk BETWEEN 1e100 AND 1e120' + check_contains 'ref: 245' -run_sql 'SELECT count(pk), min(pk), max(pk), count(uk), min(uk), max(uk) FROM vt.datetime' -check_contains 'count(pk): 70' -check_contains 'min(pk): 1026-09-21 15:15:54.335745' -check_contains 'max(pk): 9889-01-08 08:51:03.389832' -check_contains 'count(uk): 70' -check_contains 'min(uk): 1970-11-09 19:25:45.843' -check_contains 'max(uk): 2036-10-14 10:48:28.620' -run_sql "SELECT ref FROM vt.datetime WHERE pk BETWEEN '2882-01-01' AND '2882-12-31'" -check_contains 'ref: 7' -run_sql "SELECT ref FROM vt.datetime WHERE uk BETWEEN '2001-01-01' AND '2001-12-31'" -check_contains 'ref: 32' + run_sql 'SELECT count(pk), min(pk), max(pk), count(uk), min(uk), max(uk) FROM vt.datetime' + check_contains 'count(pk): 70' + check_contains 'min(pk): 1026-09-21 15:15:54.335745' + check_contains 'max(pk): 9889-01-08 08:51:03.389832' + check_contains 'count(uk): 70' + check_contains 'min(uk): 1970-11-09 19:25:45.843' + check_contains 'max(uk): 2036-10-14 10:48:28.620' + run_sql "SELECT ref FROM vt.datetime WHERE pk BETWEEN '2882-01-01' AND '2882-12-31'" + check_contains 'ref: 7' + run_sql "SELECT ref FROM vt.datetime WHERE uk BETWEEN '2001-01-01' AND '2001-12-31'" + check_contains 'ref: 32' -run_sql 'SELECT count(pk), min(pk), max(pk) FROM vt.char' -check_contains 'count(pk): 50' -check_contains 'min(pk): 090abbb2-f22e-4f97-a4fe-a52eb1a80a0b' -check_contains 'max(pk): fde1328c-409c-43a8-b1b0-8c35c8000f92' -run_sql "SELECT ref FROM vt.char WHERE pk = '55dc0343-db6a-4208-9872-9096305b8c07'" -check_contains 'ref: 41' + run_sql 'SELECT count(pk), min(pk), max(pk) FROM vt.char' + check_contains 'count(pk): 50' + check_contains 'min(pk): 090abbb2-f22e-4f97-a4fe-a52eb1a80a0b' + check_contains 'max(pk): fde1328c-409c-43a8-b1b0-8c35c8000f92' + run_sql "SELECT ref FROM vt.char WHERE pk = '55dc0343-db6a-4208-9872-9096305b8c07'" + check_contains 'ref: 41' -run_sql 'SELECT count(pk), hex(min(pk)), hex(max(pk)) FROM vt.binary' -check_contains 'count(pk): 50' -check_contains 'hex(min(pk)): 090ABBB2F22E4F97A4FEA52EB1A80A0B' -check_contains 'hex(max(pk)): FDE1328C409C43A8B1B08C35C8000F92' -run_sql "SELECT ref FROM vt.binary WHERE pk = x'55dc0343db6a420898729096305b8c07'" -check_contains 'ref: 41' + run_sql 'SELECT count(pk), hex(min(pk)), hex(max(pk)) FROM vt.binary' + check_contains 'count(pk): 50' + check_contains 'hex(min(pk)): 090ABBB2F22E4F97A4FEA52EB1A80A0B' + check_contains 'hex(max(pk)): FDE1328C409C43A8B1B08C35C8000F92' + run_sql "SELECT ref FROM vt.binary WHERE pk = x'55dc0343db6a420898729096305b8c07'" + check_contains 'ref: 41' -run_sql 'SELECT count(pk), count(DISTINCT js) FROM vt.json' -check_contains 'count(pk): 92' -check_contains 'count(DISTINCT js): 92' -run_sql 'SELECT pk FROM vt.json WHERE js = json_array(1, 2, 3)' -check_contains 'pk: 1089' -run_sql 'SELECT js FROM vt.json WHERE pk = 2000' -check_contains 'js: {' -check_contains '"52": 1' -check_contains '"54": 1' -check_contains '"68": 1' -check_contains '"126": 1' + run_sql 'SELECT count(pk), count(DISTINCT js) FROM vt.json' + check_contains 'count(pk): 92' + check_contains 'count(DISTINCT js): 92' + run_sql 'SELECT pk FROM vt.json WHERE js = json_array(1, 2, 3)' + check_contains 'pk: 1089' + run_sql 'SELECT js FROM vt.json WHERE pk = 2000' + check_contains 'js: {' + check_contains '"52": 1' + check_contains '"54": 1' + check_contains '"68": 1' + check_contains '"126": 1' -run_sql 'SELECT count(*) FROM vt.`enum-set`' -check_contains 'count(*): 26' -run_sql 'SELECT count(*) FROM vt.`enum-set` WHERE find_in_set("x50", `set`) > 0' -check_contains 'count(*): 10' -run_sql 'SELECT `set` FROM vt.`enum-set` WHERE `enum` = "gcc"' -check_contains 'set: x00,x06,x07,x09,x17,x20,x23,x24,x27,x37,x44,x46,x49,x54,x55,x58,x61,x62' -run_sql 'SELECT `set` FROM vt.`enum-set` WHERE `enum` = "g99"' -check_contains 'set: x07,x08,x09,x10,x11,x12,x14,x16,x17,x18,x19,x22,x25,x26,x28,x29,x30,x31,x32,x33,x35,x38,x39,x41,x44,x46,x49,x51,x53,x55,x56,x58,x61,x63' + run_sql 'SELECT count(*) FROM vt.`enum-set`' + check_contains 'count(*): 26' + run_sql 'SELECT count(*) FROM vt.`enum-set` WHERE find_in_set("x50", `set`) > 0' + check_contains 'count(*): 10' + run_sql 'SELECT `set` FROM vt.`enum-set` WHERE `enum` = "gcc"' + check_contains 'set: x00,x06,x07,x09,x17,x20,x23,x24,x27,x37,x44,x46,x49,x54,x55,x58,x61,x62' + run_sql 'SELECT `set` FROM vt.`enum-set` WHERE `enum` = "g99"' + check_contains 'set: x07,x08,x09,x10,x11,x12,x14,x16,x17,x18,x19,x22,x25,x26,x28,x29,x30,x31,x32,x33,x35,x38,x39,x41,x44,x46,x49,x51,x53,x55,x56,x58,x61,x63' -run_sql 'SELECT count(*) FROM vt.empty_strings' -check_contains 'count(*): 6' -run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = ""' -check_contains 'sum(pk): 5' -run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = 0x22' -check_contains 'sum(pk): 18' -run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = 0x27' -check_contains 'sum(pk): 40' + run_sql 'SELECT count(*) FROM vt.empty_strings' + check_contains 'count(*): 6' + run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = ""' + check_contains 'sum(pk): 5' + run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = 0x22' + check_contains 'sum(pk): 18' + run_sql 'SELECT sum(pk) FROM vt.empty_strings WHERE a = 0x27' + check_contains 'sum(pk): 40' -run_sql 'SELECT a, b, c, d FROM vt.precise_types' -check_contains 'a: 18446744073709551614' -check_contains 'b: -9223372036854775806' -check_contains 'c: 99999999999999999999.0' -check_contains 'd: 18446744073709551616.0' + run_sql 'SELECT a, b, c, d FROM vt.precise_types' + check_contains 'a: 18446744073709551614' + check_contains 'b: -9223372036854775806' + check_contains 'c: 99999999999999999999.0' + check_contains 'd: 18446744073709551616.0' done