Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

*: support mysql compatible auto_increment, the client side changes #38809

Merged
merged 22 commits into from
Nov 4, 2022
Merged

*: support mysql compatible auto_increment, the client side changes #38809

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
076a2af
*: support mysql compatible auto_increment, the client side changes
tiancaiamao Nov 2, 2022
206ed6a
*: support mysql compatible auto_increment, the server side changes
tiancaiamao Nov 2, 2022
5d68692
update go.mod
tiancaiamao Nov 3, 2022
79833ba
update go.mod
tiancaiamao Nov 3, 2022
3982bb3
make bazel_prepare
tiancaiamao Nov 3, 2022
64dd578
make bazel_prepare
tiancaiamao Nov 3, 2022
49df4ea
Merge branch 'master' into autoid-service
tiancaiamao Nov 3, 2022
324346a
make bazel_prepare again
tiancaiamao Nov 3, 2022
4a2f1c9
make golint happy
tiancaiamao Nov 3, 2022
3736de9
make lint check happy
tiancaiamao Nov 3, 2022
0688e4e
make bazel_prepare
tiancaiamao Nov 3, 2022
25810e6
Merge branch 'master' into autoid-client
tiancaiamao Nov 3, 2022
516202c
Merge branch 'master' into autoid-service
ti-chi-bot Nov 3, 2022
0fadc21
Merge branch 'autoid-service' into autoid-client
tiancaiamao Nov 3, 2022
c165725
make bazel_prepare
tiancaiamao Nov 3, 2022
623af29
Merge branch 'master' into autoid-client
tiancaiamao Nov 3, 2022
450bdba
make lint happy
tiancaiamao Nov 3, 2022
23a26f1
fix lint
tiancaiamao Nov 3, 2022
b5e77c0
fix the GetClient() grpc conn leak
tiancaiamao Nov 3, 2022
a62d96c
tiny clean up
tiancaiamao Nov 3, 2022
073f839
address comment
tiancaiamao Nov 3, 2022
79d19b9
Merge branch 'master' into autoid-client
ti-chi-bot Nov 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions autoid_service/autoid.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package autoid

import (
"context"

"github.com/pingcap/kvproto/pkg/autoid"
"github.com/pingcap/tidb/kv"
"google.golang.org/grpc"
)

type mockClient struct {
}

func (m *mockClient) AllocAutoID(ctx context.Context, in *autoid.AutoIDRequest, opts ...grpc.CallOption) (*autoid.AutoIDResponse, error) {
panic("TODO")
tiancaiamao marked this conversation as resolved.
Show resolved Hide resolved
}

func (m *mockClient) Rebase(ctx context.Context, in *autoid.RebaseRequest, opts ...grpc.CallOption) (*autoid.RebaseResponse, error) {
panic("TODO")
}

var global = make(map[string]*mockClient)

func MockForTest(store kv.Storage) *mockClient {
uuid := store.UUID()
ret, ok := global[uuid]
if !ok {
ret = &mockClient{}
global[uuid] = ret
}
return ret
}
22 changes: 21 additions & 1 deletion ddl/schematracker/checker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package schematracker
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"strings"
"time"
Expand Down Expand Up @@ -553,15 +554,34 @@ func (d Checker) MoveJobFromTable2Queue() error {
// StorageDDLInjector wraps kv.Storage to inject checker to domain's DDL in bootstrap time.
type StorageDDLInjector struct {
kv.Storage
kv.EtcdBackend
Injector func(ddl.DDL) *Checker
}

var _ kv.EtcdBackend = StorageDDLInjector{}

func (s StorageDDLInjector) EtcdAddrs() ([]string, error) {
return s.EtcdBackend.EtcdAddrs()
}

func (s StorageDDLInjector) TLSConfig() *tls.Config {
return s.EtcdBackend.TLSConfig()
}

func (s StorageDDLInjector) StartGCWorker() error {
return s.EtcdBackend.StartGCWorker()
}

// NewStorageDDLInjector creates a new StorageDDLInjector to inject Checker.
func NewStorageDDLInjector(s kv.Storage) kv.Storage {
return StorageDDLInjector{
ret := StorageDDLInjector{
Storage: s,
Injector: NewChecker,
}
if ebd, ok := s.(kv.EtcdBackend); ok {
ret.EtcdBackend = ebd
}
return ret
}

// UnwrapStorage unwraps StorageDDLInjector for one level.
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,3 +251,5 @@ replace (
github.com/pingcap/tidb/parser => ./parser
go.opencensus.io => go.opencensus.io v0.23.1-0.20220331163232-052120675fac
)

replace github.com/pingcap/kvproto => github.com/tiancaiamao/kvproto v0.0.0-20221101131513-753c3dd721c9
10 changes: 2 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,6 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
Expand Down Expand Up @@ -367,7 +366,6 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -774,9 +772,6 @@ github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZL
github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrrJDZ0gUfMIvKLAMU1tu1UKJib8bEQ=
github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
Expand Down Expand Up @@ -928,6 +923,8 @@ github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpR
github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY=
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ=
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
github.com/tiancaiamao/kvproto v0.0.0-20221101131513-753c3dd721c9 h1:+sVrTp/1hSRuQM+dLDl4OnjqPANDXuNFZu+EhheAWuw=
github.com/tiancaiamao/kvproto v0.0.0-20221101131513-753c3dd721c9/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/tikv/client-go/v2 v2.0.1-0.20221031063202-30e803b7082c h1:NvQHWk0GeXSLEBbmGMPnDMc0to0a3ogzgIRbTKw8MHI=
github.com/tikv/client-go/v2 v2.0.1-0.20221031063202-30e803b7082c/go.mod h1:X9s4ct/MLk1sFqe5mU79KClKegLFDTa/FCx3hzexGtk=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
Expand Down Expand Up @@ -1131,7 +1128,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -1487,7 +1483,6 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
Expand Down Expand Up @@ -1568,7 +1563,6 @@ google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljW
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f h1:P8EiVSxZwC6xH2niv2N66aqwMtYFg+D54gbjpcqKJtM=
google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE=
google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
Expand Down
3 changes: 2 additions & 1 deletion infoschema/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,8 @@ func (b *Builder) applyCreateTable(m *meta.Meta, dbInfo *model.DBInfo, tableID i
tblVer := autoid.AllocOptionTableInfoVersion(tblInfo.Version)
switch tp {
case model.ActionRebaseAutoID, model.ActionModifyTableAutoIdCache:
newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.ID, tblInfo.IsAutoIncColUnsigned(), autoid.RowIDAllocType, tblVer)
idCacheOpt := autoid.CustomAutoIncCacheOption(tblInfo.AutoIdCache)
newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.ID, tblInfo.IsAutoIncColUnsigned(), autoid.RowIDAllocType, tblVer, idCacheOpt)
allocs = append(allocs, newAlloc)
case model.ActionRebaseAutoRandomBase:
newAlloc := autoid.NewAllocator(b.store, dbInfo.ID, tblInfo.ID, tblInfo.IsAutoRandomBitColUnsigned(), autoid.AutoRandomType, tblVer)
Expand Down
51 changes: 51 additions & 0 deletions meta/autoid/autoid.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/opentracing/opentracing-go"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
autoid "github.com/pingcap/tidb/autoid_service"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/metrics"
Expand All @@ -38,6 +39,7 @@ import (
"github.com/pingcap/tidb/util/mathutil"
"github.com/tikv/client-go/v2/txnkv/txnsnapshot"
tikvutil "github.com/tikv/client-go/v2/util"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
)

Expand Down Expand Up @@ -466,6 +468,7 @@ func (alloc *allocator) ForceRebase(requiredBase int64) error {
if requiredBase == -1 {
return ErrAutoincReadFailed.GenWithStack("Cannot force rebase the next global ID to '0'")
}

tiancaiamao marked this conversation as resolved.
Show resolved Hide resolved
alloc.mu.Lock()
defer alloc.mu.Unlock()
startTime := time.Now()
Expand Down Expand Up @@ -533,6 +536,45 @@ func NextStep(curStep int64, consumeDur time.Duration) int64 {
return res
}

func newSinglePointAlloc(store kv.Storage, dbID, tblID int64, isUnsigned bool) *singlePointAlloc {
ebd, ok := store.(kv.EtcdBackend)
if !ok {
// newSinglePointAlloc fail because not etcd background
// This could happen in the server package unit test
return nil
}

addrs, err := ebd.EtcdAddrs()
if err != nil {
panic(err)
}
spa := &singlePointAlloc{
dbID: dbID,
tblID: tblID,
isUnsigned: isUnsigned,
}
if len(addrs) > 0 {
etcdCli, err := clientv3.New(clientv3.Config{
Endpoints: addrs,
TLS: ebd.TLSConfig(),
})
if err != nil {
panic(err)
}
spa.clientDiscover = clientDiscover{etcdCli: etcdCli}
} else {
spa.clientDiscover = clientDiscover{}
spa.mu.AutoIDAllocClient = autoid.MockForTest(store)
}

failpoint.Inject("mockAutoIDChange", func(val failpoint.Value) {
if val.(bool) {
spa = nil
}
})
return spa
}

// NewAllocator returns a new auto increment id generator on the store.
func NewAllocator(store kv.Storage, dbID, tbID int64, isUnsigned bool,
allocType AllocatorType, opts ...AllocOption) Allocator {
Expand All @@ -548,6 +590,15 @@ func NewAllocator(store kv.Storage, dbID, tbID int64, isUnsigned bool,
for _, fn := range opts {
fn.ApplyOn(alloc)
}

// Use the MySQL compatible AUTO_INCREMENT mode.
if allocType == RowIDAllocType && alloc.customStep && alloc.step == 1 {
alloc1 := newSinglePointAlloc(store, dbID, tbID, isUnsigned)
if alloc1 != nil {
return alloc1
}
}

return alloc
}

Expand Down
Loading