Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

playground: Add tidb-cse mode #2386

Merged
merged 5 commits into from
Mar 27, 2024
Merged

playground: Add tidb-cse mode #2386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c4b12d3
playground: Add tidb-cse mode
breezewish Mar 20, 2024
61d9693
Fix lint
breezewish Mar 20, 2024
707af59
Reserve more for remote cache
breezewish Mar 27, 2024
6018b89
Merge remote-tracking branch 'origin/master' into wenxuan/cse-mode
breezewish Mar 27, 2024
2f10eed
Merge remote-tracking branch 'origin/master' into wenxuan/cse-mode
breezewish Mar 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions components/playground/instance/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,14 @@ type Config struct {
Version string `yaml:"version"`
}

// CSEOptions contains configs to run TiDB cluster in CSE mode.
type CSEOptions struct {
S3Endpoint string `yaml:"s3_endpoint"`
Bucket string `yaml:"bucket"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
}

type instance struct {
ID int
Dir string
Expand Down
18 changes: 8 additions & 10 deletions components/playground/instance/pd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,11 @@ type PDInstance struct {
joinEndpoints []*PDInstance
pds []*PDInstance
Process
isCSEMode bool
}

// NewPDInstance return a PDInstance
func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int) *PDInstance {
func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int, isCSEMode bool) *PDInstance {
if port <= 0 {
port = 2379
}
Expand All @@ -63,8 +64,9 @@ func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, p
StatusPort: utils.MustGetFreePort(host, port),
ConfigPath: configPath,
},
Role: role,
pds: pds,
Role: role,
pds: pds,
isCSEMode: isCSEMode,
}
}

Expand Down Expand Up @@ -112,8 +114,8 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)),
fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
fmt.Sprintf("--log-file=%s", inst.LogFile()),
fmt.Sprintf("--config=%s", configPath),
}...)

switch {
case len(inst.initEndpoints) > 0:
endpoints := make([]string, 0)
Expand All @@ -140,9 +142,7 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")),
fmt.Sprintf("--log-file=%s", inst.LogFile()),
}
if inst.ConfigPath != "" {
args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
fmt.Sprintf("--config=%s", configPath),
}
case PDRoleScheduling:
endpoints := pdEndpoints(inst.pds, true)
Expand All @@ -153,9 +153,7 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")),
fmt.Sprintf("--log-file=%s", inst.LogFile()),
}
if inst.ConfigPath != "" {
args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
fmt.Sprintf("--config=%s", configPath),
}
}

Expand Down
10 changes: 10 additions & 0 deletions components/playground/instance/pd_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,15 @@ package instance
func (inst *PDInstance) getConfig() map[string]any {
config := make(map[string]any)
config["schedule.patrol-region-interval"] = "100ms"

if inst.isCSEMode {
config["keyspace.pre-alloc"] = []string{"mykeyspace"}
config["replication.enable-placement-rules"] = true
config["replication.max-replica"] = 1
config["schedule.merge-schedule-limit"] = 0
config["schedule.low-space-ration"] = 1.0
config["schedule.replica-schedule-limit"] = 500
}

return config
}
6 changes: 3 additions & 3 deletions components/playground/instance/tidb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ type TiDBInstance struct {
Process
tiproxyCertDir string
enableBinlog bool
isDisaggMode bool
isCSEMode bool
}

// NewTiDBInstance return a TiDBInstance
func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isDisaggMode bool) *TiDBInstance {
func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isCSEMode bool) *TiDBInstance {
if port <= 0 {
port = 4000
}
Expand All @@ -52,7 +52,7 @@ func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int,
tiproxyCertDir: tiproxyCertDir,
pds: pds,
enableBinlog: enableBinlog,
isDisaggMode: isDisaggMode,
isCSEMode: isCSEMode,
}
}

Expand Down
29 changes: 28 additions & 1 deletion components/playground/instance/tidb_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,36 @@ func (inst *TiDBInstance) getConfig() map[string]any {
config := make(map[string]any)
config["security.auto-tls"] = true

if inst.isDisaggMode {
if inst.isCSEMode {
config["keyspace-name"] = "mykeyspace"
config["enable-safe-point-v2"] = true
config["force-enable-vector-type"] = true
config["use-autoscaler"] = false
config["disaggregated-tiflash"] = true
config["ratelimit.full-speed"] = 1048576000
config["ratelimit.full-speed-capacity"] = 1048576000
config["ratelimit.low-speed-watermark"] = 1048576000000
config["ratelimit.block-write-watermark"] = 1048576000000
config["security.enable-sem"] = false
config["tiflash-replicas.constraints"] = []any{
map[string]any{
"key": "engine",
"op": "in",
"values": []string{
"tiflash",
},
},
map[string]any{
"key": "engine_role",
"op": "in",
"values": []string{
"write",
},
},
}
config["tiflash-replicas.group-id"] = "enable_s3_wn_region"
config["tiflash-replicas.extra-s3-rule"] = false
config["tiflash-replicas.min-count"] = 1
}

tiproxyCrtPath := filepath.Join(inst.tiproxyCertDir, "tiproxy.crt")
Expand Down
14 changes: 3 additions & 11 deletions components/playground/instance/tiflash.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,11 @@ const (
TiFlashRoleDisaggCompute TiFlashRole = "compute"
)

// DisaggOptions contains configs to run TiFlash in disaggregated mode.
type DisaggOptions struct {
S3Endpoint string `yaml:"s3_endpoint"`
Bucket string `yaml:"bucket"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
}

// TiFlashInstance represent a running TiFlash
type TiFlashInstance struct {
instance
Role TiFlashRole
DisaggOpts DisaggOptions
cseOpts CSEOptions
TCPPort int
ServicePort int
ProxyPort int
Expand All @@ -62,7 +54,7 @@ type TiFlashInstance struct {
}

// NewTiFlashInstance return a TiFlashInstance
func NewTiFlashInstance(role TiFlashRole, disaggOptions DisaggOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance {
func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance {
if role != TiFlashRoleNormal && role != TiFlashRoleDisaggWrite && role != TiFlashRoleDisaggCompute {
panic(fmt.Sprintf("Unknown TiFlash role %s", role))
}
Expand All @@ -82,7 +74,7 @@ func NewTiFlashInstance(role TiFlashRole, disaggOptions DisaggOptions, binPath,
ConfigPath: configPath,
},
Role: role,
DisaggOpts: disaggOptions,
cseOpts: cseOptions,
TCPPort: utils.MustGetFreePort(host, 9100), // 9000 for default object store port
ServicePort: utils.MustGetFreePort(host, 3930),
ProxyPort: utils.MustGetFreePort(host, 20170),
Expand Down
41 changes: 28 additions & 13 deletions components/playground/instance/tiflash_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,18 @@ func (inst *TiFlashInstance) getProxyConfig() map[string]any {
config["raftdb.max-open-files"] = 256
config["storage.reserve-space"] = 0
config["storage.reserve-raft-space"] = 0

if inst.Role == TiFlashRoleDisaggWrite {
config["storage.api-version"] = 2
config["storage.enable-ttl"] = true
config["dfs.prefix"] = "tikv"
config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint
config["dfs.s3-key-id"] = inst.cseOpts.AccessKey
config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey
config["dfs.s3-bucket"] = inst.cseOpts.Bucket
config["dfs.s3-region"] = "local"
}

return config
}

Expand All @@ -31,23 +43,26 @@ func (inst *TiFlashInstance) getConfig() map[string]any {
config["logger.level"] = "debug"

if inst.Role == TiFlashRoleDisaggWrite {
config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint
config["storage.s3.bucket"] = inst.DisaggOpts.Bucket
config["storage.s3.root"] = "/"
config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey
config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey
config["enable_safe_point_v2"] = true
config["storage.api_version"] = 2
config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint
config["storage.s3.bucket"] = inst.cseOpts.Bucket
config["storage.s3.root"] = "/tiflash-cse/"
config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey
config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey
config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")}
config["flash.disaggregated_mode"] = "tiflash_write"
config["flash.use_autoscaler"] = false
} else if inst.Role == TiFlashRoleDisaggCompute {
config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint
config["storage.s3.bucket"] = inst.DisaggOpts.Bucket
config["storage.s3.root"] = "/"
config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey
config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey
config["enable_safe_point_v2"] = true
config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint
config["storage.s3.bucket"] = inst.cseOpts.Bucket
config["storage.s3.root"] = "/tiflash-cse/"
config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey
config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey
config["storage.remote.cache.dir"] = filepath.Join(inst.Dir, "remote_cache")
config["storage.remote.cache.capacity"] = 1000000000 // 1GB
config["storage.remote.cache.capacity"] = 20000000000 // 20GB
config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")}
config["flash.disaggregated_mode"] = "tiflash_compute"
config["flash.use_autoscaler"] = false
}

return config
Expand Down
8 changes: 6 additions & 2 deletions components/playground/instance/tikv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ type TiKVInstance struct {
instance
pds []*PDInstance
Process
isCSEMode bool
cseOpts CSEOptions
}

// NewTiKVInstance return a TiKVInstance
func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiKVInstance {
func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance, isCSEMode bool, cseOptions CSEOptions) *TiKVInstance {
if port <= 0 {
port = 20160
}
Expand All @@ -45,7 +47,9 @@ func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port
StatusPort: utils.MustGetFreePort(host, 20180),
ConfigPath: configPath,
},
pds: pds,
pds: pds,
isCSEMode: isCSEMode,
cseOpts: cseOptions,
}
}

Expand Down
12 changes: 12 additions & 0 deletions components/playground/instance/tikv_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,17 @@ func (inst *TiKVInstance) getConfig() map[string]any {
config["raftdb.max-open-files"] = 256
config["storage.reserve-space"] = 0
config["storage.reserve-raft-space"] = 0

if inst.isCSEMode {
config["storage.api-version"] = 2
config["storage.enable-ttl"] = true
config["dfs.prefix"] = "tikv"
config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint
config["dfs.s3-key-id"] = inst.cseOpts.AccessKey
config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey
config["dfs.s3-bucket"] = inst.cseOpts.Bucket
config["dfs.s3-region"] = "local"
}

return config
}
Loading
Loading