From c4b12d3bbdfd2c3db9c786bfe8543687c0c5fa67 Mon Sep 17 00:00:00 2001 From: Wish Date: Wed, 20 Mar 2024 14:20:23 +0800 Subject: [PATCH 1/3] playground: Add tidb-cse mode Signed-off-by: Wish --- components/playground/instance/instance.go | 8 +++ components/playground/instance/pd.go | 22 +++--- components/playground/instance/pd_config.go | 10 +++ components/playground/instance/tidb.go | 6 +- components/playground/instance/tidb_config.go | 29 +++++++- components/playground/instance/tiflash.go | 14 +--- .../playground/instance/tiflash_config.go | 39 ++++++---- components/playground/instance/tikv.go | 8 ++- components/playground/instance/tikv_config.go | 12 ++++ components/playground/main.go | 72 +++++++++---------- components/playground/playground.go | 46 +++++++----- pkg/tidbver/tidbver.go | 6 -- 12 files changed, 170 insertions(+), 102 deletions(-) diff --git a/components/playground/instance/instance.go b/components/playground/instance/instance.go index 620f2a502d..fe161d2e89 100644 --- a/components/playground/instance/instance.go +++ b/components/playground/instance/instance.go @@ -37,6 +37,14 @@ type Config struct { Version string `yaml:"version"` } +// CSEOptions contains configs to run TiDB cluster in CSE mode. +type CSEOptions struct { + S3Endpoint string `yaml:"s3_endpoint"` + Bucket string `yaml:"bucket"` + AccessKey string `yaml:"access_key"` + SecretKey string `yaml:"secret_key"` +} + type instance struct { ID int Dir string diff --git a/components/playground/instance/pd.go b/components/playground/instance/pd.go index 38a7e4f950..49c01aa357 100644 --- a/components/playground/instance/pd.go +++ b/components/playground/instance/pd.go @@ -48,10 +48,11 @@ type PDInstance struct { joinEndpoints []*PDInstance pds []*PDInstance Process + isCSEMode bool } // NewPDInstance return a PDInstance -func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int) *PDInstance { +func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int, isCSEMode bool) *PDInstance { if port <= 0 { port = 2379 } @@ -65,8 +66,9 @@ func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, p StatusPort: utils.MustGetFreePort(host, port), ConfigPath: configPath, }, - Role: role, - pds: pds, + Role: role, + pds: pds, + isCSEMode: isCSEMode, } } @@ -114,8 +116,8 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)), fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), fmt.Sprintf("--log-file=%s", inst.LogFile()), + fmt.Sprintf("--config=%s", configPath), }...) - switch { case len(inst.initEndpoints) > 0: endpoints := make([]string, 0) @@ -142,9 +144,7 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")), fmt.Sprintf("--log-file=%s", inst.LogFile()), - } - if inst.ConfigPath != "" { - args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) + fmt.Sprintf("--config=%s", configPath), } case PDRoleScheduling: endpoints := pdEndpoints(inst.pds, true) @@ -155,9 +155,7 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")), fmt.Sprintf("--log-file=%s", inst.LogFile()), - } - if inst.ConfigPath != "" { - args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) + fmt.Sprintf("--config=%s", configPath), } case PDRoleResourceManager: endpoints := pdEndpoints(inst.pds, true) @@ -168,9 +166,7 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")), fmt.Sprintf("--log-file=%s", inst.LogFile()), - } - if inst.ConfigPath != "" { - args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) + fmt.Sprintf("--config=%s", configPath), } } diff --git a/components/playground/instance/pd_config.go b/components/playground/instance/pd_config.go index 3ef80b6f4d..d951299606 100644 --- a/components/playground/instance/pd_config.go +++ b/components/playground/instance/pd_config.go @@ -16,5 +16,15 @@ package instance func (inst *PDInstance) getConfig() map[string]any { config := make(map[string]any) config["schedule.patrol-region-interval"] = "100ms" + + if inst.isCSEMode { + config["keyspace.pre-alloc"] = []string{"mykeyspace"} + config["replication.enable-placement-rules"] = true + config["replication.max-replica"] = 1 + config["schedule.merge-schedule-limit"] = 0 + config["schedule.low-space-ration"] = 1.0 + config["schedule.replica-schedule-limit"] = 500 + } + return config } diff --git a/components/playground/instance/tidb.go b/components/playground/instance/tidb.go index 93acc95e24..ab349b95bd 100644 --- a/components/playground/instance/tidb.go +++ b/components/playground/instance/tidb.go @@ -30,11 +30,11 @@ type TiDBInstance struct { pds []*PDInstance Process enableBinlog bool - isDisaggMode bool + isCSEMode bool } // NewTiDBInstance return a TiDBInstance -func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, enableBinlog bool, isDisaggMode bool) *TiDBInstance { +func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, enableBinlog bool, isCSEMode bool) *TiDBInstance { if port <= 0 { port = 4000 } @@ -50,7 +50,7 @@ func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, }, pds: pds, enableBinlog: enableBinlog, - isDisaggMode: isDisaggMode, + isCSEMode: isCSEMode, } } diff --git a/components/playground/instance/tidb_config.go b/components/playground/instance/tidb_config.go index 74731b8c39..959c76681c 100644 --- a/components/playground/instance/tidb_config.go +++ b/components/playground/instance/tidb_config.go @@ -17,9 +17,36 @@ func (inst *TiDBInstance) getConfig() map[string]any { config := make(map[string]any) config["security.auto-tls"] = true - if inst.isDisaggMode { + if inst.isCSEMode { + config["keyspace-name"] = "mykeyspace" + config["enable-safe-point-v2"] = true + config["force-enable-vector-type"] = true config["use-autoscaler"] = false config["disaggregated-tiflash"] = true + config["ratelimit.full-speed"] = 1048576000 + config["ratelimit.full-speed-capacity"] = 1048576000 + config["ratelimit.low-speed-watermark"] = 1048576000000 + config["ratelimit.block-write-watermark"] = 1048576000000 + config["security.enable-sem"] = false + config["tiflash-replicas.constraints"] = []interface{}{ + map[string]interface{}{ + "key": "engine", + "op": "in", + "values": []string{ + "tiflash", + }, + }, + map[string]interface{}{ + "key": "engine_role", + "op": "in", + "values": []string{ + "write", + }, + }, + } + config["tiflash-replicas.group-id"] = "enable_s3_wn_region" + config["tiflash-replicas.extra-s3-rule"] = false + config["tiflash-replicas.min-count"] = 1 } return config diff --git a/components/playground/instance/tiflash.go b/components/playground/instance/tiflash.go index 84175b3ab1..618dd9a881 100644 --- a/components/playground/instance/tiflash.go +++ b/components/playground/instance/tiflash.go @@ -39,19 +39,11 @@ const ( TiFlashRoleDisaggCompute TiFlashRole = "compute" ) -// DisaggOptions contains configs to run TiFlash in disaggregated mode. -type DisaggOptions struct { - S3Endpoint string `yaml:"s3_endpoint"` - Bucket string `yaml:"bucket"` - AccessKey string `yaml:"access_key"` - SecretKey string `yaml:"secret_key"` -} - // TiFlashInstance represent a running TiFlash type TiFlashInstance struct { instance Role TiFlashRole - DisaggOpts DisaggOptions + cseOpts CSEOptions TCPPort int ServicePort int ProxyPort int @@ -62,7 +54,7 @@ type TiFlashInstance struct { } // NewTiFlashInstance return a TiFlashInstance -func NewTiFlashInstance(role TiFlashRole, disaggOptions DisaggOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance { +func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance { if role != TiFlashRoleNormal && role != TiFlashRoleDisaggWrite && role != TiFlashRoleDisaggCompute { panic(fmt.Sprintf("Unknown TiFlash role %s", role)) } @@ -82,7 +74,7 @@ func NewTiFlashInstance(role TiFlashRole, disaggOptions DisaggOptions, binPath, ConfigPath: configPath, }, Role: role, - DisaggOpts: disaggOptions, + cseOpts: cseOptions, TCPPort: utils.MustGetFreePort(host, 9100), // 9000 for default object store port ServicePort: utils.MustGetFreePort(host, 3930), ProxyPort: utils.MustGetFreePort(host, 20170), diff --git a/components/playground/instance/tiflash_config.go b/components/playground/instance/tiflash_config.go index 15c8870653..257dda49a3 100644 --- a/components/playground/instance/tiflash_config.go +++ b/components/playground/instance/tiflash_config.go @@ -21,6 +21,18 @@ func (inst *TiFlashInstance) getProxyConfig() map[string]any { config["raftdb.max-open-files"] = 256 config["storage.reserve-space"] = 0 config["storage.reserve-raft-space"] = 0 + + if inst.Role == TiFlashRoleDisaggWrite { + config["storage.api-version"] = 2 + config["storage.enable-ttl"] = true + config["dfs.prefix"] = "tikv" + config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint + config["dfs.s3-key-id"] = inst.cseOpts.AccessKey + config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey + config["dfs.s3-bucket"] = inst.cseOpts.Bucket + config["dfs.s3-region"] = "local" + } + return config } @@ -31,23 +43,26 @@ func (inst *TiFlashInstance) getConfig() map[string]any { config["logger.level"] = "debug" if inst.Role == TiFlashRoleDisaggWrite { - config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint - config["storage.s3.bucket"] = inst.DisaggOpts.Bucket - config["storage.s3.root"] = "/" - config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey - config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey + config["enable_safe_point_v2"] = true + config["storage.api_version"] = 2 + config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint + config["storage.s3.bucket"] = inst.cseOpts.Bucket + config["storage.s3.root"] = "/tiflash-cse/" + config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey + config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey + config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")} config["flash.disaggregated_mode"] = "tiflash_write" - config["flash.use_autoscaler"] = false } else if inst.Role == TiFlashRoleDisaggCompute { - config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint - config["storage.s3.bucket"] = inst.DisaggOpts.Bucket - config["storage.s3.root"] = "/" - config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey - config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey + config["enable_safe_point_v2"] = true + config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint + config["storage.s3.bucket"] = inst.cseOpts.Bucket + config["storage.s3.root"] = "/tiflash-cse/" + config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey + config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey config["storage.remote.cache.dir"] = filepath.Join(inst.Dir, "remote_cache") config["storage.remote.cache.capacity"] = 1000000000 // 1GB + config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")} config["flash.disaggregated_mode"] = "tiflash_compute" - config["flash.use_autoscaler"] = false } return config diff --git a/components/playground/instance/tikv.go b/components/playground/instance/tikv.go index e767ec61a2..1448714874 100644 --- a/components/playground/instance/tikv.go +++ b/components/playground/instance/tikv.go @@ -28,10 +28,12 @@ type TiKVInstance struct { instance pds []*PDInstance Process + isCSEMode bool + cseOpts CSEOptions } // NewTiKVInstance return a TiKVInstance -func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiKVInstance { +func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance, isCSEMode bool, cseOptions CSEOptions) *TiKVInstance { if port <= 0 { port = 20160 } @@ -45,7 +47,9 @@ func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port StatusPort: utils.MustGetFreePort(host, 20180), ConfigPath: configPath, }, - pds: pds, + pds: pds, + isCSEMode: isCSEMode, + cseOpts: cseOptions, } } diff --git a/components/playground/instance/tikv_config.go b/components/playground/instance/tikv_config.go index 197a9492c7..45fab76a0e 100644 --- a/components/playground/instance/tikv_config.go +++ b/components/playground/instance/tikv_config.go @@ -19,5 +19,17 @@ func (inst *TiKVInstance) getConfig() map[string]any { config["raftdb.max-open-files"] = 256 config["storage.reserve-space"] = 0 config["storage.reserve-raft-space"] = 0 + + if inst.isCSEMode { + config["storage.api-version"] = 2 + config["storage.enable-ttl"] = true + config["dfs.prefix"] = "tikv" + config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint + config["dfs.s3-key-id"] = inst.cseOpts.AccessKey + config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey + config["dfs.s3-bucket"] = inst.cseOpts.Bucket + config["dfs.s3-region"] = "local" + } + return config } diff --git a/components/playground/main.go b/components/playground/main.go index 713f6c8e33..e80097fb53 100644 --- a/components/playground/main.go +++ b/components/playground/main.go @@ -55,27 +55,27 @@ import ( // BootOptions is the topology and options used to start a playground cluster type BootOptions struct { - Mode string `yaml:"mode"` - PDMode string `yaml:"pd_mode"` - Version string `yaml:"version"` - PD instance.Config `yaml:"pd"` // ignored when pd_mode == ms - PDAPI instance.Config `yaml:"pd_api"` // Only available when pd_mode == ms - PDTSO instance.Config `yaml:"pd_tso"` // Only available when pd_mode == ms - PDScheduling instance.Config `yaml:"pd_scheduling"` // Only available when pd_mode == ms - PDRM instance.Config `yaml:"pd_rm"` // Only available when pd_mode == ms - TiProxy instance.Config `yaml:"tiproxy"` - TiDB instance.Config `yaml:"tidb"` - TiKV instance.Config `yaml:"tikv"` - TiFlash instance.Config `yaml:"tiflash"` // ignored when mode == tidb-disagg - TiFlashWrite instance.Config `yaml:"tiflash_write"` // Only available when mode == tidb-disagg - TiFlashCompute instance.Config `yaml:"tiflash_compute"` // Only available when mode == tidb-disagg - TiCDC instance.Config `yaml:"ticdc"` - TiKVCDC instance.Config `yaml:"tikv_cdc"` - Pump instance.Config `yaml:"pump"` - Drainer instance.Config `yaml:"drainer"` - Host string `yaml:"host"` - Monitor bool `yaml:"monitor"` - DisaggOpts instance.DisaggOptions `yaml:"disagg"` // Only available when mode == tidb-disagg + Mode string `yaml:"mode"` + PDMode string `yaml:"pd_mode"` + Version string `yaml:"version"` + PD instance.Config `yaml:"pd"` // ignored when pd_mode == ms + PDAPI instance.Config `yaml:"pd_api"` // Only available when pd_mode == ms + PDTSO instance.Config `yaml:"pd_tso"` // Only available when pd_mode == ms + PDScheduling instance.Config `yaml:"pd_scheduling"` // Only available when pd_mode == ms + PDRM instance.Config `yaml:"pd_rm"` // Only available when pd_mode == ms + TiProxy instance.Config `yaml:"tiproxy"` + TiDB instance.Config `yaml:"tidb"` + TiKV instance.Config `yaml:"tikv"` + TiFlash instance.Config `yaml:"tiflash"` // ignored when mode == tidb-cse + TiFlashWrite instance.Config `yaml:"tiflash_write"` // Only available when mode == tidb-cse + TiFlashCompute instance.Config `yaml:"tiflash_compute"` // Only available when mode == tidb-cse + TiCDC instance.Config `yaml:"ticdc"` + TiKVCDC instance.Config `yaml:"tikv_cdc"` + Pump instance.Config `yaml:"pump"` + Drainer instance.Config `yaml:"drainer"` + Host string `yaml:"host"` + Monitor bool `yaml:"monitor"` + CSEOpts instance.CSEOptions `yaml:"cse"` // Only available when mode == tidb-cse } var ( @@ -277,7 +277,7 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. }, } - rootCmd.Flags().StringVar(&options.Mode, "mode", "tidb", "TiUP playground mode: 'tidb', 'tidb-disagg', 'tikv-slim'") + rootCmd.Flags().StringVar(&options.Mode, "mode", "tidb", "TiUP playground mode: 'tidb', 'tidb-cse', 'tikv-slim'") rootCmd.Flags().StringVar(&options.PDMode, "pd.mode", "pd", "PD mode: 'pd', 'ms'") rootCmd.PersistentFlags().StringVarP(&tag, "tag", "T", "", "Specify a tag for playground") // Use `PersistentFlags()` to make it available to subcommands. rootCmd.Flags().Bool("without-monitor", false, "Don't start prometheus and grafana component") @@ -290,9 +290,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().IntVar(&options.TiKV.Num, "kv", 0, "TiKV instance number") rootCmd.Flags().IntVar(&options.PD.Num, "pd", 0, "PD instance number") rootCmd.Flags().IntVar(&options.TiProxy.Num, "tiproxy", 0, "TiProxy instance number") - rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-disagg this will set instance number for both Write Node and Compute Node") - rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-disagg, take precedence over --tiflash") - rootCmd.Flags().IntVar(&options.TiFlashCompute.Num, "tiflash.compute", 0, "TiFlash Compute instance number, available when --mode=tidb-disagg, take precedence over --tiflash") + rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-cse this will set instance number for both Write Node and Compute Node") + rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-cse, take precedence over --tiflash") + rootCmd.Flags().IntVar(&options.TiFlashCompute.Num, "tiflash.compute", 0, "TiFlash Compute instance number, available when --mode=tidb-cse, take precedence over --tiflash") rootCmd.Flags().IntVar(&options.TiCDC.Num, "ticdc", 0, "TiCDC instance number") rootCmd.Flags().IntVar(&options.TiKVCDC.Num, "kvcdc", 0, "TiKV-CDC instance number") rootCmd.Flags().IntVar(&options.Pump.Num, "pump", 0, "Pump instance number") @@ -323,9 +323,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.TiKV.ConfigPath, "kv.config", "", "TiKV instance configuration file") rootCmd.Flags().StringVar(&options.PD.ConfigPath, "pd.config", "", "PD instance configuration file") rootCmd.Flags().StringVar(&options.TiProxy.ConfigPath, "tiproxy.config", "", "TiProxy instance configuration file") - rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-disagg this will set config file for both Write Node and Compute Node") - rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-disagg, take precedence over --tiflash.config") - rootCmd.Flags().StringVar(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", "", "TiFlash Compute instance configuration file, available when --mode=tidb-disagg, take precedence over --tiflash.config") + rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-cse this will set config file for both Write Node and Compute Node") + rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-cse, take precedence over --tiflash.config") + rootCmd.Flags().StringVar(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", "", "TiFlash Compute instance configuration file, available when --mode=tidb-cse, take precedence over --tiflash.config") rootCmd.Flags().StringVar(&options.Pump.ConfigPath, "pump.config", "", "Pump instance configuration file") rootCmd.Flags().StringVar(&options.Drainer.ConfigPath, "drainer.config", "", "Drainer instance configuration file") rootCmd.Flags().StringVar(&options.TiCDC.ConfigPath, "ticdc.config", "", "TiCDC instance configuration file") @@ -341,9 +341,9 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.PD.BinPath, "pd.binpath", "", "PD instance binary path") rootCmd.Flags().StringVar(&options.TiProxy.BinPath, "tiproxy.binpath", "", "TiProxy instance binary path") rootCmd.Flags().StringVar(&options.TiProxy.Version, "tiproxy.version", "", "TiProxy instance version") - rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-disagg this will set binary path for both Write Node and Compute Node") - rootCmd.Flags().StringVar(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", "", "TiFlash Write instance binary path, available when --mode=tidb-disagg, take precedence over --tiflash.binpath") - rootCmd.Flags().StringVar(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", "", "TiFlash Compute instance binary path, available when --mode=tidb-disagg, take precedence over --tiflash.binpath") + rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-cse this will set binary path for both Write Node and Compute Node") + rootCmd.Flags().StringVar(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", "", "TiFlash Write instance binary path, available when --mode=tidb-cse, take precedence over --tiflash.binpath") + rootCmd.Flags().StringVar(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", "", "TiFlash Compute instance binary path, available when --mode=tidb-cse, take precedence over --tiflash.binpath") rootCmd.Flags().StringVar(&options.TiCDC.BinPath, "ticdc.binpath", "", "TiCDC instance binary path") rootCmd.Flags().StringVar(&options.TiKVCDC.BinPath, "kvcdc.binpath", "", "TiKV-CDC instance binary path") rootCmd.Flags().StringVar(&options.Pump.BinPath, "pump.binpath", "", "Pump instance binary path") @@ -356,10 +356,10 @@ Note: Version constraint [bold]%s[reset] is resolved to [green][bold]%s[reset]. rootCmd.Flags().StringVar(&options.TiKVCDC.Version, "kvcdc.version", "", "TiKV-CDC instance version") - rootCmd.Flags().StringVar(&options.DisaggOpts.S3Endpoint, "disagg.s3_endpoint", "127.0.0.1:9000", "Object store URL for the disaggregated TiFlash, available when --mode=tidb-disagg") - rootCmd.Flags().StringVar(&options.DisaggOpts.Bucket, "disagg.bucket", "tiflash", "Object store bucket for the disaggregated TiFlash, available when --mode=tidb-disagg") - rootCmd.Flags().StringVar(&options.DisaggOpts.AccessKey, "disagg.access_key", "minioadmin", "Object store access key, available when --mode=tidb-disagg") - rootCmd.Flags().StringVar(&options.DisaggOpts.SecretKey, "disagg.secret_key", "minioadmin", "Object store secret key, available when --mode=tidb-disagg") + rootCmd.Flags().StringVar(&options.CSEOpts.S3Endpoint, "cse.s3_endpoint", "http://127.0.0.1:9000", "Object store URL for the disaggregated TiFlash, available when --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.Bucket, "cse.bucket", "tiflash", "Object store bucket for the disaggregated TiFlash, available when --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.AccessKey, "cse.access_key", "minioadmin", "Object store access key, available when --mode=tidb-cse") + rootCmd.Flags().StringVar(&options.CSEOpts.SecretKey, "cse.secret_key", "minioadmin", "Object store secret key, available when --mode=tidb-cse") rootCmd.AddCommand(newDisplay()) rootCmd.AddCommand(newScaleOut()) @@ -393,7 +393,7 @@ func populateDefaultOpt(flagSet *pflag.FlagSet) error { defaultInt(&options.TiFlash.Num, "tiflash", 1) case "tikv-slim": defaultInt(&options.TiKV.Num, "kv", 1) - case "tidb-disagg": + case "tidb-cse": defaultInt(&options.TiDB.Num, "db", 1) defaultInt(&options.TiKV.Num, "kv", 1) defaultInt(&options.TiFlash.Num, "tiflash", 1) diff --git a/components/playground/playground.go b/components/playground/playground.go index d4d6438b53..c027f4c0b5 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -487,7 +487,7 @@ func (p *Playground) handleScaleOut(w io.Writer, cmd *Command) error { if err != nil { return err } - // TODO: Support scale-out in disaggregated mode + // TODO: Support scale-out in CSE mode inst, err := p.addInstance(cmd.ComponentID, instance.PDRoleNormal, instance.TiFlashRoleNormal, cmd.Config) if err != nil { return err @@ -720,7 +720,7 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif switch componentID { case spec.ComponentPD: - inst := instance.NewPDInstance(pdRole, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, cfg.Port) + inst := instance.NewPDInstance(pdRole, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, cfg.Port, p.bootOptions.Mode == "tidb-cse") ins = inst if pdRole == instance.PDRoleNormal || pdRole == instance.PDRoleAPI { if p.booted { @@ -740,15 +740,15 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif p.rms = append(p.rms, inst) } case spec.ComponentTiDB: - inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, p.enableBinlog(), p.bootOptions.Mode == "tidb-disagg") + inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, p.enableBinlog(), p.bootOptions.Mode == "tidb-cse") ins = inst p.tidbs = append(p.tidbs, inst) case spec.ComponentTiKV: - inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds) + inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, p.bootOptions.Mode == "tidb-cse", p.bootOptions.CSEOpts) ins = inst p.tikvs = append(p.tikvs, inst) case spec.ComponentTiFlash: - inst := instance.NewTiFlashInstance(tiflashRole, p.bootOptions.DisaggOpts, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, p.tidbs, cfg.Version) + inst := instance.NewTiFlashInstance(tiflashRole, p.bootOptions.CSEOpts, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, p.tidbs, cfg.Version) ins = inst p.tiflashs = append(p.tiflashs, inst) case spec.ComponentTiProxy: @@ -967,37 +967,47 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme instances = append(instances, InstancePair{spec.ComponentTiFlash, instance.PDRoleNormal, instance.TiFlashRoleNormal, options.TiFlash}, ) - } else if options.Mode == "tidb-disagg" { - if !tidbver.TiDBSupportDisagg(options.Version) { - return fmt.Errorf("TiDB cluster doesn't support disaggregated mode in version %s", options.Version) - } + } else if options.Mode == "tidb-cse" { if !tidbver.TiFlashPlaygroundNewStartMode(options.Version) { // For simplicity, currently we only implemented disagg mode when TiFlash can run without config. - return fmt.Errorf("TiUP playground only supports disaggregated mode for TiDB cluster >= v7.1.0 (or nightly)") + return fmt.Errorf("TiUP playground only supports CSE mode for TiDB cluster >= v7.1.0 (or nightly)") + } + + if !strings.HasPrefix(options.CSEOpts.S3Endpoint, "https://") && !strings.HasPrefix(options.CSEOpts.S3Endpoint, "http://") { + return fmt.Errorf("CSE mode requires S3 endpoint to start with http:// or https://") + } + + isSecure := strings.HasPrefix(options.CSEOpts.S3Endpoint, "https://") + rawEndpoint := strings.TrimPrefix(options.CSEOpts.S3Endpoint, "https://") + rawEndpoint = strings.TrimPrefix(rawEndpoint, "http://") + + // Currently we always assign region=local. Other regions are not supported. + if strings.Contains(rawEndpoint, "amazonaws.com") { + return fmt.Errorf("Currently TiUP playground CSE mode only supports local S3 (like minio). S3 on AWS Regions are not supported. Contributions are welcome!") } // Preflight check whether specified object storage is available. - s3Client, err := minio.New(options.DisaggOpts.S3Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(options.DisaggOpts.AccessKey, options.DisaggOpts.SecretKey, ""), - Secure: false, + s3Client, err := minio.New(rawEndpoint, &minio.Options{ + Creds: credentials.NewStaticV4(options.CSEOpts.AccessKey, options.CSEOpts.SecretKey, ""), + Secure: isSecure, }) if err != nil { - return errors.Annotate(err, "Disaggregate mode preflight check failed") + return errors.Annotate(err, "CSE mode preflight check failed") } ctxCheck, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - bucketExists, err := s3Client.BucketExists(ctxCheck, options.DisaggOpts.Bucket) + bucketExists, err := s3Client.BucketExists(ctxCheck, options.CSEOpts.Bucket) if err != nil { - return errors.Annotate(err, "Disaggregate mode preflight check failed") + return errors.Annotate(err, "CSE mode preflight check failed") } if !bucketExists { // Try to create bucket. - err := s3Client.MakeBucket(ctxCheck, options.DisaggOpts.Bucket, minio.MakeBucketOptions{}) + err := s3Client.MakeBucket(ctxCheck, options.CSEOpts.Bucket, minio.MakeBucketOptions{}) if err != nil { - return fmt.Errorf("Disaggregate mode preflight check failed: Bucket %s doesn't exist", options.DisaggOpts.Bucket) + return fmt.Errorf("CSE mode preflight check failed: Bucket %s doesn't exist", options.CSEOpts.Bucket) } } diff --git a/pkg/tidbver/tidbver.go b/pkg/tidbver/tidbver.go index eae857f78e..f4ffd22889 100644 --- a/pkg/tidbver/tidbver.go +++ b/pkg/tidbver/tidbver.go @@ -94,12 +94,6 @@ func TiFlashPlaygroundNewStartMode(version string) bool { return semver.Compare(version, "v7.1.0") >= 0 || strings.Contains(version, "nightly") } -// TiDBSupportDisagg returns true if the given version of TiDB and TiFlash supports -// disaggregated mode. -func TiDBSupportDisagg(version string) bool { - return semver.Compare(version, "v7.0.0") >= 0 || strings.Contains(version, "nightly") -} - // PDSupportMicroServices returns true if the given version of PD supports micro services. func PDSupportMicroServices(version string) bool { return semver.Compare(version, "v7.3.0") >= 0 || strings.Contains(version, "nightly") From 61d9693e0e059a73ebb7c2836cbb2cd10b62b105 Mon Sep 17 00:00:00 2001 From: Wish Date: Wed, 20 Mar 2024 15:44:15 +0800 Subject: [PATCH 2/3] Fix lint Signed-off-by: Wish --- components/playground/instance/tidb_config.go | 6 +++--- components/playground/playground.go | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/playground/instance/tidb_config.go b/components/playground/instance/tidb_config.go index 959c76681c..b613f9d74b 100644 --- a/components/playground/instance/tidb_config.go +++ b/components/playground/instance/tidb_config.go @@ -28,15 +28,15 @@ func (inst *TiDBInstance) getConfig() map[string]any { config["ratelimit.low-speed-watermark"] = 1048576000000 config["ratelimit.block-write-watermark"] = 1048576000000 config["security.enable-sem"] = false - config["tiflash-replicas.constraints"] = []interface{}{ - map[string]interface{}{ + config["tiflash-replicas.constraints"] = []any{ + map[string]any{ "key": "engine", "op": "in", "values": []string{ "tiflash", }, }, - map[string]interface{}{ + map[string]any{ "key": "engine_role", "op": "in", "values": []string{ diff --git a/components/playground/playground.go b/components/playground/playground.go index c027f4c0b5..9238bad914 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -904,6 +904,8 @@ func (p *Playground) bindVersion(comp string, version string) (bindVersion strin return } +//revive:disable:cognitive-complexity +//revive:disable:error-strings func (p *Playground) bootCluster(ctx context.Context, env *environment.Environment, options *BootOptions) error { for _, cfg := range []*instance.Config{ &options.PD, From 707af5974c45a8996bc5f482c624397362a6c3ad Mon Sep 17 00:00:00 2001 From: Wish Date: Wed, 27 Mar 2024 10:13:33 +0800 Subject: [PATCH 3/3] Reserve more for remote cache Signed-off-by: Wish --- components/playground/instance/tiflash_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/playground/instance/tiflash_config.go b/components/playground/instance/tiflash_config.go index 257dda49a3..869a1571fe 100644 --- a/components/playground/instance/tiflash_config.go +++ b/components/playground/instance/tiflash_config.go @@ -60,7 +60,7 @@ func (inst *TiFlashInstance) getConfig() map[string]any { config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey config["storage.remote.cache.dir"] = filepath.Join(inst.Dir, "remote_cache") - config["storage.remote.cache.capacity"] = 1000000000 // 1GB + config["storage.remote.cache.capacity"] = 20000000000 // 20GB config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")} config["flash.disaggregated_mode"] = "tiflash_compute" }