diff --git a/etc/manualTestEnv/.gitignore b/etc/manualTestEnv/.gitignore new file mode 100644 index 0000000000..2f8e58cd1e --- /dev/null +++ b/etc/manualTestEnv/.gitignore @@ -0,0 +1,2 @@ +.vagrant/ +tiup-cluster-*.log diff --git a/etc/manualTestEnv/_shared/Vagrantfile.partial.pubKey.ruby b/etc/manualTestEnv/_shared/Vagrantfile.partial.pubKey.ruby new file mode 100644 index 0000000000..3af2b17570 --- /dev/null +++ b/etc/manualTestEnv/_shared/Vagrantfile.partial.pubKey.ruby @@ -0,0 +1,12 @@ +Vagrant.configure("2") do |config| + ssh_pub_key = File.readlines("#{File.dirname(__FILE__)}/vagrant_key.pub").first.strip + + config.vm.box = "hashicorp/bionic64" + config.vm.provision "shell", privileged: false, inline: <<-SHELL + sudo apt install -y zsh + sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" + sudo chsh -s /usr/bin/zsh vagrant + + echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys + SHELL +end diff --git a/etc/manualTestEnv/_shared/vagrant_key b/etc/manualTestEnv/_shared/vagrant_key new file mode 100644 index 0000000000..7b55495744 --- /dev/null +++ b/etc/manualTestEnv/_shared/vagrant_key @@ -0,0 +1,27 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAQEAxboZzYumqNoVOQ/hKKhIZHxNhf5tmnkLZry8i6Xur4FPLDiRxos/ +xVVDx0ynTPOyQVVaXtNxZnAmbR4HuNBzRvNoklwSXazt5YgWeiKCHtPpKFt3PJeE2cn6FJ +p6F6qFChG0NSPbZxJWWxv4noX0U3PLKgHNIehYK2Fu0E6plhSZazzJEVWapwo9d7aGnAsz +bBCd5TNZ5ogrXn+3bSFcdCbAfWOwYg54a+PzTQlzgt6JmhlEjpFfPhhpBW92pQXxmQ2c17 +iPCbA8G++FiaEwA5teex8k1+HzmHf7YjyhPr+I67EzEiIueJg2+0PYbM1p06S8kVTNDXsf +0eJx4Dr8qQAAA9iFPcpVhT3KVQAAAAdzc2gtcnNhAAABAQDFuhnNi6ao2hU5D+EoqEhkfE +2F/m2aeQtmvLyLpe6vgU8sOJHGiz/FVUPHTKdM87JBVVpe03FmcCZtHge40HNG82iSXBJd +rO3liBZ6IoIe0+koW3c8l4TZyfoUmnoXqoUKEbQ1I9tnElZbG/iehfRTc8sqAc0h6FgrYW +7QTqmWFJlrPMkRVZqnCj13toacCzNsEJ3lM1nmiCtef7dtIVx0JsB9Y7BiDnhr4/NNCXOC +3omaGUSOkV8+GGkFb3alBfGZDZzXuI8JsDwb74WJoTADm157HyTX4fOYd/tiPKE+v4jrsT +MSIi54mDb7Q9hszWnTpLyRVM0Nex/R4nHgOvypAAAAAwEAAQAAAQBtk0+/YDgQ9SKzx8AQ +xwmvXk+cBT76T0BpRAj9HwziiDe3GvZ2YC8MDc+NAEbq11ae7E0zpdv/WAGDkRPYcPShij +0Wdx3aef4wqLVEJCGWMfvRWLcAhjuiclM73cvxl5c42EzU8jUhrsDapuql9zhKky4w7mSe ++OL7z3gYyq8isvcQMe+1eXJqiv27AJJfAir+rLJZO/gDW36hOowhnZxYRlVYPgZ8GwetxD +VdCrgwUgR/2HYmbXYdVxI0PwswGc6rEqs5XXOYRzwvPTvRKdD3J5MxmsvJljT7FMr4kCLT +X1+aWysk1cgAUIdzzwQL8DLE/N9PFFYdZyNBkZMgedl9AAAAgCtP3F8XYFR18gQLPGLDyQ +FFg8+JHN9b/yIg2pymC6SI8qEp+GnuEK9IKhqh/Uw14KEKcs/9sgbZo0K9uTBTDG5F6Qmp +hADVbWXJ/97Xeya6kH2Sa56UKLCQ/uQWBKwLQ0auU/qwxATIZowh31XUXjzVBg6wgUjT7Q ++3Fk1zGYxnAAAAgQD5USIRUNwkI+htv+f1g8QdmrFAGymcGEkXAixKvBTon9cWQb2iyiK+ +2IO8EwFwRdL5kw2foILCnlp/4FevfxHU7wTcoFEp3PItUlcxYqO8vY2VCZ913oNLKBIt9p +uFfG2BZM5szMRNMh0svelu61FePsfN5Z8J0ltPrS8UKB95ywAAAIEAywbyNbjz1AxEjWIX +2Vbk4/MjQyjui8Wi7H0F+LDWyMfPJHzhnbr79Z/lIZmDAo++3EYU9J9s0C+wJ6vXGK+gvC +7e5qGfT/0J0DwBfLbpeTdDELCa/LmfLWVPzZ9Q+9Fq0AjmW9YXFZ/+qT9xfY1v9XfztFRS +xR1iXJ42q6ff5NsAAAAeYnJlZXpld2lzaEBCcmVlemV3aXNoTUJQLmxvY2FsAQIDBAU= +-----END OPENSSH PRIVATE KEY----- diff --git a/etc/manualTestEnv/_shared/vagrant_key.pub b/etc/manualTestEnv/_shared/vagrant_key.pub new file mode 100644 index 0000000000..e9962c03b1 --- /dev/null +++ b/etc/manualTestEnv/_shared/vagrant_key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDFuhnNi6ao2hU5D+EoqEhkfE2F/m2aeQtmvLyLpe6vgU8sOJHGiz/FVUPHTKdM87JBVVpe03FmcCZtHge40HNG82iSXBJdrO3liBZ6IoIe0+koW3c8l4TZyfoUmnoXqoUKEbQ1I9tnElZbG/iehfRTc8sqAc0h6FgrYW7QTqmWFJlrPMkRVZqnCj13toacCzNsEJ3lM1nmiCtef7dtIVx0JsB9Y7BiDnhr4/NNCXOC3omaGUSOkV8+GGkFb3alBfGZDZzXuI8JsDwb74WJoTADm157HyTX4fOYd/tiPKE+v4jrsTMSIi54mDb7Q9hszWnTpLyRVM0Nex/R4nHgOvyp diff --git a/etc/manualTestEnv/multiHost/README.md b/etc/manualTestEnv/multiHost/README.md new file mode 100644 index 0000000000..a747f7db4a --- /dev/null +++ b/etc/manualTestEnv/multiHost/README.md @@ -0,0 +1,36 @@ +# multiHost + +TiDB, PD, TiKV, TiFlash each in different hosts. + +## Usage + +1. Start the box: + + ```bash + vagrant up + ``` + +1. Use [TiUP](https://tiup.io/) to deploy the cluster to the box (only need to do it once): + + ```bash + tiup cluster deploy multiHost v4.0.0 topology.yaml -i ../_shared/vagrant_key -y --user vagrant + ``` + +1. Start the cluster in the box: + + ```bash + tiup cluster start multiHost + ``` + +1. Start TiDB Dashboard server: + + ```bash + bin/tidb-dashboard --pd http://10.0.1.11:2379 + ``` + +## Cleanup + +```bash +tiup cluster destroy multiHost -y +vagrant destroy --force +``` diff --git a/etc/manualTestEnv/multiHost/Vagrantfile b/etc/manualTestEnv/multiHost/Vagrantfile new file mode 100644 index 0000000000..9f4890fdd1 --- /dev/null +++ b/etc/manualTestEnv/multiHost/Vagrantfile @@ -0,0 +1,14 @@ +load "#{File.dirname(__FILE__)}/../_shared/Vagrantfile.partial.pubKey.ruby" + +Vagrant.configure("2") do |config| + config.vm.provider "virtualbox" do |v| + v.memory = 1024 + v.cpus = 1 + end + + (1..4).each do |i| + config.vm.define "node#{i}" do |node| + node.vm.network "private_network", ip: "10.0.1.#{i+10}" + end + end +end diff --git a/etc/manualTestEnv/multiHost/topology.yaml b/etc/manualTestEnv/multiHost/topology.yaml new file mode 100644 index 0000000000..f695125db3 --- /dev/null +++ b/etc/manualTestEnv/multiHost/topology.yaml @@ -0,0 +1,42 @@ +global: + user: tidb + deploy_dir: tidb-deploy + data_dir: tidb-data + +server_configs: + tikv: + server.grpc-concurrency: 1 + raftstore.apply-pool-size: 1 + raftstore.store-pool-size: 1 + readpool.unified.max-thread-count: 1 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + storage.block-cache.capacity: 256MB + raftstore.capacity: 10GB + pd: + replication.enable-placement-rules: true + +pd_servers: + - host: 10.0.1.11 + - host: 10.0.1.12 + - host: 10.0.1.13 + +tikv_servers: + - host: 10.0.1.12 + +tidb_servers: + - host: 10.0.1.11 + - host: 10.0.1.12 + - host: 10.0.1.13 + +tiflash_servers: + - host: 10.0.1.14 + +grafana_servers: + - host: 10.0.1.11 + +monitoring_servers: + - host: 10.0.1.11 + +alertmanager_servers: + - host: 10.0.1.11 diff --git a/etc/manualTestEnv/multiReplica/README.md b/etc/manualTestEnv/multiReplica/README.md new file mode 100644 index 0000000000..b1a027e644 --- /dev/null +++ b/etc/manualTestEnv/multiReplica/README.md @@ -0,0 +1,36 @@ +# multiReplica + +Multiple TiKV nodes in different labels. + +## Usage + +1. Start the box: + + ```bash + vagrant up + ``` + +1. Use [TiUP](https://tiup.io/) to deploy the cluster to the box (only need to do it once): + + ```bash + tiup cluster deploy multiReplica v4.0.0 topology.yaml -i ../_shared/vagrant_key -y --user vagrant + ``` + +1. Start the cluster in the box: + + ```bash + tiup cluster start multiReplica + ``` + +1. Start TiDB Dashboard server: + + ```bash + bin/tidb-dashboard --pd http://10.0.1.20:2379 + ``` + +## Cleanup + +```bash +tiup cluster destroy multiReplica -y +vagrant destroy --force +``` diff --git a/etc/manualTestEnv/multiReplica/Vagrantfile b/etc/manualTestEnv/multiReplica/Vagrantfile new file mode 100644 index 0000000000..82098283c1 --- /dev/null +++ b/etc/manualTestEnv/multiReplica/Vagrantfile @@ -0,0 +1,10 @@ +load "#{File.dirname(__FILE__)}/../_shared/Vagrantfile.partial.pubKey.ruby" + +Vagrant.configure("2") do |config| + config.vm.provider "virtualbox" do |v| + v.memory = 4 * 1024 + v.cpus = 2 + end + + config.vm.network "private_network", ip: "10.0.1.20" +end diff --git a/etc/manualTestEnv/multiReplica/topology.yaml b/etc/manualTestEnv/multiReplica/topology.yaml new file mode 100644 index 0000000000..cad40a20ea --- /dev/null +++ b/etc/manualTestEnv/multiReplica/topology.yaml @@ -0,0 +1,64 @@ +global: + user: tidb + deploy_dir: tidb-deploy + data_dir: tidb-data + +server_configs: + tikv: + server.grpc-concurrency: 1 + raftstore.apply-pool-size: 1 + raftstore.store-pool-size: 1 + readpool.unified.max-thread-count: 1 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + storage.block-cache.capacity: 256MB + raftstore.capacity: 10GB + pd: + replication.location-labels: + - zone + - rack + - host + +pd_servers: + - host: 10.0.1.20 + +tikv_servers: + - host: 10.0.1.20 + port: 20160 + status_port: 20180 + config: + server.labels: { host: tikv1, rack: rack1 } + - host: 10.0.1.20 + port: 20161 + status_port: 20181 + config: + server.labels: { host: tikv1, rack: rack1 } + - host: 10.0.1.20 + port: 20162 + status_port: 20182 + config: + server.labels: { host: tikv2, rack: rack1 } + - host: 10.0.1.20 + port: 20163 + status_port: 20183 + config: + server.labels: { host: tikv2, rack: rack1 } + - host: 10.0.1.20 + port: 20164 + status_port: 20184 + config: + server.labels: { host: tikv3, rack: rack2 } + - host: 10.0.1.20 + port: 20165 + status_port: 20185 + config: + server.labels: { host: tikv3, rack: rack2 } + +tidb_servers: + - host: 10.0.1.20 + +grafana_servers: + - host: 10.0.1.20 + +monitoring_servers: + - host: 10.0.1.20 diff --git a/etc/manualTestEnv/singleHost/README.md b/etc/manualTestEnv/singleHost/README.md new file mode 100644 index 0000000000..27e1e2995c --- /dev/null +++ b/etc/manualTestEnv/singleHost/README.md @@ -0,0 +1,36 @@ +# singleHost + +TiDB, PD, TiKV, TiFlash in the same host. + +## Usage + +1. Start the box: + + ```bash + vagrant up + ``` + +1. Use [TiUP](https://tiup.io/) to deploy the cluster to the box (only need to do it once): + + ```bash + tiup cluster deploy singleHost v4.0.0 topology.yaml -i ../_shared/vagrant_key -y --user vagrant + ``` + +1. Start the cluster in the box: + + ```bash + tiup cluster start singleHost + ``` + +1. Start TiDB Dashboard server: + + ```bash + bin/tidb-dashboard --pd http://10.0.1.2:2379 + ``` + +## Cleanup + +```bash +tiup cluster destroy singleHost -y +vagrant destroy --force +``` diff --git a/etc/manualTestEnv/singleHost/Vagrantfile b/etc/manualTestEnv/singleHost/Vagrantfile new file mode 100644 index 0000000000..77d49ffdea --- /dev/null +++ b/etc/manualTestEnv/singleHost/Vagrantfile @@ -0,0 +1,10 @@ +load "#{File.dirname(__FILE__)}/../_shared/Vagrantfile.partial.pubKey.ruby" + +Vagrant.configure("2") do |config| + config.vm.provider "virtualbox" do |v| + v.memory = 3 * 1024 + v.cpus = 2 + end + + config.vm.network "private_network", ip: "10.0.1.2" +end diff --git a/etc/manualTestEnv/singleHost/topology.yaml b/etc/manualTestEnv/singleHost/topology.yaml new file mode 100644 index 0000000000..bf79c98bff --- /dev/null +++ b/etc/manualTestEnv/singleHost/topology.yaml @@ -0,0 +1,37 @@ +global: + user: tidb + deploy_dir: tidb-deploy + data_dir: tidb-data + +server_configs: + tikv: + server.grpc-concurrency: 1 + raftstore.apply-pool-size: 1 + raftstore.store-pool-size: 1 + readpool.unified.max-thread-count: 1 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + storage.block-cache.capacity: 256MB + pd: + replication.enable-placement-rules: true + +pd_servers: + - host: 10.0.1.2 + +tikv_servers: + - host: 10.0.1.2 + +tidb_servers: + - host: 10.0.1.2 + +tiflash_servers: + - host: 10.0.1.2 + +grafana_servers: + - host: 10.0.1.2 + +monitoring_servers: + - host: 10.0.1.2 + +alertmanager_servers: + - host: 10.0.1.2 diff --git a/etc/manualTestEnv/singleHostMultiDisk/.gitignore b/etc/manualTestEnv/singleHostMultiDisk/.gitignore new file mode 100644 index 0000000000..8fce603003 --- /dev/null +++ b/etc/manualTestEnv/singleHostMultiDisk/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/etc/manualTestEnv/singleHostMultiDisk/README.md b/etc/manualTestEnv/singleHostMultiDisk/README.md new file mode 100644 index 0000000000..0a591b450b --- /dev/null +++ b/etc/manualTestEnv/singleHostMultiDisk/README.md @@ -0,0 +1,36 @@ +# singleHostMultiDisk + +All instances in a single host, but on different disks. + +## Usage + +1. Start the box: + + ```bash + vagrant up + ``` + +1. Use [TiUP](https://tiup.io/) to deploy the cluster to the box (only need to do it once): + + ```bash + tiup cluster deploy singleHostMultiDisk v4.0.0 topology.yaml -i ../_shared/vagrant_key -y --user vagrant + ``` + +1. Start the cluster in the box: + + ```bash + tiup cluster start singleHostMultiDisk + ``` + +1. Start TiDB Dashboard server: + + ```bash + bin/tidb-dashboard --pd http://10.0.1.3:2379 + ``` + +## Cleanup + +```bash +tiup cluster destroy singleHostMultiDisk -y +vagrant destroy --force +``` diff --git a/etc/manualTestEnv/singleHostMultiDisk/Vagrantfile b/etc/manualTestEnv/singleHostMultiDisk/Vagrantfile new file mode 100644 index 0000000000..971db113cc --- /dev/null +++ b/etc/manualTestEnv/singleHostMultiDisk/Vagrantfile @@ -0,0 +1,10 @@ +load "#{File.dirname(__FILE__)}/../_shared/Vagrantfile.partial.pubKey.ruby" + +Vagrant.configure("2") do |config| + config.vm.provider "virtualbox" do |v| + v.memory = 3 * 1024 + v.cpus = 2 + end + + config.vm.network "private_network", ip: "10.0.1.3" +end diff --git a/etc/manualTestEnv/singleHostMultiDisk/topology.yaml b/etc/manualTestEnv/singleHostMultiDisk/topology.yaml new file mode 100644 index 0000000000..516de7cb5c --- /dev/null +++ b/etc/manualTestEnv/singleHostMultiDisk/topology.yaml @@ -0,0 +1,39 @@ +global: + user: vagrant + deploy_dir: tidb-deploy + data_dir: tidb-data + +server_configs: + tikv: + server.grpc-concurrency: 1 + raftstore.apply-pool-size: 1 + raftstore.store-pool-size: 1 + readpool.unified.max-thread-count: 1 + readpool.storage.use-unified-pool: false + readpool.coprocessor.use-unified-pool: true + storage.block-cache.capacity: 256MB + pd: + replication.enable-placement-rules: true + +pd_servers: + - host: 10.0.1.3 + +tikv_servers: + - host: 10.0.1.3 + +tidb_servers: + - host: 10.0.1.3 + deploy_dir: /vagrant/data/tidb + log_dir: /vagrant/data/tidb/log + +tiflash_servers: + - host: 10.0.1.3 + +grafana_servers: + - host: 10.0.1.3 + +monitoring_servers: + - host: 10.0.1.3 + +alertmanager_servers: + - host: 10.0.1.3 diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 4c41c9e5ad..3914bb2fe0 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -107,6 +107,7 @@ func (s *Service) Start(ctx context.Context) error { s.newPDDataProvider, dbstore.NewDBStore, pd.NewEtcdClient, + pd.NewPDClient, config.NewDynamicConfigManager, tidb.NewForwarderConfig, tidb.NewForwarder, diff --git a/pkg/apiserver/clusterinfo/fetch.go b/pkg/apiserver/clusterinfo/fetch.go deleted file mode 100644 index 182660fc52..0000000000 --- a/pkg/apiserver/clusterinfo/fetch.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package clusterinfo - -import ( - "context" - - "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/clusterinfo" -) - -type ClusterInfo struct { - TiDB struct { - Nodes []clusterinfo.TiDBInfo `json:"nodes"` - Err *string `json:"err"` - } `json:"tidb"` - - TiKV struct { - Nodes []clusterinfo.TiKVInfo `json:"nodes"` - Err *string `json:"err"` - } `json:"tikv"` - - PD struct { - Nodes []clusterinfo.PDInfo `json:"nodes"` - Err *string `json:"err"` - } `json:"pd"` - - TiFlash struct { - Nodes []clusterinfo.TiFlashInfo `json:"nodes"` - Err *string `json:"err"` - } `json:"tiflash"` - - Grafana *clusterinfo.GrafanaInfo `json:"grafana"` - AlertManager *clusterinfo.AlertManagerInfo `json:"alert_manager"` -} - -// fetches etcd, and parses the ns below: -// * /topology/grafana -// * /topology/alertmanager -// * /topology/tidb -func fillTopologyUnderEtcd(ctx context.Context, service *Service, fillTargetInfo *ClusterInfo) { - tidb, grafana, alertManager, err := clusterinfo.GetTopologyUnderEtcd(ctx, service.etcdClient) - if err != nil { - // Note: GetTopology return error only when fetch etcd failed. - // So it's ok to fill all of them err - errString := err.Error() - fillTargetInfo.TiDB.Err = &errString - return - } - if grafana != nil { - fillTargetInfo.Grafana = grafana - } - if alertManager != nil { - fillTargetInfo.AlertManager = alertManager - } - //if len(tidb) == 0 { - // tidb, err = clusterinfo.GetTiDBTopologyFromOld(ctx, service.etcdClient) - // if err != nil { - // errString := err.Error() - // fillTargetInfo.TiDB.Err = &errString - // return - // } - //} - fillTargetInfo.TiDB.Nodes = tidb -} - -func fillPDTopology(ctx context.Context, service *Service, fillTargetInfo *ClusterInfo) { - pdPeers, err := clusterinfo.GetPDTopology(service.config.PDEndPoint, service.httpClient) - if err != nil { - errString := err.Error() - fillTargetInfo.PD.Err = &errString - return - } - fillTargetInfo.PD.Nodes = pdPeers -} - -func fillStoreTopology(ctx context.Context, service *Service, fillTargetInfo *ClusterInfo) { - kv, tiflashes, err := clusterinfo.GetStoreTopology(service.config.PDEndPoint, service.httpClient) - if err != nil { - errString := err.Error() - fillTargetInfo.TiKV.Err = &errString - fillTargetInfo.TiFlash.Err = &errString - return - } - fillTargetInfo.TiKV.Nodes = kv - fillTargetInfo.TiFlash.Nodes = tiflashes -} diff --git a/pkg/apiserver/clusterinfo/service.go b/pkg/apiserver/clusterinfo/service.go index 290380936b..adf597846f 100644 --- a/pkg/apiserver/clusterinfo/service.go +++ b/pkg/apiserver/clusterinfo/service.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net/http" + "sort" "sync" "time" @@ -29,30 +30,30 @@ import ( "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/utils" - "github.com/pingcap-incubator/tidb-dashboard/pkg/config" + "github.com/pingcap-incubator/tidb-dashboard/pkg/pd" "github.com/pingcap-incubator/tidb-dashboard/pkg/tidb" - "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/clusterinfo" + "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/topology" ) type Service struct { - ctx context.Context + lifecycleCtx context.Context - config *config.Config + pdClient *pd.Client etcdClient *clientv3.Client httpClient *http.Client tidbForwarder *tidb.Forwarder } -func NewService(lc fx.Lifecycle, config *config.Config, etcdClient *clientv3.Client, httpClient *http.Client, tidbForwarder *tidb.Forwarder) *Service { +func NewService(lc fx.Lifecycle, pdClient *pd.Client, etcdClient *clientv3.Client, httpClient *http.Client, tidbForwarder *tidb.Forwarder) *Service { s := &Service{ - config: config, + pdClient: pdClient, etcdClient: etcdClient, httpClient: httpClient, tidbForwarder: tidbForwarder, } lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { - s.ctx = ctx + s.lifecycleCtx = ctx return nil }, }) @@ -62,14 +63,18 @@ func NewService(lc fx.Lifecycle, config *config.Config, etcdClient *clientv3.Cli func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) { endpoint := r.Group("/topology") endpoint.Use(auth.MWAuthRequired()) - endpoint.GET("/all", s.topologyHandler) - endpoint.DELETE("/tidb/:address", s.deleteTiDBTopologyHandler) - endpoint.GET("/alertmanager/:address/count", s.topologyGetAlertCount) + endpoint.GET("/tidb", s.getTiDBTopology) + endpoint.DELETE("/tidb/:address", s.deleteTiDBTopology) + endpoint.GET("/store", s.getStoreTopology) + endpoint.GET("/pd", s.getPDTopology) + endpoint.GET("/alertmanager", s.getAlertManagerTopology) + endpoint.GET("/alertmanager/:address/count", s.getAlertManagerCounts) + endpoint.GET("/grafana", s.getGrafanaTopology) endpoint = r.Group("/host") endpoint.Use(auth.MWAuthRequired()) endpoint.Use(utils.MWConnectTiDB(s.tidbForwarder)) - endpoint.GET("/all", s.hostHandler) + endpoint.GET("/all", s.getHostsInfo) } // @Summary Delete etcd's tidb key. @@ -80,12 +85,12 @@ func Register(r *gin.RouterGroup, auth *user.AuthService, s *Service) { // @Failure 401 {object} utils.APIError "Unauthorized failure" // @Security JwtAuth // @Router /topology/tidb/{address} [delete] -func (s *Service) deleteTiDBTopologyHandler(c *gin.Context) { +func (s *Service) deleteTiDBTopology(c *gin.Context) { address := c.Param("address") errorChannel := make(chan error, 2) ttlKey := fmt.Sprintf("/topology/tidb/%v/ttl", address) nonTTLKey := fmt.Sprintf("/topology/tidb/%v/info", address) - ctx, cancel := context.WithTimeout(s.ctx, time.Second*5) + ctx, cancel := context.WithTimeout(s.lifecycleCtx, time.Second*5) defer cancel() var wg sync.WaitGroup @@ -113,50 +118,111 @@ func (s *Service) deleteTiDBTopologyHandler(c *gin.Context) { c.JSON(http.StatusOK, nil) } -// @Summary Get all Dashboard topology and liveness. -// @Description Get information about the dashboard topology. +// @ID getTiDBTopology +// @Summary Get TiDB instances +// @Description Get TiDB instances topology // @Produce json -// @Success 200 {object} clusterinfo.ClusterInfo -// @Router /topology/all [get] +// @Success 200 {array} topology.TiDBInfo +// @Router /topology/tidb [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) topologyHandler(c *gin.Context) { - var returnObject ClusterInfo +func (s *Service) getTiDBTopology(c *gin.Context) { + instances, err := topology.FetchTiDBTopology(s.lifecycleCtx, s.etcdClient) + if err != nil { + _ = c.Error(err) + return + } + c.JSON(http.StatusOK, instances) +} - ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second) - defer cancel() +type StoreTopologyResponse struct { + TiKV []topology.StoreInfo `json:"tikv"` + TiFlash []topology.StoreInfo `json:"tiflash"` +} - fetchers := []func(ctx context.Context, service *Service, info *ClusterInfo){ - fillTopologyUnderEtcd, - fillStoreTopology, - fillPDTopology, +// @ID getStoreTopology +// @Summary Get TiKV / TiFlash instances +// @Description Get TiKV / TiFlash instances topology +// @Produce json +// @Success 200 {object} StoreTopologyResponse +// @Router /topology/store [get] +// @Security JwtAuth +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) getStoreTopology(c *gin.Context) { + tikvInstances, tiFlashInstances, err := topology.FetchStoreTopology(s.pdClient) + if err != nil { + _ = c.Error(err) + return } + c.JSON(http.StatusOK, StoreTopologyResponse{ + TiKV: tikvInstances, + TiFlash: tiFlashInstances, + }) +} - var wg sync.WaitGroup - for _, fetcher := range fetchers { - wg.Add(1) - currentFetcher := fetcher - go func() { - defer wg.Done() - currentFetcher(ctx, s, &returnObject) - }() +// @ID getPDTopology +// @Summary Get PD instances +// @Description Get PD instances topology +// @Produce json +// @Success 200 {array} topology.PDInfo +// @Router /topology/pd [get] +// @Security JwtAuth +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) getPDTopology(c *gin.Context) { + instances, err := topology.FetchPDTopology(s.pdClient) + if err != nil { + _ = c.Error(err) + return } - wg.Wait() + c.JSON(http.StatusOK, instances) +} + +// @ID getAlertManagerTopology +// @Summary Get AlertManager instance +// @Description Get AlertManager instance topology +// @Produce json +// @Success 200 {object} topology.AlertManagerInfo +// @Router /topology/alertmanager [get] +// @Security JwtAuth +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) getAlertManagerTopology(c *gin.Context) { + instance, err := topology.FetchAlertManagerTopology(s.lifecycleCtx, s.etcdClient) + if err != nil { + _ = c.Error(err) + return + } + c.JSON(http.StatusOK, instance) +} - c.JSON(http.StatusOK, returnObject) +// @ID getGrafanaTopology +// @Summary Get Grafana instance +// @Description Get Grafana instance topology +// @Produce json +// @Success 200 {object} topology.GrafanaInfo +// @Router /topology/grafana [get] +// @Security JwtAuth +// @Failure 401 {object} utils.APIError "Unauthorized failure" +func (s *Service) getGrafanaTopology(c *gin.Context) { + instance, err := topology.FetchGrafanaTopology(s.lifecycleCtx, s.etcdClient) + if err != nil { + _ = c.Error(err) + return + } + c.JSON(http.StatusOK, instance) } -// @Summary Get the count of alert -// @Description Get alert number of the alert manager. +// @ID getAlertManagerCounts +// @Summary Get alert count +// @Description Get alert count from alert manager // @Produce json // @Success 200 {object} int // @Param address path string true "ip:port" // @Router /topology/alertmanager/{address}/count [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) topologyGetAlertCount(c *gin.Context) { +func (s *Service) getAlertManagerCounts(c *gin.Context) { address := c.Param("address") - cnt, err := clusterinfo.GetAlertCountByAddress(address, s.httpClient) + cnt, err := fetchAlertManagerCounts(s.lifecycleCtx, address, s.httpClient) if err != nil { _ = c.Error(err) return @@ -164,6 +230,7 @@ func (s *Service) topologyGetAlertCount(c *gin.Context) { c.JSON(http.StatusOK, cnt) } +// @ID getHostsInfo // @Summary Get all host information in the cluster // @Description Get information about host in the cluster // @Produce json @@ -171,80 +238,72 @@ func (s *Service) topologyGetAlertCount(c *gin.Context) { // @Router /host/all [get] // @Security JwtAuth // @Failure 401 {object} utils.APIError "Unauthorized failure" -func (s *Service) hostHandler(c *gin.Context) { +func (s *Service) getHostsInfo(c *gin.Context) { db := utils.GetTiDBConnection(c) - ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second) - defer cancel() - - var wg sync.WaitGroup - var clusterInfo ClusterInfo - fetchers := []func(ctx context.Context, service *Service, info *ClusterInfo){ - fillTopologyUnderEtcd, - fillStoreTopology, - fillPDTopology, - } - for _, fetcher := range fetchers { - wg.Add(1) - currentFetcher := fetcher - go func() { - defer wg.Done() - currentFetcher(ctx, s, &clusterInfo) - }() + allHostsMap, err := s.fetchAllInstanceHostsMap() + if err != nil { + _ = c.Error(err) + return } - - var infos []HostInfo - var err error - wg.Add(1) - go func() { - defer wg.Done() - infos, err = GetAllHostInfo(db) - }() - wg.Wait() - + hostsInfo, err := GetAllHostInfo(db) if err != nil { _ = c.Error(err) return } - allHosts := loadUnavailableHosts(clusterInfo) + hostsInfoMap := make(map[string]HostInfo) + for _, hi := range hostsInfo { + hostsInfoMap[hi.IP] = hi + } -OuterLoop: - for _, host := range allHosts { - for _, info := range infos { - if host == info.IP { - continue OuterLoop - } + hiList := make([]HostInfo, 0, len(hostsInfo)) + for hostIP := range allHostsMap { + if hi, ok := hostsInfoMap[hostIP]; ok { + hiList = append(hiList, hi) + } else { + hiList = append(hiList, HostInfo{ + IP: hostIP, + Unavailable: true, + }) } - infos = append(infos, HostInfo{ - IP: host, - Unavailable: true, - }) } - c.JSON(http.StatusOK, infos) + sort.Slice(hiList, func(i, j int) bool { + return hiList[i].IP > hiList[j].IP + }) + + c.JSON(http.StatusOK, hiList) } -func loadUnavailableHosts(info ClusterInfo) []string { - var allNodes []string - for _, node := range info.TiDB.Nodes { - if node.Status != clusterinfo.ComponentStatusUp { - allNodes = append(allNodes, node.IP) - } +func (s *Service) fetchAllInstanceHostsMap() (map[string]struct{}, error) { + allHosts := make(map[string]struct{}) + pdInfo, err := topology.FetchPDTopology(s.pdClient) + if err != nil { + return nil, err } - for _, node := range info.TiKV.Nodes { - switch node.Status { - case clusterinfo.ComponentStatusUp, - clusterinfo.ComponentStatusOffline, - clusterinfo.ComponentStatusTombstone: - default: - allNodes = append(allNodes, node.IP) - } + for _, i := range pdInfo { + allHosts[i.IP] = struct{}{} } - for _, node := range info.PD.Nodes { - if node.Status != clusterinfo.ComponentStatusUp { - allNodes = append(allNodes, node.IP) - } + + tikvInfo, tiFlashInfo, err := topology.FetchStoreTopology(s.pdClient) + if err != nil { + return nil, err + } + for _, i := range tikvInfo { + allHosts[i.IP] = struct{}{} } - return allNodes + for _, i := range tiFlashInfo { + allHosts[i.IP] = struct{}{} + } + + tidbInfo, err := topology.FetchTiDBTopology(s.lifecycleCtx, s.etcdClient) + if err != nil { + return nil, err + } + for _, i := range tidbInfo { + allHosts[i.IP] = struct{}{} + } + + return allHosts, nil } diff --git a/pkg/apiserver/clusterinfo/topology.go b/pkg/apiserver/clusterinfo/topology.go new file mode 100644 index 0000000000..446af76e24 --- /dev/null +++ b/pkg/apiserver/clusterinfo/topology.go @@ -0,0 +1,40 @@ +package clusterinfo + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +func fetchAlertManagerCounts(ctx context.Context, alertManagerAddr string, httpClient *http.Client) (int, error) { + uri := fmt.Sprintf("http://%s/api/v2/alerts", alertManagerAddr) + req, err := http.NewRequestWithContext(ctx, "GET", uri, nil) + if err != nil { + return 0, err + } + + resp, err := httpClient.Do(req) + if err != nil { + return 0, err + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("alert manager API returns non success status code") + } + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, err + } + + var alerts []struct{} + err = json.Unmarshal(data, &alerts) + if err != nil { + return 0, err + } + + return len(alerts), nil +} diff --git a/pkg/apiserver/metrics/metrics.go b/pkg/apiserver/metrics/metrics.go index e58dfb70c4..be99433ead 100644 --- a/pkg/apiserver/metrics/metrics.go +++ b/pkg/apiserver/metrics/metrics.go @@ -2,7 +2,6 @@ package metrics import ( "context" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -16,25 +15,25 @@ import ( "go.uber.org/fx" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver/user" - "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/clusterinfo" + "github.com/pingcap-incubator/tidb-dashboard/pkg/utils/topology" ) var ( - ErrNS = errorx.NewNamespace("error.api.metrics") - ErrEtcdAccessFailed = ErrNS.NewType("etcd_access_failed") - ErrPrometheusNotFound = ErrNS.NewType("prometheus_not_found") - ErrPrometheusRegistryInvalid = ErrNS.NewType("prometheus_registry_invalid") - ErrPrometheusQueryFailed = ErrNS.NewType("prometheus_query_failed") + ErrNS = errorx.NewNamespace("error.api.metrics") + ErrPrometheusNotFound = ErrNS.NewType("prometheus_not_found") + ErrPrometheusQueryFailed = ErrNS.NewType("prometheus_query_failed") ) type Service struct { ctx context.Context + httpClient *http.Client etcdClient *clientv3.Client } -func NewService(lc fx.Lifecycle, etcdClient *clientv3.Client) *Service { +func NewService(lc fx.Lifecycle, httpClient *http.Client, etcdClient *clientv3.Client) *Service { s := &Service{ + httpClient: httpClient, etcdClient: etcdClient, } @@ -81,22 +80,15 @@ func (s *Service) queryHandler(c *gin.Context) { return } - ctx, cancel := context.WithTimeout(s.ctx, 2*time.Second) - defer cancel() - resp, err := s.etcdClient.Get(ctx, "/topology/prometheus", clientv3.WithPrefix()) + pi, err := topology.FetchPrometheusTopology(s.ctx, s.etcdClient) if err != nil { - _ = c.Error(ErrEtcdAccessFailed.NewWithNoMessage()) + _ = c.Error(err) return } - if resp.Count == 0 { + if pi == nil { _ = c.Error(ErrPrometheusNotFound.NewWithNoMessage()) return } - info := clusterinfo.PrometheusInfo{} - if err = json.Unmarshal(resp.Kvs[0].Value, &info); err != nil { - _ = c.Error(ErrPrometheusRegistryInvalid.NewWithNoMessage()) - return - } params := url.Values{} params.Add("query", req.Query) @@ -104,23 +96,32 @@ func (s *Service) queryHandler(c *gin.Context) { params.Add("end", strconv.Itoa(req.EndTimeSec)) params.Add("step", strconv.Itoa(req.StepSec)) - client := http.Client{ - Timeout: 10 * time.Second, + uri := fmt.Sprintf("http://%s:%d/api/v1/query_range?%s", pi.IP, pi.Port, params.Encode()) + promReq, err := http.NewRequestWithContext(s.ctx, http.MethodGet, uri, nil) + if err != nil { + _ = c.Error(ErrPrometheusQueryFailed.Wrap(err, "failed to build Prometheus request")) + return } - promResp, err := client.Get(fmt.Sprintf("http://%s:%d/api/v1/query_range?%s", info.IP, info.Port, params.Encode())) + + newHTTPClient := *s.httpClient + newHTTPClient.Timeout = 10 * time.Second + promResp, err := newHTTPClient.Do(promReq) if err != nil { - _ = c.Error(ErrPrometheusQueryFailed.Wrap(err, "failed to query Prometheus")) + _ = c.Error(ErrPrometheusQueryFailed.Wrap(err, "failed to send requests to Prometheus")) return } + defer promResp.Body.Close() if promResp.StatusCode != http.StatusOK { _ = c.Error(ErrPrometheusQueryFailed.New("failed to query Prometheus")) return } + body, err := ioutil.ReadAll(promResp.Body) if err != nil { - _ = c.Error(ErrPrometheusQueryFailed.Wrap(err, "failed to query Prometheus")) + _ = c.Error(ErrPrometheusQueryFailed.Wrap(err, "failed to read Prometheus query result")) return } + c.Data(promResp.StatusCode, promResp.Header.Get("content-type"), body) } diff --git a/pkg/apiserver/profiling/profile.go b/pkg/apiserver/profiling/profile.go index 15a4992c38..db73d88f60 100644 --- a/pkg/apiserver/profiling/profile.go +++ b/pkg/apiserver/profiling/profile.go @@ -125,11 +125,10 @@ func (f *fetcher) Fetch(src string, duration, timeout time.Duration) (*profile.P } func (f *fetcher) getProfile(target *model.RequestTargetNode, source string) (*profile.Profile, error) { - req, err := http.NewRequest(http.MethodGet, source, nil) + req, err := http.NewRequestWithContext(f.ctx, http.MethodGet, source, nil) if err != nil { return nil, fmt.Errorf("failed to create a new request %s: %v", source, err) } - req = req.WithContext(f.ctx) if target.Kind == model.NodeKindPD { // forbidden PD follower proxy req.Header.Add("PD-Allow-follower-handle", "true") @@ -161,19 +160,18 @@ func profileAndWriteSVG(ctx context.Context, target *model.RequestTargetNode, fi } func fetchTiKVFlameGraphSVG(ctx context.Context, httpClient *http.Client, target *model.RequestTargetNode, fileNameWithoutExt string, profileDurationSecs uint, schema string) (string, error) { - url := fmt.Sprintf("%s://%s:%d/debug/pprof/profile?seconds=%d", schema, target.IP, target.Port, profileDurationSecs) - req, err := http.NewRequest(http.MethodGet, url, nil) + uri := fmt.Sprintf("%s://%s:%d/debug/pprof/profile?seconds=%d", schema, target.IP, target.Port, profileDurationSecs) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) if err != nil { - return "", fmt.Errorf("failed to create a new request %s: %v", url, err) + return "", fmt.Errorf("failed to create a new request %s: %v", uri, err) } - req = req.WithContext(ctx) resp, err := httpClient.Do(req) if err != nil { - return "", fmt.Errorf("request %s failed: %v", url, err) + return "", fmt.Errorf("request %s failed: %v", uri, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("response of request %s is not ok: %s", url, resp.Status) + return "", fmt.Errorf("response of request %s is not ok: %s", uri, resp.Status) } svgFilePath, err := writePprofRsSVG(resp.Body, fileNameWithoutExt) if err != nil { diff --git a/pkg/keyvisual/decorator/tidb_requests.go b/pkg/keyvisual/decorator/tidb_requests.go index c465d8bbe6..bc3d260c37 100644 --- a/pkg/keyvisual/decorator/tidb_requests.go +++ b/pkg/keyvisual/decorator/tidb_requests.go @@ -69,7 +69,7 @@ func (s *tidbLabelStrategy) updateMap(ctx context.Context) { hostname, port := s.forwarder.GetStatusConnProps() tidbEndpoint := fmt.Sprintf("%s://%s:%d", reqScheme, hostname, port) if err := request(tidbEndpoint, "schema", &dbInfos, s.HTTPClient); err != nil { - log.Error("fail to send schema request to tidb", zap.String("endpoint", tidbEndpoint), zap.Error(err)) + log.Error("fail to send schema request to TiDB", zap.String("endpoint", tidbEndpoint), zap.Error(err)) return } @@ -81,7 +81,7 @@ func (s *tidbLabelStrategy) updateMap(ctx context.Context) { continue } if err := request(tidbEndpoint, fmt.Sprintf("schema/%s", db.Name.O), &tableInfos, s.HTTPClient); err != nil { - log.Error("fail to send schema request to tidb", zap.String("endpoint", tidbEndpoint), zap.Error(err)) + log.Error("fail to send schema request to TiDB", zap.String("endpoint", tidbEndpoint), zap.Error(err)) updateSuccess = false continue } @@ -117,19 +117,20 @@ func (s *tidbLabelStrategy) updateMap(ctx context.Context) { } } -func request(endpoint string, uri string, v interface{}, httpClient *http.Client) error { - url := fmt.Sprintf("%s/%s", endpoint, uri) - resp, err := httpClient.Get(url) //nolint:gosec - if err == nil { - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - err = ErrTiDBHTTPRequestFailed.New("http status code: %d", resp.StatusCode) - } - } +func request(endpoint string, path string, v interface{}, httpClient *http.Client) error { + uri := fmt.Sprintf("%s/%s", endpoint, path) + + // FIXME: Better to assign a context + resp, err := httpClient.Get(uri) //nolint:gosec if err != nil { - log.Warn("request failed", zap.String("url", url), zap.Error(err)) - return err + return ErrTiDBHTTPRequestFailed.Wrap(err, "TiDB HTTP API request failed") } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ErrTiDBHTTPRequestFailed.New("TiDB HTTP API returns non success status code") + } + decoder := json.NewDecoder(resp.Body) return decoder.Decode(v) } diff --git a/pkg/pd/pd.go b/pkg/pd/pd.go new file mode 100644 index 0000000000..0abee5dacb --- /dev/null +++ b/pkg/pd/pd.go @@ -0,0 +1,18 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import "github.com/joomcode/errorx" + +var ErrNS = errorx.NewNamespace("error.pd") diff --git a/pkg/pd/pd_client.go b/pkg/pd/pd_client.go new file mode 100644 index 0000000000..affbd27fdc --- /dev/null +++ b/pkg/pd/pd_client.go @@ -0,0 +1,74 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "context" + "io/ioutil" + "net/http" + + "go.uber.org/fx" + + "github.com/pingcap-incubator/tidb-dashboard/pkg/config" +) + +var ( + ErrPDClientRequestFailed = ErrNS.NewType("client_request_failed") +) + +type Client struct { + endpointAddr string + httpClient *http.Client + lifecycleCtx context.Context +} + +func NewPDClient(lc fx.Lifecycle, httpClient *http.Client, config *config.Config) *Client { + client := &Client{ + httpClient: httpClient, + endpointAddr: config.PDEndPoint, + } + + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + client.lifecycleCtx = ctx + return nil + }, + }) + + return client +} + +func (pd *Client) SendRequest(path string) ([]byte, error) { + uri := pd.endpointAddr + path + req, err := http.NewRequestWithContext(pd.lifecycleCtx, "GET", uri, nil) + if err != nil { + return nil, ErrPDClientRequestFailed.Wrap(err, "failed to build request for PD API %s", path) + } + + resp, err := pd.httpClient.Do(req) + if err != nil { + return nil, ErrPDClientRequestFailed.Wrap(err, "failed to send request to PD API %s", path) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, ErrPDClientRequestFailed.New("received non success status code %d from PD API %s", resp.StatusCode, path) + } + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, ErrPDClientRequestFailed.Wrap(err, "failed to read response from PD API %s", path) + } + + return data, nil +} diff --git a/pkg/utils/clusterinfo/fetcher.go b/pkg/utils/clusterinfo/fetcher.go deleted file mode 100644 index 0315837de6..0000000000 --- a/pkg/utils/clusterinfo/fetcher.go +++ /dev/null @@ -1,518 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package clusterinfo - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/pingcap/log" - "go.etcd.io/etcd/clientv3" - "go.uber.org/zap" -) - -// GetTopology return error only when fetch etcd failed. -func GetTopologyUnderEtcd(ctx context.Context, etcdClient *clientv3.Client) (tidbNodes []TiDBInfo, grafanaNode *GrafanaInfo, alertManagerNode *AlertManagerInfo, e error) { - resp, err := etcdClient.Get(ctx, "/topology", clientv3.WithPrefix()) - if err != nil { - return nil, nil, nil, err - } - tidbTTLMap := map[string][]byte{} - tidbEntryMap := map[string]*TiDBInfo{} - for _, kv := range resp.Kvs { - keyParts := strings.Split(string(kv.Key), "/")[1:] - if len(keyParts) < 2 { - continue - } - // There can be four kinds of keys: - // * /topology/grafana: stores grafana topology info. - // * /topology/alertmanager: stores alertmanager topology info. - // * /topology/tidb/ip:port/info: stores tidb topology info. - // * /topology/tidb/ip:port/ttl : stores tidb last update ttl time. - switch keyParts[1] { - case "grafana": - r := GrafanaInfo{} - if err = json.Unmarshal(kv.Value, &r); err != nil { - continue - } - grafanaNode = &r - case "alertmanager": - r := AlertManagerInfo{} - if err = json.Unmarshal(kv.Value, &r); err != nil { - continue - } - alertManagerNode = &r - case "tidb": - // the key should be like /topology/tidb/ip:port/info or /ttl - if len(keyParts) != 4 { - continue - } - address, fieldType := keyParts[2], keyParts[3] - fillDBMap(address, fieldType, kv.Value, tidbEntryMap, tidbTTLMap) - } - } - - tidbNodes = genDBList(tidbEntryMap, tidbTTLMap) - - return tidbNodes, grafanaNode, alertManagerNode, nil -} - -// address should be like "ip:port" -// fieldType should be "ttl" or "info" -// value is field value. -func fillDBMap(address, fieldType string, value []byte, infoMap map[string]*TiDBInfo, ttlMap map[string][]byte) { - if fieldType == "ttl" { - ttlMap[address] = value - } else if fieldType == "info" { - ds := struct { - Version string `json:"version"` - GitHash string `json:"git_hash"` - StatusPort uint `json:"status_port"` - DeployPath string `json:"deploy_path"` - StartTimestamp int64 `json:"start_timestamp"` - }{} - - //var currentInfo TiDB - err := json.Unmarshal(value, &ds) - if err != nil { - return - } - host, port, err := parseHostAndPortFromAddress(address) - if err != nil { - return - } - - infoMap[address] = &TiDBInfo{ - GitHash: ds.GitHash, - Version: ds.Version, - IP: host, - Port: port, - DeployPath: ds.DeployPath, - Status: ComponentStatusUnreachable, - StatusPort: ds.StatusPort, - StartTimestamp: ds.StartTimestamp, - } - } -} - -func genDBList(infoMap map[string]*TiDBInfo, ttlMap map[string][]byte) []TiDBInfo { - nodes := make([]TiDBInfo, 0) - - // Note: it means this TiDB has non-ttl key, but ttl-key not exists. - for address, info := range infoMap { - if ttlFreshUnixNanoSec, ok := ttlMap[address]; ok { - unixNano, err := strconv.ParseInt(string(ttlFreshUnixNanoSec), 10, 64) - if err != nil { - info.Status = ComponentStatusUnreachable - } else { - ttlFreshTime := time.Unix(0, unixNano) - if time.Since(ttlFreshTime) > time.Second*45 { - info.Status = ComponentStatusUnreachable - } else { - info.Status = ComponentStatusUp - } - } - } else { - info.Status = ComponentStatusUnreachable - } - nodes = append(nodes, *info) - } - - return nodes -} - -type store struct { - Address string `json:"address"` - ID int `json:"id"` - Labels []struct { - Key string `json:"key"` - Value string `json:"value"` - } - StateName string `json:"state_name"` - Version string `json:"version"` - StatusAddress string `json:"status_address"` - GitHash string `json:"git_hash"` - DeployPath string `json:"deploy_path"` - StartTimestamp int64 `json:"start_timestamp"` -} - -func getAllStoreNodes(endpoint string, httpClient *http.Client) ([]store, error) { - resp, err := httpClient.Get(endpoint + "/pd/api/v1/stores") - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("fetch stores got wrong status code") - } - defer resp.Body.Close() - storeResp := struct { - Count int `json:"count"` - Stores []struct { - Store store - } `json:"stores"` - }{} - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - err = json.Unmarshal(data, &storeResp) - if err != nil { - return nil, err - } - ret := make([]store, storeResp.Count) - for i, s := range storeResp.Stores { - ret[i] = s.Store - } - return ret, nil -} - -type tikvStore struct { - store -} - -func getAllTiKVNodes(stores []store) []tikvStore { - tikvs := make([]tikvStore, len(stores)) - for i := range stores { - isTiFlash := false - for _, label := range stores[i].Labels { - if label.Key == "engine" && label.Value == "tiflash" { - isTiFlash = true - } - } - if !isTiFlash { - tikvs = append(tikvs, tikvStore{stores[i]}) - } - } - return tikvs -} - -func getTiKVTopology(stores []tikvStore) ([]TiKVInfo, error) { - nodes := make([]TiKVInfo, 0) - for _, v := range stores { - // parse ip and port - host, port, err := parseHostAndPortFromAddress(v.Address) - if err != nil { - continue - } - _, statusPort, err := parseHostAndPortFromAddress(v.StatusAddress) - if err != nil { - continue - } - // In current TiKV, it's version may not start with 'v', - // so we may need to add a prefix 'v' for it. - version := strings.Trim(v.Version, "\n ") - if !strings.HasPrefix(version, "v") { - version = "v" + version - } - node := TiKVInfo{ - Version: version, - IP: host, - Port: port, - GitHash: v.GitHash, - DeployPath: v.DeployPath, - Status: storeStateToStatus(v.StateName), - StatusPort: statusPort, - Labels: map[string]string{}, - StartTimestamp: v.StartTimestamp, - } - for _, v := range v.Labels { - node.Labels[v.Key] = node.Labels[v.Value] - } - nodes = append(nodes, node) - } - - return nodes, nil -} - -type tiflashStore struct { - store -} - -func getAllTiFlashNodes(stores []store) []tiflashStore { - tiflashes := make([]tiflashStore, len(stores)) - for i := range stores { - for _, label := range stores[i].Labels { - if label.Key == "engine" && label.Value == "tiflash" { - tiflashes = append(tiflashes, tiflashStore{stores[i]}) - } - } - } - - return tiflashes -} - -func getTiFlashTopology(stores []tiflashStore) ([]TiFlashInfo, error) { - nodes := make([]TiFlashInfo, 0) - for _, v := range stores { - // parse ip and port - host, port, err := parseHostAndPortFromAddress(v.Address) - if err != nil { - continue - } - _, statusPort, err := parseHostAndPortFromAddress(v.StatusAddress) - if err != nil { - continue - } - version := strings.Trim(v.Version, "\n ") - node := TiFlashInfo{ - Version: version, - IP: host, - Port: port, - DeployPath: v.DeployPath, // TiFlash hasn't BinaryPath for now, so it would be empty - Status: storeStateToStatus(v.StateName), - StatusPort: statusPort, - Labels: map[string]string{}, - StartTimestamp: v.StartTimestamp, - } - for _, v := range v.Labels { - node.Labels[v.Key] = node.Labels[v.Value] - } - nodes = append(nodes, node) - } - - return nodes, nil -} - -func GetStoreTopology(endpoint string, httpClient *http.Client) ([]TiKVInfo, []TiFlashInfo, error) { - stores, err := getAllStoreNodes(endpoint, httpClient) - if err != nil { - return nil, nil, err - } - - tikvStores := getAllTiKVNodes(stores) - tikvInfos, err := getTiKVTopology(tikvStores) - if err != nil { - return nil, nil, err - } - - tiflashStores := getAllTiFlashNodes(stores) - tiflashInfos, err := getTiFlashTopology(tiflashStores) - if err != nil { - return nil, nil, err - } - - return tikvInfos, tiflashInfos, nil -} - -func GetPDTopology(pdEndPoint string, httpClient *http.Client) ([]PDInfo, error) { - nodes := make([]PDInfo, 0) - healthMapChan := make(chan map[string]struct{}) - go func() { - var err error - healthMap, err := getPDNodesHealth(pdEndPoint, httpClient) - if err != nil { - healthMap = map[string]struct{}{} - } - healthMapChan <- healthMap - }() - - resp, err := httpClient.Get(pdEndPoint + "/pd/api/v1/members") - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("fetch PD members got wrong status code") - } - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return nil, err - } - - ds := struct { - Count int `json:"count"` - Members []struct { - GitHash string `json:"git_hash"` - ClientUrls []string `json:"client_urls"` - DeployPath string `json:"deploy_path"` - BinaryVersion string `json:"binary_version"` - MemberID json.Number `json:"member_id"` - } `json:"members"` - }{} - - err = json.Unmarshal(data, &ds) - if err != nil { - return nil, err - } - - healthMap := <-healthMapChan - close(healthMapChan) - for _, ds := range ds.Members { - u := ds.ClientUrls[0] - ts, err := getPDStartTimestamp(u, httpClient) - if err != nil { - log.Warn("failed to get PD node status", zap.Error(err)) - continue - } - host, port, err := parseHostAndPortFromAddressURL(u) - if err != nil { - continue - } - var storeStatus ComponentStatus - if _, ok := healthMap[ds.MemberID.String()]; ok { - storeStatus = ComponentStatusUp - } else { - storeStatus = ComponentStatusUnreachable - } - - nodes = append(nodes, PDInfo{ - GitHash: ds.GitHash, - Version: ds.BinaryVersion, - IP: host, - Port: port, - DeployPath: ds.DeployPath, - Status: storeStatus, - StartTimestamp: ts, - }) - } - return nodes, nil -} - -func getPDStartTimestamp(pdEndPoint string, httpClient *http.Client) (int64, error) { - resp, err := httpClient.Get(pdEndPoint + "/pd/api/v1/status") - if err != nil { - return 0, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return 0, fmt.Errorf("fetch PD %s status got wrong status code", pdEndPoint) - } - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return 0, err - } - - ds := struct { - StartTimestamp int64 `json:"start_timestamp"` - }{} - err = json.Unmarshal(data, &ds) - if err != nil { - return 0, err - } - - return ds.StartTimestamp, nil -} - -// address should be like "ip:port" as "127.0.0.1:2379". -// return error if string is not like "ip:port". -func parseHostAndPortFromAddress(address string) (string, uint, error) { - addresses := strings.Split(address, ":") - if len(addresses) != 2 { - return "", 0, fmt.Errorf("invalid address %s", address) - } - port, err := strconv.Atoi(addresses[1]) - if err != nil { - return "", 0, err - } - return addresses[0], uint(port), nil -} - -// address should be like "protocol://ip:port" as "http://127.0.0.1:2379". -func parseHostAndPortFromAddressURL(urlString string) (string, uint, error) { - u, err := url.Parse(urlString) - if err != nil { - return "", 0, err - } - port, err := strconv.Atoi(u.Port()) - if err != nil { - return "", 0, err - } - return u.Hostname(), uint(port), nil -} - -func storeStateToStatus(state string) ComponentStatus { - state = strings.Trim(strings.ToLower(state), "\n ") - switch state { - case "up": - return ComponentStatusUp - case "tombstone": - return ComponentStatusTombstone - case "offline": - return ComponentStatusOffline - case "down": - return ComponentStatusDown - case "disconnected": - return ComponentStatusUnreachable - default: - return ComponentStatusUnreachable - } -} - -func getPDNodesHealth(pdEndPoint string, httpClient *http.Client) (map[string]struct{}, error) { - // health member set - memberHealth := map[string]struct{}{} - resp, err := httpClient.Get(pdEndPoint + "/pd/api/v1/health") - if err != nil { - return nil, err - } - defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return nil, err - } - - var healths []struct { - MemberID json.Number `json:"member_id"` - } - - err = json.Unmarshal(data, &healths) - if err != nil { - return nil, err - } - - for _, v := range healths { - memberHealth[v.MemberID.String()] = struct{}{} - } - return memberHealth, nil -} - -// GetAlertCountByAddress receives alert manager's address like "ip:port", and it returns the -// alert number of the alert manager. -func GetAlertCountByAddress(address string, httpClient *http.Client) (int, error) { - ip, port, err := parseHostAndPortFromAddress(address) - if err != nil { - return 0, err - } - - apiAddress := fmt.Sprintf("http://%s:%d/api/v2/alerts", ip, port) - resp, err := httpClient.Get(apiAddress) - if err != nil { - return 0, err - } - - defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) - - if err != nil { - return 0, err - } - - var alerts []struct{} - - err = json.Unmarshal(data, &alerts) - if err != nil { - return 0, err - } - - return len(alerts), nil -} diff --git a/pkg/utils/clusterinfo/clusterinfo.go b/pkg/utils/topology/models.go similarity index 67% rename from pkg/utils/clusterinfo/clusterinfo.go rename to pkg/utils/topology/models.go index 8dfea81240..8a9104c88c 100644 --- a/pkg/utils/clusterinfo/clusterinfo.go +++ b/pkg/utils/topology/models.go @@ -11,19 +11,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package clusterinfo +package topology type ComponentStatus uint const ( - // ComponentStatusUnreachable means unreachable or disconnected ComponentStatusUnreachable ComponentStatus = 0 ComponentStatusUp ComponentStatus = 1 ComponentStatusTombstone ComponentStatus = 2 ComponentStatusOffline ComponentStatus = 3 - - // PD's Store may have state name down. - ComponentStatusDown ComponentStatus = 4 + ComponentStatusDown ComponentStatus = 4 ) type PDInfo struct { @@ -33,7 +30,7 @@ type PDInfo struct { Port uint `json:"port"` DeployPath string `json:"deploy_path"` Status ComponentStatus `json:"status"` - StartTimestamp int64 `json:"start_timestamp"` + StartTimestamp int64 `json:"start_timestamp"` // Ts = 0 means unknown } type TiDBInfo struct { @@ -47,7 +44,8 @@ type TiDBInfo struct { StartTimestamp int64 `json:"start_timestamp"` } -type TiKVInfo struct { +// Store may be a TiKV store or TiFlash store +type StoreInfo struct { GitHash string `json:"git_hash"` Version string `json:"version"` IP string `json:"ip"` @@ -59,31 +57,19 @@ type TiKVInfo struct { StartTimestamp int64 `json:"start_timestamp"` } -type TiFlashInfo struct { - Version string `json:"version"` - IP string `json:"ip"` - Port uint `json:"port"` - DeployPath string `json:"deploy_path"` - Status ComponentStatus `json:"status"` - StatusPort uint `json:"status_port"` - Labels map[string]string `json:"labels"` - StartTimestamp int64 `json:"start_timestamp"` +type StandardComponentInfo struct { + IP string `json:"ip"` + Port uint `json:"port"` } type AlertManagerInfo struct { - IP string `json:"ip"` - Port uint `json:"port"` - DeployPath string `json:"deploy_path"` + StandardComponentInfo } type GrafanaInfo struct { - IP string `json:"ip"` - Port uint `json:"port"` - DeployPath string `json:"deploy_path"` + StandardComponentInfo } type PrometheusInfo struct { - IP string `json:"ip"` - Port uint `json:"port"` - BinaryPath string `json:"binary_path"` + StandardComponentInfo } diff --git a/pkg/utils/topology/monitor.go b/pkg/utils/topology/monitor.go new file mode 100644 index 0000000000..2b18656178 --- /dev/null +++ b/pkg/utils/topology/monitor.go @@ -0,0 +1,53 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "context" + + "go.etcd.io/etcd/clientv3" +) + +func FetchAlertManagerTopology(ctx context.Context, etcdClient *clientv3.Client) (*AlertManagerInfo, error) { + i, err := fetchStandardComponentTopology(ctx, "alertmanager", etcdClient) + if err != nil { + return nil, err + } + if i == nil { + return nil, nil + } + return &AlertManagerInfo{StandardComponentInfo: *i}, nil +} + +func FetchGrafanaTopology(ctx context.Context, etcdClient *clientv3.Client) (*GrafanaInfo, error) { + i, err := fetchStandardComponentTopology(ctx, "grafana", etcdClient) + if err != nil { + return nil, err + } + if i == nil { + return nil, nil + } + return &GrafanaInfo{StandardComponentInfo: *i}, nil +} + +func FetchPrometheusTopology(ctx context.Context, etcdClient *clientv3.Client) (*PrometheusInfo, error) { + i, err := fetchStandardComponentTopology(ctx, "prometheus", etcdClient) + if err != nil { + return nil, err + } + if i == nil { + return nil, nil + } + return &PrometheusInfo{StandardComponentInfo: *i}, nil +} diff --git a/pkg/utils/topology/pd.go b/pkg/utils/topology/pd.go new file mode 100644 index 0000000000..1c75d5170e --- /dev/null +++ b/pkg/utils/topology/pd.go @@ -0,0 +1,131 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "encoding/json" + "sort" + + "github.com/pingcap/log" + "go.uber.org/zap" + + "github.com/pingcap-incubator/tidb-dashboard/pkg/pd" +) + +func FetchPDTopology(pdClient *pd.Client) ([]PDInfo, error) { + nodes := make([]PDInfo, 0) + healthMap, err := fetchPDHealth(pdClient) + if err != nil { + return nil, err + } + + data, err := pdClient.SendRequest("/pd/api/v1/members") + if err != nil { + return nil, err + } + ds := struct { + Count int `json:"count"` + Members []struct { + GitHash string `json:"git_hash"` + ClientUrls []string `json:"client_urls"` + DeployPath string `json:"deploy_path"` + BinaryVersion string `json:"binary_version"` + MemberID uint64 `json:"member_id"` + } `json:"members"` + }{} + + err = json.Unmarshal(data, &ds) + if err != nil { + return nil, ErrInvalidTopologyData.Wrap(err, "PD members API unmarshal failed") + } + + for _, ds := range ds.Members { + u := ds.ClientUrls[0] + host, port, err := parseHostAndPortFromAddressURL(u) + if err != nil { + continue + } + + ts, err := fetchPDStartTimestamp(pdClient) + if err != nil { + log.Warn("Failed to fetch PD start timestamp", zap.String("targetPdNode", u), zap.Error(err)) + ts = 0 + } + + var storeStatus ComponentStatus + if _, ok := healthMap[ds.MemberID]; ok { + storeStatus = ComponentStatusUp + } else { + storeStatus = ComponentStatusUnreachable + } + + nodes = append(nodes, PDInfo{ + GitHash: ds.GitHash, + Version: ds.BinaryVersion, + IP: host, + Port: port, + DeployPath: ds.DeployPath, + Status: storeStatus, + StartTimestamp: ts, + }) + } + + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].IP > nodes[j].IP && nodes[i].Port > nodes[j].Port + }) + + return nodes, nil +} + +func fetchPDStartTimestamp(pdClient *pd.Client) (int64, error) { + data, err := pdClient.SendRequest("/pd/api/v1/status") + if err != nil { + return 0, err + } + + ds := struct { + StartTimestamp int64 `json:"start_timestamp"` + }{} + err = json.Unmarshal(data, &ds) + if err != nil { + return 0, ErrInvalidTopologyData.Wrap(err, "PD status API unmarshal failed") + } + + return ds.StartTimestamp, nil +} + +func fetchPDHealth(pdClient *pd.Client) (map[uint64]struct{}, error) { + data, err := pdClient.SendRequest("/pd/api/v1/health") + if err != nil { + return nil, err + } + + var healths []struct { + MemberID uint64 `json:"member_id"` + Health bool `json:"health"` + } + + err = json.Unmarshal(data, &healths) + if err != nil { + return nil, ErrInvalidTopologyData.Wrap(err, "PD health API unmarshal failed") + } + + memberHealth := map[uint64]struct{}{} + for _, v := range healths { + if v.Health { + memberHealth[v.MemberID] = struct{}{} + } + } + return memberHealth, nil +} diff --git a/pkg/utils/topology/store.go b/pkg/utils/topology/store.go new file mode 100644 index 0000000000..0aee4328da --- /dev/null +++ b/pkg/utils/topology/store.go @@ -0,0 +1,152 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "encoding/json" + "sort" + "strings" + + "github.com/pingcap/log" + "go.uber.org/zap" + + "github.com/pingcap-incubator/tidb-dashboard/pkg/pd" +) + +// FetchStoreTopology returns TiKV info and TiFlash info. +func FetchStoreTopology(pdClient *pd.Client) ([]StoreInfo, []StoreInfo, error) { + stores, err := fetchStores(pdClient) + if err != nil { + return nil, nil, err + } + + tiKVStores := make([]store, 0, len(stores)) + tiFlashStores := make([]store, 0, len(stores)) + for _, store := range stores { + isTiFlash := false + for _, label := range store.Labels { + if label.Key == "engine" && label.Value == "tiflash" { + isTiFlash = true + } + } + if isTiFlash { + tiFlashStores = append(tiFlashStores, store) + } else { + tiKVStores = append(tiKVStores, store) + } + } + + return buildStoreTopology(tiKVStores), buildStoreTopology(tiFlashStores), nil +} + +func buildStoreTopology(stores []store) []StoreInfo { + nodes := make([]StoreInfo, 0, len(stores)) + for _, v := range stores { + host, port, err := parseHostAndPortFromAddress(v.Address) + if err != nil { + log.Warn("Failed to parse store address", zap.Any("store", v)) + continue + } + _, statusPort, err := parseHostAndPortFromAddress(v.StatusAddress) + if err != nil { + log.Warn("Failed to parse store status address", zap.Any("store", v)) + continue + } + // In current TiKV, it's version may not start with 'v', + // so we may need to add a prefix 'v' for it. + version := strings.Trim(v.Version, "\n ") + if !strings.HasPrefix(version, "v") { + version = "v" + version + } + node := StoreInfo{ + Version: version, + IP: host, + Port: port, + GitHash: v.GitHash, + DeployPath: v.DeployPath, + Status: parseStoreState(v.StateName), + StatusPort: statusPort, + Labels: map[string]string{}, + StartTimestamp: v.StartTimestamp, + } + for _, v := range v.Labels { + node.Labels[v.Key] = node.Labels[v.Value] + } + nodes = append(nodes, node) + } + + return nodes +} + +type store struct { + Address string `json:"address"` + ID int `json:"id"` + Labels []struct { + Key string `json:"key"` + Value string `json:"value"` + } + StateName string `json:"state_name"` + Version string `json:"version"` + StatusAddress string `json:"status_address"` + GitHash string `json:"git_hash"` + DeployPath string `json:"deploy_path"` + StartTimestamp int64 `json:"start_timestamp"` +} + +func fetchStores(pdClient *pd.Client) ([]store, error) { + data, err := pdClient.SendRequest("/pd/api/v1/stores") + if err != nil { + return nil, err + } + + storeResp := struct { + Count int `json:"count"` + Stores []struct { + Store store + } `json:"stores"` + }{} + err = json.Unmarshal(data, &storeResp) + if err != nil { + return nil, ErrInvalidTopologyData.Wrap(err, "PD stores API unmarshal failed") + } + + ret := make([]store, 0, storeResp.Count) + for _, s := range storeResp.Stores { + ret = append(ret, s.Store) + } + + sort.Slice(ret, func(i, j int) bool { + return ret[i].Address > ret[j].Address + }) + + return ret, nil +} + +func parseStoreState(state string) ComponentStatus { + state = strings.Trim(strings.ToLower(state), "\n ") + switch state { + case "up": + return ComponentStatusUp + case "tombstone": + return ComponentStatusTombstone + case "offline": + return ComponentStatusOffline + case "down": + return ComponentStatusDown + case "disconnected": + return ComponentStatusUnreachable + default: + return ComponentStatusUnreachable + } +} diff --git a/pkg/utils/topology/tidb.go b/pkg/utils/topology/tidb.go new file mode 100644 index 0000000000..e6b31cbb53 --- /dev/null +++ b/pkg/utils/topology/tidb.go @@ -0,0 +1,138 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "context" + "encoding/json" + "sort" + "strconv" + "strings" + "time" + + "github.com/pingcap/log" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" +) + +const tidbTopologyKeyPrefix = "/topology/tidb/" + +func FetchTiDBTopology(ctx context.Context, etcdClient *clientv3.Client) ([]TiDBInfo, error) { + ctx2, cancel := context.WithTimeout(ctx, defaultFetchTimeout) + defer cancel() + + resp, err := etcdClient.Get(ctx2, tidbTopologyKeyPrefix, clientv3.WithPrefix()) + if err != nil { + return nil, ErrEtcdRequestFailed.Wrap(err, "failed to get key %s from PD etcd", tidbTopologyKeyPrefix) + } + + nodesAlive := map[string]bool{} + nodesInfo := map[string]*TiDBInfo{} + + for _, kv := range resp.Kvs { + key := string(kv.Key) + if !strings.HasPrefix(key, tidbTopologyKeyPrefix) { + continue + } + // remainingKey looks like `ip:port/info` or `ip:port/ttl`. + remainingKey := key[len(tidbTopologyKeyPrefix):] + keyParts := strings.Split(remainingKey, "/") + if len(keyParts) != 2 { + log.Warn("Ignored invalid TiDB topology key", zap.String("key", key)) + continue + } + + switch keyParts[1] { + case "info": + node, err := parseTiDBInfo(keyParts[0], kv.Value) + if err == nil { + nodesInfo[keyParts[0]] = node + } else { + log.Warn("Ignored invalid TiDB topology info entry", + zap.String("key", key), + zap.String("value", string(kv.Value)), + zap.Error(err)) + } + case "ttl": + alive, err := parseTiDBAliveness(kv.Value) + if err == nil { + nodesAlive[keyParts[0]] = alive + } else { + log.Warn("Ignored invalid TiDB topology TTL entry", + zap.String("key", key), + zap.String("value", string(kv.Value)), + zap.Error(err)) + } + } + } + + nodes := make([]TiDBInfo, 0) + + for addr, info := range nodesInfo { + if alive, ok := nodesAlive[addr]; ok { + if alive { + info.Status = ComponentStatusUp + } + } + nodes = append(nodes, *info) + } + + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].IP > nodes[j].IP && nodes[i].Port > nodes[j].Port + }) + + return nodes, nil +} + +func parseTiDBInfo(address string, value []byte) (*TiDBInfo, error) { + ds := struct { + Version string `json:"version"` + GitHash string `json:"git_hash"` + StatusPort uint `json:"status_port"` + DeployPath string `json:"deploy_path"` + StartTimestamp int64 `json:"start_timestamp"` + }{} + + err := json.Unmarshal(value, &ds) + if err != nil { + return nil, ErrInvalidTopologyData.Wrap(err, "TiDB info unmarshal failed") + } + host, port, err := parseHostAndPortFromAddress(address) + if err != nil { + return nil, ErrInvalidTopologyData.Wrap(err, "TiDB info address parse failed") + } + + return &TiDBInfo{ + GitHash: ds.GitHash, + Version: ds.Version, + IP: host, + Port: port, + DeployPath: ds.DeployPath, + Status: ComponentStatusUnreachable, + StatusPort: ds.StatusPort, + StartTimestamp: ds.StartTimestamp, + }, nil +} + +func parseTiDBAliveness(value []byte) (bool, error) { + unixTimestampNano, err := strconv.ParseUint(string(value), 10, 64) + if err != nil { + return false, ErrInvalidTopologyData.Wrap(err, "TiDB TTL info parse failed") + } + t := time.Unix(0, int64(unixTimestampNano)) + if time.Since(t) > time.Second*45 { + return false, nil + } + return true, nil +} diff --git a/pkg/utils/topology/topology.go b/pkg/utils/topology/topology.go new file mode 100644 index 0000000000..d3b904750a --- /dev/null +++ b/pkg/utils/topology/topology.go @@ -0,0 +1,87 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package topology + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strconv" + "strings" + "time" + + "github.com/joomcode/errorx" + "github.com/pingcap/log" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" +) + +var ( + ErrNS = errorx.NewNamespace("error.topology") + ErrEtcdRequestFailed = ErrNS.NewType("pd_etcd_request_failed") + ErrInvalidTopologyData = ErrNS.NewType("invalid_topology_data") +) + +const defaultFetchTimeout = 2 * time.Second + +// address should be like "ip:port" as "127.0.0.1:2379". +// return error if string is not like "ip:port". +func parseHostAndPortFromAddress(address string) (string, uint, error) { + addresses := strings.Split(address, ":") + if len(addresses) != 2 { + return "", 0, fmt.Errorf("invalid address %s", address) + } + port, err := strconv.Atoi(addresses[1]) + if err != nil { + return "", 0, err + } + return addresses[0], uint(port), nil +} + +// address should be like "protocol://ip:port" as "http://127.0.0.1:2379". +func parseHostAndPortFromAddressURL(urlString string) (string, uint, error) { + u, err := url.Parse(urlString) + if err != nil { + return "", 0, err + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + return "", 0, err + } + return u.Hostname(), uint(port), nil +} + +func fetchStandardComponentTopology(ctx context.Context, componentName string, etcdClient *clientv3.Client) (*StandardComponentInfo, error) { + ctx2, cancel := context.WithTimeout(ctx, defaultFetchTimeout) + defer cancel() + + key := "/topology/" + componentName + resp, err := etcdClient.Get(ctx2, key, clientv3.WithPrefix()) + if err != nil { + return nil, ErrEtcdRequestFailed.Wrap(err, "failed to get key %s from PD etcd", key) + } + if resp.Count == 0 { + return nil, nil + } + info := StandardComponentInfo{} + kv := resp.Kvs[0] + if err = json.Unmarshal(kv.Value, &info); err != nil { + log.Warn("Failed to unmarshal topology value", + zap.String("key", string(kv.Key)), + zap.String("value", string(kv.Value))) + return nil, nil + } + return &info, nil +} diff --git a/ui/dashboardApp/index.js b/ui/dashboardApp/index.js index f97471f348..c3a26d08e8 100644 --- a/ui/dashboardApp/index.js +++ b/ui/dashboardApp/index.js @@ -11,6 +11,7 @@ import * as client from '@dashboard/client' import LayoutMain from '@dashboard/layout/main' import LayoutSignIn from '@dashboard/layout/signin' +import AppDebugPlayground from '@lib/apps/DebugPlayground/index.meta' import AppDashboardSettings from '@lib/apps/DashboardSettings/index.meta' import AppUserProfile from '@lib/apps/UserProfile/index.meta' import AppOverview from '@lib/apps/Overview/index.meta' @@ -50,6 +51,7 @@ async function main() { ) registry + .register(AppDebugPlayground) .register(AppDashboardSettings) .register(AppUserProfile) .register(AppOverview) diff --git a/ui/dashboardApp/layout/main/Sider/index.js b/ui/dashboardApp/layout/main/Sider/index.js index 02dc3f36a6..ab3ee416a7 100644 --- a/ui/dashboardApp/layout/main/Sider/index.js +++ b/ui/dashboardApp/layout/main/Sider/index.js @@ -4,44 +4,26 @@ import { Layout, Menu } from 'antd' import { Link } from 'react-router-dom' import { useEventListener } from '@umijs/hooks' import { useTranslation } from 'react-i18next' -import { useTrail, useSpring, animated } from 'react-spring' +import { useSpring, animated } from 'react-spring' import client from '@lib/client' import Banner from './Banner' import styles from './index.module.less' -const AnimatedMenuItem = animated(Menu.Item) -const AnimatedSubMenu = animated(Menu.SubMenu) - -function TrailMenu({ items, delay, ...props }) { - const trail = useTrail(items.length, { - opacity: 1, - transform: 'translate3d(0, 0, 0)', - from: { opacity: 0, transform: 'translate3d(0, 60px, 0)' }, - delay, - config: { mass: 1, tension: 5000, friction: 200 }, - }) - return ( - {trail.map((style, idx) => items[idx]({ style }))} - ) -} - -function useAnimatedAppMenuItem(registry, appId, title) { +function useAppMenuItem(registry, appId, title) { const { t } = useTranslation() - return (animationProps) => { - const app = registry.apps[appId] - if (!app) { - return null - } - return ( - - - {app.icon ? : null} - {title ? title : t(`${appId}.nav_title`, appId)} - - - ) + const app = registry.apps[appId] + if (!app) { + return null } + return ( + + + {app.icon ? : null} + {title ? title : t(`${appId}.nav_title`, appId)} + + + ) } function useActiveAppId(registry) { @@ -69,7 +51,7 @@ function useCurrentLogin() { return login } -export default function Sider({ +function Sider({ registry, fullWidth, defaultCollapsed, @@ -82,11 +64,9 @@ export default function Sider({ const activeAppId = useActiveAppId(registry) const currentLogin = useCurrentLogin() - const debugSubMenuItems = [ - useAnimatedAppMenuItem(registry, 'instance_profiling'), - ] - const debugSubMenu = (animationProps) => ( - @@ -94,26 +74,26 @@ export default function Sider({ {t('nav.sider.debug')} } - {...animationProps} > - {debugSubMenuItems.map((r) => r())} - + {debugSubMenuItems} + ) const menuItems = [ - useAnimatedAppMenuItem(registry, 'overview'), - useAnimatedAppMenuItem(registry, 'cluster_info'), - useAnimatedAppMenuItem(registry, 'keyviz'), - useAnimatedAppMenuItem(registry, 'statement'), - useAnimatedAppMenuItem(registry, 'slow_query'), - useAnimatedAppMenuItem(registry, 'diagnose'), - useAnimatedAppMenuItem(registry, 'search_logs'), + useAppMenuItem(registry, 'debug_playground'), + useAppMenuItem(registry, 'overview'), + useAppMenuItem(registry, 'cluster_info'), + useAppMenuItem(registry, 'keyviz'), + useAppMenuItem(registry, 'statement'), + useAppMenuItem(registry, 'slow_query'), + useAppMenuItem(registry, 'diagnose'), + useAppMenuItem(registry, 'search_logs'), debugSubMenu, ] const extraMenuItems = [ - useAnimatedAppMenuItem(registry, 'dashboard_settings'), - useAnimatedAppMenuItem( + useAppMenuItem(registry, 'dashboard_settings'), + useAppMenuItem( registry, 'user_profile', currentLogin ? currentLogin.username : '...' @@ -142,20 +122,24 @@ export default function Sider({ fullWidth={fullWidth} collapsedWidth={collapsedWidth} /> - - + {menuItems} + + + > + {extraMenuItems} + ) } + +export default Sider diff --git a/ui/lib/antd.global.less b/ui/lib/antd.global.less index 292c0dfb39..5b81af33f7 100644 --- a/ui/lib/antd.global.less +++ b/ui/lib/antd.global.less @@ -1,3 +1,5 @@ +// Slightly modified from https://github.com/ant-design/ant-design/blob/master/components/style/core/base.less + /* stylelint-disable at-rule-no-unknown */ // Reboot diff --git a/ui/lib/apps/ClusterInfo/components/HostTable.tsx b/ui/lib/apps/ClusterInfo/components/HostTable.tsx index e1215a6846..76c219f365 100644 --- a/ui/lib/apps/ClusterInfo/components/HostTable.tsx +++ b/ui/lib/apps/ClusterInfo/components/HostTable.tsx @@ -21,7 +21,7 @@ export default function HostTable() { const { t } = useTranslation() const { data: tableData, isLoading } = useClientRequest((cancelToken) => - client.getInstance().hostAllGet({ cancelToken }) + client.getInstance().getHostsInfo({ cancelToken }) ) const columns = [ diff --git a/ui/lib/apps/ClusterInfo/components/InstanceTable.tsx b/ui/lib/apps/ClusterInfo/components/InstanceTable.tsx index 45573d2861..bb0ff1bd7d 100644 --- a/ui/lib/apps/ClusterInfo/components/InstanceTable.tsx +++ b/ui/lib/apps/ClusterInfo/components/InstanceTable.tsx @@ -1,230 +1,205 @@ -import { Badge, Divider, Popconfirm, Tooltip } from 'antd' -import React, { ReactNode } from 'react' +import { Divider, Popconfirm, Tooltip, Alert } from 'antd' +import React, { useMemo, useCallback } from 'react' import { useTranslation } from 'react-i18next' import { DeleteOutlined } from '@ant-design/icons' - -import { - STATUS_DOWN, - STATUS_OFFLINE, - STATUS_TOMBSTONE, - STATUS_UP, -} from '@lib/apps/ClusterInfo/status/status' -import client from '@lib/client' -import { CardTableV2 } from '@lib/components' +import { CardTableV2, InstanceStatusBadge } from '@lib/components' import DateTime from '@lib/components/DateTime' import { useClientRequest } from '@lib/utils/useClientRequest' +import { usePersistFn } from '@umijs/hooks' +import { + buildInstanceTable, + IInstanceTableItem, + InstanceStatus, +} from '@lib/utils/instanceTable' +import client from '@lib/client' -function useStatusColumnRender(handleHideTiDB) { +function StatusColumn({ + node, + onHideTiDB, +}: { + node: IInstanceTableItem + onHideTiDB: (node) => void +}) { const { t } = useTranslation() - return (node) => { - if (node.status == null) { - // Tree node - return - } - let statusNode: ReactNode = null - switch (node.status) { - case STATUS_DOWN: - statusNode = ( - - ) - break - case STATUS_UP: - statusNode = ( - - ) - break - case STATUS_TOMBSTONE: - statusNode = ( - - ) - break - case STATUS_OFFLINE: - statusNode = ( - - ) - break - default: - statusNode = ( - - ) - break - } - return ( - - {statusNode} - {node.nodeKind === 'tidb' && node.status !== STATUS_UP && ( - <> - - { + onHideTiDB && onHideTiDB(node) + }) + + return ( + + {node.instanceKind === 'tidb' && node.status !== InstanceStatus.Up && ( + <> + + handleHideTiDB(node)} > - - - - - - - - )} - - ) - } -} - -function useHideTiDBHandler(updateData) { - return async (node) => { - await client - .getInstance() - .topologyTidbAddressDelete(`${node.ip}:${node.port}`) - updateData() - } -} - -function buildData(data) { - if (data === undefined) { - return {} - } - const tableData: any[] = [] // FIXME - const groupData: any[] = [] // FIXME - let startIndex = 0 - const kinds = ['tidb', 'tikv', 'pd', 'tiflash'] - kinds.forEach((nodeKind) => { - const nodes = data[nodeKind] - if (nodes.err) { - return - } - const count = nodes.nodes.length - groupData.push({ - key: nodeKind, - name: nodeKind, - startIndex: startIndex, - count: count, - level: 0, - }) - startIndex += count - const children = nodes.nodes.map((node) => { - return { - key: `${node.ip}:${node.port}`, - ...node, - nodeKind, - } - }) - tableData.push(...children) - }) - return { tableData, groupData } + + + + + + + + )} + + + ) } export default function ListPage() { const { t } = useTranslation() + const { + data: dataTiDB, + isLoading: loadingTiDB, + error: errTiDB, + sendRequest, + } = useClientRequest((cancelToken) => + client.getInstance().getTiDBTopology({ cancelToken }) + ) + const { + data: dataStores, + isLoading: loadingStores, + error: errStores, + } = useClientRequest((cancelToken) => + client.getInstance().getStoreTopology({ cancelToken }) + ) + const { + data: dataPD, + isLoading: loadingPD, + error: errPD, + } = useClientRequest((cancelToken) => + client.getInstance().getPDTopology({ cancelToken }) + ) - const { data, isLoading, sendRequest } = useClientRequest((cancelToken) => - client.getInstance().topologyAllGet({ cancelToken }) + const [tableData, groupData] = useMemo( + () => + buildInstanceTable({ + dataPD, + dataTiDB, + dataTiKV: dataStores?.tikv, + dataTiFlash: dataStores?.tiflash, + includeTiFlash: true, + }), + [dataTiDB, dataStores, dataPD] ) - const { tableData, groupData } = buildData(data) - const handleHideTiDB = useHideTiDBHandler(sendRequest) - const renderStatusColumn = useStatusColumnRender(handleHideTiDB) + const handleHideTiDB = useCallback( + async (node) => { + await client + .getInstance() + .topologyTidbAddressDelete(`${node.ip}:${node.port}`) + sendRequest() + }, + [sendRequest] + ) - const columns = [ - { - name: t('cluster_info.list.instance_table.columns.node'), - key: 'node', - minWidth: 100, - maxWidth: 160, - onRender: ({ ip, port }) => { - const fullName = `${ip}:${port}` - return ( - - {fullName} + const columns = useMemo( + () => [ + { + name: t('cluster_info.list.instance_table.columns.node'), + key: 'node', + minWidth: 100, + maxWidth: 160, + onRender: ({ ip, port }) => { + const fullName = `${ip}:${port}` + return ( + + {fullName} + + ) + }, + }, + { + name: t('cluster_info.list.instance_table.columns.status'), + key: 'status', + minWidth: 100, + maxWidth: 120, + onRender: (node) => ( + + ), + }, + { + name: t('cluster_info.list.instance_table.columns.up_time'), + key: 'start_timestamp', + minWidth: 100, + maxWidth: 150, + onRender: ({ start_timestamp: ts }) => { + if (ts !== undefined && ts !== 0) { + return + } + }, + }, + { + name: t('cluster_info.list.instance_table.columns.version'), + fieldName: 'version', + key: 'version', + minWidth: 100, + maxWidth: 150, + onRender: ({ version }) => ( + + {version} - ) + ), }, - }, - { - name: t('cluster_info.list.instance_table.columns.status'), - key: 'status', - minWidth: 80, - maxWidth: 100, - onRender: renderStatusColumn, - }, - { - name: t('cluster_info.list.instance_table.columns.up_time'), - key: 'start_timestamp', - minWidth: 100, - maxWidth: 150, - onRender: ({ start_timestamp: ts }) => { - if (ts !== undefined && ts !== 0) { - return - } + { + name: t('cluster_info.list.instance_table.columns.git_hash'), + fieldName: 'git_hash', + key: 'git_hash', + minWidth: 100, + maxWidth: 200, + onRender: ({ git_hash }) => ( + + {git_hash} + + ), }, - }, - { - name: t('cluster_info.list.instance_table.columns.version'), - fieldName: 'version', - key: 'version', - minWidth: 100, - maxWidth: 250, - onRender: ({ version }) => ( - - {version} - - ), - }, - { - name: t('cluster_info.list.instance_table.columns.deploy_path'), - fieldName: 'deploy_path', - key: 'deploy_path', - minWidth: 100, - maxWidth: 200, - onRender: ({ deploy_path }) => ( - - {deploy_path} - - ), - }, - { - name: t('cluster_info.list.instance_table.columns.git_hash'), - fieldName: 'git_hash', - key: 'git_hash', - minWidth: 100, - maxWidth: 150, - onRender: ({ git_hash }) => ( - - {git_hash} - - ), - }, - ] + { + name: t('cluster_info.list.instance_table.columns.deploy_path'), + fieldName: 'deploy_path', + key: 'deploy_path', + minWidth: 150, + maxWidth: 300, + onRender: ({ deploy_path }) => ( + + {deploy_path} + + ), + }, + ], + [t, handleHideTiDB] + ) return ( - + <> + {errTiDB && ( + + )} + {errStores && ( + + )} + {errPD && ( + + )} + + ) } diff --git a/ui/lib/apps/ClusterInfo/translations/en.yaml b/ui/lib/apps/ClusterInfo/translations/en.yaml index 56d4b58987..b2cbc1d260 100644 --- a/ui/lib/apps/ClusterInfo/translations/en.yaml +++ b/ui/lib/apps/ClusterInfo/translations/en.yaml @@ -14,13 +14,6 @@ cluster_info: hide_db: tooltip: Hide confirm: Do you want to hide this TiDB instance? - status: - up: Up - down: Down - tombstone: Tombstone - offline: Leaving - unknown: Unknown - unreachable: Unreachable host_table: title: Hosts columns: diff --git a/ui/lib/apps/ClusterInfo/translations/zh-CN.yaml b/ui/lib/apps/ClusterInfo/translations/zh-CN.yaml index f475c01358..464b0bb095 100644 --- a/ui/lib/apps/ClusterInfo/translations/zh-CN.yaml +++ b/ui/lib/apps/ClusterInfo/translations/zh-CN.yaml @@ -14,13 +14,6 @@ cluster_info: hide_db: tooltip: 隐藏 confirm: 您确认要隐藏该 TiDB 实例吗? - status: - up: 在线 - down: 离线 - tombstone: 已缩容下线 - offline: 下线中 - unknown: 未知 - unreachable: 无法访问 host_table: title: 主机 columns: diff --git a/ui/lib/apps/DebugPlayground/index.meta.ts b/ui/lib/apps/DebugPlayground/index.meta.ts new file mode 100644 index 0000000000..ece87f1f28 --- /dev/null +++ b/ui/lib/apps/DebugPlayground/index.meta.ts @@ -0,0 +1,8 @@ +import { BugOutlined } from '@ant-design/icons' + +export default { + id: 'debug_playground', + routerPrefix: '/debug_playground', + icon: BugOutlined, + reactRoot: () => import(/* webpackChunkName: "debug_playground" */ '.'), +} diff --git a/ui/lib/apps/DebugPlayground/index.tsx b/ui/lib/apps/DebugPlayground/index.tsx new file mode 100644 index 0000000000..4677c98fc1 --- /dev/null +++ b/ui/lib/apps/DebugPlayground/index.tsx @@ -0,0 +1,81 @@ +import React, { useState, useRef } from 'react' +import { + Root, + BaseSelect, + InstanceSelect, + IInstanceSelectRefProps, + Pre, +} from '@lib/components' +import { Select, Button } from 'antd' + +const InstanceSelectRegion = () => { + const [instanceSelectValue, setInstanceSelectValue] = useState([]) + const s = useRef(null) + + return ( + <> +

Instance Select

+ +
Instance select value = {JSON.stringify(instanceSelectValue)}
+
+        Instance select value instances ={' '}
+        {JSON.stringify(
+          s.current && s.current.getInstanceByKeys(instanceSelectValue)
+        )}
+      
+ + ) +} + +const App = () => { + return ( + +

Debug Playground

+

Base Select

+
Content
} + valueRender={() => Short} + /> +
Content
} + valueRender={() => Very Lonnnnnnnnng Value} + /> +
Content
} + valueRender={() => Disabled} + /> + +

Antd Select

+ + + +

Misc

+
{ + e.preventDefault() + }} + > + Prevent Default Area +
+
+ ) +} + +export default App diff --git a/ui/lib/apps/InstanceProfiling/pages/Detail.tsx b/ui/lib/apps/InstanceProfiling/pages/Detail.tsx index 6977ab5807..d0c93dc8c1 100644 --- a/ui/lib/apps/InstanceProfiling/pages/Detail.tsx +++ b/ui/lib/apps/InstanceProfiling/pages/Detail.tsx @@ -8,6 +8,7 @@ import { usePersistFn } from '@umijs/hooks' import client from '@lib/client' import { CardTableV2, Head } from '@lib/components' import { useClientRequestWithPolling } from '@lib/utils/useClientRequest' +import { InstanceKindName } from '@lib/utils/instanceTable' function mapData(data) { if (!data) { @@ -64,7 +65,9 @@ export default function Page() { key: 'kind', minWidth: 100, maxWidth: 150, - onRender: (record) => record.target.kind, + onRender: (record) => { + return InstanceKindName[record.target.kind] + }, }, { name: t('instance_profiling.detail.table.columns.status'), diff --git a/ui/lib/apps/InstanceProfiling/pages/List.tsx b/ui/lib/apps/InstanceProfiling/pages/List.tsx index ba90c1bd68..b069dafd97 100644 --- a/ui/lib/apps/InstanceProfiling/pages/List.tsx +++ b/ui/lib/apps/InstanceProfiling/pages/List.tsx @@ -1,137 +1,91 @@ -import { Badge, Button, Form, message, Select, TreeSelect } from 'antd' +import { Badge, Button, Form, Select, Modal } from 'antd' import { ScrollablePane } from 'office-ui-fabric-react/lib/ScrollablePane' -import React, { useMemo, useState } from 'react' +import React, { useMemo, useState, useCallback, useRef } from 'react' import { useTranslation } from 'react-i18next' import { useNavigate } from 'react-router-dom' import { usePersistFn } from '@umijs/hooks' - -import client from '@lib/client' -import { Card, CardTableV2 } from '@lib/components' +import client, { + ProfilingStartRequest, + ModelRequestTargetNode, +} from '@lib/client' +import { + Card, + CardTableV2, + InstanceSelect, + IInstanceSelectRefProps, +} from '@lib/components' import DateTime from '@lib/components/DateTime' import openLink from '@lib/utils/openLink' import { useClientRequest } from '@lib/utils/useClientRequest' -// FIXME: The following logic should be extracted into a common component. -function getTreeData(topologyMap) { - const treeDataByKind = { - tidb: [], - tikv: [], - pd: [], - } - Object.values(topologyMap).forEach((target: any) => { - if (!(target.kind in treeDataByKind)) { - return - } - treeDataByKind[target.kind].push({ - title: target.display_name, - value: target.display_name, - key: target.display_name, - }) - }) - const kindTitleMap = { - tidb: 'TiDB', - tikv: 'TiKV', - pd: 'PD', - } - return Object.keys(treeDataByKind) - .filter((kind) => treeDataByKind[kind].length > 0) - .map((kind) => ({ - title: kindTitleMap[kind], - value: kind, - key: kind, - children: treeDataByKind[kind], - })) -} - -function filterTreeNode(inputValue, treeNode) { - const name = treeNode.key - return name.includes(inputValue) -} - -function useTargetsMap() { - const { data } = useClientRequest((cancelToken) => - client.getInstance().topologyAllGet({ cancelToken }) - ) - return useMemo(() => { - const map = {} - if (!data) { - return map - } - // FIXME, declare type - data.tidb?.nodes?.forEach((node) => { - const display = `${node.ip}:${node.port}` - const target = { - kind: 'tidb', - display_name: display, - ip: node.ip, - port: node.status_port, - } - map[display] = target - }) - data.tikv?.nodes?.forEach((node) => { - const display = `${node.ip}:${node.port}` - const target = { - kind: 'tikv', - display_name: display, - ip: node.ip, - port: node.status_port, - } - map[display] = target - }) - data.pd?.nodes?.forEach((node) => { - const display = `${node.ip}:${node.port}` - const target = { - kind: 'pd', - display_name: display, - ip: node.ip, - port: node.port, - } - map[display] = target - }) - return map - }, [data]) -} - const profilingDurationsSec = [10, 30, 60, 120] const defaultProfilingDuration = 30 export default function Page() { - const targetsMap = useTargetsMap() - - // FIXME: Use Antd form - const [selectedTargets, setSelectedTargets] = useState([]) - const [duration, setDuration] = useState(defaultProfilingDuration) - - const [submitting, setSubmitting] = useState(false) const { data: historyTable, isLoading: listLoading, } = useClientRequest((cancelToken) => client.getInstance().getProfilingGroups({ cancelToken }) ) - const { t } = useTranslation() const navigate = useNavigate() + const instanceSelect = useRef(null) + const [submitting, setSubmitting] = useState(false) - async function handleStart() { - if (selectedTargets.length === 0) { - // TODO: Show notification - return - } - setSubmitting(true) - const req = { - targets: selectedTargets.map((k) => targetsMap[k]), - duration_secs: duration, - } - try { - const res = await client.getInstance().startProfiling(req) - navigate(`/instance_profiling/${res.data.id}`) - } catch (e) { - // FIXME - message.error(e.message) - } - setSubmitting(false) - } + const handleFinish = useCallback( + async (fieldsValue) => { + if (!fieldsValue.instances || fieldsValue.instances.length === 0) { + Modal.error({ + content: 'Some required fields are not filled', + }) + return + } + if (!instanceSelect.current) { + Modal.error({ + content: 'Internal error: Instance select is not ready', + }) + return + } + setSubmitting(true) + const targets: ModelRequestTargetNode[] = instanceSelect + .current!.getInstanceByKeys(fieldsValue.instances) + .map((instance) => { + let port + switch (instance.instanceKind) { + case 'pd': + port = instance.port + break + case 'tidb': + case 'tikv': + port = instance.status_port + break + } + return { + kind: instance.instanceKind, + display_name: instance.key, + ip: instance.ip, + port, + } + }) + .filter((i) => i.port != null) + const req: ProfilingStartRequest = { + targets, + duration_secs: fieldsValue.duration, + } + try { + const res = await client.getInstance().startProfiling(req) + navigate(`/instance_profiling/${res.data.id}`) + } catch (e) { + // FIXME + Modal.error({ + content: e.message, + }) + } + setSubmitting(false) + }, + [navigate] + ) const handleRowClick = usePersistFn( (rec, _idx, ev: React.MouseEvent) => { @@ -139,98 +93,95 @@ export default function Page() { } ) - const historyTableColumns = [ - { - name: t('instance_profiling.list.table.columns.targets'), - key: 'targets', - minWidth: 150, - maxWidth: 250, - onRender: (rec) => { - // TODO: Extract to utility function - const r: string[] = [] - if (rec.target_stats.num_tidb_nodes) { - r.push(`${rec.target_stats.num_tidb_nodes} TiDB`) - } - if (rec.target_stats.num_tikv_nodes) { - r.push(`${rec.target_stats.num_tikv_nodes} TiKV`) - } - if (rec.target_stats.num_pd_nodes) { - r.push(`${rec.target_stats.num_pd_nodes} PD`) - } - return {r.join(', ')} + const historyTableColumns = useMemo( + () => [ + { + name: t('instance_profiling.list.table.columns.targets'), + key: 'targets', + minWidth: 150, + maxWidth: 250, + onRender: (rec) => { + // TODO: Extract to utility function + const r: string[] = [] + if (rec.target_stats.num_tidb_nodes) { + r.push(`${rec.target_stats.num_tidb_nodes} TiDB`) + } + if (rec.target_stats.num_tikv_nodes) { + r.push(`${rec.target_stats.num_tikv_nodes} TiKV`) + } + if (rec.target_stats.num_pd_nodes) { + r.push(`${rec.target_stats.num_pd_nodes} PD`) + } + return {r.join(', ')} + }, }, - }, - { - name: t('instance_profiling.list.table.columns.status'), - key: 'status', - minWidth: 100, - maxWidth: 150, - onRender: (rec) => { - if (rec.state === 1) { - return ( - - ) - } else if (rec.state === 2) { - return ( - - ) - } + { + name: t('instance_profiling.list.table.columns.status'), + key: 'status', + minWidth: 100, + maxWidth: 150, + onRender: (rec) => { + if (rec.state === 1) { + return ( + + ) + } else if (rec.state === 2) { + return ( + + ) + } + }, }, - }, - { - name: t('instance_profiling.list.table.columns.start_at'), - key: 'started_at', - minWidth: 160, - maxWidth: 220, - onRender: (rec) => { - return + { + name: t('instance_profiling.list.table.columns.start_at'), + key: 'started_at', + minWidth: 160, + maxWidth: 220, + onRender: (rec) => { + return + }, }, - }, - { - name: t('instance_profiling.list.table.columns.duration'), - key: 'duration', - minWidth: 100, - maxWidth: 150, - fieldName: 'profile_duration_secs', - }, - ] + { + name: t('instance_profiling.list.table.columns.duration'), + key: 'duration', + minWidth: 100, + maxWidth: 150, + fieldName: 'profile_duration_secs', + }, + ], + [t] + ) return (
-
+ - + - {profilingDurationsSec.map((sec) => ( {sec}s @@ -239,7 +190,7 @@ export default function Page() { - diff --git a/ui/lib/apps/InstanceProfiling/translations/en.yaml b/ui/lib/apps/InstanceProfiling/translations/en.yaml index 35561eb241..0ac5a047fa 100644 --- a/ui/lib/apps/InstanceProfiling/translations/en.yaml +++ b/ui/lib/apps/InstanceProfiling/translations/en.yaml @@ -3,9 +3,8 @@ instance_profiling: list: control_form: title: Start Profiling Instances - nodes: + instances: label: Select instances - placeholder: Please select the instance to profile duration: label: Duration submit: Start Profiling diff --git a/ui/lib/apps/InstanceProfiling/translations/zh-CN.yaml b/ui/lib/apps/InstanceProfiling/translations/zh-CN.yaml index b4568b2ace..cc136ec2df 100644 --- a/ui/lib/apps/InstanceProfiling/translations/zh-CN.yaml +++ b/ui/lib/apps/InstanceProfiling/translations/zh-CN.yaml @@ -3,9 +3,8 @@ instance_profiling: list: control_form: title: 开始性能分析 - nodes: + instances: label: 选择实例 - placeholder: 请选择需要进行性能分析的目标实例 duration: label: 分析时长 submit: 开始分析 diff --git a/ui/lib/apps/KeyViz/components/KeyVizSettingForm.tsx b/ui/lib/apps/KeyViz/components/KeyVizSettingForm.tsx index fe8ba826af..7c8bf2c658 100644 --- a/ui/lib/apps/KeyViz/components/KeyVizSettingForm.tsx +++ b/ui/lib/apps/KeyViz/components/KeyVizSettingForm.tsx @@ -12,7 +12,6 @@ import { import { ExclamationCircleOutlined } from '@ant-design/icons' import { useTranslation } from 'react-i18next' import client, { ConfigKeyVisualConfig } from '@lib/client' -import { setHidden } from '@lib/utils/form' const policyConfigurable = process.env.NODE_ENV === 'development' @@ -175,17 +174,23 @@ function KeyVizSettingForm({ onClose, onConfigUpdated }: Props) { {policyOptions} - client.getInstance().topologyAllGet({ cancelToken }) - ) +import { RightOutlined, WarningOutlined } from '@ant-design/icons' +import { Stack } from 'office-ui-fabric-react/lib/Stack' - const statusMap = useMemo(() => { - if (!data) { - return [] +function ComponentItem(props: { + name: string + resp: { data?: { status?: number }[]; isLoading: boolean; error?: any } +}) { + const { name, resp } = props + const [upNums, allNums] = useMemo(() => { + if (!resp.data) { + return [0, 0] } - const r: any[] = [] - const components = ['tidb', 'tikv', 'tiflash', 'pd'] - components.forEach((componentName) => { - if (!data[componentName]) { - return - } - if (data[componentName].err) { - r.push({ name: componentName, error: true }) - return + let up = 0 + let all = 0 + for (const instance of resp.data) { + all++ + if ( + instance.status === STATUS_UP || + instance.status === STATUS_TOMBSTONE || + instance.status === STATUS_OFFLINE + ) { + up++ } + } + return [up, all] + }, [resp]) - let normals = 0, - abnormals = 0 - data[componentName].nodes.forEach((n) => { - if ( - n.status === STATUS_UP || - n.status === STATUS_TOMBSTONE || - n.status === STATUS_OFFLINE - ) { - normals++ - } else { - abnormals++ - } - }) + return ( + + {!resp.error && ( + + + + {upNums} + / {allNums} + + + + )} + {resp.error && ( + + + Error + + + )} + + ) +} - if (normals > 0 || abnormals > 0) { - r.push({ name: componentName, normals, abnormals }) - } - }) - return r - }, [data]) +export default function Nodes() { + const { t } = useTranslation() + const tidbResp = useClientRequest((cancelToken) => + client.getInstance().getTiDBTopology({ cancelToken }) + ) + const storeResp = useClientRequest((cancelToken) => + client.getInstance().getStoreTopology({ cancelToken }) + ) + const tiKVResp = { + ...storeResp, + data: storeResp.data?.tikv, + } + const tiFlashResp = { + ...storeResp, + data: storeResp.data?.tiflash, + } + const pdResp = useClientRequest((cancelToken) => + client.getInstance().getPDTopology({ cancelToken }) + ) return ( - - {error && } - {data && - statusMap.map((s) => { - return ( -

- {t(`overview.instances.component.${s.name}`)}: - {s.error && ( - Error - )} - {!s.error && ( - - {s.normals} Up /{' '} - 0 ? 'danger' : undefined} - > - {s.abnormals} Down - - - )} -

- ) - })} -
+ + + + + + + + + + + + + + + + + +
) } diff --git a/ui/lib/apps/Overview/components/MonitorAlert.module.less b/ui/lib/apps/Overview/components/MonitorAlert.module.less deleted file mode 100644 index 36d1b45f41..0000000000 --- a/ui/lib/apps/Overview/components/MonitorAlert.module.less +++ /dev/null @@ -1,8 +0,0 @@ -@import '~antd/es/style/themes/default.less'; - -.warn { - color: red; - &:hover { - text-decoration: black; - } -} diff --git a/ui/lib/apps/Overview/components/MonitorAlert.tsx b/ui/lib/apps/Overview/components/MonitorAlert.tsx index 5f18d79d53..67b8f5bad3 100644 --- a/ui/lib/apps/Overview/components/MonitorAlert.tsx +++ b/ui/lib/apps/Overview/components/MonitorAlert.tsx @@ -1,69 +1,100 @@ import React, { useEffect, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { Link } from 'react-router-dom' -import { RightOutlined } from '@ant-design/icons' - +import { RightOutlined, WarningOutlined } from '@ant-design/icons' +import { Card, AnimatedSkeleton } from '@lib/components' import client from '@lib/client' -import { AnimatedSkeleton, Card } from '@lib/components' - -import styles from './MonitorAlert.module.less' +import { Link } from 'react-router-dom' +import { useTranslation } from 'react-i18next' +import { useClientRequest } from '@lib/utils/useClientRequest' +import { Space, Typography } from 'antd' +import { Stack } from 'office-ui-fabric-react/lib/Stack' -export default function MonitorAlert({ cluster }) { +export default function MonitorAlert() { const { t } = useTranslation() const [alertCounter, setAlertCounter] = useState(0) + const { + data: amData, + isLoading: amIsLoading, + } = useClientRequest((cancelToken) => + client.getInstance().getAlertManagerTopology({ cancelToken }) + ) + const { + data: grafanaData, + isLoading: grafanaIsLoading, + } = useClientRequest((cancelToken) => + client.getInstance().getGrafanaTopology({ cancelToken }) + ) + useEffect(() => { - const fetchNum = async () => { - if (!cluster || !cluster.alert_manager) { - return - } + if (!amData) { + return + } + async function fetch() { let resp = await client .getInstance() - .topologyAlertmanagerAddressCountGet( - `${cluster.alert_manager.ip}:${cluster.alert_manager.port}` - ) + .getAlertManagerCounts(`${amData!.ip}:${amData!.port}`) setAlertCounter(resp.data) } - fetchNum() - }, [cluster]) + fetch() + }, [amData]) return ( - -

- {!cluster || !cluster.grafana ? ( - t('overview.monitor_alert.view_monitor_warn') - ) : ( - - {t('overview.monitor_alert.view_monitor')} - + + + {!grafanaData && ( + + + + {t('overview.monitor_alert.view_monitor_warn')} + + + )} + {grafanaData && ( + + + {t('overview.monitor_alert.view_monitor')} + + )} -

-

- {!cluster || !cluster.alert_manager ? ( - t('overview.monitor_alert.view_alerts_warn') - ) : ( - - {alertCounter === 0 - ? t('overview.monitor_alert.view_zero_alerts') - : t('overview.monitor_alert.view_alerts', { - alertCount: alertCounter, - })} - + + + {!amData && ( + + + + {t('overview.monitor_alert.view_alerts_warn')} + + + )} + {amData && ( + + + 0 ? 'danger' : undefined}> + {alertCounter === 0 + ? t('overview.monitor_alert.view_zero_alerts') + : t('overview.monitor_alert.view_alerts', { + alertCount: alertCounter, + })} + + + )} -

-
-

- - {t('overview.monitor_alert.run_diagnose')} - - -

+ +
+ + + {t('overview.monitor_alert.run_diagnose')} + + + +
+
) } diff --git a/ui/lib/apps/Overview/index.tsx b/ui/lib/apps/Overview/index.tsx index 7ab932cc82..f009349e2d 100644 --- a/ui/lib/apps/Overview/index.tsx +++ b/ui/lib/apps/Overview/index.tsx @@ -1,11 +1,10 @@ import { Col, Row } from 'antd' -import React, { useEffect, useState } from 'react' +import React from 'react' import { useTranslation } from 'react-i18next' import { HashRouter as Router, Link } from 'react-router-dom' import { RightOutlined } from '@ant-design/icons' import { StatementsTable, useStatement } from '@lib/apps/Statement' -import client, { ClusterinfoClusterInfo } from '@lib/client' import { DateTime, MetricChart, Root } from '@lib/components' import SlowQueriesTable from '../SlowQuery/components/SlowQueriesTable' @@ -18,7 +17,6 @@ import Instances from './components/Instances' export default function App() { const { t } = useTranslation() - const [cluster, setCluster] = useState(null) const { orderOptions: stmtOrderOptions, changeOrder: changeStmtOrder, @@ -41,18 +39,6 @@ export default function App() { errorMsg, } = useSlowQuery({ ...DEF_SLOW_QUERY_OPTIONS, limit: 10 }, false) - useEffect(() => { - const fetchLoad = async () => { - try { - let res = await client.getInstance().topologyAllGet() - setCluster(res.data) - } catch (error) { - setCluster(null) - } - } - fetchLoad() - }, []) - return ( @@ -155,7 +141,7 @@ export default function App() { - + diff --git a/ui/lib/apps/Overview/translations/en.yaml b/ui/lib/apps/Overview/translations/en.yaml index 429c8594c5..e94469c516 100644 --- a/ui/lib/apps/Overview/translations/en.yaml +++ b/ui/lib/apps/Overview/translations/en.yaml @@ -4,6 +4,8 @@ overview: title: Top SQL Statements recent_slow_query: title: Recent Slow Queries + instances: + title: Alive Instances monitor_alert: title: Monitor & Alert view_monitor: View Metrics @@ -15,10 +17,3 @@ overview: metrics: total_requests: QPS latency: Latency - instances: - title: Instances - component: - tidb: TiDB - tikv: TiKV - pd: PD - tiflash: TiFlash diff --git a/ui/lib/apps/Overview/translations/zh-CN.yaml b/ui/lib/apps/Overview/translations/zh-CN.yaml index 0989e5f45f..0a6574d777 100644 --- a/ui/lib/apps/Overview/translations/zh-CN.yaml +++ b/ui/lib/apps/Overview/translations/zh-CN.yaml @@ -4,6 +4,8 @@ overview: title: Top SQL 语句 recent_slow_query: title: 最近慢查询 + instances: + title: 在线实例 monitor_alert: title: 监控和告警 view_monitor: 查看监控 @@ -15,10 +17,3 @@ overview: metrics: total_requests: QPS latency: 延迟 - instances: - title: 实例 - component: - tidb: TiDB - tikv: TiKV - pd: PD - tiflash: TiFlash diff --git a/ui/lib/apps/SearchLogs/components/SearchHeader.tsx b/ui/lib/apps/SearchLogs/components/SearchHeader.tsx index 699003d246..9eeef2cdc1 100644 --- a/ui/lib/apps/SearchLogs/components/SearchHeader.tsx +++ b/ui/lib/apps/SearchLogs/components/SearchHeader.tsx @@ -1,212 +1,190 @@ -import { Button, Form, Input, Select, TreeSelect } from 'antd' -import { LegacyDataNode } from 'rc-tree-select/lib/interface' -import React, { ChangeEvent, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { useNavigate } from 'react-router-dom' -import { useMount } from '@umijs/hooks' - -import client, { +import client from '@lib/client' +import { LogsearchCreateTaskGroupRequest, ModelRequestTargetNode, } from '@lib/client' +import { Button, Form, Input, Select, Modal } from 'antd' +import React, { useState, useCallback, useRef } from 'react' +import { useTranslation } from 'react-i18next' +import { useNavigate } from 'react-router-dom' +import { useMount } from '@umijs/hooks' import { - calcTimeRange, - DEF_TIME_RANGE, - TimeRange, TimeRangeSelector, + TimeRange, + calcTimeRange, + InstanceSelect, + IInstanceSelectRefProps, } from '@lib/components' -import { - namingMap, - NodeKind, - NodeKindList, - parseClusterInfo, - parseSearchingParams, -} from '../utils' - -import styles from './Styles.module.less' - -const { SHOW_CHILD } = TreeSelect -const { Option } = Select -function buildTreeData(targets: ModelRequestTargetNode[]) { - const servers = { - [NodeKind.TiDB]: [], - [NodeKind.TiKV]: [], - [NodeKind.PD]: [], - [NodeKind.TiFlash]: [], - } - - targets.forEach((item) => { - if (item === undefined || item.kind === undefined) { - return - } - servers[item.kind].push(item) - }) - - return NodeKindList.filter((kind) => servers[kind].length > 0).map( - (kind) => ({ - title: namingMap[kind], - value: kind, - key: kind, - children: servers[kind].map((item: ModelRequestTargetNode) => { - const addr = item.display_name! - return { - title: addr, - value: addr, - key: addr, - } - }), - }) - ) -} +import { ValidLogLevels, LogLevelText } from '../utils' interface Props { taskGroupID?: number } -const LOG_LEVELS = ['debug', 'info', 'warn', 'trace', 'critical', 'error'] +interface IFormProps { + timeRange?: TimeRange + logLevel?: number + instances?: string[] + keywords?: string +} export default function SearchHeader({ taskGroupID }: Props) { const { t } = useTranslation() const navigate = useNavigate() - - const [timeRange, setTimeRange] = useState(DEF_TIME_RANGE) - const [logLevel, setLogLevel] = useState(2) - const [selectedComponents, setComponents] = useState([]) - const [searchValue, setSearchValue] = useState('') - const [allTargets, setAllTargets] = useState([]) + const [form] = Form.useForm() + const [isSubmitting, setSubmitting] = useState(false) + const instanceSelect = useRef(null) useMount(() => { async function fetchData() { - const res = await client.getInstance().topologyAllGet() - const targets = parseClusterInfo(res.data) - setAllTargets(targets) - setComponents(targets.map((item) => item.display_name!)) if (!taskGroupID) { return } - const res2 = await client + const res = await client .getInstance() - .logsTaskgroupsIdGet(taskGroupID + '') - const { - timeRange, - logLevel, - components, - searchValue, - } = parseSearchingParams(res2.data) - setTimeRange(timeRange) - setLogLevel(logLevel === 0 ? 2 : logLevel) - setComponents(components.map((item) => item.display_name ?? '')) - setSearchValue(searchValue) + .logsTaskgroupsIdGet(String(taskGroupID)) + const { task_group, tasks } = res.data + const { start_time, end_time, min_level, patterns } = + task_group?.search_request ?? {} + const fieldsValue: IFormProps = { + timeRange: { + type: 'absolute', + value: [start_time! / 1000, end_time! / 1000], + }, + logLevel: min_level || 2, + instances: (tasks ?? []) + .filter((t) => t.target && t.target!.display_name) + .map((t) => t.target!.display_name!), + keywords: (patterns ?? []).join(' '), + } + form.setFieldsValue(fieldsValue) } fetchData() }) - async function createTaskGroup() { - // TODO: check select at least one component - const targets: ModelRequestTargetNode[] = allTargets.filter((item) => - selectedComponents.some((addr) => addr === item.display_name ?? '') - ) - - const [startTime, endTime] = calcTimeRange(timeRange) - const params: LogsearchCreateTaskGroupRequest = { - targets: targets, - request: { - start_time: startTime * 1000, // unix millionsecond - end_time: endTime * 1000, // unix millionsecond - min_level: logLevel, - patterns: searchValue.split(/\s+/), // 'foo boo' => ['foo', 'boo'] - }, - } - const result = await client.getInstance().logsTaskgroupPut(params) - const id = result.data.task_group?.id - if (!id) { - // promp error here - return - } - navigate('/search_logs/detail/' + id) - } - - function handleTimeRangeChange(value: TimeRange) { - setTimeRange(value) - } - - function handleLogLevelChange(value: number) { - setLogLevel(value) - } - - function handleComponentChange(values: string[]) { - setComponents(values) - } - - function handleSearchPatternChange(e: ChangeEvent) { - setSearchValue(e.target.value) - } - - function handleSearch() { - createTaskGroup() - } + const handleSearch = useCallback( + async (fieldsValue: IFormProps) => { + if ( + !fieldsValue.instances || + fieldsValue.instances.length === 0 || + !fieldsValue.logLevel || + !fieldsValue.timeRange + ) { + Modal.error({ + content: 'Some required fields are not filled', + }) + return + } + if (!instanceSelect.current) { + Modal.error({ + content: 'Internal error: Instance select is not ready', + }) + return + } + setSubmitting(true) + + const targets: ModelRequestTargetNode[] = instanceSelect + .current!.getInstanceByKeys(fieldsValue.instances) + .map((instance) => { + let port + switch (instance.instanceKind) { + case 'pd': + case 'tikv': + case 'tiflash': + port = instance.port + break + case 'tidb': + port = instance.status_port + break + } + return { + kind: instance.instanceKind, + display_name: instance.key, + ip: instance.ip, + port, + } + }) + .filter((i) => i.port != null) + + const [startTime, endTime] = calcTimeRange(fieldsValue.timeRange) + + const req: LogsearchCreateTaskGroupRequest = { + targets, + request: { + start_time: startTime * 1000, // unix millionsecond + end_time: endTime * 1000, // unix millionsecond + min_level: fieldsValue.logLevel, + patterns: (fieldsValue.keywords ?? '').split(/\s+/), // 'foo boo' => ['foo', 'boo'] + }, + } - function filterTreeNode( - inputValue: string, - legacyDataNode?: LegacyDataNode - ): boolean { - const name = legacyDataNode?.key as string - return name.includes(inputValue) - } + try { + const result = await client.getInstance().logsTaskgroupPut(req) + const id = result.data.task_group?.id + if (!id) { + throw new Error('Invalid server response') + } + navigate(`/search_logs/detail/${id}`) + } catch (e) { + // FIXME + Modal.error({ + content: e.message, + }) + } + setSubmitting(false) + }, + [navigate] + ) return ( - - + + - - + {ValidLogLevels.map((val) => ( + +
{LogLevelText[val]}
+
))}
- - - 0 ? '' : 'error'} + rules={[{ required: true }]} > - + + + - diff --git a/ui/lib/apps/SearchLogs/components/SearchProgress.tsx b/ui/lib/apps/SearchLogs/components/SearchProgress.tsx index 17c6f7badb..40172f122b 100644 --- a/ui/lib/apps/SearchLogs/components/SearchProgress.tsx +++ b/ui/lib/apps/SearchLogs/components/SearchProgress.tsx @@ -1,40 +1,38 @@ import { Button, Modal, Tree } from 'antd' import _ from 'lodash' -import React, { useEffect, useState } from 'react' +import React, { useEffect, useState, useMemo, useCallback } from 'react' import { useTranslation } from 'react-i18next' import { getValueFormat } from '@baurine/grafana-value-formats' import client, { LogsearchTaskModel } from '@lib/client' import { AnimatedSkeleton, Card } from '@lib/components' import { FailIcon, LoadingIcon, SuccessIcon } from './Icon' -import { namingMap, NodeKind, NodeKindList, TaskState } from '../utils' +import { TaskState } from '../utils' import styles from './Styles.module.less' +import { InstanceKindName, InstanceKinds } from '@lib/utils/instanceTable' const { confirm } = Modal -const { TreeNode } = Tree const taskStateIcons = { [TaskState.Running]: LoadingIcon, [TaskState.Finished]: SuccessIcon, [TaskState.Error]: FailIcon, } -function leafNodeProps(task: LogsearchTaskModel) { - return { - icon: taskStateIcons[task.state || TaskState.Error], - disableCheckbox: !task.size || task.state !== TaskState.Finished, - } -} - -function renderLeafNodes(tasks: LogsearchTaskModel[]) { +function getLeafNodes(tasks: LogsearchTaskModel[]) { return tasks.map((task) => { - let title = task.target?.display_name ?? '' - if (task.size) { - title += ' (' + getValueFormat('bytes')(task.size!, 1) + ')' - } - return ( - + const title = ( + + {task.target?.display_name ?? ''}{' '} + ({getValueFormat('bytes')(task.size!, 1)}) + ) + return { + key: String(task.id), + title, + icon: taskStateIcons[task.state || TaskState.Error], + disableCheckbox: !task.size || task.state !== TaskState.Finished, + } }) } @@ -81,79 +79,65 @@ export default function SearchProgress({ } }, [tasks]) - const descriptionArray = [ - t('search_logs.progress.running'), - t('search_logs.progress.success'), - t('search_logs.progress.failed'), - ] - - function progressDescription(tasks: LogsearchTaskModel[]) { - const arr = [0, 0, 0] - tasks.forEach((task) => { - const state = task.state - if (state !== undefined) { - arr[state - 1]++ - } - }) - const res: string[] = [] - arr.forEach((count, index) => { - if (index < 1 || count <= 0) { - return - } - const str = `${count} ${descriptionArray[index]}` - res.push(str) - }) - return ( - res.join(',') + - ' (' + - getValueFormat('bytes')(_.sumBy(tasks, 'size'), 1) + - ')' - ) - } + const descriptionArray = useMemo( + () => [ + t('search_logs.progress.running'), + t('search_logs.progress.success'), + t('search_logs.progress.failed'), + ], + [t] + ) - function renderTreeNodes(tasks: LogsearchTaskModel[]) { - const servers = { - [NodeKind.TiDB]: [], - [NodeKind.TiKV]: [], - [NodeKind.PD]: [], - [NodeKind.TiFlash]: [], - } + const describeProgress = useCallback( + (tasks: LogsearchTaskModel[]) => { + const arr = [0, 0, 0] + tasks.forEach((task) => { + const state = task.state + if (state !== undefined) { + arr[state - 1]++ + } + }) + const res: string[] = [] + arr.forEach((count, index) => { + if (index < 1 || count <= 0) { + return + } + const str = `${count} ${descriptionArray[index]}` + res.push(str) + }) + return ( + res.join(', ') + + ' (' + + getValueFormat('bytes')(_.sumBy(tasks, 'size'), 1) + + ')' + ) + }, + [descriptionArray] + ) - tasks.forEach((task) => { - if (task.target?.kind === undefined) { + const treeData = useMemo(() => { + const data: any[] = [] + const tasksByIK = _.groupBy(tasks, (t) => t.target?.kind) + InstanceKinds.forEach((ik) => { + const tasks = tasksByIK[ik] + if (!tasks) { return } - servers[task.target.kind].push(task) + const title = ( + + {InstanceKindName[ik]} {describeProgress(tasks)} + + ) + data.push({ + title, + key: ik, + icon: parentNodeIcon(tasks), + disableCheckbox: !parentNodeCheckable(tasks), + children: getLeafNodes(tasks), + }) }) - - return NodeKindList.filter((kind) => servers[kind].length > 0).map( - (kind) => { - const tasks: LogsearchTaskModel[] = servers[kind] - const title = ( - - {namingMap[kind]} - - {progressDescription(tasks)} - - - ) - return ( - - ) - } - ) - } + return data + }, [tasks, describeProgress]) async function handleDownload() { if (taskGroupID < 0) { @@ -161,7 +145,7 @@ export default function SearchProgress({ } // filter out all parent node const keys = checkedKeys.filter( - (key) => !Object.keys(namingMap).some((name) => name === key) + (key) => !InstanceKinds.some((ik) => ik === key) ) const res = await client.getInstance().logsDownloadAcquireTokenGet(keys) @@ -199,9 +183,9 @@ export default function SearchProgress({ }) } - const handleCheck = (checkedKeys) => { + const handleCheck = useCallback((checkedKeys) => { setCheckedKeys(checkedKeys as string[]) - } + }, []) return ( {tasks && ( <> -
{progressDescription(tasks)}
+
{describeProgress(tasks)}