diff --git a/build/charts/theia/templates/theia-manager/service.yaml b/build/charts/theia/templates/theia-manager/service.yaml index 66ec3931..3dfe6ee9 100644 --- a/build/charts/theia/templates/theia-manager/service.yaml +++ b/build/charts/theia/templates/theia-manager/service.yaml @@ -9,7 +9,6 @@ metadata: spec: ports: - port: {{ .Values.theiaManager.apiServer.apiPort }} - name: tcp protocol: TCP targetPort: theia-api-http selector: diff --git a/pkg/apis/stats/v1alpha1/types.go b/pkg/apis/stats/v1alpha1/types.go index 718cb90b..d8320966 100644 --- a/pkg/apis/stats/v1alpha1/types.go +++ b/pkg/apis/stats/v1alpha1/types.go @@ -14,7 +14,9 @@ package v1alpha1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // +genclient // +genclient:nonNamespaced @@ -24,5 +26,39 @@ type ClickHouseStats struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Stat [][]string `json:"stat,omitempty"` + DiskInfos []DiskInfo `json:"diskInfos,omitempty"` + TableInfos []TableInfo `json:"tableInfos,omitempty"` + InsertRates []InsertRate `json:"insertRates,omitempty"` + StackTraces []StackTrace `json:"stackTraces,omitempty"` + ErrorMsg []string `json:"errorMsg,omitempty"` +} + +type DiskInfo struct { + Shard string `json:"shard,omitempty"` + Database string `json:"name,omitempty"` + Path string `json:"path,omitempty"` + FreeSpace string `json:"freeSpace,omitempty"` + TotalSpace string `json:"totalSpace,omitempty"` + UsedPercentage string `json:"usedPercentage,omitempty"` +} + +type TableInfo struct { + Shard string `json:"shard,omitempty"` + Database string `json:"database,omitempty"` + TableName string `json:"tableName,omitempty"` + TotalRows string `json:"totalRows,omitempty"` + TotalBytes string `json:"totalBytes,omitempty"` + TotalCols string `json:"totalCols,omitempty"` +} + +type InsertRate struct { + Shard string `json:"shard,omitempty"` + RowsPerSec string `json:"rowsPerSec,omitempty"` + BytesPerSec string `json:"bytesPerSec,omitempty"` +} + +type StackTrace struct { + Shard string `json:"shard,omitempty"` + TraceFunctions string `json:"traceFunctions,omitempty"` + Count string `json:"count,omitempty"` } diff --git a/pkg/apis/stats/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/stats/v1alpha1/zz_generated.deepcopy.go index a6c9feb3..c8d5b64b 100644 --- a/pkg/apis/stats/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/stats/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,26 @@ func (in *ClickHouseStats) DeepCopyInto(out *ClickHouseStats) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.DiskInfos != nil { + in, out := &in.DiskInfos, &out.DiskInfos + *out = make([]DiskInfo, len(*in)) + copy(*out, *in) + } + if in.TableInfos != nil { + in, out := &in.TableInfos, &out.TableInfos + *out = make([]TableInfo, len(*in)) + copy(*out, *in) + } + if in.InsertRates != nil { + in, out := &in.InsertRates, &out.InsertRates + *out = make([]InsertRate, len(*in)) + copy(*out, *in) + } + if in.StackTraces != nil { + in, out := &in.StackTraces, &out.StackTraces + *out = make([]StackTrace, len(*in)) + copy(*out, *in) + } return } @@ -48,3 +68,67 @@ func (in *ClickHouseStats) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DiskInfo) DeepCopyInto(out *DiskInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DiskInfo. +func (in *DiskInfo) DeepCopy() *DiskInfo { + if in == nil { + return nil + } + out := new(DiskInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InsertRate) DeepCopyInto(out *InsertRate) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InsertRate. +func (in *InsertRate) DeepCopy() *InsertRate { + if in == nil { + return nil + } + out := new(InsertRate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StackTrace) DeepCopyInto(out *StackTrace) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StackTrace. +func (in *StackTrace) DeepCopy() *StackTrace { + if in == nil { + return nil + } + out := new(StackTrace) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TableInfo) DeepCopyInto(out *TableInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TableInfo. +func (in *TableInfo) DeepCopy() *TableInfo { + if in == nil { + return nil + } + out := new(TableInfo) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apiserver/registry/stats/clickhouse/rest.go b/pkg/apiserver/registry/stats/clickhouse/rest.go index d82fad74..5a5919b4 100644 --- a/pkg/apiserver/registry/stats/clickhouse/rest.go +++ b/pkg/apiserver/registry/stats/clickhouse/rest.go @@ -48,24 +48,36 @@ func (r *REST) New() runtime.Object { } func (r *REST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { - var stats [][]string + var status v1alpha1.ClickHouseStats var err error switch name { case "diskInfo": - stats, err = r.clickHouseStatusQuerier.GetDiskInfo(defaultNameSpace) + err = r.clickHouseStatusQuerier.GetDiskInfo(defaultNameSpace, &status) + if status.DiskInfos == nil { + return nil, fmt.Errorf("no diskInfo data is returned by database") + } case "tableInfo": - stats, err = r.clickHouseStatusQuerier.GetTableInfo(defaultNameSpace) + err = r.clickHouseStatusQuerier.GetTableInfo(defaultNameSpace, &status) + if status.TableInfos == nil { + return nil, fmt.Errorf("no tableInfo data is returned by database") + } case "insertRate": - stats, err = r.clickHouseStatusQuerier.GetInsertRate(defaultNameSpace) - case "stackTraces": - stats, err = r.clickHouseStatusQuerier.GetStackTraces(defaultNameSpace) + err = r.clickHouseStatusQuerier.GetInsertRate(defaultNameSpace, &status) + if status.InsertRates == nil { + return nil, fmt.Errorf("no insertRate data is returned by database") + } + case "stackTrace": + err = r.clickHouseStatusQuerier.GetStackTrace(defaultNameSpace, &status) + if status.StackTraces == nil { + return nil, fmt.Errorf("no stackTrace data is returned by database") + } default: return nil, fmt.Errorf("cannot recognize the statua name: %s", name) } if err != nil { return nil, fmt.Errorf("error when sending query to ClickHouse: %s", err) } - return &v1alpha1.ClickHouseStats{Stat: stats}, nil + return &status, nil } func (r *REST) NamespaceScoped() bool { diff --git a/pkg/apiserver/registry/stats/clickhouse/rest_test.go b/pkg/apiserver/registry/stats/clickhouse/rest_test.go index e2a142a2..26245d38 100644 --- a/pkg/apiserver/registry/stats/clickhouse/rest_test.go +++ b/pkg/apiserver/registry/stats/clickhouse/rest_test.go @@ -32,31 +32,47 @@ func TestREST_Get(t *testing.T) { name string queryName string expectErr error - expectResult [][]string + expectResult *stats.ClickHouseStats }{ { - name: "Get diskInfo", - queryName: "diskInfo", - expectErr: nil, - expectResult: [][]string{{"diskInfo_test"}}, + name: "Get diskInfo", + queryName: "diskInfo", + expectErr: nil, + expectResult: &stats.ClickHouseStats{ + DiskInfos: []stats.DiskInfo{{ + Shard: "Shard_test", + }}, + }, }, { - name: "Get tableInfo", - queryName: "tableInfo", - expectErr: nil, - expectResult: [][]string{{"tableInfo_test"}}, + name: "Get tableInfo", + queryName: "tableInfo", + expectErr: nil, + expectResult: &stats.ClickHouseStats{ + TableInfos: []stats.TableInfo{{ + Shard: "Shard_test", + }}, + }, }, { - name: "Get insertRate", - queryName: "insertRate", - expectErr: nil, - expectResult: [][]string{{"insertRate_test"}}, + name: "Get insertRate", + queryName: "insertRate", + expectErr: nil, + expectResult: &stats.ClickHouseStats{ + InsertRates: []stats.InsertRate{{ + Shard: "Shard_test", + }}, + }, }, { - name: "Get stackTraces", - queryName: "stackTraces", - expectErr: nil, - expectResult: [][]string{{"stackTraces_test"}}, + name: "Get stackTraces", + queryName: "stackTrace", + expectErr: nil, + expectResult: &stats.ClickHouseStats{ + StackTraces: []stats.StackTrace{{ + Shard: "Shard_test", + }}, + }, }, { name: "not found", @@ -73,7 +89,7 @@ func TestREST_Get(t *testing.T) { assert.NoError(t, err) status, ok := result.(*stats.ClickHouseStats) assert.True(t, ok) - assert.ElementsMatch(t, tt.expectResult, status.Stat) + assert.EqualValues(t, tt.expectResult, status) } else { assert.Error(t, err) @@ -83,15 +99,27 @@ func TestREST_Get(t *testing.T) { } } -func (c *fakeQuerier) GetDiskInfo(namespace string) ([][]string, error) { - return [][]string{{"diskInfo_test"}}, nil +func (c *fakeQuerier) GetDiskInfo(namespace string, status *stats.ClickHouseStats) error { + status.DiskInfos = []stats.DiskInfo{{ + Shard: "Shard_test", + }} + return nil } -func (c *fakeQuerier) GetTableInfo(namespace string) ([][]string, error) { - return [][]string{{"tableInfo_test"}}, nil +func (c *fakeQuerier) GetTableInfo(namespace string, status *stats.ClickHouseStats) error { + status.TableInfos = []stats.TableInfo{{ + Shard: "Shard_test", + }} + return nil } -func (c *fakeQuerier) GetInsertRate(namespace string) ([][]string, error) { - return [][]string{{"insertRate_test"}}, nil +func (c *fakeQuerier) GetInsertRate(namespace string, status *stats.ClickHouseStats) error { + status.InsertRates = []stats.InsertRate{{ + Shard: "Shard_test", + }} + return nil } -func (c *fakeQuerier) GetStackTraces(namespace string) ([][]string, error) { - return [][]string{{"stackTraces_test"}}, nil +func (c *fakeQuerier) GetStackTrace(namespace string, status *stats.ClickHouseStats) error { + status.StackTraces = []stats.StackTrace{{ + Shard: "Shard_test", + }} + return nil } diff --git a/pkg/apiserver/utils/stats/clickhouse_stats.go b/pkg/apiserver/utils/stats/clickhouse_stats.go index 43bc525f..41a5a12e 100644 --- a/pkg/apiserver/utils/stats/clickhouse_stats.go +++ b/pkg/apiserver/utils/stats/clickhouse_stats.go @@ -20,45 +20,16 @@ import ( "k8s.io/client-go/kubernetes" + "antrea.io/theia/pkg/apis/stats/v1alpha1" "antrea.io/theia/pkg/util" ) -type diskInfo struct { - shard string - name string - path string - freeSpace string - totalSpace string - usedPercentage string -} - -type tableInfo struct { - shard string - database string - tableName string - totalRows sql.NullString - totalBytes sql.NullString - totalCols string -} - -type insertRate struct { - shard string - rowsPerSec string - bytesPerSec string -} - -type stackTraces struct { - shard string - traceFunctions string - count string -} - const ( diskQuery int = iota tableInfoQuery // average writing rate for all tables per second insertRateQuery - stackTracesQuery + stackTraceQuery ) var queryMap = map[int]string{ @@ -117,7 +88,7 @@ FROM ( ORDER BY t DESC, shardNum() ) sd WHERE sd.rowNumber=1`, - stackTracesQuery: ` + stackTraceQuery: ` SELECT shardNum() as Shard, arrayStringConcat(arrayMap(x -> demangle(addressToSymbol(x)), trace), '\\n') AS trace_function, @@ -142,87 +113,96 @@ func NewClickHouseStatQuerierImpl( return c } -func (c *ClickHouseStatQuerierImpl) GetDiskInfo(namespace string) ([][]string, error) { - data, err := c.getDataFromClickHouse(diskQuery, namespace) +func (c *ClickHouseStatQuerierImpl) GetDiskInfo(namespace string, stats *v1alpha1.ClickHouseStats) error { + err := c.getDataFromClickHouse(diskQuery, namespace, stats) if err != nil { - return nil, fmt.Errorf("error when getting diskInfo from clickhouse: %v", err) + return fmt.Errorf("error when getting diskInfo from clickhouse: %v", err) } - return data, nil + return nil } -func (c *ClickHouseStatQuerierImpl) GetTableInfo(namespace string) ([][]string, error) { - data, err := c.getDataFromClickHouse(tableInfoQuery, namespace) +func (c *ClickHouseStatQuerierImpl) GetTableInfo(namespace string, stats *v1alpha1.ClickHouseStats) error { + err := c.getDataFromClickHouse(tableInfoQuery, namespace, stats) if err != nil { - return nil, fmt.Errorf("error when getting tableInfo from clickhouse: %v", err) + return fmt.Errorf("error when getting tableInfo from clickhouse: %v", err) } - return data, nil + return nil } -func (c *ClickHouseStatQuerierImpl) GetInsertRate(namespace string) ([][]string, error) { - data, err := c.getDataFromClickHouse(insertRateQuery, namespace) +func (c *ClickHouseStatQuerierImpl) GetInsertRate(namespace string, stats *v1alpha1.ClickHouseStats) error { + err := c.getDataFromClickHouse(insertRateQuery, namespace, stats) if err != nil { - return nil, fmt.Errorf("error when getting insertRate from clickhouse: %v", err) + return fmt.Errorf("error when getting insertRate from clickhouse: %v", err) } - return data, nil + return nil } -func (c *ClickHouseStatQuerierImpl) GetStackTraces(namespace string) ([][]string, error) { - data, err := c.getDataFromClickHouse(stackTracesQuery, namespace) +func (c *ClickHouseStatQuerierImpl) GetStackTrace(namespace string, stats *v1alpha1.ClickHouseStats) error { + err := c.getDataFromClickHouse(stackTraceQuery, namespace, stats) if err != nil { - return nil, fmt.Errorf("error when getting stackTraces from clickhouse: %v", err) + return fmt.Errorf("error when getting stackTrace from clickhouse: %v", err) } - return data, nil + return nil } -func (c *ClickHouseStatQuerierImpl) getDataFromClickHouse(query int, namespace string) ([][]string, error) { +func (c *ClickHouseStatQuerierImpl) getDataFromClickHouse(query int, namespace string, stats *v1alpha1.ClickHouseStats) error { var err error if c.clickhouseConnect == nil { c.clickhouseConnect, err = util.SetupClickHouseConnection(c.kubeClient, namespace) if err != nil { - return nil, err + return err } } result, err := c.clickhouseConnect.Query(queryMap[query]) if err != nil { c.clickhouseConnect = nil - return nil, fmt.Errorf("failed to get data from clickhouse: %v", err) + return fmt.Errorf("failed to get data from clickhouse: %v", err) } defer result.Close() - columnName, err := result.Columns() - if err != nil { - return nil, fmt.Errorf("failed to get the name of columns: %v", err) - } - var data [][]string - data = append(data, columnName) for result.Next() { var err error switch query { case diskQuery: - var res diskInfo - err = result.Scan(&res.shard, &res.name, &res.path, &res.freeSpace, &res.totalSpace, &res.usedPercentage) - data = append(data, []string{res.shard, res.name, res.path, res.freeSpace, res.totalSpace, res.usedPercentage + " %"}) + res := v1alpha1.DiskInfo{} + err = result.Scan(&res.Shard, &res.Database, &res.Path, &res.FreeSpace, &res.TotalSpace, &res.UsedPercentage) + if err != nil { + stats.ErrorMsg = append(stats.ErrorMsg, fmt.Sprintf("failed to parse the data returned by database: %v", err)) + continue + } + res.UsedPercentage = res.UsedPercentage + " %" + stats.DiskInfos = append(stats.DiskInfos, res) case tableInfoQuery: - res := tableInfo{} - err = result.Scan(&res.shard, &res.database, &res.tableName, &res.totalRows, &res.totalBytes, &res.totalCols) - if !res.totalRows.Valid || !res.totalBytes.Valid { + res := v1alpha1.TableInfo{} + var totalRows sql.NullString + var totalBytes sql.NullString + err = result.Scan(&res.Shard, &res.Database, &res.TableName, &totalRows, &totalBytes, &res.TotalCols) + if err != nil { + stats.ErrorMsg = append(stats.ErrorMsg, fmt.Sprintf("failed to parse the data returned by database: %v", err)) + continue + } + if !totalRows.Valid || !totalBytes.Valid { continue } - data = append(data, []string{res.shard, res.database, res.tableName, res.totalRows.String, res.totalBytes.String, res.totalCols}) + res.TotalRows = totalRows.String + res.TotalBytes = totalBytes.String + stats.TableInfos = append(stats.TableInfos, res) case insertRateQuery: - res := insertRate{} - err = result.Scan(&res.shard, &res.rowsPerSec, &res.bytesPerSec) - data = append(data, []string{res.shard, res.rowsPerSec, res.bytesPerSec}) - case stackTracesQuery: - res := stackTraces{} - err = result.Scan(&res.shard, &res.traceFunctions, &res.count) - data = append(data, []string{res.shard, res.traceFunctions, res.count}) - } - if err != nil { - return nil, fmt.Errorf("failed to parse the data returned by database: %v", err) + res := v1alpha1.InsertRate{} + err = result.Scan(&res.Shard, &res.RowsPerSec, &res.BytesPerSec) + if err != nil { + stats.ErrorMsg = append(stats.ErrorMsg, fmt.Sprintf("failed to parse the data returned by database: %v", err)) + continue + } + stats.InsertRates = append(stats.InsertRates, res) + case stackTraceQuery: + res := v1alpha1.StackTrace{} + err = result.Scan(&res.Shard, &res.TraceFunctions, &res.Count) + if err != nil { + stats.ErrorMsg = append(stats.ErrorMsg, fmt.Sprintf("failed to parse the data returned by database: %v", err)) + continue + } + stats.StackTraces = append(stats.StackTraces, res) } } - if len(data) <= 1 { - return nil, fmt.Errorf("no data is returned by database") - } - return data, nil + return nil } diff --git a/pkg/apiserver/utils/stats/clickhouse_stats_test.go b/pkg/apiserver/utils/stats/clickhouse_stats_test.go index 4af146f1..6f988e29 100644 --- a/pkg/apiserver/utils/stats/clickhouse_stats_test.go +++ b/pkg/apiserver/utils/stats/clickhouse_stats_test.go @@ -15,13 +15,14 @@ package stats import ( - "fmt" "regexp" "testing" "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "antrea.io/theia/pkg/apis/stats/v1alpha1" "antrea.io/theia/pkg/theia/commands/config" ) @@ -29,44 +30,89 @@ func TestGetDataFromClickHouse(t *testing.T) { testCases := []struct { name string query int - expectedError error - expectedResult [][]string + expectedResult *v1alpha1.ClickHouseStats returnedRow *sqlmock.Rows }{ { - name: "Get diskInfo", - query: diskQuery, - expectedError: nil, - returnedRow: sqlmock.NewRows([]string{"Shard", "DatabaseName", "Path", "Free", "Total", "Used_Percentage"}).AddRow("a", "b", "c", "d", "e", "f"), - expectedResult: [][]string{{"Shard", "DatabaseName", "Path", "Free", "Total", "Used_Percentage"}, {"a", "b", "c", "d", "e", "f %"}}, + name: "Get diskInfo", + query: diskQuery, + returnedRow: sqlmock.NewRows([]string{"Shard", "DatabaseName", "Path", "Free", "Total", "Used_Percentage"}).AddRow("a", "b", "c", "d", "e", "f"), + expectedResult: &v1alpha1.ClickHouseStats{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + DiskInfos: []v1alpha1.DiskInfo{ + {Shard: "a", Database: "b", Path: "c", FreeSpace: "d", TotalSpace: "e", UsedPercentage: "f %"}, + }, + TableInfos: nil, + InsertRates: nil, + StackTraces: nil, + }, }, { - name: "Get tableInfo", - query: tableInfoQuery, - expectedError: nil, - returnedRow: sqlmock.NewRows([]string{"Shard", "DatabaseName", "TableName", "TotalRows", "TotalBytes", "TotalCols"}).AddRow("a", "b", "c", "d", "e", "f"), - expectedResult: [][]string{{"Shard", "DatabaseName", "TableName", "TotalRows", "TotalBytes", "TotalCols"}, {"a", "b", "c", "d", "e", "f"}}, + name: "Get tableInfo", + query: tableInfoQuery, + returnedRow: sqlmock.NewRows([]string{"Shard", "DatabaseName", "TableName", "TotalRows", "TotalBytes", "TotalCols"}).AddRow("a", "b", "c", "d", "e", "f"), + expectedResult: &v1alpha1.ClickHouseStats{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + DiskInfos: nil, + TableInfos: []v1alpha1.TableInfo{{Shard: "a", Database: "b", TableName: "c", TotalRows: "d", TotalBytes: "e", TotalCols: "f"}}, + InsertRates: nil, + StackTraces: nil, + }, }, { - name: "Get insertRate", - query: insertRateQuery, - expectedError: nil, - returnedRow: sqlmock.NewRows([]string{"Shard", "RowsPerSecond", "BytesPerSecond"}).AddRow("a", "b", "c"), - expectedResult: [][]string{{"Shard", "RowsPerSecond", "BytesPerSecond"}, {"a", "b", "c"}}, + name: "Get insertRate", + query: insertRateQuery, + returnedRow: sqlmock.NewRows([]string{"Shard", "RowsPerSecond", "BytesPerSecond"}).AddRow("a", "b", "c"), + expectedResult: &v1alpha1.ClickHouseStats{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + DiskInfos: nil, + TableInfos: nil, + InsertRates: []v1alpha1.InsertRate{{Shard: "a", RowsPerSec: "b", BytesPerSec: "c"}}, + StackTraces: nil, + }, }, { - name: "Get stackTraces", - query: stackTracesQuery, - expectedError: nil, - returnedRow: sqlmock.NewRows([]string{"Shard", "trace_function", "count()"}).AddRow("a", "b", "c"), - expectedResult: [][]string{{"Shard", "trace_function", "count()"}, {"a", "b", "c"}}, + name: "Get stackTraces", + query: stackTraceQuery, + returnedRow: sqlmock.NewRows([]string{"Shard", "trace_function", "count()"}).AddRow("a", "b", "c"), + expectedResult: &v1alpha1.ClickHouseStats{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + DiskInfos: nil, + TableInfos: nil, + InsertRates: nil, + StackTraces: []v1alpha1.StackTrace{{Shard: "a", TraceFunctions: "b", Count: "c"}}, + }, }, { - name: "Empty result", - query: stackTracesQuery, - expectedError: fmt.Errorf("no data is returned by database"), - returnedRow: sqlmock.NewRows([]string{"Shard", "trace_function", "count()"}), - expectedResult: nil, + name: "Empty result", + query: stackTraceQuery, + returnedRow: sqlmock.NewRows([]string{"Shard", "trace_function", "count()"}), + expectedResult: &v1alpha1.ClickHouseStats{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + DiskInfos: nil, + TableInfos: nil, + InsertRates: nil, + StackTraces: nil, + }, + }, + { + name: "Scan error", + query: stackTraceQuery, + returnedRow: sqlmock.NewRows([]string{"Shard", "trace_function", "count()"}).AddRow(nil, "b", "c").AddRow("test1", "test2", "test3"), + expectedResult: &v1alpha1.ClickHouseStats{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + DiskInfos: nil, + TableInfos: nil, + InsertRates: nil, + StackTraces: []v1alpha1.StackTrace{{Shard: "test1", TraceFunctions: "test2", Count: "test3"}}, + ErrorMsg: []string{"failed to parse the data returned by database: sql: Scan error on column index 0, name \"Shard\": converting NULL to string is unsupported"}, + }, }, } for _, tc := range testCases { @@ -75,15 +121,12 @@ func TestGetDataFromClickHouse(t *testing.T) { assert.NoError(t, err) mock.ExpectQuery(regexp.QuoteMeta(queryMap[tc.query])).WillReturnRows(tc.returnedRow) controller := ClickHouseStatQuerierImpl{clickhouseConnect: db} - result, err := controller.getDataFromClickHouse(tc.query, config.FlowVisibilityNS) + var result v1alpha1.ClickHouseStats + err = controller.getDataFromClickHouse(tc.query, config.FlowVisibilityNS, &result) + + assert.NoError(t, err) + assert.EqualValues(t, tc.expectedResult, &result) - if tc.expectedError == nil { - assert.NoError(t, err) - assert.ElementsMatch(t, result, tc.expectedResult) - } else { - assert.Error(t, err) - assert.Contains(t, err.Error(), tc.expectedError.Error()) - } }) } } diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 319c577d..412933bd 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -16,6 +16,7 @@ package querier import ( "antrea.io/theia/pkg/apis/crd/v1alpha1" + statsV1 "antrea.io/theia/pkg/apis/stats/v1alpha1" ) type NPRecommendationQuerier interface { @@ -26,8 +27,8 @@ type NPRecommendationQuerier interface { } type ClickHouseStatQuerier interface { - GetDiskInfo(namespace string) ([][]string, error) - GetTableInfo(namespace string) ([][]string, error) - GetInsertRate(namespace string) ([][]string, error) - GetStackTraces(namespace string) ([][]string, error) + GetDiskInfo(namespace string, stats *statsV1.ClickHouseStats) error + GetTableInfo(namespace string, stats *statsV1.ClickHouseStats) error + GetInsertRate(namespace string, stats *statsV1.ClickHouseStats) error + GetStackTrace(namespace string, stats *statsV1.ClickHouseStats) error } diff --git a/pkg/theia/commands/clickhouse_status.go b/pkg/theia/commands/clickhouse_status.go index fd7a5b37..b8bb47ee 100644 --- a/pkg/theia/commands/clickhouse_status.go +++ b/pkg/theia/commands/clickhouse_status.go @@ -22,10 +22,10 @@ import ( ) type chOptions struct { - diskInfo bool - tableInfo bool - insertRate bool - stackTraces bool + diskInfo bool + tableInfo bool + insertRate bool + stackTrace bool } var options *chOptions @@ -50,11 +50,11 @@ func init() { clickHouseStatusCmd.Flags().BoolVar(&options.diskInfo, "diskInfo", false, "check disk usage information") clickHouseStatusCmd.Flags().BoolVar(&options.tableInfo, "tableInfo", false, "check basic table information") clickHouseStatusCmd.Flags().BoolVar(&options.insertRate, "insertRate", false, "check the insertion-rate of clickhouse") - clickHouseStatusCmd.Flags().BoolVar(&options.stackTraces, "stackTraces", false, "check stacktrace of clickhouse") + clickHouseStatusCmd.Flags().BoolVar(&options.stackTrace, "stackTrace", false, "check stacktrace of clickhouse") } func getStatus(cmd *cobra.Command, args []string) error { - if !options.diskInfo && !options.tableInfo && !options.insertRate && !options.stackTraces { + if !options.diskInfo && !options.tableInfo && !options.insertRate && !options.stackTrace { return fmt.Errorf("no metric related flag is specified") } useClusterIP, err := cmd.Flags().GetBool("use-cluster-ip") @@ -78,18 +78,46 @@ func getStatus(cmd *cobra.Command, args []string) error { if options.insertRate { names = append(names, "insertRate") } - if options.stackTraces { - names = append(names, "stackTraces") + if options.stackTrace { + names = append(names, "stackTrace") } for _, name := range names { data, err := getClickHouseStatusByCategory(theiaClient, name) if err != nil { return fmt.Errorf("error when getting clickhouse %v status: %s", name, err) } - if name == "stackTraces" { - TableOutputVertical(data.Stat) + if len(data.ErrorMsg) != 0 { + for _, errorMsg := range data.ErrorMsg { + fmt.Printf("Error message: %s\n", errorMsg) + } + } + var result [][]string + switch name { + case "diskInfo": + result = append(result, []string{"Shard", "DatabaseName", "Path", "Free", "Total", "Used_Percentage"}) + for _, diskInfo := range data.DiskInfos { + result = append(result, []string{diskInfo.Shard, diskInfo.Database, diskInfo.Path, diskInfo.FreeSpace, diskInfo.TotalSpace, diskInfo.UsedPercentage}) + } + case "tableInfo": + result = append(result, []string{"Shard", "DatabaseName", "TableName", "TotalRows", "TotalBytes", "TotalCols"}) + for _, tableInfo := range data.TableInfos { + result = append(result, []string{tableInfo.Shard, tableInfo.Database, tableInfo.TableName, tableInfo.TotalRows, tableInfo.TotalBytes, tableInfo.TotalCols}) + } + case "insertRate": + result = append(result, []string{"Shard", "RowsPerSecond", "BytesPerSecond"}) + for _, insertRate := range data.InsertRates { + result = append(result, []string{insertRate.Shard, insertRate.RowsPerSec, insertRate.BytesPerSec}) + } + case "stackTrace": + result = append(result, []string{"Shard", "TraceFunctions", "Count()"}) + for _, stackTrace := range data.StackTraces { + result = append(result, []string{stackTrace.Shard, stackTrace.TraceFunctions, stackTrace.Count}) + } + } + if name == "stackTrace" { + TableOutputVertical(result) } else { - TableOutput(data.Stat) + TableOutput(result) } } return nil diff --git a/pkg/theia/commands/clickhouse_status_test.go b/pkg/theia/commands/clickhouse_status_test.go index 3d52c2e2..b6d9207b 100644 --- a/pkg/theia/commands/clickhouse_status_test.go +++ b/pkg/theia/commands/clickhouse_status_test.go @@ -37,7 +37,7 @@ func TestGetStatus(t *testing.T) { name string testServer *httptest.Server expectedErrorMsg string - expectedMsg string + expectedMsg []string options *chOptions }{ { @@ -46,7 +46,14 @@ func TestGetStatus(t *testing.T) { switch strings.TrimSpace(r.URL.Path) { case fmt.Sprintf("/apis/stats.theia.antrea.io/v1alpha1/clickhouse/diskInfo"): status := &stats.ClickHouseStats{ - Stat: [][]string{{"test_diskInfo"}}, + DiskInfos: []stats.DiskInfo{{ + Shard: "Shard_test", + Database: "Database_test", + Path: "Path_test", + FreeSpace: "FreeSpace_test", + TotalSpace: "TotalSpace", + UsedPercentage: "UsedPercentage_test", + }}, } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -55,7 +62,8 @@ func TestGetStatus(t *testing.T) { })), options: &chOptions{diskInfo: true}, expectedErrorMsg: "", - expectedMsg: "test_diskInfo", + expectedMsg: []string{"Shard", "DatabaseName", "Path", "Free", "Total", "Used_Percentage", + "Shard_test", "Database_test", "Path_test", "FreeSpace_test", "TotalSpace", "UsedPercentage_test"}, }, { name: "Get tableInfo", @@ -63,7 +71,14 @@ func TestGetStatus(t *testing.T) { switch strings.TrimSpace(r.URL.Path) { case fmt.Sprintf("/apis/stats.theia.antrea.io/v1alpha1/clickhouse/tableInfo"): status := &stats.ClickHouseStats{ - Stat: [][]string{{"test_tableInfo"}}, + TableInfos: []stats.TableInfo{{ + Shard: "Shard_test", + Database: "Database_test", + TableName: "TableName_test", + TotalRows: "TotalRows_test", + TotalBytes: "TotalBytes_test", + TotalCols: "TotalCols_test", + }}, } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -72,7 +87,8 @@ func TestGetStatus(t *testing.T) { })), options: &chOptions{tableInfo: true}, expectedErrorMsg: "", - expectedMsg: "test_tableInfo", + expectedMsg: []string{"Shard", "DatabaseName", "TableName", "TotalRows", "TotalBytes", "TotalCols", + "Shard_test", "Database_test", "TableName_test", "TotalRows_test", "TotalBytes_test", "TotalCols_test"}, }, { name: "Get insertRate", @@ -80,7 +96,11 @@ func TestGetStatus(t *testing.T) { switch strings.TrimSpace(r.URL.Path) { case fmt.Sprintf("/apis/stats.theia.antrea.io/v1alpha1/clickhouse/insertRate"): status := &stats.ClickHouseStats{ - Stat: [][]string{{"test_insertRate"}}, + InsertRates: []stats.InsertRate{{ + Shard: "Shard_test", + RowsPerSec: "RowsPerSec_test", + BytesPerSec: "RowsPerSec_test", + }}, } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -89,31 +109,61 @@ func TestGetStatus(t *testing.T) { })), options: &chOptions{insertRate: true}, expectedErrorMsg: "", - expectedMsg: "test_insertRate", + expectedMsg: []string{"Shard", "RowsPerSecond", "RowsPerSecond", + "Shard_test", "RowsPerSec_test", "RowsPerSec_test"}, }, { name: "Get stackTraces", testServer: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch strings.TrimSpace(r.URL.Path) { - case fmt.Sprintf("/apis/stats.theia.antrea.io/v1alpha1/clickhouse/stackTraces"): + case fmt.Sprintf("/apis/stats.theia.antrea.io/v1alpha1/clickhouse/stackTrace"): status := &stats.ClickHouseStats{ - Stat: [][]string{{"test_stackTraces"}, {"fakeData"}}, + StackTraces: []stats.StackTrace{{ + Shard: "Shard_test", + TraceFunctions: "TraceFunctions_test", + Count: "Count_test", + }}, } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(status) } })), - options: &chOptions{stackTraces: true}, + options: &chOptions{stackTrace: true}, expectedErrorMsg: "", - expectedMsg: "test_stackTraces: fakeData", + expectedMsg: []string{"Shard", "TraceFunctions", "Count()", + "Shard_test", "TraceFunctions_test", "Count_test"}, }, { name: "No metrics specified", testServer: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})), options: &chOptions{}, expectedErrorMsg: "no metric related flag is specified", - expectedMsg: "", + expectedMsg: nil, + }, + { + name: "ErrorMsg in response is not empty", + testServer: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch strings.TrimSpace(r.URL.Path) { + case fmt.Sprintf("/apis/stats.theia.antrea.io/v1alpha1/clickhouse/stackTrace"): + status := &stats.ClickHouseStats{ + StackTraces: []stats.StackTrace{{ + Shard: "Shard_test", + TraceFunctions: "TraceFunctions_test", + Count: "Count_test", + }}, + ErrorMsg: []string{"converting NULL to string is unsupported"}, + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(status) + } + })), + options: &chOptions{stackTrace: true}, + expectedErrorMsg: "", + expectedMsg: []string{"Shard", "TraceFunctions", "Count()", + "Shard_test", "TraceFunctions_test", "Count_test", + "converting NULL to string is unsupported"}, }, } for _, tt := range testCases { @@ -140,7 +190,9 @@ func TestGetStatus(t *testing.T) { assert.NoError(t, err) outcome := readStdout(t, r, w) os.Stdout = orig - assert.Contains(t, outcome, tt.expectedMsg) + for _, msg := range tt.expectedMsg { + assert.Contains(t, outcome, msg) + } } else { assert.Error(t, err) assert.Contains(t, err.Error(), tt.expectedErrorMsg)