Skip to content

Commit

Permalink
feature: db_type support (#2049)
Browse files Browse the repository at this point in the history 
* db_type support

* metric expression

* rm service_node from jvm_memory and nodejs_memory
  • Loading branch information
@innerpeacez
innerpeacez authored Sep 23, 2021
1 parent adc81ff commit 01e3901
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 36 deletions.
15 changes: 15 additions & 0 deletions .erda/migrations/monitor/20210923-sp_metric_expression.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
UPDATE `sp_metric_expression` SET `enable` = 0 WHERE `id` = (SELECT `id` FROM (SELECT `id` FROM `sp_metric_expression` WHERE `enable` = 1 AND `expression` LIKE "%\"alias\":\"application_cache_service\"%") AS `ids`);
UPDATE `sp_metric_expression` SET `enable` = 0 WHERE `id` = (SELECT `id` FROM (SELECT `id` FROM `sp_metric_expression` WHERE `enable` = 1 AND `expression` LIKE "%\"alias\":\"application_db_service\"%") AS `ids`);
UPDATE `sp_metric_expression` SET `enable` = 0 WHERE `id` = (SELECT `id` FROM (SELECT `id` FROM `sp_metric_expression` WHERE `enable` = 1 AND `expression` LIKE "%\"alias\":\"application_http_service\"%") AS `ids`);
UPDATE `sp_metric_expression` SET `enable` = 0 WHERE `id` = (SELECT `id` FROM (SELECT `id` FROM `sp_metric_expression` WHERE `enable` = 1 AND `expression` LIKE "%\"alias\":\"application_mq_service\"%") AS `ids`);
UPDATE `sp_metric_expression` SET `enable` = 0 WHERE `id` = (SELECT `id` FROM (SELECT `id` FROM `sp_metric_expression` WHERE `enable` = 1 AND `expression` LIKE "%\"alias\":\"application_rpc_service\"%") AS `ids`);

UPDATE `sp_metric_expression` SET `enable` = 0 WHERE `id` = (SELECT `id` FROM (SELECT `id` FROM `sp_metric_expression` WHERE `enable` = 1 AND `expression` LIKE "%\"metric\":\"jvm_memory\"%") AS `ids`);
UPDATE `sp_metric_expression` SET `enable` = 0 WHERE `id` = (SELECT `id` FROM (SELECT `id` FROM `sp_metric_expression` WHERE `enable` = 1 AND `expression` LIKE "%\"metric\":\"nodejs_memory\"%") AS `ids`);

INSERT `sp_metric_expression`(`attributes`,`expression`,`version`) VALUES("{}","{\"alias\":\"application_cache_service\",\"filter\":{},\"functions\":[{\"aggregator\":\"sum\",\"field\":\"elapsed\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_mean * field.elapsed_count; }\"},{\"aggregator\":\"sum\",\"field\":\"count\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_count; }\"}],\"group\":[\"host\",\"source_terminus_key\",\"source_application_id\",\"source_runtime_name\",\"source_service_name\",\"source_addon_id\"],\"metric\":\"application_cache\",\"outputs\":[\"metric\"],\"select\":{\"cluster_name\":\"#cluster_name\",\"component\":\"#component\",\"db_type\":\"#db_type\",\"host\":\"#host\",\"source_addon_id\":\"#source_addon_id\",\"source_addon_type\":\"#source_addon_type\",\"source_application_id\":\"#source_application_id\",\"source_application_name\":\"#source_application_name\",\"source_project_id\":\"#source_project_id\",\"source_project_name\":\"#source_project_name\",\"source_runtime_id\":\"#source_runtime_id\",\"source_runtime_name\":\"#source_runtime_name\",\"source_service_id\":\"#source_service_id\",\"source_service_name\":\"#source_service_name\",\"source_terminus_key\":\"#source_terminus_key\",\"source_workspace\":\"#source_workspace\"},\"window\":3}","3.0");
INSERT `sp_metric_expression`(`attributes`,`expression`,`version`) VALUES("{}","{\"alias\":\"application_db_service\",\"filter\":{},\"functions\":[{\"aggregator\":\"sum\",\"field\":\"elapsed\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_mean * field.elapsed_count; }\"},{\"aggregator\":\"sum\",\"field\":\"count\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_count; }\"}],\"group\":[\"host\",\"source_terminus_key\",\"source_application_id\",\"source_runtime_name\",\"source_service_name\",\"source_addon_id\"],\"metric\":\"application_db\",\"outputs\":[\"metric\"],\"select\":{\"cluster_name\":\"#cluster_name\",\"component\":\"#component\",\"db_type\":\"#db_type\",\"host\":\"#host\",\"source_addon_id\":\"#source_addon_id\",\"source_addon_type\":\"#source_addon_type\",\"source_application_id\":\"#source_application_id\",\"source_application_name\":\"#source_application_name\",\"source_project_id\":\"#source_project_id\",\"source_project_name\":\"#source_project_name\",\"source_runtime_id\":\"#source_runtime_id\",\"source_runtime_name\":\"#source_runtime_name\",\"source_service_id\":\"#source_service_id\",\"source_service_name\":\"#source_service_name\",\"source_terminus_key\":\"#source_terminus_key\",\"source_workspace\":\"#source_workspace\"},\"window\":3}","3.0");
INSERT `sp_metric_expression`(`attributes`,`expression`,`version`) VALUES("{}","{\"alias\":\"application_http_service\",\"filter\":{},\"functions\":[{\"aggregator\":\"sum\",\"field\":\"elapsed\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_mean * field.elapsed_count; }\"},{\"aggregator\":\"sum\",\"field\":\"count\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_count; }\"},{\"aggregator\":\"sum\",\"field\":\"errors\",\"field_script\":\"function invoke(field, tag){ return tag.error=='true'?field.elapsed_count:0; }\"}],\"group\":[\"host\",\"req_host\",\"target_terminus_key\",\"target_application_id\",\"target_runtime_name\",\"target_service_name\",\"target_addon_id\",\"source_terminus_key\",\"source_application_id\",\"source_runtime_name\",\"source_service_name\",\"source_addon_id\"],\"metric\":\"application_http\",\"outputs\":[\"metric\"],\"select\":{\"cluster_name\":\"#cluster_name\",\"component\":\"#component\",\"db_type\":\"#db_type\",\"host\":\"#host\",\"peer_service_scope\":\"#peer_service_scope\",\"req_host\":\"#req_host\",\"source_addon_id\":\"#source_addon_id\",\"source_addon_type\":\"#source_addon_type\",\"source_application_id\":\"#source_application_id\",\"source_application_name\":\"#source_application_name\",\"source_project_id\":\"#source_project_id\",\"source_project_name\":\"#source_project_name\",\"source_runtime_id\":\"#source_runtime_id\",\"source_runtime_name\":\"#source_runtime_name\",\"source_service_id\":\"#source_service_id\",\"source_service_name\":\"#source_service_name\",\"source_terminus_key\":\"#source_terminus_key\",\"source_workspace\":\"#source_workspace\",\"target_addon_id\":\"#target_addon_id\",\"target_addon_type\":\"#target_addon_type\",\"target_application_id\":\"#target_application_id\",\"target_application_name\":\"#target_application_name\",\"target_instance_id\":\"#target_instance_id\",\"target_project_id\":\"#target_project_id\",\"target_project_name\":\"#target_project_name\",\"target_runtime_name\":\"#target_runtime_name\",\"target_service_id\":\"#target_service_id\",\"target_service_name\":\"#target_service_name\",\"target_terminus_key\":\"#target_terminus_key\",\"target_workspace\":\"#target_workspace\"},\"window\":3}","3.0");
INSERT `sp_metric_expression`(`attributes`,`expression`,`version`) VALUES("{}","{\"alias\":\"application_mq_service\",\"filter\":{},\"functions\":[{\"aggregator\":\"sum\",\"field\":\"elapsed\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_mean * field.elapsed_count; }\"},{\"aggregator\":\"sum\",\"field\":\"count\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_count; }\"}],\"group\":[\"host\",\"target_terminus_key\",\"target_application_id\",\"target_runtime_name\",\"target_service_name\",\"target_addon_id\",\"source_terminus_key\",\"source_application_id\",\"source_runtime_name\",\"source_service_name\",\"source_addon_id\"],\"metric\":\"application_mq\",\"outputs\":[\"metric\"],\"select\":{\"cluster_name\":\"#cluster_name\",\"component\":\"#component\",\"db_type\":\"#db_type\",\"host\":\"#host\",\"source_addon_id\":\"#source_addon_id\",\"source_addon_type\":\"#source_addon_type\",\"source_application_id\":\"#source_application_id\",\"source_application_name\":\"#source_application_name\",\"source_project_id\":\"#source_project_id\",\"source_project_name\":\"#source_project_name\",\"source_runtime_id\":\"#source_runtime_id\",\"source_runtime_name\":\"#source_runtime_name\",\"source_service_id\":\"#source_service_id\",\"source_service_name\":\"#source_service_name\",\"source_terminus_key\":\"#source_terminus_key\",\"source_workspace\":\"#source_workspace\",\"target_addon_id\":\"#target_addon_id\",\"target_addon_type\":\"#target_addon_type\",\"target_application_id\":\"#target_application_id\",\"target_application_name\":\"#target_application_name\",\"target_instance_id\":\"#target_instance_id\",\"target_project_id\":\"#target_project_id\",\"target_project_name\":\"#target_project_name\",\"target_runtime_name\":\"#target_runtime_name\",\"target_service_id\":\"#target_service_id\",\"target_service_name\":\"#target_service_name\",\"target_terminus_key\":\"#target_terminus_key\",\"target_workspace\":\"#target_workspace\"},\"window\":3}","3.0");
INSERT `sp_metric_expression`(`attributes`,`expression`,`version`) VALUES("{}","{\"alias\":\"application_rpc_service\",\"filter\":{},\"functions\":[{\"aggregator\":\"sum\",\"field\":\"elapsed\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_mean * field.elapsed_count; }\"},{\"aggregator\":\"sum\",\"field\":\"count\",\"field_script\":\"function invoke(field, tag){ return field.elapsed_count; }\"},{\"aggregator\":\"sum\",\"field\":\"errors\",\"field_script\":\"function invoke(field, tag){ return tag.error=='true'?field.elapsed_count:0; }\"}],\"group\":[\"host\",\"target_terminus_key\",\"target_application_id\",\"target_runtime_name\",\"target_service_name\",\"target_addon_id\",\"source_terminus_key\",\"source_application_id\",\"source_runtime_name\",\"source_service_name\",\"source_addon_id\"],\"metric\":\"application_rpc\",\"outputs\":[\"metric\"],\"select\":{\"cluster_name\":\"#cluster_name\",\"component\":\"#component\",\"db_type\":\"#db_type\",\"host\":\"#host\",\"peer_service_scope\":\"#peer_service_scope\",\"source_addon_id\":\"#source_addon_id\",\"source_addon_type\":\"#source_addon_type\",\"source_application_id\":\"#source_application_id\",\"source_application_name\":\"#source_application_name\",\"source_project_id\":\"#source_project_id\",\"source_project_name\":\"#source_project_name\",\"source_runtime_id\":\"#source_runtime_id\",\"source_runtime_name\":\"#source_runtime_name\",\"source_service_id\":\"#source_service_id\",\"source_service_name\":\"#source_service_name\",\"source_terminus_key\":\"#source_terminus_key\",\"source_workspace\":\"#source_workspace\",\"target_addon_id\":\"#target_addon_id\",\"target_addon_type\":\"#target_addon_type\",\"target_application_id\":\"#target_application_id\",\"target_application_name\":\"#target_application_name\",\"target_instance_id\":\"#target_instance_id\",\"target_project_id\":\"#target_project_id\",\"target_project_name\":\"#target_project_name\",\"target_runtime_name\":\"#target_runtime_name\",\"target_service_id\":\"#target_service_id\",\"target_service_name\":\"#target_service_name\",\"target_terminus_key\":\"#target_terminus_key\",\"target_workspace\":\"#target_workspace\"},\"window\":3}","3.0");
INSERT `sp_metric_expression`(`attributes`,`expression`,`version`) VALUES("{}","{\"alias\":\"service_node\",\"filter\":{},\"functions\":[{\"aggregator\":\"sum\",\"field\":\"count\",\"field_script\":\"function invoke(field, tag){ return 1; }\"}],\"group\":[\"terminus_key\",\"msp_env_id\",\"service_id\"],\"metric\":\"application_service_node\",\"outputs\":[\"metric\"],\"select\":{\"application_name\":\"#application_name\",\"instrumentation_library\":\"#instrumentation_library\",\"instrumentation_library_version\":\"#instrumentation_library_version\",\"msp_env_id\":\"#msp_env_id\",\"project_name\":\"#project_name\",\"runtime_id\":\"#runtime_id\",\"runtime_name\":\"#runtime_name\",\"service_id\":\"#service_id\",\"service_name\":\"#service_name\",\"terminus_key\":\"#terminus_key\",\"workspace\":\"#workspace\"},\"window\":2}","3.0");
1 change: 1 addition & 0 deletions modules/monitor/apm/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const (
TagsSourceAddonGroup = Tags + Sep4 + "source_addon_group"
TagsSourceTerminusKey = Tags + Sep4 + "source_terminus_key"
TagsComponent = Tags + Sep4 + "component"
TagsDBType = Tags + Sep4 + "db_type"
TagsHost = Tags + Sep4 + "host"
TagsApplicationId = Tags + Sep4 + "application_id"
TagsApplicationName = Tags + Sep4 + "application_name"
Expand Down
42 changes: 6 additions & 36 deletions modules/monitor/apm/topology/topology.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ type Field struct {

type Tag struct {
Component string `json:"component,omitempty"`
DBType string `json:"db_type,omitempty"`
Host string `json:"host,omitempty"`
SourceProjectId string `json:"source_project_id,omitempty"`
SourceProjectName string `json:"source_project_name,omitempty"`
Expand Down Expand Up @@ -410,8 +411,8 @@ func init() {
}
TargetComponentNodeType = &NodeType{
Type: TargetComponentNode,
GroupByField: &GroupByField{Name: apm.TagsComponent, SubField: &GroupByField{Name: apm.TagsHost}},
SourceFields: []string{apm.TagsComponent, apm.TagsHost, apm.TagsTargetAddonGroup},
GroupByField: &GroupByField{Name: apm.TagsDBType, SubField: &GroupByField{Name: apm.TagsHost}},
SourceFields: []string{apm.TagsComponent, apm.TagsHost, apm.TagsTargetAddonGroup, apm.TagsDBType},
Filter: elastic.NewBoolQuery().MustNot(elastic.NewExistsQuery(apm.TagsTargetAddonType),
elastic.NewExistsQuery(apm.TagsTargetApplicationId)),
Aggregation: NodeAggregation,
Expand Down Expand Up @@ -1506,40 +1507,6 @@ func (topology *provider) GetTopology(param Vo) []*Node {
return nodes
}

// FilterNodeByTags
func (topology *provider) FilterNodeByTags(tags []string, nodes []*Node) []*Node {
if tags != nil && len(tags) > 0 {
for _, v := range tags {
tagInfo := strings.Split(v, ":")
tag := tagInfo[0]
value := tagInfo[1]
switch tag {
case ApplicationSearchTag.Tag:
for i, node := range nodes {
if strings.ToLower(node.Name) == strings.ToLower(TypeGateway) {
continue
}
for _, parentNode := range node.Parents {
if node.ApplicationName != value && parentNode.ApplicationName != value {
nodes = append(nodes[:i], nodes[i+1:]...)
i--
}
}

}
case ServiceSearchTag.Tag:
for i, node := range nodes {
if node.ServiceName != value {
nodes = append(nodes[:i], nodes[i+1:]...)
i--
}
}
}
}
}
return nodes
}

func selectRelation(indexType string) (*AggregationCondition, []*NodeRelation) {
var aggregationConditions *AggregationCondition
var relations []*NodeRelation
Expand Down Expand Up @@ -1774,6 +1741,9 @@ func columnsParser(nodeType string, nodeRelation *TopologyNodeRelation) *Node {
case TargetComponentNode:
node.Type = tags.Component
node.Name = tags.Host
if tags.DBType != "" {
node.Type = tags.DBType
}
node.Id = encodeTypeToKey(node.Type + apm.Sep1 + node.Name)
case SourceMQNode:
node.Type = tags.Component
Expand Down

0 comments on commit 01e3901

Please sign in to comment.