Merge pull request #46 from KunfengHe/br_release_sdk_v3

CC支持显示资源标签；AGT指标展示实际维度信息

README_cn.md

@@ -1,4 +1,4 @@
-
+
 # 华为云 Exporter
 
 [华为云](https://www.huaweicloud.com/)云监控的 Prometheus Exporter.
@@ -64,8 +64,8 @@ Prometheus是用于展示大型测量数据的开源可视化工具，在工业
 # 参考命令：
 mkdir cloudeye-exporter
 cd cloudeye-exporter
-wget https://github.com/huaweicloud/cloudeye-exporter/releases/download/v2.0.2/cloudeye-exporter.v2.0.2.tar.gz
-tar -xzvf cloudeye-exporter.v2.0.2.tar.gz
+wget https://github.com/huaweicloud/cloudeye-exporter/releases/download/v2.0.1/cloudeye-exporter.v2.0.1.tar.gz
+tar -xzvf cloudeye-exporter.v2.0.1.tar.gz
 ```
 2. 编辑clouds.yml文件配置公有云信息
 ```

collector/bms.go

@@ -25,6 +25,7 @@ func (getter BMSInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metric
 		sysConfigMap := getMetricConfigMap("SYS.BMS")
 		if metricNames, ok := sysConfigMap["instance_id"]; ok {
 			for _, instance := range services {
+				loadAgentDimensions(instance.ID)
 				metrics := buildSingleDimensionMetrics(metricNames, "SYS.BMS", "instance_id", instance.ID)
 				filterMetrics = append(filterMetrics, metrics...)
 				info := labelInfo{

collector/cc.go

@@ -0,0 +1,181 @@
+package collector
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/global"
+	cc "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3"
+	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3/model"
+	region "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3/region"
+	cesmodel "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model"
+
+	"github.com/huaweicloud/cloudeye-exporter/logs"
+)
+
+var (
+	ccInfo serversInfo
+	limit  = int32(2000)
+)
+
+const (
+	CCNamespace      = "SYS.CC"
+	CCConfigDimNames = "cloud_connect_id,bwp_id,region_bandwidth_id"
+)
+
+type CCInfo struct{}
+
+func (getter CCInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.MetricInfoList) {
+	ccInfo.Lock()
+	defer ccInfo.Unlock()
+	if ccInfo.LabelInfo == nil || time.Now().Unix() > ccInfo.TTL {
+		sysConfigMap := getMetricConfigMap(CCNamespace)
+		metricNames := sysConfigMap[CCConfigDimNames]
+		if len(metricNames) == 0 {
+			logs.Logger.Warn("Metric config is empty of SYS.CC.")
+			return ccInfo.LabelInfo, ccInfo.FilterMetrics
+		}
+
+		connections, err := listCCConnections()
+		if err != nil {
+			logs.Logger.Errorf("Get all connections error: %s", err.Error())
+			return ccInfo.LabelInfo, ccInfo.FilterMetrics
+		}
+
+		packages, err := listBandwidthPackages()
+		if err != nil {
+			logs.Logger.Errorf("Get all bandwidth packages error: %s", err.Error())
+			return ccInfo.LabelInfo, ccInfo.FilterMetrics
+		}
+
+		bandwidths, err := listInterRegionBandwidths()
+		if err != nil {
+			logs.Logger.Errorf("Get all inter region bandwidths error: %s", err.Error())
+			return ccInfo.LabelInfo, ccInfo.FilterMetrics
+		}
+		ccInfo.LabelInfo, ccInfo.FilterMetrics = buildResourceInfoAndMetrics(metricNames, connections, packages, bandwidths)
+		ccInfo.TTL = time.Now().Add(TTL).Unix()
+	}
+	return ccInfo.LabelInfo, ccInfo.FilterMetrics
+}
+
+func buildResourceInfoAndMetrics(metricNames []string, connections map[string]model.CloudConnection, packages map[string]model.BandwidthPackage, bandwidths []model.InterRegionBandwidth) (map[string]labelInfo, []cesmodel.MetricInfoList) {
+	resourceInfos := map[string]labelInfo{}
+	filterMetrics := make([]cesmodel.MetricInfoList, 0)
+	for _, bandwidth := range bandwidths {
+		if *bandwidth.CloudConnectionId == "" || *bandwidth.BandwidthPackageId == "" {
+			continue
+		}
+		metrics := buildDimensionMetrics(metricNames, CCNamespace,
+			[]cesmodel.MetricsDimension{{Name: "cloud_connect_id", Value: *bandwidth.CloudConnectionId},
+				{Name: "bwp_id", Value: *bandwidth.BandwidthPackageId},
+				{Name: "region_bandwidth_id", Value: *bandwidth.Id}})
+		filterMetrics = append(filterMetrics, metrics...)
+
+		var info labelInfo
+		connectionName, connectionValue := getConnectionInfo(connections, *bandwidth.CloudConnectionId)
+		info.Name = append(info.Name, connectionName...)
+		info.Value = append(info.Value, connectionValue...)
+
+		pkgName, pkgValue := getBandwidthPackageInfo(packages, *bandwidth.BandwidthPackageId)
+		info.Name = append(info.Name, pkgName...)
+		info.Value = append(info.Value, pkgValue...)
+
+		if bandwidth.InterRegions != nil && len(*bandwidth.InterRegions) != 0 {
+			info.Name = append(info.Name, "interRegions", "bandwidthName")
+			info.Value = append(info.Value, fmt.Sprintf("%s<->%s",
+				getDefaultString((*bandwidth.InterRegions)[0].LocalRegionId), getDefaultString((*bandwidth.InterRegions)[0].RemoteRegionId)),
+				getDefaultString(bandwidth.Name))
+		}
+		resourceInfos[GetResourceKeyFromMetricInfo(metrics[0])] = info
+	}
+	return resourceInfos, filterMetrics
+}
+
+func getConnectionInfo(connections map[string]model.CloudConnection, connectionId string) ([]string, []string) {
+	connection, ok := connections[connectionId]
+	if ok {
+		return []string{"connectionName", "connectionEpId"}, []string{*connection.Name, *connection.EnterpriseProjectId}
+	}
+	return nil, nil
+}
+
+func getBandwidthPackageInfo(packages map[string]model.BandwidthPackage, connectionId string) ([]string, []string) {
+	pkg, ok := packages[connectionId]
+	if !ok {
+		return nil, nil
+	}
+	name := []string{"packageName", "packageEpId"}
+	vale := []string{getDefaultString(pkg.Name), getDefaultString(pkg.EnterpriseProjectId)}
+	if pkg.Tags != nil {
+		keys, values := getTags(fmtTags(pkg.Tags))
+		name = append(name, keys...)
+		vale = append(vale, values...)
+	}
+	return name, vale
+}
+
+func listCCConnections() (map[string]model.CloudConnection, error) {
+	request := &model.ListCloudConnectionsRequest{Limit: &limit}
+	client := getCCClient()
+	connections := make(map[string]model.CloudConnection, 0)
+	for {
+		response, err := client.ListCloudConnections(request)
+		if err != nil {
+			return connections, err
+		}
+		for _, connection := range *response.CloudConnections {
+			connections[*connection.Id] = connection
+		}
+		if response.PageInfo.NextMarker == nil {
+			break
+		}
+		request.Marker = response.PageInfo.NextMarker
+	}
+	return connections, nil
+}
+
+func listBandwidthPackages() (map[string]model.BandwidthPackage, error) {
+	request := &model.ListBandwidthPackagesRequest{Limit: &limit}
+	client := getCCClient()
+	bandwidthPackages := make(map[string]model.BandwidthPackage, 0)
+	for {
+		response, err := client.ListBandwidthPackages(request)
+		if err != nil {
+			fmt.Println(err.Error())
+			return bandwidthPackages, err
+		}
+		for _, bandwidthPackage := range *response.BandwidthPackages {
+			bandwidthPackages[*bandwidthPackage.Id] = bandwidthPackage
+		}
+		if response.PageInfo.NextMarker == nil {
+			break
+		}
+		request.Marker = response.PageInfo.NextMarker
+	}
+	return bandwidthPackages, nil
+}
+
+func getCCClient() *cc.CcClient {
+	return cc.NewCcClient(cc.CcClientBuilder().WithRegion(region.ValueOf("cn-north-4")).
+		WithCredential(global.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).Build()).Build())
+}
+
+func listInterRegionBandwidths() ([]model.InterRegionBandwidth, error) {
+	request := &model.ListInterRegionBandwidthsRequest{Limit: &limit}
+	client := getCCClient()
+	var resources []model.InterRegionBandwidth
+	for {
+		response, err := client.ListInterRegionBandwidths(request)
+		if err != nil {
+			fmt.Println(err.Error())
+			return resources, err
+		}
+		resources = append(resources, *response.InterRegionBandwidths...)
+		if response.PageInfo.NextMarker == nil {
+			break
+		}
+		request.Marker = response.PageInfo.NextMarker
+	}
+	return resources, nil
+}

collector/collector.go

@@ -107,13 +107,34 @@ func getDimLabel(metric model.BatchMetricData) labelInfo {
 	var label labelInfo
 	for _, dim := range *metric.Dimensions {
 		label.Name = append(label.Name, strings.ReplaceAll(dim.Name, "-", "_"))
-		label.Value = append(label.Value, dim.Value)
+		label.Value = append(label.Value, getDimValue(*metric.Namespace, dim.Name, dim.Value))
 	}
 	label.Name = append(label.Name, "unit")
 	label.Value = append(label.Value, *metric.Unit)
 	return label
 }
 
+func getDimValue(namespaces, dimName, dimValue string) string {
+	if !isContainsInStringArr(namespaces, []string{"AGT.ECS", "SERVICE.BMS"}) {
+		return dimValue
+	}
+
+	if !isContainsInStringArr(dimName, []string{"mount_point", "disk", "proc", "gpu", "raid"}) {
+		return dimValue
+	}
+
+	return getAgentOriginValue(dimValue)
+}
+
+func isContainsInStringArr(target string, array []string) bool {
+	for index := range array {
+		if target == array[index] {
+			return true
+		}
+	}
+	return false
+}
+
 func (exporter *BaseHuaweiCloudExporter) collectMetricByNamespace(ctx context.Context, ch chan<- prometheus.Metric, namespace string) {
 	defer func() {
 		if err := recover(); err != nil {

collector/ecs.go

@@ -38,6 +38,7 @@ func (getter ECSInfo) GetResourceInfo() (map[string]labelInfo, []model.MetricInf
 		sysConfigMap := getMetricConfigMap("SYS.ECS")
 		for _, server := range servers {
 			if metricNames, ok := sysConfigMap["instance_id"]; ok {
+				loadAgentDimensions(server.ID)
 				metrics := buildSingleDimensionMetrics(metricNames, "SYS.ECS", "instance_id", server.ID)
 				filterMetrics = append(filterMetrics, metrics...)
 				info := labelInfo{

collector/elb.go

@@ -98,7 +98,9 @@ func (getter ELBInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.Metric
 	if elbInfo.LabelInfo == nil || time.Now().Unix() > elbInfo.TTL {
 		getResourceMap()
 		sysConfigMap := getMetricConfigMap("SYS.ELB")
-		for _, loadBalancer := range listLoadBalancers() {
+		loadBalancers := listLoadBalancers()
+		for index := range loadBalancers {
+			loadBalancer := loadBalancers[index]
 			if loadBalancerMetricNames, ok := sysConfigMap["lbaas_instance_id"]; ok {
 				metrics := buildSingleDimensionMetrics(loadBalancerMetricNames, "SYS.ELB", "lbaas_instance_id", loadBalancer.Id)
 				filterMetrics = append(filterMetrics, metrics...)

collector/extensions.go

@@ -46,6 +46,7 @@ var (
 		"SYS.ModelArts":     ModelArtsInfo{},
 		"SYS.GES":           GESInfo{},
 		"SYS.DBSS":          DBSSInfo{},
+		"SYS.CC":            CCInfo{},
 	}
 )
 

collector/metrics.go

@@ -5,12 +5,15 @@ import (
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config"
 	ces "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model"
+	cesv2 "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v2"
+	cesv2model "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v2/model"
 
 	"github.com/huaweicloud/cloudeye-exporter/logs"
 )
 
 var (
 	host string
+	agentDimensions = make(map[string]string, 0)
 )
 
 func getCESClient() *ces.CesClient {
@@ -63,3 +66,39 @@ func listAllMetrics(namespace string) ([]model.MetricInfoList, error) {
 
 	return metricData, nil
 }
+
+func getCESClientV2() *cesv2.CesClient {
+	return cesv2.NewCesClient(ces.CesClientBuilder().WithCredential(
+		basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
+		WithHttpConfig(config.DefaultHttpConfig().WithIgnoreSSLVerification(true)).
+		WithEndpoint(getEndpoint("ces", "v2")).Build())
+}
+
+func getAgentOriginValue(value string) string {
+	originValue, ok := agentDimensions[value]
+	if ok {
+		return originValue
+	}
+	return value
+}
+
+func loadAgentDimensions(instanceID string) {
+	dimName := cesv2model.GetListAgentDimensionInfoRequestDimNameEnum()
+	dimNames := []cesv2model.ListAgentDimensionInfoRequestDimName{dimName.DISK,
+		dimName.MOUNT_POINT, dimName.GPU, dimName.PROC, dimName.RAID}
+	for _, name := range dimNames {
+		request := &cesv2model.ListAgentDimensionInfoRequest{
+			ContentType: "application/json",
+			InstanceId:  instanceID,
+			DimName:     name,
+		}
+		response, err := getCESClientV2().ListAgentDimensionInfo(request)
+		if err != nil {
+			logs.Logger.Errorf("Failed to list agentDimensions: %s", err.Error())
+			return
+		}
+		for _, dimension := range *response.Dimensions {
+			agentDimensions[*dimension.Value] = *dimension.OriginValue
+		}
+	}
+}

collector/utils.go

@@ -67,7 +67,7 @@ func GetResourceKeyFromMetricData(metric model.BatchMetricData) string {
 	if *metric.Namespace == "SYS.DMS" {
 		return getDmsResourceKey(metric)
 	}
-	if *metric.Namespace == "AGT.ECS" || *metric.Namespace == "SERVICE.BMS"{
+	if *metric.Namespace == "AGT.ECS" || *metric.Namespace == "SERVICE.BMS" {
 		return getServerResourceKey(metric)
 	}
 	sort.Slice(*metric.Dimensions, func(i, j int) bool {
@@ -183,7 +183,7 @@ func buildDimensionMetrics(metricNames []string, namespace string, dimensions []
 func getHcClient(endpoint string) *core.HcHttpClient {
 	return core.NewHcHttpClient(impl.NewDefaultHttpClient(config.DefaultHttpConfig().WithIgnoreSSLVerification(true))).
 		WithCredential(basic.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithProjectId(conf.ProjectID).Build()).
-		WithEndpoint(endpoint)
+		WithEndpoints([]string{endpoint})
 }
 
 func genDefaultReqDefWithOffsetAndLimit(path string, response interface{}) *def.HttpRequestDef {

go.mod

@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/cihub/seelog v0.0.0-20191126193922-f561c5e57575
-	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.0.99
+	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.35
 	github.com/prometheus/client_golang v1.12.2
 	gopkg.in/yaml.v3 v3.0.1
 )