SonarCloud API support (#8132)

* feat(sonarcloud): add org to the connections table

* feat: add sonarqube cloud connection config

* fix: sonarcloud now working

* fix: linting

* fix: copywriting for sonarqube cloud

* feat: added cq_issue_impacts table

* fix: unit test

* fix: adjust sonarqube cloud config

* feat: add sonarcloud dashboard (#8140)

Co-authored-by: Startrekzky <kaiyun.zhang@merico.dev>

---------

Co-authored-by: mintsweet <0x1304570@gmail.com>
Co-authored-by: Louis.z <louis.s4372121@gmail.com>
Co-authored-by: Startrekzky <kaiyun.zhang@merico.dev>

backend/core/models/domainlayer/codequality/cq_issues.go

@@ -52,3 +52,14 @@ type CqIssue struct {
 func (CqIssue) TableName() string {
 	return "cq_issues"
 }
+
+type CqIssueImpact struct {
+	common.NoPKModel
+	CqIssueId       string `gorm:"primaryKey;type:varchar(255)"`
+	SoftwareQuality string `gorm:"primaryKey;type:varchar(255)"`
+	Severity        string `gorm:"type:varchar(100)"`
+}
+
+func (CqIssueImpact) TableName() string {
+	return "cq_issue_impacts"
+}

backend/core/models/domainlayer/domaininfo/domaininfo.go

@@ -53,6 +53,7 @@ func GetDomainTablesInfo() []dal.Tabler {
 		&codequality.CqFileMetrics{},
 		&codequality.CqIssueCodeBlock{},
 		&codequality.CqIssue{},
+		&codequality.CqIssueImpact{},
 		&codequality.CqProject{},
 		// crossdomain
 		&crossdomain.Account{},

backend/core/models/migrationscripts/20241010_add_ca_issue_impacts.go

@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addCqIssueImpacts)(nil)
+
+type cqIssueImpacts struct {
+	archived.NoPKModel
+	CqIssueId       string `gorm:"primaryKey;type:varchar(255)"`
+	SoftwareQuality string `gorm:"primaryKey;type:varchar(255)"`
+	Severity        string `gorm:"type:varchar(100)"`
+}
+
+type addCqIssueImpacts struct {
+}
+
+func (script *addCqIssueImpacts) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(&cqIssueImpacts{})
+}
+
+func (*addCqIssueImpacts) Version() uint64 {
+	return 20241010162658
+}
+
+func (*addCqIssueImpacts) Name() string {
+	return "add cq_issue_impacts table"
+}

backend/core/models/migrationscripts/register.go

@@ -133,5 +133,6 @@ func All() []plugin.MigrationScript {
 		new(addAssigneeToIncident),
 		new(addIsSubtaskToIssue),
 		new(addIsChildToCicdPipeline),
+		new(addCqIssueImpacts),
 	}
 }

backend/helpers/pluginhelper/api/api_client.go

@@ -62,6 +62,7 @@ type ApiClient struct {
 	data       map[string]interface{}
 	data_mutex sync.Mutex
 
+	authFunc      plugin.ApiClientBeforeRequest
 	beforeRequest plugin.ApiClientBeforeRequest
 	afterResponse plugin.ApiClientAfterResponse
 	ctx           gocontext.Context
@@ -92,7 +93,7 @@ func NewApiClientFromConnection(
 
 	// if connection requires authorization
 	if authenticator, ok := connection.(plugin.ApiAuthenticator); ok {
-		apiClient.SetBeforeFunction(func(req *http.Request) errors.Error {
+		apiClient.SetAuthFunction(func(req *http.Request) errors.Error {
 			return authenticator.SetupAuthentication(req)
 		})
 	}
@@ -255,6 +256,16 @@ func (apiClient *ApiClient) SetBeforeFunction(callback plugin.ApiClientBeforeReq
 	apiClient.beforeRequest = callback
 }
 
+// GetAuthFunction
+func (apiClient *ApiClient) GetAuthFunction() plugin.ApiClientBeforeRequest {
+	return apiClient.authFunc
+}
+
+// SetAuthFunction
+func (apiClient *ApiClient) SetAuthFunction(callback plugin.ApiClientBeforeRequest) {
+	apiClient.authFunc = callback
+}
+
 // GetAfterFunction return afterResponseFunction
 func (apiClient *ApiClient) GetAfterFunction() plugin.ApiClientAfterResponse {
 	return apiClient.afterResponse
@@ -345,6 +356,14 @@ func (apiClient *ApiClient) Do(
 	}
 
 	var res *http.Response
+	// authFunc
+	if apiClient.authFunc != nil {
+		err = apiClient.authFunc(req)
+		if err != nil {
+			apiClient.logError(err, "[api-client] authFunc returned error for %s", req.URL.String())
+			return nil, err
+		}
+	}
 	// before send
 	if apiClient.beforeRequest != nil {
 		err = apiClient.beforeRequest(req)

backend/plugins/sonarqube/impl/impl.go

@@ -80,6 +80,7 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler {
 		&models.SonarqubeConnection{},
 		&models.SonarqubeProject{},
 		&models.SonarqubeIssue{},
+		&models.SonarqubeIssueImpact{},
 		&models.SonarqubeIssueCodeBlock{},
 		&models.SonarqubeHotspot{},
 		&models.SonarqubeFileMetrics{},
@@ -102,6 +103,7 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
 		tasks.ExtractAccountsMeta,
 		tasks.ConvertProjectsMeta,
 		tasks.ConvertIssuesMeta,
+		tasks.ConvertIssueImpactsMeta,
 		tasks.ConvertIssueCodeBlocksMeta,
 		tasks.ConvertHotspotsMeta,
 		tasks.ConvertFileMetricsMeta,

backend/plugins/sonarqube/models/connection.go

@@ -20,9 +20,10 @@ package models
 import (
 	"encoding/base64"
 	"fmt"
-	"github.com/apache/incubator-devlake/core/utils"
 	"net/http"
 
+	"github.com/apache/incubator-devlake/core/utils"
+
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -49,6 +50,7 @@ func (sat SonarqubeAccessToken) GetEncodedToken() string {
 type SonarqubeConn struct {
 	helper.RestConnection `mapstructure:",squash"`
 	SonarqubeAccessToken  `mapstructure:",squash"`
+	Organization          string `gorm:"serializer:json" json:"org" mapstructure:"org"`
 }
 
 func (connection SonarqubeConn) Sanitize() SonarqubeConn {
@@ -89,3 +91,24 @@ func (connection *SonarqubeConnection) MergeFromRequest(target *SonarqubeConnect
 	}
 	return nil
 }
+
+func (connection *SonarqubeConnection) IsCloud() bool {
+	return connection.Endpoint == "https://sonarcloud.io/api/"
+}
+
+const ORG = "org"
+
+func (connection *SonarqubeConn) PrepareApiClient(apiClient plugin.ApiClient) errors.Error {
+	apiClient.SetData(ORG, connection.Organization)
+	apiClient.SetBeforeFunction(func(req *http.Request) errors.Error {
+		org := apiClient.GetData(ORG).(string)
+		if org != "" {
+			query := req.URL.Query()
+			query.Add("organization", org)
+			req.URL.RawQuery = query.Encode()
+		}
+		return nil
+	})
+
+	return nil
+}

backend/plugins/sonarqube/models/migrationscripts/20240930_add_connection_orgs.go

@@ -0,0 +1,50 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addOrgToConn)(nil)
+
+type connection20240930 struct {
+	Organization string
+}
+
+func (connection20240930) TableName() string {
+	return "_tool_sonarqube_connections"
+}
+
+type addOrgToConn struct {
+}
+
+func (script *addOrgToConn) Up(basicRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(basicRes, &connection20240930{})
+}
+
+func (*addOrgToConn) Version() uint64 {
+	return 20240930151715
+}
+
+func (*addOrgToConn) Name() string {
+	return "add organizations to the connections table"
+}

backend/plugins/sonarqube/models/migrationscripts/20241010_add_issue_impacts.go

@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addIssueImpacts)(nil)
+
+type issueImpacts20241010 struct {
+	ConnectionId    uint64 `gorm:"primaryKey"`
+	IssueKey        string `gorm:"primaryKey;type:varchar(100)"`
+	SoftwareQuality string `gorm:"primaryKey;type:varchar(255)"`
+	Severity        string `gorm:"type:varchar(100)"`
+	archived.NoPKModel
+}
+
+func (issueImpacts20241010) TableName() string {
+	return "_tool_sonarqube_issue_impacts"
+}
+
+type addIssueImpacts struct {
+}
+
+func (script *addIssueImpacts) Up(basicRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(basicRes, &issueImpacts20241010{})
+}
+
+func (*addIssueImpacts) Version() uint64 {
+	return 20241010162943
+}
+
+func (*addIssueImpacts) Name() string {
+	return "add issue_impacts table for sonarcloud"
+}

backend/plugins/sonarqube/models/migrationscripts/register.go

@@ -36,5 +36,7 @@ func All() []plugin.MigrationScript {
 		new(modifyNameLength),
 		new(changeIssueComponentType),
 		new(increaseProjectKeyLength),
+		new(addOrgToConn),
+		new(addIssueImpacts),
 	}
 }

backend/plugins/sonarqube/models/sonarqube_issue.go

@@ -50,3 +50,15 @@ type SonarqubeIssue struct {
 func (SonarqubeIssue) TableName() string {
 	return "_tool_sonarqube_issues"
 }
+
+type SonarqubeIssueImpact struct {
+	ConnectionId    uint64 `gorm:"primaryKey"`
+	IssueKey        string `gorm:"primaryKey;type:varchar(100)"`
+	SoftwareQuality string `gorm:"primaryKey;type:varchar(255)"`
+	Severity        string `gorm:"type:varchar(100)"`
+	common.NoPKModel
+}
+
+func (SonarqubeIssueImpact) TableName() string {
+	return "_tool_sonarqube_issue_impacts"
+}

backend/plugins/sonarqube/tasks/issue_impacts_convertor.go

@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"reflect"
+
+	"github.com/apache/incubator-devlake/core/dal"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	sonarqubeModels "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+)
+
+var ConvertIssueImpactsMeta = plugin.SubTaskMeta{
+	Name:             "convertIssueImpacts",
+	EntryPoint:       ConvertIssueImpacts,
+	EnabledByDefault: true,
+	Description:      "Convert tool layer table sonarqube_issue_impacts into  domain layer table cq_issue_impacts",
+	DomainTypes:      []string{plugin.DOMAIN_TYPE_CODE_QUALITY},
+}
+
+func ConvertIssueImpacts(taskCtx plugin.SubTaskContext) errors.Error {
+	db := taskCtx.GetDal()
+	rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUES_TABLE)
+	cursor, err := db.Cursor(
+		dal.From("_tool_sonarqube_issue_impacts p"),
+		dal.Join("LEFT JOIN _tool_sonarqube_issues i ON (i.connection_id = p.connection_id AND i.issue_key = p.issue_key)"),
+		dal.Where("i.connection_id = ? AND i.project_key = ?", data.Options.ConnectionId, data.Options.ProjectKey))
+	if err != nil {
+		return err
+	}
+	defer cursor.Close()
+
+	issueIdGen := didgen.NewDomainIdGenerator(&sonarqubeModels.SonarqubeIssue{})
+	converter, err := api.NewDataConverter(api.DataConverterArgs{
+		InputRowType:       reflect.TypeOf(sonarqubeModels.SonarqubeIssueImpact{}),
+		Input:              cursor,
+		RawDataSubTaskArgs: *rawDataSubTaskArgs,
+		Convert: func(inputRow interface{}) ([]interface{}, errors.Error) {
+			impact := inputRow.(*sonarqubeModels.SonarqubeIssueImpact)
+			domainIssueImpact := &codequality.CqIssueImpact{
+				CqIssueId:       issueIdGen.Generate(data.Options.ConnectionId, impact.IssueKey),
+				SoftwareQuality: impact.SoftwareQuality,
+				Severity:        impact.Severity,
+			}
+			return []interface{}{
+				domainIssueImpact,
+			}, nil
+		},
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return converter.Execute()
+}