feature: cmp support new import method (#806)

.erda/migrations/cluster-manager/20210708-add-manageconfig.sql

@@ -0,0 +1,2 @@
+-- add manage config field to store cluster credential info
+ALTER TABLE co_clusters ADD COLUMN `manage_config` text NOT NULL COMMENT "cluster crendtial config";

apistructs/cluster.go

@@ -99,6 +99,12 @@ type ClusterUpdateRequest struct {
 	URLs map[string]string `json:"urls"`
 }
 
+type CMPClusterUpdateRequest struct {
+	ClusterUpdateRequest
+	CredentialType string       `json:"credentialType"`
+	Credential     ICCredential `json:"credential"`
+}
+
 // ClusterUpdateResponse 集群更新响应
 type ClusterUpdateResponse struct {
 	Header
@@ -324,10 +330,14 @@ type DereferenceClusterResponse struct {
 }
 
 type ManageConfig struct {
-	Type     string `json:"type"`
-	Address  string `json:"address"`
-	CaData   string `json:"caData"`
-	CertData string `json:"certData"`
-	KeyData  string `json:"keyData"`
-	Token    string `json:"token"`
+	// manage type, support proxy,token,cert
+	Type      string `json:"type"`
+	Address   string `json:"address"`
+	CaData    string `json:"caData"`
+	CertData  string `json:"certData"`
+	KeyData   string `json:"keyData"`
+	Token     string `json:"token"`
+	AccessKey string `json:"accessKey"`
+	// credential content from, support kubeconfig, serviceAccount
+	CredentialSource string `json:"credentialSource"`
 }

apistructs/cmp.go

@@ -19,6 +19,7 @@ import (
 
 const (
 	AddNodesEssSource = "ess-autoscale"
+	ComClusterKey     = "DICE_CLUSTER_NAME"
 )
 
 type AddNodesRequest struct {
@@ -1580,3 +1581,36 @@ type CloudAccountResponse struct {
 	Header
 	Data CloudAccount `json:"data"`
 }
+
+// ImportCluster cluster import request body
+type ImportCluster struct {
+	ClusterName    string             `json:"name"`
+	ScheduleConfig ClusterSchedConfig `json:"scheduler"`
+	Credential     ICCredential       `json:"credential"`
+	CredentialType string             `json:"credentialType"`
+	OrgID          uint64             `json:"orgId"`
+	ClusterType    string             `json:"type"`
+	WildcardDomain string             `json:"wildcardDomain"`
+	DisplayName    string             `json:"displayName"`
+	Description    string             `json:"description"`
+}
+
+// ICCredential import cluster credential
+type ICCredential struct {
+	Address string `json:"address"`
+	Content string `json:"content"`
+}
+
+type ImportClusterResponse struct {
+	Header
+	Data string `json:"data"`
+}
+
+type ClusterInitRetry struct {
+	ClusterName string `json:"clusterName"`
+}
+
+type InitClusterResponse struct {
+	Header
+	Data string `json:"data"`
+}

apistructs/cmp_cloud_resource.go

@@ -188,14 +188,14 @@ type CloudClusterNewCreateInfo struct {
 	CloudBasicRsc   string // ecs\ack
 
 	// 云环境vpc配置信息
-	Region       string      `json:"region"`                         //区域
-	ClusterType  string      `json:"clusterType" default:"Edge"`     //集群类型，默认边缘集群
-	ClusterSpec  ClusterSpec `json:"clusterSpec" default:"Standard"` //集群规格，Standard, Small, Test
-	ChargeType   string      `json:"chargeType" default:"PrePaid"`   //付费类型，PrePaid, PostPaid
-	ChargePeriod int         `json:"chargePeriod" default:"1"`       //付费周期
-	AppNodeNum   int         `json:"appNodeNum" default:"-1"`        //平台节点数
-	AccessKey    string      `json:"accessKey"`
-	SecretKey    string      `json:"secretKey"`
+	Region       string               `json:"region"`                         //区域
+	ClusterType  string               `json:"clusterType" default:"Edge"`     //集群类型，默认边缘集群
+	ClusterSpec  ClusterSpecification `json:"clusterSpec" default:"Standard"` //集群规格，Standard, Small, Test
+	ChargeType   string               `json:"chargeType" default:"PrePaid"`   //付费类型，PrePaid, PostPaid
+	ChargePeriod int                  `json:"chargePeriod" default:"1"`       //付费周期
+	AppNodeNum   int                  `json:"appNodeNum" default:"-1"`        //平台节点数
+	AccessKey    string               `json:"accessKey"`
+	SecretKey    string               `json:"secretKey"`
 	// 从已有vpc创建，指定该值；否则新建vpc，指定VpcCIDR
 	VpcID   string `json:"vpcID"`
 	VpcCIDR string `json:"vpcCIDR"`
@@ -298,15 +298,15 @@ const (
 	NasSpec = "1TB"
 )
 
-type ClusterSpec string
+type ClusterSpecification string
 
 const (
-	ClusterSpecStandard ClusterSpec = "Standard"
-	ClusterSpecSmall    ClusterSpec = "Small"
-	ClusterSpecTest     ClusterSpec = "Test"
+	ClusterSpecStandard ClusterSpecification = "Standard"
+	ClusterSpecSmall    ClusterSpecification = "Small"
+	ClusterSpecTest     ClusterSpecification = "Test"
 )
 
-func (spec ClusterSpec) GetSpecNum() int {
+func (spec ClusterSpecification) GetSpecNum() int {
 	switch spec {
 	case ClusterSpecStandard:
 		return EdgeStandardNum

apistructs/erda_operator.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2021 Terminus, Inc.
+//
+// This program is free software: you can use, redistribute, and/or modify
+// it under the terms of the GNU Affero General Public License, version 3
+// or later ("AGPL"), as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package apistructs
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/erda-project/erda/pkg/parser/diceyml"
+)
+
+const (
+	ClusterPhaseNone     ClusterPhase = ""
+	ClusterPhaseInitJobs ClusterPhase = "InitJobs"
+	ClusterPhaseCreating ClusterPhase = "Creating"
+	ClusterPhaseUpdating ClusterPhase = "Updating"
+	ClusterPhaseRunning  ClusterPhase = "Running"
+	ClusterPhaseFailed   ClusterPhase = "Failed"
+	ClusterPhasePending  ClusterPhase = "Pending"
+)
+
+type ClusterPhase string
+type ComponentStatus string
+type ClusterSize string
+
+type DiceClusterList struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Items             []DiceCluster `json:"items"`
+}
+
+type DiceCluster struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              ClusterSpec   `json:"spec"`
+	Status            ClusterStatus `json:"status"`
+}
+
+type ClusterSpec struct {
+	ResetStatus          bool        `json:"resetStatus"`
+	AddonConfigMap       string      `json:"addonConfigMap"`
+	ClusterinfoConfigMap string      `json:"clusterinfoConfigMap"`
+	PlatformDomain       string      `json:"platformDomain"`
+	CookieDomain         string      `json:"cookieDomain"`
+	Size                 ClusterSize `json:"size"`
+	DiceCluster          string      `json:"diceCluster"`
+	// collector, openapi
+	MainPlatform map[string]string `json:"mainPlatform"`
+	// key: dice-service-name(e.g. ui), value: domain
+	// customDomain:
+	//   ui: dice.terminus.io,*.terminus.io
+	CustomDomain map[string]string `json:"customDomain"`
+	// deployment affinity labels for specific dice-service
+	// key: dice-service-name(e.g. gittar), value: label
+	// e.g.
+	// gittar: dice/gittar
+	CustomAffinity map[string]string `json:"customAffinity"`
+
+	InitJobs diceyml.Object `json:"initJobs"`
+
+	Dice           diceyml.Object `json:"dice"`
+	AddonPlatform  diceyml.Object `json:"addonPlatform"`
+	Gittar         diceyml.Object `json:"gittar"`
+	Pandora        diceyml.Object `json:"pandora"`
+	DiceUI         diceyml.Object `json:"diceUI"`
+	UC             diceyml.Object `json:"uc"`
+	SpotAnalyzer   diceyml.Object `json:"spotAnalyzer"`
+	SpotCollector  diceyml.Object `json:"spotCollector"`
+	SpotDashboard  diceyml.Object `json:"spotDashboard"`
+	SpotFilebeat   diceyml.Object `json:"spotFilebeat"`
+	SpotStatus     diceyml.Object `json:"spotStatus"`
+	SpotTelegraf   diceyml.Object `json:"spotTelegraf"`
+	Tmc            diceyml.Object `json:"tmc"`
+	Hepa           diceyml.Object `json:"hepa"`
+	SpotMonitor    diceyml.Object `json:"spotMonitor"`
+	Fdp            diceyml.Object `json:"fdp"`
+	MeshController diceyml.Object `json:"meshController"`
+}
+type ClusterStatus struct {
+	Phase      ClusterPhase               `json:"phase"`
+	Conditions []ErdaCondition            `json:"conditions"`
+	Components map[string]ComponentStatus `json:"components"`
+}
+
+type ErdaCondition struct {
+	Reason         string `json:"reason"`
+	TransitionTime string `json:"transitionTime"`
+}

apistructs/org.go

@@ -228,6 +228,11 @@ type OrgClusterRelationDTOResponse struct {
 	Data []OrgClusterRelationDTO `json:"data"`
 }
 
+type OrgClusterRelationDTOCreateResponse struct {
+	Header
+	Data string `json:"data"`
+}
+
 // OrgClusterRelationDTO 企业对应集群关系结构
 type OrgClusterRelationDTO struct {
 	ID          uint64    `json:"id"`

bundle/cluster.go

@@ -140,7 +140,7 @@ func (b *Bundle) CreateCluster(req *apistructs.ClusterCreateRequest, header ...h
 
 	var createResp apistructs.ClusterCreateResponse
 
-	q := hc.Post(host).Path("/api/clusters")
+	q := hc.Post(host).Path("/api/clusters").Header(httputil.InternalHeader, "bundle")
 	if len(header) > 0 {
 		q.Headers(header[0])
 	}
@@ -156,6 +156,18 @@ func (b *Bundle) CreateCluster(req *apistructs.ClusterCreateRequest, header ...h
 	return nil
 }
 
+func (b *Bundle) CreateClusterWithOrg(userID string, orgID uint64, req *apistructs.ClusterCreateRequest, header ...http.Header) error {
+	if err := b.CreateCluster(req, header...); err != nil {
+		return err
+	}
+
+	if err := b.CreateOrgClusterRelationsByOrg(req.Name, userID, orgID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // PatchCluster patch cluster with event
 func (b *Bundle) PatchCluster(req *apistructs.ClusterPatchRequest, header ...http.Header) error {
 	host, err := b.urls.ClusterManager()

bundle/org.go

@@ -350,3 +350,40 @@ func (b *Bundle) DereferenceCluster(orgID uint64, clusterName, userID string) (s
 	}
 	return resp.Data, nil
 }
+
+// CreateOrgClusterRelationsByOrg create orgClusters relation by orgID
+func (b *Bundle) CreateOrgClusterRelationsByOrg(clusterName string, userID string, orgID uint64) error {
+	host, err := b.urls.CoreServices()
+	if err != nil {
+		return err
+	}
+	hc := b.hc
+
+	org, err := b.GetOrg(orgID)
+	if err != nil {
+		return err
+	}
+
+	var createResp apistructs.OrgClusterRelationDTOCreateResponse
+
+	req := &apistructs.OrgClusterRelationCreateRequest{
+		OrgID:       orgID,
+		OrgName:     org.Name,
+		ClusterName: clusterName,
+	}
+
+	resp, err := hc.Post(host).Path("/api/orgs/actions/relate-cluster").
+		Header(httputil.UserHeader, userID).
+		JSONBody(req).
+		Do().
+		JSON(&createResp)
+
+	if err != nil {
+		return apierrors.ErrInvoke.InternalError(err)
+	}
+	if !resp.IsOK() || !createResp.Success {
+		return toAPIError(resp.StatusCode(), createResp.Error)
+	}
+
+	return nil
+}

cmd/cluster-init/main.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2021 Terminus, Inc.
+//
+// This program is free software: you can use, redistribute, and/or modify
+// it under the terms of the GNU Affero General Public License, version 3
+// or later ("AGPL"), as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"github.com/erda-project/erda-infra/modcom"
+	_ "github.com/erda-project/erda-infra/providers"
+	_ "github.com/erda-project/erda/modules/cluster-init"
+)
+
+func main() {
+	modcom.RunWithCfgDir("conf/cluster-init", "cluster-init")
+}

conf/cluster-init/cluster-init.yaml

@@ -0,0 +1 @@
+cluster-init:

erda.yml

@@ -506,6 +506,9 @@ services:
       exec: {}
   cmp:
     cmd: "/app/cmp"
+    envs:
+      UC_CLIENT_ID: "dice"
+      UC_CLIENT_SECRET: "secret"
     ports:
       - port: 9027
         protocol: "TCP"

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/clbanning/mxj v1.8.4 // indirect
 	github.com/cockroachdb/apd v1.1.0 // indirect
 	github.com/confluentinc/confluent-kafka-go v1.5.2
-	github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa
+	github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1
 	github.com/coreos/etcd v3.3.25+incompatible
 	github.com/creack/pty v1.1.11 // indirect
 	github.com/davecgh/go-spew v1.1.1
@@ -59,6 +59,7 @@ require (
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a // indirect
 	github.com/gocql/gocql v0.0.0-20210401103645-80ab1e13e309
+	github.com/gofrs/flock v0.7.1
 	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/gogap/errors v0.0.0-20200228125012-531a6449b28c
 	github.com/gogap/stack v0.0.0-20150131034635-fef68dddd4f8 // indirect
@@ -85,7 +86,6 @@ require (
 	github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570 // indirect
 	github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f
 	github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042 // indirect
-	github.com/lib/pq v1.3.0 // indirect
 	github.com/libgit2/git2go/v30 v30.0.5
 	github.com/magiconair/properties v1.8.4
 	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
@@ -152,10 +152,12 @@ require (
 	gorm.io/driver/mysql v1.0.5
 	gorm.io/gorm v1.21.8
 	gotest.tools v2.2.0+incompatible
+	helm.sh/helm/v3 v3.3.4
 	howett.net/plist v0.0.0-20201203080718-1454fab16a06
 	istio.io/api v0.0.0-20200715212100-dbf5277541ef
 	istio.io/client-go v0.0.0-20201005161859-d8818315d678
 	k8s.io/api v0.19.1
+	k8s.io/apiextensions-apiserver v0.18.8
 	k8s.io/apimachinery v0.19.1
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/kubernetes v1.18.3