feat: deployment order support show batch status (#4113)

apistructs/deployment_order.go

@@ -59,7 +59,7 @@ type DeploymentOrderListConditions struct {
 
 type DeploymentOrderDetail struct {
 	DeploymentOrderItem
-	ApplicationsInfo []*ApplicationInfo `json:"applicationsInfo"`
+	ApplicationsInfo [][]*ApplicationInfo `json:"applicationsInfo"`
 }
 
 type ApplicationInfo struct {
@@ -73,7 +73,6 @@ type ApplicationInfo struct {
 	CommitId       string                `json:"commitId,omitempty"`
 	PreCheckResult *PreCheckResult       `json:"preCheckResult,omitempty"`
 	DiceYaml       string                `json:"diceYaml,omitempty"`
-	Batch          int                   `json:"batch"`
 	Status         DeploymentStatus      `json:"status,omitempty"`
 }
 

conf/orchestrator/orchestrator.yaml

@@ -15,8 +15,9 @@ grpc-server:
 service-register:
 
 i18n:
-  files:
+  common:
     - conf/orchestrator/i18n/log-trans.yaml
+    - conf/orchestrator/i18n/deployment-order-precheck.yaml
 
 erda.orchestrator.runtime:
 erda.orchestrator.events:

modules/orchestrator/dbclient/deployment_order.go

@@ -170,7 +170,19 @@ func (db *DBClient) ListReleases(releasesId []string) ([]*Release, error) {
 	return releases, nil
 }
 
-func (db *DBClient) UpdateDeploymentOrderAppsStatus(orderId string, newOrderStatusMap apistructs.DeploymentOrderStatusMap) error {
+func (db *DBClient) ListReleasesMap(releasesId []string) (map[string]*Release, error) {
+	releases := make([]*Release, 0)
+	releaseMap := make(map[string]*Release, 0)
+	if err := db.Where("release_id in (?)", releasesId).Find(&releases).Error; err != nil {
+		return nil, errors.Wrapf(err, "failed to list release %+v", releasesId)
+	}
+	for _, r := range releases {
+		releaseMap[r.ReleaseId] = r
+	}
+	return releaseMap, nil
+}
+
+func (db *DBClient) UpdateDeploymentOrderStatusDetail(orderId string, newOrderStatusMap apistructs.DeploymentOrderStatusMap) error {
 	var (
 		deploymentOrder   DeploymentOrder
 		curOrderStatusMap apistructs.DeploymentOrderStatusMap
@@ -209,7 +221,7 @@ func (db *DBClient) UpdateDeploymentOrderAppsStatus(orderId string, newOrderStat
 	return nil
 }
 
-func (db *DBClient) UpdateDeploymentOrderStatus(id string, appName string,
+func (db *DBClient) UpdateDeploymentOrderAppStatus(id string, appName string,
 	appStatus apistructs.DeploymentOrderStatusItem) error {
 	return db.Transaction(func(tx *gorm.DB) error {
 		var (

modules/orchestrator/endpoints/deployment_order.go

@@ -58,6 +58,7 @@ func (e *Endpoints) CreateDeploymentOrder(ctx context.Context, r *http.Request,
 		return apierrors.ErrCreateDeploymentOrder.InvalidParameter(strutil.Concat("illegal workspace ", req.Workspace)).ToResp(), nil
 	}
 
+	req.Workspace = strings.ToUpper(req.Workspace)
 	req.Operator = userID.String()
 
 	data, err := e.deploymentOrder.Create(&req)

modules/orchestrator/initialize.go

@@ -91,7 +91,7 @@ func (p *provider) Initialize(ctx servicehub.Context) error {
 	// start cron jobs to sync addon & project infos
 	go initCron(ep, ctx)
 
-	i18n.SetSingle(p.LogTrans)
+	i18n.SetSingle(p.Trans)
 
 	return nil
 }

modules/orchestrator/provider.go

@@ -29,7 +29,7 @@ type provider struct {
 	Orm          *gorm.DB             `autowired:"mysql-client"`
 	EventManager *events.EventManager `autowired:"erda.orchestrator.events.event-manager"`
 	PusherQueue  *queue.PusherQueue   `autowired:"erda.orchestrator.events.pusher-queue"`
-	LogTrans     i18n.Translator      `translator:"log-trans"`
+	Trans        i18n.Translator      `autowired:"i18n" translator:"common"`
 }
 
 func (p *provider) Init(ctx servicehub.Context) error {

modules/orchestrator/services/deployment/deployment_context.go

@@ -711,7 +711,7 @@ func (fsm *DeployFSMContext) UpdateDeploymentStatusToRuntimeAndOrder() error {
 	}
 	logrus.Infof("update deployment(%+v) status for app (%+v) to deployment_order (%+v) detail is: %+v",
 		fsm.deploymentID, app.Name, DeploymentOrderID, appDeploymentStatus)
-	if err := fsm.db.UpdateDeploymentOrderStatus(DeploymentOrderID,
+	if err := fsm.db.UpdateDeploymentOrderAppStatus(DeploymentOrderID,
 		app.Name, appDeploymentStatus); err != nil {
 		errMsg := fmt.Sprintf("failed to update deployment order status of deployment[%s]: %v",
 			DeploymentOrderID, err)

modules/orchestrator/services/deployment_order/deployment_order_create.go

@@ -125,8 +125,10 @@ func (d *DeploymentOrder) Deploy(req *apistructs.DeploymentOrderDeployRequest) (
 	}
 
 	// deploy interface will means execute from the first batch
-	// TODO: continue deploying need front function design
-	order.CurrentBatch = FirstBatch
+	// if current batch is not zero, means it is a retry at current batch, deploy will continue from the current batch
+	if order.CurrentBatch == 0 {
+		order.CurrentBatch = FirstBatch
+	}
 
 	if _, err := d.executeDeploy(order, releaseResp, apistructs.SourceDeployCenter, false); err != nil {
 		logrus.Errorf("failed to execute deploy, order id: %s, err: %v", req.DeploymentOrderId, err)
@@ -168,12 +170,10 @@ func (d *DeploymentOrder) executeDeploy(order *dbclient.DeploymentOrder, release
 
 	applicationsStatus := make(apistructs.DeploymentOrderStatusMap)
 	// redeploy
-	if order.CurrentBatch != FirstBatch {
-		if order.StatusDetail != "" {
-			if err := json.Unmarshal([]byte(order.StatusDetail), &applicationsStatus); err != nil {
-				return nil, fmt.Errorf("failed to unmarshal to deployment order status (%s), err: %v",
-					order.ID, err)
-			}
+	if order.CurrentBatch != FirstBatch && order.StatusDetail != "" {
+		if err := json.Unmarshal([]byte(order.StatusDetail), &applicationsStatus); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal to deployment order status (%s), err: %v",
+				order.ID, err)
 		}
 	} else {
 		order.StartedAt = time.Now()

modules/orchestrator/services/deployment_order/deployment_order_cron.go

@@ -15,11 +15,14 @@
 package deployment_order
 
 import (
+	"encoding/json"
+
 	"github.com/jinzhu/gorm"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
 	"github.com/erda-project/erda/apistructs"
+	"github.com/erda-project/erda/modules/orchestrator/dbclient"
 	"github.com/erda-project/erda/modules/orchestrator/queue"
 	"github.com/erda-project/erda/modules/orchestrator/utils"
 )
@@ -90,7 +93,7 @@ func (d *DeploymentOrder) PushOnDeploymentOrderPolling() (abort bool, err0 error
 		}
 
 		// status update, only update status of current batch
-		if err := d.db.UpdateDeploymentOrderAppsStatus(order.ID, statusMap); err != nil {
+		if err := inspectDeploymentStatusDetail(&order, statusMap); err != nil {
 			logrus.Errorf("failed to update deployment order %s status, (%v)", order.ID, err)
 			continue
 		}
@@ -132,3 +135,30 @@ func (d *DeploymentOrder) PushOnDeploymentOrderPolling() (abort bool, err0 error
 
 	return
 }
+
+func inspectDeploymentStatusDetail(order *dbclient.DeploymentOrder, newOrderStatusMap apistructs.DeploymentOrderStatusMap) error {
+	curOrderStatusMap := make(apistructs.DeploymentOrderStatusMap, 0)
+
+	if order.StatusDetail != "" {
+		if err := json.Unmarshal([]byte(order.StatusDetail), &curOrderStatusMap); err != nil {
+			return errors.Wrapf(err, "failed to unmarshal to deployment order status (%s)",
+				order.ID)
+		}
+	}
+
+	for appName, status := range newOrderStatusMap {
+		if status.DeploymentID == 0 || status.DeploymentStatus == "" {
+			continue
+		}
+		curOrderStatusMap[appName] = status
+	}
+
+	orderStatusMapJson, err := json.Marshal(curOrderStatusMap)
+	if err != nil {
+		return errors.Wrapf(err, "failed to marshal to deployment order status (%s)",
+			order.ID)
+	}
+
+	order.StatusDetail = string(orderStatusMapJson)
+	return nil
+}

modules/orchestrator/services/deployment_order/deployment_order_cron_test.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2021 Terminus, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deployment_order
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/erda-project/erda/apistructs"
+	"github.com/erda-project/erda/modules/orchestrator/dbclient"
+)
+
+func TestInspectDeploymentStatusDetail(t *testing.T) {
+	type args struct {
+		DeploymentOrder *dbclient.DeploymentOrder
+		StatusMap       apistructs.DeploymentOrderStatusMap
+	}
+
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "first-batch",
+			args: args{
+				DeploymentOrder: &dbclient.DeploymentOrder{},
+				StatusMap: apistructs.DeploymentOrderStatusMap{
+					"java-demo": apistructs.DeploymentOrderStatusItem{
+						AppID:            1,
+						DeploymentID:     1,
+						DeploymentStatus: apistructs.DeploymentStatusInit,
+						RuntimeID:        1,
+					},
+				},
+			},
+			want: "{\"java-demo\":{\"appId\":1,\"deploymentId\":1,\"deploymentStatus\":\"INIT\",\"runtimeId\":1}}",
+		},
+		{
+			name: "status-appending",
+			args: args{
+				DeploymentOrder: &dbclient.DeploymentOrder{
+					StatusDetail: "{\"go-demo\":{\"appId\":0,\"deploymentId\":0,\"deploymentStatus\":\"INIT\",\"runtimeId\":0}}",
+				},
+				StatusMap: apistructs.DeploymentOrderStatusMap{
+					"java-demo": apistructs.DeploymentOrderStatusItem{
+						AppID:            1,
+						DeploymentID:     1,
+						DeploymentStatus: apistructs.DeploymentStatusDeploying,
+						RuntimeID:        1,
+					},
+				},
+			},
+			want: "{\"go-demo\":{\"appId\":0,\"deploymentId\":0,\"deploymentStatus\":\"INIT\",\"runtimeId\":0},\"" +
+				"java-demo\":{\"appId\":1,\"deploymentId\":1,\"deploymentStatus\":\"DEPLOYING\",\"runtimeId\":1}}",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := inspectDeploymentStatusDetail(tt.args.DeploymentOrder, tt.args.StatusMap)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, tt.args.DeploymentOrder.StatusDetail)
+		})
+	}
+}

modules/orchestrator/services/deployment_order/deployment_order_get.go

@@ -61,7 +61,7 @@ func (d *DeploymentOrder) Get(userId string, orderId string) (*apistructs.Deploy
 		return nil, fmt.Errorf("failed to get release, err: %v", err)
 	}
 
-	releases := make([]*dbclient.Release, 0)
+	releases := make([][]*dbclient.Release, 0)
 
 	if curRelease.IsProjectRelease {
 		subReleasesId := make([][]string, 0)
@@ -74,15 +74,21 @@ func (d *DeploymentOrder) Get(userId string, orderId string) (*apistructs.Deploy
 			conditionData = append(conditionData, id...)
 		}
 
-		subReleases, err := d.db.ListReleases(conditionData)
+		subReleaseMap, err := d.db.ListReleasesMap(conditionData)
 		if err != nil {
 			return nil, fmt.Errorf("failed to list sub release, err: %v", err)
 		}
-		for _, subRelease := range subReleases {
-			releases = append(releases, subRelease)
+
+		for _, sr := range subReleasesId {
+			tmp := make([]*dbclient.Release, 0)
+			for _, r := range sr {
+				tmp = append(tmp, subReleaseMap[r])
+			}
+			releases = append(releases, tmp)
 		}
+
 	} else {
-		releases = append(releases, curRelease)
+		releases = append(releases, []*dbclient.Release{curRelease})
 	}
 
 	// compose applications info
@@ -117,56 +123,60 @@ func (d *DeploymentOrder) Get(userId string, orderId string) (*apistructs.Deploy
 	}, nil
 }
 
-func composeApplicationsInfo(releases []*dbclient.Release, params map[string]apistructs.DeploymentOrderParam,
-	appsStatus apistructs.DeploymentOrderStatusMap) ([]*apistructs.ApplicationInfo, error) {
-
-	asi := make([]*apistructs.ApplicationInfo, 0)
-
-	for _, subRelease := range releases {
-		applicationName := subRelease.ApplicationName
-
-		// parse deployment order
-		orderParamsData := make(apistructs.DeploymentOrderParam, 0)
-
-		param, ok := params[applicationName]
-		if ok {
-			for _, data := range param {
-				if data.Encrypt {
-					data.Value = ""
+func composeApplicationsInfo(releases [][]*dbclient.Release, params map[string]apistructs.DeploymentOrderParam,
+	appsStatus apistructs.DeploymentOrderStatusMap) ([][]*apistructs.ApplicationInfo, error) {
+
+	asi := make([][]*apistructs.ApplicationInfo, 0)
+
+	for _, sr := range releases {
+		ai := make([]*apistructs.ApplicationInfo, 0)
+		for _, r := range sr {
+			applicationName := r.ApplicationName
+
+			// parse deployment order
+			orderParamsData := make(apistructs.DeploymentOrderParam, 0)
+
+			param, ok := params[applicationName]
+			if ok {
+				for _, data := range param {
+					if data.Encrypt {
+						data.Value = ""
+					}
+					orderParamsData = append(orderParamsData, &apistructs.DeploymentOrderParamData{
+						Key:     data.Key,
+						Value:   data.Value,
+						Encrypt: data.Encrypt,
+						Type:    convertConfigType(data.Type),
+						Comment: data.Comment,
+					})
 				}
-				orderParamsData = append(orderParamsData, &apistructs.DeploymentOrderParamData{
-					Key:     data.Key,
-					Value:   data.Value,
-					Encrypt: data.Encrypt,
-					Type:    convertConfigType(data.Type),
-					Comment: data.Comment,
-				})
 			}
-		}
 
-		var status apistructs.DeploymentStatus
-		app, ok := appsStatus[subRelease.ApplicationName]
-		if ok {
-			status = app.DeploymentStatus
-		}
+			var status apistructs.DeploymentStatus = apistructs.OrderStatusWaitDeploy
+			app, ok := appsStatus[r.ApplicationName]
+			if ok {
+				status = app.DeploymentStatus
+			}
 
-		labels := make(map[string]string)
-		if err := json.Unmarshal([]byte(subRelease.Labels), &labels); err != nil {
-			return nil, fmt.Errorf("failed to unmarshal release labels, err: %v", err)
-		}
+			labels := make(map[string]string)
+			if err := json.Unmarshal([]byte(r.Labels), &labels); err != nil {
+				return nil, fmt.Errorf("failed to unmarshal release labels, err: %v", err)
+			}
 
-		asi = append(asi, &apistructs.ApplicationInfo{
-			Id:             subRelease.ApplicationId,
-			Name:           applicationName,
-			DeploymentId:   app.DeploymentID,
-			Params:         &orderParamsData,
-			ReleaseId:      subRelease.ReleaseId,
-			ReleaseVersion: subRelease.Version,
-			Branch:         labels["gitBranch"],
-			DiceYaml:       subRelease.DiceYaml,
-			CommitId:       labels["gitCommitId"],
-			Status:         status,
-		})
+			ai = append(ai, &apistructs.ApplicationInfo{
+				Id:             r.ApplicationId,
+				Name:           applicationName,
+				DeploymentId:   app.DeploymentID,
+				Params:         &orderParamsData,
+				ReleaseId:      r.ReleaseId,
+				ReleaseVersion: r.Version,
+				Branch:         labels["gitBranch"],
+				DiceYaml:       r.DiceYaml,
+				CommitId:       labels["gitCommitId"],
+				Status:         status,
+			})
+		}
+		asi = append(asi, ai)
 	}
 
 	return asi, nil