test: promotion

Signed-off-by: francois.samin <francois.samin@corp.ovh.com>

contrib/grpcplugins/action/docker-push/main.go

@@ -85,7 +85,7 @@ func (actPlugin *dockerPushPlugin) perform(ctx context.Context, image string, ta
 		return sdk.Errorf("unable to get instanciate docker client: %v", err)
 	}
 
-	imageSummaries, err := cli.ImageList(ctx, types.ImageListOptions{All: true})
+	imageSummaries, err := cli.ImageList(ctx, types.ImageListOptions{All: false})
 	if err != nil {
 		return sdk.Errorf("unable to get docker image %q: %v", image, err)
 	}

contrib/grpcplugins/action/downloadArtifact/main.go

@@ -87,6 +87,8 @@ func (actPlugin *runActionDownloadArtifactlugin) perform(ctx context.Context, na
 	grpcplugins.Logf("Total number of files that will be downloaded: %d", len(response.RunResults))
 
 	for _, r := range response.RunResults {
+		// TODO: some of the run results are not downloadable
+
 		t0 := time.Now()
 
 		switch {

contrib/grpcplugins/action/helm-push/main.go

@@ -116,8 +116,9 @@ func (p *helmPushPlugin) perform(
 			Status:   sdk.V2WorkflowRunResultStatusPending,
 			Detail: sdk.V2WorkflowRunResultDetail{
 				Data: sdk.V2WorkflowRunResultHelmDetail{
-					Name:       chartFolder,
-					AppVersion: chartVersion,
+					Name:         chartFolder,
+					AppVersion:   appVersion,
+					ChartVersion: chartFolder,
 				},
 			},
 		},

contrib/grpcplugins/grpcplugins.go

@@ -12,10 +12,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/jfrog/jfrog-client-go/artifactory/services/utils"
 	"github.com/pkg/errors"
 
+	art "github.com/ovh/cds/contrib/integrations/artifactory"
 	"github.com/ovh/cds/engine/worker/pkg/workerruntime"
 	"github.com/ovh/cds/sdk"
+	"github.com/ovh/cds/sdk/artifact_manager"
 	"github.com/ovh/cds/sdk/grpcplugin/actionplugin"
 )
 
@@ -379,3 +382,89 @@ func GetArtifactoryFolderInfo(ctx context.Context, c *actionplugin.Common, confi
 
 	return &res, nil
 }
+
+func GetArtifactoryRunResults(ctx context.Context, c *actionplugin.Common, pattern string) (*workerruntime.V2GetResultResponse, error) {
+	response, err := GetV2RunResults(ctx, c, workerruntime.V2FilterRunResult{Pattern: pattern, WithClearIntegration: true})
+	if err != nil {
+		return nil, err
+	}
+	var final []sdk.V2WorkflowRunResult
+	for i := range response.RunResults {
+		if response.RunResults[i].ArtifactManagerIntegrationName != nil {
+			final = append(final, response.RunResults[i])
+		}
+	}
+	return &workerruntime.V2GetResultResponse{
+		RunResults: final,
+	}, nil
+}
+
+func PromoteArtifactoryRunResult(ctx context.Context, c *actionplugin.Common, r sdk.V2WorkflowRunResult, promotionType sdk.WorkflowRunResultPromotionType, maturity string, props *utils.Properties) error {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Minute)
+	defer cancel()
+
+	integration, err := GetIntegrationByName(ctx, c, *r.ArtifactManagerIntegrationName)
+	if err != nil {
+		return err
+	}
+
+	rtConfig := ArtifactoryConfig{
+		URL:   integration.Config[sdk.ArtifactoryConfigURL].Value,
+		Token: integration.Config[sdk.ArtifactoryConfigToken].Value,
+	}
+
+	artifactClient, err := artifact_manager.NewClient("artifactory", rtConfig.URL, rtConfig.Token)
+	if err != nil {
+		return errors.Errorf("Failed to create artifactory client: %v", err)
+	}
+
+	latestPromotion := r.DataSync.LatestPromotionOrRelease()
+	currentMaturity := integration.Config[sdk.ArtifactoryConfigPromotionLowMaturity].Value
+	if latestPromotion != nil {
+		currentMaturity = latestPromotion.ToMaturity
+	}
+
+	if maturity == "" {
+		maturity = integration.Config[sdk.ArtifactoryConfigPromotionHighMaturity].Value
+	}
+
+	newPromotion := sdk.WorkflowRunResultPromotion{
+		Date:         time.Now(),
+		FromMaturity: currentMaturity,
+		ToMaturity:   maturity,
+	}
+
+	data := art.FileToPromote{
+		RepoType: r.ArtifactManagerMetadata.Get("type"),
+		RepoName: r.ArtifactManagerMetadata.Get("repository"),
+		Name:     r.ArtifactManagerMetadata.Get("name"),
+		Path:     r.ArtifactManagerMetadata.Get("path"),
+	}
+
+	switch r.Type {
+	case "docker":
+		if err := art.PromoteDockerImage(ctx, artifactClient, data, latestPromotion.FromMaturity, latestPromotion.ToMaturity, props, false); err != nil {
+			return errors.Errorf("unable to promote docker image: %s: %v", data.Name+"-"+latestPromotion.ToMaturity, err)
+		}
+	default:
+		Logf("promoting %s (%s) from %s-%s to %s", data.Name, data.Path, data.RepoName, latestPromotion.FromMaturity, latestPromotion.ToMaturity)
+		if err := art.PromoteFile(artifactClient, data, latestPromotion.FromMaturity, latestPromotion.ToMaturity, props, false); err != nil {
+			return errors.Errorf("unable to promote file: %s: %v", data.Name, err)
+		}
+	}
+
+	switch promotionType {
+	case sdk.WorkflowRunResultPromotionTypePromote:
+		r.DataSync.Promotions = append(r.DataSync.Promotions, newPromotion)
+	case sdk.WorkflowRunResultPromotionTypeRelease:
+		r.DataSync.Releases = append(r.DataSync.Releases, newPromotion)
+	}
+
+	// TODO: update metadata
+
+	if _, err := UpdateRunResult(ctx, c, &workerruntime.V2RunResultRequest{RunResult: &r}); err != nil {
+		return err
+	}
+
+	return nil
+}

contrib/integrations/artifactory/artifactory-promote-plugin/main.go

@@ -126,11 +126,11 @@ func (e *artifactoryPromotePlugin) Run(ctx context.Context, opts *integrationplu
 		}
 		switch rData.RepoType {
 		case "docker":
-			if err := art.PromoteDockerImage(ctx, artifactClient, rData, latestPromotion.FromMaturity, latestPromotion.ToMaturity, props, false); err != nil {
+			if err := art.PromoteDockerImage(ctx, artifactClient, art.FileToPromote{RepoType: rData.RepoType, RepoName: rData.RepoName, Name: rData.Name, Path: rData.Path}, latestPromotion.FromMaturity, latestPromotion.ToMaturity, props, false); err != nil {
 				return fail("unable to promote docker image: %s: %v", rData.Name+"-"+latestPromotion.ToMaturity, err)
 			}
 		default:
-			if err := art.PromoteFile(artifactClient, rData, latestPromotion.FromMaturity, latestPromotion.ToMaturity, props, false); err != nil {
+			if err := art.PromoteFile(artifactClient, art.FileToPromote{RepoType: rData.RepoType, RepoName: rData.RepoName, Name: rData.Name, Path: rData.Path}, latestPromotion.FromMaturity, latestPromotion.ToMaturity, props, false); err != nil {
 				return fail("unable to promote file: %s: %v", rData.Name, err)
 			}
 		}

contrib/integrations/artifactory/artifactory.go

@@ -71,7 +71,14 @@ func CreateArtifactoryClient(ctx context.Context, url, token string) (artifactor
 	return artifactory.New(serviceConfig)
 }
 
-func PromoteFile(artiClient artifact_manager.ArtifactManager, data sdk.WorkflowRunResultArtifactManager, lowMaturity, highMaturity string, props *utils.Properties, skipExistingArtifacts bool) error {
+type FileToPromote struct {
+	RepoType string
+	RepoName string
+	Name     string
+	Path     string
+}
+
+func PromoteFile(artiClient artifact_manager.ArtifactManager, data FileToPromote, lowMaturity, highMaturity string, props *utils.Properties, skipExistingArtifacts bool) error {
 	// artifactory does not manage virtual cargo repositories
 	var srcRepo, targetRepo string
 	switch data.RepoType {
@@ -106,7 +113,7 @@ func PromoteFile(artiClient artifact_manager.ArtifactManager, data sdk.WorkflowR
 			// Get the properties of the source reposiytory
 			maturity, err := artiClient.GetRepositoryMaturity(srcRepo)
 			if err != nil {
-				return fmt.Errorf("unable to get repository maturity: %v\n", err)
+				return fmt.Errorf("unable to get repository maturity: %v", err)
 			}
 
 			if maturity == "release" {
@@ -143,7 +150,7 @@ func PromoteFile(artiClient artifact_manager.ArtifactManager, data sdk.WorkflowR
 	return nil
 }
 
-func PromoteDockerImage(ctx context.Context, artiClient artifact_manager.ArtifactManager, data sdk.WorkflowRunResultArtifactManager, lowMaturity, highMaturity string, props *utils.Properties, skipExistingArtifacts bool) error {
+func PromoteDockerImage(ctx context.Context, artiClient artifact_manager.ArtifactManager, data FileToPromote, lowMaturity, highMaturity string, props *utils.Properties, skipExistingArtifacts bool) error {
 	sourceRepo := fmt.Sprintf("%s-%s", data.RepoName, lowMaturity)
 	targetRepo := fmt.Sprintf("%s-%s", data.RepoName, highMaturity)
 	params := services.NewDockerPromoteParams(data.Path, sourceRepo, targetRepo)

engine/api/api.go

@@ -63,6 +63,7 @@ import (
 	"github.com/ovh/cds/sdk"
 	"github.com/ovh/cds/sdk/cdsclient"
 	"github.com/ovh/cds/sdk/jws"
+	cdslog "github.com/ovh/cds/sdk/log"
 )
 
 // Configuration is the configuration structure for CDS API
@@ -483,6 +484,9 @@ type StartupConfigConsumer struct {
 // Serve will start the http api server
 func (a *API) Serve(ctx context.Context) error {
 
+	// Skip this verbose log
+	log.Skip(cdslog.Handler, "api.(*API).postServiceHearbeatHandler-fm.(*API).postServiceHearbeatHandler")
+
 	log.Info(ctx, "Starting CDS API Server %s", sdk.VERSION)
 
 	a.StartupTime = time.Now()

engine/sql/api/107_primary_keys_set.sql

@@ -10,7 +10,7 @@ select create_primary_key('pipeline_trigger', 'ID');
 select create_primary_key('application_key', 'ID');
 select create_primary_key('template_action', 'template_id,action_id');
 select create_primary_key('action_audit', 'action_id,user_id,versionned');
-select create_primary_key('gorp_migrations_lock', 'ID');
+--- select create_primary_key('gorp_migrations_lock', 'ID');
 
 DROP TABLE IF EXISTS sla;
 

engine/worker/internal/runtime_v2.go

@@ -69,6 +69,7 @@ func (wk *CurrentWorker) V2AddRunResult(ctx context.Context, req workerruntime.V
 		response.CDNSignature = signature
 		response.CDNAddress = wk.CDNHttpURL()
 	} else {
+		// TODO: filter by run result type
 		log.Info(ctx, "enabling integration %q for run result %s", integ.Name, response.RunResult.ID)
 		response.RunResult.ArtifactManagerIntegrationName = &integ.Name
 	}
@@ -132,13 +133,18 @@ func (wk *CurrentWorker) V2GetRunResult(ctx context.Context, filter workerruntim
 	}
 	pattern := glob.New(filter.Pattern)
 	for _, r := range resp {
-		if r.Type != filter.Type {
+		if filter.Type != "" && r.Type != filter.Type {
 			continue
 		}
 		switch r.Detail.Type {
 		case "V2WorkflowRunResultGenericDetail":
-			x, _ := r.GetDetailAsV2WorkflowRunResultGenericDetail()
-			res, err := pattern.MatchString(x.Name)
+			var res *glob.Result
+			if filter.Type == "V2WorkflowRunResultGenericDetail" { // If the filter is set to "V2WorkflowRunResultGenericDetail" we can directly check the artifact name. This is the usecase of plugin "downloadArtifact"
+				x, _ := r.GetDetailAsV2WorkflowRunResultGenericDetail()
+				res, err = pattern.MatchString(x.Name)
+			} else {
+				res, err = pattern.MatchString(r.Name())
+			}
 			if err != nil {
 				log.Error(ctx, "unable to perform glob expression on %s (%s): %v", r.Name(), r.ID, err)
 				continue
@@ -147,7 +153,14 @@ func (wk *CurrentWorker) V2GetRunResult(ctx context.Context, filter workerruntim
 				result.RunResults = append(result.RunResults, r)
 			}
 		default:
-			log.Error(ctx, "unsupported run result detail %q type", r.Detail.Type)
+			res, err := pattern.MatchString(r.Name()) // We match with the implementation of the Name function that depends on V2WorkflowRunResult.Type (docker:image/latest, generic:foo.txt, etc...)
+			if err != nil {
+				log.Error(ctx, "unable to perform glob expression on %s (%s): %v", r.Name(), r.ID, err)
+				continue
+			}
+			if res != nil {
+				result.RunResults = append(result.RunResults, r)
+			}
 		}
 	}
 

engine/worker/pkg/workerruntime/handlers_v2.go

@@ -100,10 +100,6 @@ func V2_runResultHandler(ctx context.Context, wk Runtime) http.HandlerFunc {
 				writeError(w, r, sdk.NewError(sdk.ErrWrongRequest, err))
 				return
 			}
-			if filter.Type == "" {
-				writeError(w, r, sdk.ErrWrongRequest)
-				return
-			}
 			response, err := wk.V2GetRunResult(ctx, filter)
 			if err != nil {
 				writeError(w, r, err)

sdk/cdsclient/client_queue.go

@@ -119,8 +119,8 @@ func (c *client) QueuePolling(ctx context.Context, goRoutines *sdk.GoRoutines, h
 						log.Debug(ctx, "skipping job %s", id)
 						continue
 					}
-					lenqueue := pendingWorkerCreation.SetJobInPendingWorkerCreation(id)
-					log.Debug(ctx, "v1_len_queue: %v", lenqueue)
+					_ = pendingWorkerCreation.SetJobInPendingWorkerCreation(id)
+					//log.Debug(ctx, "v1_len_queue: %v", lenqueue)
 					telemetry.Record(ctx, hatcheryMetrics.ChanV1JobAdd, 1)
 					jobs <- *job
 				}
@@ -151,7 +151,7 @@ func (c *client) QueuePolling(ctx context.Context, goRoutines *sdk.GoRoutines, h
 				}
 				queueFiltered = append(queueFiltered, job)
 			}
-			log.Debug(ctx, "v1_job_queue_from_api: %v job_queue_filtered: %v len_queue: %v", len(queue), len(queueFiltered), pendingWorkerCreation.NbJobInPendingWorkerCreation())
+			// log.Debug(ctx, "v1_job_queue_from_api: %v job_queue_filtered: %v len_queue: %v", len(queue), len(queueFiltered), pendingWorkerCreation.NbJobInPendingWorkerCreation())
 
 			shrinkQueue(&queueFiltered, cap(jobs))
 			for _, j := range queueFiltered {

sdk/cdsclient/client_queue_V2.go

@@ -148,8 +148,8 @@ func (c *client) V2QueuePolling(ctx context.Context, regionName string, goRoutin
 					log.Debug(ctx, "skipping job %s", wsEvent.JobRunID)
 					continue
 				}
-				lenqueue := pendingWorkerCreation.SetJobInPendingWorkerCreation(wsEvent.JobRunID)
-				log.Debug(ctx, "v2_len_queue: %v", lenqueue)
+				_ = pendingWorkerCreation.SetJobInPendingWorkerCreation(wsEvent.JobRunID)
+				//log.Debug(ctx, "v2_len_queue: %v", lenqueue)
 				telemetry.Record(ctx, hatcheryMetrics.ChanV2JobAdd, 1)
 				jobs <- *j
 			}
@@ -179,7 +179,7 @@ func (c *client) V2QueuePolling(ctx context.Context, regionName string, goRoutin
 				queueFiltered = append(queueFiltered, job)
 			}
 
-			log.Debug(ctx, "v2_job_queue_from_api: %v job_queue_filtered: %v len_queue: %v", len(queue), len(queueFiltered), pendingWorkerCreation.NbJobInPendingWorkerCreation())
+			// log.Debug(ctx, "v2_job_queue_from_api: %v job_queue_filtered: %v len_queue: %v", len(queue), len(queueFiltered), pendingWorkerCreation.NbJobInPendingWorkerCreation())
 
 			max := cap(jobs) * 2
 			if len(queueFiltered) < max {

sdk/glob/glob_test.go

@@ -101,6 +101,18 @@ func TestGlob_Match(t *testing.T) {
 				},
 			},
 		},
+		{
+			"With colon",
+			`docker:path/to/image:* helm:**/*`, testcase{
+				args: []string{
+					"docker:path/to/image:latest", "helm:chart",
+				},
+				want: []string{
+					"image:latest",
+					"helm:chart",
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

sdk/v2_worker.go

@@ -26,5 +26,5 @@ type V2TakeJobResponse struct {
 	AsCodeActions  map[string]V2Action    `json:"actions"`
 	SigningKey     string                 `json:"signing_key"`
 	Contexts       WorkflowRunJobsContext `json:"contexts"`
-	SensitiveDatas []string               `json:"sensitive_datas`
+	SensitiveDatas []string               `json:"sensitive_datas"`
 }

sdk/v2_workflow_run.go

@@ -502,6 +502,11 @@ func (r *V2WorkflowRunResult) Name() string {
 		if ok {
 			return string(r.Type) + ":" + detail.Name
 		}
+	case V2WorkflowRunResultTypeHelm:
+		detail, ok := r.Detail.Data.(*V2WorkflowRunResultHelmDetail)
+		if ok {
+			return string(r.Type) + ":" + detail.Name
+		}
 	}
 	return string(r.Type) + ":" + r.ID
 }
@@ -680,8 +685,9 @@ type V2WorkflowRunResultDockerDetail struct {
 }
 
 type V2WorkflowRunResultHelmDetail struct {
-	Name       string `json:"name" mapstructure:"name"`
-	AppVersion string `json:"appVersion" mapstructure:"appVersion"`
+	Name         string `json:"name" mapstructure:"name"`
+	AppVersion   string `json:"appVersion" mapstructure:"appVersion"`
+	ChartVersion string `json:"chartVersion" mapstructure:"chartVersion"`
 }
 
 type V2WorkflowRunResultVariableDetail struct {