From 9e132d656d60d07d42749c16952be51373eff4bc Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Wed, 3 Jan 2024 16:03:46 +0100 Subject: [PATCH 01/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- .../artifactory-build-info-plugin/go.mod | 1 + .../artifactory-build-info-plugin/go.sum | 2 + .../artifactory-promote-plugin/go.mod | 1 + .../artifactory-promote-plugin/go.sum | 2 + .../artifactory-release-plugin/go.mod | 1 + .../artifactory-release-plugin/go.sum | 2 + engine/api/operation/operation.go | 55 +++++++------- engine/api/v2_hooks.go | 16 ++-- engine/api/v2_repository_analyze.go | 2 - engine/api/v2_workflow_run.go | 12 +-- engine/api/v2_workflow_run_craft.go | 41 ++++++++-- .../scheduler_repository_event_callback.go | 76 ++++++++++--------- engine/hooks/trigger_workflow.go | 20 ++--- engine/repositories/processor_checkout.go | 13 ++++ go.mod | 3 +- go.sum | 6 +- sdk/contexts.go | 2 + sdk/hooks_repository_event.go | 11 ++- sdk/repositories_operation.go | 5 ++ sdk/v2_workflow_run.go | 25 +++++- 20 files changed, 194 insertions(+), 102 deletions(-) diff --git a/contrib/integrations/artifactory/artifactory-build-info-plugin/go.mod b/contrib/integrations/artifactory/artifactory-build-info-plugin/go.mod index 91c28f3910..21a01e2371 100644 --- a/contrib/integrations/artifactory/artifactory-build-info-plugin/go.mod +++ b/contrib/integrations/artifactory/artifactory-build-info-plugin/go.mod @@ -43,6 +43,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gookit/color v1.5.4 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect diff --git a/contrib/integrations/artifactory/artifactory-build-info-plugin/go.sum b/contrib/integrations/artifactory/artifactory-build-info-plugin/go.sum index 31908529bf..ce3dc0faeb 100644 --- a/contrib/integrations/artifactory/artifactory-build-info-plugin/go.sum +++ b/contrib/integrations/artifactory/artifactory-build-info-plugin/go.sum @@ -257,6 +257,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/contrib/integrations/artifactory/artifactory-promote-plugin/go.mod b/contrib/integrations/artifactory/artifactory-promote-plugin/go.mod index a7b47698d1..c6faa8d11e 100644 --- a/contrib/integrations/artifactory/artifactory-promote-plugin/go.mod +++ b/contrib/integrations/artifactory/artifactory-promote-plugin/go.mod @@ -43,6 +43,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gookit/color v1.5.4 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect diff --git a/contrib/integrations/artifactory/artifactory-promote-plugin/go.sum b/contrib/integrations/artifactory/artifactory-promote-plugin/go.sum index 31908529bf..ce3dc0faeb 100644 --- a/contrib/integrations/artifactory/artifactory-promote-plugin/go.sum +++ b/contrib/integrations/artifactory/artifactory-promote-plugin/go.sum @@ -257,6 +257,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/contrib/integrations/artifactory/artifactory-release-plugin/go.mod b/contrib/integrations/artifactory/artifactory-release-plugin/go.mod index 278cec1245..c12dd563ee 100644 --- a/contrib/integrations/artifactory/artifactory-release-plugin/go.mod +++ b/contrib/integrations/artifactory/artifactory-release-plugin/go.mod @@ -43,6 +43,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gookit/color v1.5.4 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect diff --git a/contrib/integrations/artifactory/artifactory-release-plugin/go.sum b/contrib/integrations/artifactory/artifactory-release-plugin/go.sum index 31908529bf..ce3dc0faeb 100644 --- a/contrib/integrations/artifactory/artifactory-release-plugin/go.sum +++ b/contrib/integrations/artifactory/artifactory-release-plugin/go.sum @@ -257,6 +257,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/engine/api/operation/operation.go b/engine/api/operation/operation.go index bac7bfb075..9eedb69382 100644 --- a/engine/api/operation/operation.go +++ b/engine/api/operation/operation.go @@ -194,31 +194,32 @@ func Poll(ctx context.Context, db gorp.SqlExecutor, operationUUID string) (*sdk. } func CheckoutAndAnalyzeOperation(ctx context.Context, db gorp.SqlExecutor, proj sdk.Project, vcsWithSecret sdk.VCSProject, repoName, repoCloneURL string, commit, branch string) (*sdk.Operation, error) { - ope := &sdk.Operation{ - VCSServer: vcsWithSecret.Name, - RepoFullName: repoName, - URL: repoCloneURL, - RepositoryStrategy: sdk.RepositoryStrategy{ - SSHKey: vcsWithSecret.Auth.SSHKeyName, - User: vcsWithSecret.Auth.Username, - Password: vcsWithSecret.Auth.Token, - }, - Setup: sdk.OperationSetup{ - Checkout: sdk.OperationCheckout{ - Commit: commit, - Branch: branch, - CheckSignature: true, - }, - }, - } - if vcsWithSecret.Auth.SSHKeyName != "" { - ope.RepositoryStrategy.ConnectionType = "ssh" - } else { - ope.RepositoryStrategy.ConnectionType = "https" - } - - if err := PostRepositoryOperation(ctx, db, proj, ope, nil); err != nil { - return nil, err - } - return ope, nil + ope := &sdk.Operation{ + VCSServer: vcsWithSecret.Name, + RepoFullName: repoName, + URL: repoCloneURL, + RepositoryStrategy: sdk.RepositoryStrategy{ + SSHKey: vcsWithSecret.Auth.SSHKeyName, + User: vcsWithSecret.Auth.Username, + Password: vcsWithSecret.Auth.Token, + }, + Setup: sdk.OperationSetup{ + Checkout: sdk.OperationCheckout{ + Commit: commit, + Branch: branch, + CheckSignature: true, + ProcessSemver: true, + }, + }, + } + if vcsWithSecret.Auth.SSHKeyName != "" { + ope.RepositoryStrategy.ConnectionType = "ssh" + } else { + ope.RepositoryStrategy.ConnectionType = "https" + } + + if err := PostRepositoryOperation(ctx, db, proj, ope, nil); err != nil { + return nil, err + } + return ope, nil } diff --git a/engine/api/v2_hooks.go b/engine/api/v2_hooks.go index 0e11c001bf..32727633d2 100644 --- a/engine/api/v2_hooks.go +++ b/engine/api/v2_hooks.go @@ -138,14 +138,20 @@ func (api *API) postHookEventRetrieveSignKeyHandler() ([]service.RbacChecker, se HookEventUUID: hookRetrieveSignKey.HookEventUUID, SigningKeyCallback: &sdk.HookSigninKeyCallback{}, } - if ope.Status == sdk.OperationStatusDone && ope.Setup.Checkout.Result.CommitVerified { - callback.SigningKeyCallback.SignKey = ope.Setup.Checkout.Result.SignKeyID - } else if ope.Status == sdk.OperationStatusDone && !ope.Setup.Checkout.Result.CommitVerified { - callback.SigningKeyCallback.SignKey = ope.Setup.Checkout.Result.SignKeyID - callback.SigningKeyCallback.Error = ope.Setup.Checkout.Result.Msg + if ope.Status == sdk.OperationStatusDone { + callback.SigningKeyCallback.SemverCurrent = ope.Setup.Checkout.Result.Semver.Current + callback.SigningKeyCallback.SemverNext = ope.Setup.Checkout.Result.Semver.Next + + if ope.Setup.Checkout.Result.CommitVerified { + callback.SigningKeyCallback.SignKey = ope.Setup.Checkout.Result.SignKeyID + } else { + callback.SigningKeyCallback.SignKey = ope.Setup.Checkout.Result.SignKeyID + callback.SigningKeyCallback.Error = ope.Setup.Checkout.Result.Msg + } } else { callback.SigningKeyCallback.Error = ope.Error.Message } + if _, code, err := services.NewClient(api.mustDB(), srvs).DoJSONRequest(ctx, http.MethodPost, "/v2/repository/event/callback", callback, nil); err != nil { log.ErrorWithStackTrace(ctx, sdk.WrapError(err, "unable to send analysis call to hook [HTTP: %d]", code)) return diff --git a/engine/api/v2_repository_analyze.go b/engine/api/v2_repository_analyze.go index c6074390da..5ff5d96df3 100644 --- a/engine/api/v2_repository_analyze.go +++ b/engine/api/v2_repository_analyze.go @@ -643,8 +643,6 @@ func sendAnalysisHookCallback(ctx context.Context, db *gorp.DbMap, analysis sdk. AnalysisID: analysis.ID, Models: make([]sdk.EntityFullName, 0), Workflows: make([]sdk.EntityFullName, 0), - UserID: analysis.Data.CDSUserID, - Username: analysis.Data.CDSUserName, }, } for _, e := range entities { diff --git a/engine/api/v2_workflow_run.go b/engine/api/v2_workflow_run.go index dedebe4598..8437a9a8f2 100644 --- a/engine/api/v2_workflow_run.go +++ b/engine/api/v2_workflow_run.go @@ -731,10 +731,12 @@ func (api *API) postWorkflowRunFromHookV2Handler() ([]service.RbacChecker, servi } case sdk.WorkflowHookTypeRepository: runEvent.GitTrigger = &sdk.GitTrigger{ - Payload: runRequest.Payload, - EventName: runRequest.EventName, - Ref: runRequest.Ref, - Sha: runRequest.Sha, + Payload: runRequest.Payload, + EventName: runRequest.EventName, + Ref: runRequest.Ref, + Sha: runRequest.Sha, + SemverCurrent: runRequest.SemverCurrent, + SemverNext: runRequest.SemverNext, } default: return sdk.WrapError(sdk.ErrWrongRequest, "unknown event: %v", runRequest) @@ -1144,7 +1146,7 @@ func (api *API) postWorkflowRunV2Handler() ([]service.RbacChecker, service.Handl return err } - runEvent := sdk.V2WorkflowRunEvent{ + runEvent := sdk.V2WorkflowRunEvent{ // TODO handler semver ? Manual: &sdk.ManualTrigger{ Payload: runRequest, }, diff --git a/engine/api/v2_workflow_run_craft.go b/engine/api/v2_workflow_run_craft.go index 1129995b3e..5f6f4de9b3 100644 --- a/engine/api/v2_workflow_run_craft.go +++ b/engine/api/v2_workflow_run_craft.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/go-gorp/gorp" "github.com/rockbears/log" "github.com/rockbears/yaml" @@ -548,7 +549,7 @@ func stopRun(ctx context.Context, db *gorp.DbMap, run *sdk.V2WorkflowRun, messag func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p sdk.Project, wr sdk.V2WorkflowRun, vcsServer sdk.VCSProject, repo sdk.ProjectRepository, u sdk.AuthentifiedUser) (*sdk.WorkflowRunContext, error) { var runContext sdk.WorkflowRunContext - var ref, commit string + var ref, commit, semverNext, semverCurrent string cdsContext := sdk.CDSContext{ ProjectKey: wr.ProjectKey, @@ -583,6 +584,29 @@ func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p s cdsContext.Event = wr.RunEvent.GitTrigger.Payload ref = wr.RunEvent.GitTrigger.Ref commit = wr.RunEvent.GitTrigger.Sha + + semverNext = wr.RunEvent.GitTrigger.SemverNext + + // Compute current semver version with CDS metadata + currentVersion, _ := semver.NewVersion(wr.RunEvent.GitTrigger.SemverCurrent) + if currentVersion != nil { + suffix := currentVersion.Metadata() + splittedSuffix := strings.Split(suffix, ".") + metadataStr := strconv.FormatInt(wr.RunNumber, 10) + if len(splittedSuffix) >= 2 { + metadataStr += "." + splittedSuffix[0] + ".sha." + splittedSuffix[1] + } + for i := 2; i < len(splittedSuffix); i++ { + metadataStr += "." + splittedSuffix[i] + } + + v, _ := currentVersion.SetMetadata(metadataStr) + semverCurrent = v.String() + } else { + // If no semver found, compute it from 0.1.0 + semverCurrent = "0.1.0+" + strconv.FormatInt(wr.RunNumber, 10) + ".sha." + commit + } + case wr.RunEvent.ModelUpdateTrigger != nil: ref = wr.RunEvent.ModelUpdateTrigger.Ref case wr.RunEvent.WorkflowUpdateTrigger != nil: @@ -604,12 +628,17 @@ func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p s workflowVCSServer := *vcsTmp gitContext := sdk.GitContext{ - Server: workflowVCSServer.Name, - SSHKey: workflowVCSServer.Auth.SSHKeyName, - Username: workflowVCSServer.Auth.Username, - Ref: ref, - Sha: commit, + Server: workflowVCSServer.Name, + SSHKey: workflowVCSServer.Auth.SSHKeyName, + Username: workflowVCSServer.Auth.Username, + Ref: ref, + Sha: commit, + SemverCurrent: semverCurrent, + SemverNext: semverNext, } + + log.Warn(ctx, "git context: %+v", gitContext) + if gitContext.SSHKey != "" { gitContext.Connection = "ssh" } else { diff --git a/engine/hooks/scheduler_repository_event_callback.go b/engine/hooks/scheduler_repository_event_callback.go index 074f985550..024103856a 100644 --- a/engine/hooks/scheduler_repository_event_callback.go +++ b/engine/hooks/scheduler_repository_event_callback.go @@ -81,48 +81,52 @@ func (s *Service) updateHookEventWithCallback(ctx context.Context, callback sdk. return nil } - if hre.Status != sdk.HookEventStatusAnalysis && hre.Status != sdk.HookEventStatusSignKey { - return nil - } - - if callback.AnalysisCallback != nil { - if callback.AnalysisCallback.UserID != "" { - hre.UserID = callback.AnalysisCallback.UserID - hre.Username = callback.AnalysisCallback.Username - } - for i := range hre.Analyses { - a := &hre.Analyses[i] - if a.AnalyzeID == callback.AnalysisCallback.AnalysisID { - if a.Status == sdk.RepositoryAnalysisStatusInProgress { - a.Status = callback.AnalysisCallback.AnalysisStatus - hre.ModelUpdated = append(hre.ModelUpdated, callback.AnalysisCallback.Models...) - hre.WorkflowUpdated = append(hre.WorkflowUpdated, callback.AnalysisCallback.Workflows...) - if err := s.Dao.SaveRepositoryEvent(ctx, &hre); err != nil { - return err + switch hre.Status { + case sdk.HookEventStatusAnalysis: + if callback.AnalysisCallback != nil { + for i := range hre.Analyses { + a := &hre.Analyses[i] + if a.AnalyzeID == callback.AnalysisCallback.AnalysisID { + if a.Status == sdk.RepositoryAnalysisStatusInProgress { + a.Status = callback.AnalysisCallback.AnalysisStatus + hre.ModelUpdated = append(hre.ModelUpdated, callback.AnalysisCallback.Models...) + hre.WorkflowUpdated = append(hre.WorkflowUpdated, callback.AnalysisCallback.Workflows...) + if err := s.Dao.SaveRepositoryEvent(ctx, &hre); err != nil { + return err + } + break } - break } } + } else { + return sdk.Errorf("missing analysis callback data") } - } - // Manage signinKey - if callback.SigningKeyCallback != nil { - if callback.SigningKeyCallback.SignKey != "" && callback.SigningKeyCallback.Error != "" { - // event on error commit unverified - hre.Status = sdk.HookEventStatusSkipped - hre.LastError = callback.SigningKeyCallback.Error - hre.NbErrors++ - } else if callback.SigningKeyCallback.SignKey != "" && callback.SigningKeyCallback.Error == "" { - // commit verified - hre.SignKey = callback.SigningKeyCallback.SignKey - } else if callback.SigningKeyCallback.Error != "" { - hre.LastError = "Unable to get signing key: " + callback.SigningKeyCallback.Error - hre.NbErrors++ - } - if err := s.Dao.SaveRepositoryEvent(ctx, &hre); err != nil { - return err + case sdk.HookEventStatusSignKey: + if callback.SigningKeyCallback != nil { + hre.SemverCurrent = callback.SigningKeyCallback.SemverCurrent + hre.SemverNext = callback.SigningKeyCallback.SemverNext + if callback.SigningKeyCallback.SignKey != "" && callback.SigningKeyCallback.Error != "" { + // event on error commit unverified + hre.Status = sdk.HookEventStatusSkipped + hre.LastError = callback.SigningKeyCallback.Error + hre.NbErrors++ + } else if callback.SigningKeyCallback.SignKey != "" && callback.SigningKeyCallback.Error == "" { + // commit verified + hre.SignKey = callback.SigningKeyCallback.SignKey + } else if callback.SigningKeyCallback.Error != "" { + hre.LastError = "Unable to get signing key: " + callback.SigningKeyCallback.Error + hre.NbErrors++ + } + } else { + return sdk.Errorf("missing analysis callback data") } + default: + return nil + } + + if err := s.Dao.SaveRepositoryEvent(ctx, &hre); err != nil { + return err } if err := s.Dao.EnqueueRepositoryEvent(ctx, &hre); err != nil { diff --git a/engine/hooks/trigger_workflow.go b/engine/hooks/trigger_workflow.go index e755398f4a..1a02442ce2 100644 --- a/engine/hooks/trigger_workflow.go +++ b/engine/hooks/trigger_workflow.go @@ -54,21 +54,21 @@ func (s *Service) triggerWorkflows(ctx context.Context, hre *sdk.HookRepositoryE wh := &hre.WorkflowHooks[i] if wh.Status == sdk.HookEventWorkflowStatusScheduler { - targetCommit := "HEAD" - runRequest := sdk.V2WorkflowRunHookRequest{ - HookEventID: hre.UUID, - UserID: hre.UserID, - Ref: wh.TargetBranch, - Sha: targetCommit, - Payload: event, - EventName: hre.EventName, - HookType: wh.Type, + HookEventID: hre.UUID, + UserID: hre.UserID, + Ref: wh.TargetBranch, + Sha: "HEAD", + Payload: event, + EventName: hre.EventName, + HookType: wh.Type, + SemverCurrent: hre.SemverCurrent, + SemverNext: hre.SemverNext, } switch wh.Type { case sdk.WorkflowHookTypeRepository: - targetCommit = hre.ExtractData.Commit + runRequest.Sha = hre.ExtractData.Commit if runRequest.Ref == "" { runRequest.Ref = hre.ExtractData.Branch } diff --git a/engine/repositories/processor_checkout.go b/engine/repositories/processor_checkout.go index 350b9e5b4c..e2f6067047 100644 --- a/engine/repositories/processor_checkout.go +++ b/engine/repositories/processor_checkout.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/pkg/errors" "github.com/rockbears/log" "github.com/ovh/cds/sdk" @@ -69,6 +70,18 @@ func (s *Service) processCheckout(ctx context.Context, op *sdk.Operation) error } } + if op.Setup.Checkout.ProcessSemver { + describe, err := gitRepo.Describe(ctx, nil) + if err != nil { + log.ErrorWithStackTrace(ctx, errors.Wrap(err, "git describe failed")) + } else { + if describe.Semver != nil { + op.Setup.Checkout.Result.Semver.Current = describe.SemverString + op.Setup.Checkout.Result.Semver.Next = describe.Semver.IncMinor().String() + } + } + } + if op.Setup.Checkout.CheckSignature && op.Setup.Checkout.Commit != "" { log.Debug(ctx, "retrieve gpg key id") c, err := gitRepo.GetCommit(ctx, op.Setup.Checkout.Commit) diff --git a/go.mod b/go.mod index e1556a1ebf..eacfc173f3 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( code.gitea.io/sdk/gitea v0.15.1-0.20220530220844-359c771ce3d2 contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.4.2 + github.com/Masterminds/semver/v3 v3.2.1 github.com/Shopify/sarama v1.36.0 github.com/andygrunwald/go-gerrit v0.0.0-20181207071854-19ef3e9332a4 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 @@ -21,7 +22,7 @@ require ( github.com/eapache/go-resiliency v1.3.0 github.com/fatih/color v1.13.0 github.com/fsamin/go-dump v1.0.9 - github.com/fsamin/go-repo v0.2.3 + github.com/fsamin/go-repo v0.3.0 github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be github.com/fujiwara/shapeio v0.0.0-20170602072123-c073257dd745 github.com/go-gorp/gorp v2.0.0+incompatible diff --git a/go.sum b/go.sum index 7b7af9ba2a..36529097f9 100644 --- a/go.sum +++ b/go.sum @@ -83,6 +83,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBIdfux8gnQ= github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -250,8 +252,8 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsamin/go-dump v1.0.9 h1:3MAneAJLnGfKTJtFEAdgrD+QqqK2Hwj7EJUQMQZcDls= github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMWdvAa9o= -github.com/fsamin/go-repo v0.2.3 h1:2fhiQHfrPonIWy7cY3kOTkfH3wkmIEJzMgpJUuVEXRU= -github.com/fsamin/go-repo v0.2.3/go.mod h1:b6ydaKjjNNVOn/IlxZQK7x7fVDxKQIunAyQBgz+CYoA= +github.com/fsamin/go-repo v0.3.0 h1:pRt4r7QxdSU76PPPRMoDKAxNUOurBmjJ0KrV7GXIG+E= +github.com/fsamin/go-repo v0.3.0/go.mod h1:E4T4rNd76b3ZeobYtwRNiHFxZwfMw7mlV72amdTF7Vo= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be h1:UhjSvwE1gxUYfekK9BXZ/LL55we9Avg+2Pt0PIlMYCk= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= diff --git a/sdk/contexts.go b/sdk/contexts.go index 29bd7c41e4..556a5764ac 100644 --- a/sdk/contexts.go +++ b/sdk/contexts.go @@ -91,6 +91,8 @@ type GitContext struct { Connection string `json:"connection,omitempty"` SSHKey string `json:"ssh_key,omitempty"` Username string `json:"username,omitempty"` + SemverCurrent string `json:"semver_current,omitempty"` + SemverNext string `json:"semver_next,omitempty"` } type JobContext struct { diff --git a/sdk/hooks_repository_event.go b/sdk/hooks_repository_event.go index 6345cfeca6..b987e31872 100644 --- a/sdk/hooks_repository_event.go +++ b/sdk/hooks_repository_event.go @@ -5,6 +5,7 @@ import ( "crypto/sha512" "encoding/base64" + "golang.org/x/crypto/pbkdf2" ) @@ -41,8 +42,10 @@ type HookEventCallback struct { } type HookSigninKeyCallback struct { - SignKey string `json:"sign_key"` - Error string `json:"error"` + SignKey string `json:"sign_key"` + SemverCurrent string `json:"semver_current"` + SemverNext string `json:"semver_next"` + Error string `json:"error"` } type HookAnalysisCallback struct { @@ -50,8 +53,6 @@ type HookAnalysisCallback struct { AnalysisStatus string `json:"analysis_status"` Models []EntityFullName `json:"models"` Workflows []EntityFullName `json:"workflows"` - UserID string `json:"user_id"` - Username string `json:"username"` } type HookRepository struct { @@ -83,6 +84,8 @@ type HookRepositoryEvent struct { Username string `json:"username"` SignKey string `json:"sign_key"` SigningKeyOperation string `json:"signing_key_operation"` + SemverCurrent string `json:"semver_current"` + SemverNext string `json:"semver_next"` } type HookRepositoryEventWorkflow struct { diff --git a/sdk/repositories_operation.go b/sdk/repositories_operation.go index 8c4388b2ab..2f668531d7 100644 --- a/sdk/repositories_operation.go +++ b/sdk/repositories_operation.go @@ -87,10 +87,15 @@ type OperationCheckout struct { Branch string `json:"branch,omitempty"` Commit string `json:"commit,omitempty"` CheckSignature bool `json:"check_signature,omitempty"` + ProcessSemver bool `json:"process_semver,omitempty"` Result struct { SignKeyID string `json:"sign_key_id"` CommitVerified bool `json:"verified"` Msg string `json:"msg"` + Semver struct { + Current string `json:"current"` + Next string `json:"next"` + } `json:"semver"` } `json:"result"` } diff --git a/sdk/v2_workflow_run.go b/sdk/v2_workflow_run.go index 86c8921a70..eee8f474c5 100644 --- a/sdk/v2_workflow_run.go +++ b/sdk/v2_workflow_run.go @@ -27,6 +27,8 @@ type V2WorkflowRunHookRequest struct { Payload map[string]interface{} `json:"payload"` HookType string `json:"hook_type"` EntityUpdated string `json:"entity_updated"` + SemverCurrent string `json:"semver_current"` + SemverNext string `json:"semver_next"` } type V2WorkflowRun struct { @@ -174,10 +176,12 @@ type SchedulerTrigger struct { } type GitTrigger struct { - EventName string `json:"event_name"` - Payload map[string]interface{} `json:"payload"` - Ref string `json:"ref"` - Sha string `json:"sha"` + EventName string `json:"event_name"` + Payload map[string]interface{} `json:"payload"` + Ref string `json:"ref"` + Sha string `json:"sha"` + SemverCurrent string `json:"semver_current"` + SemverNext string `json:"sember_next"` } type WorkflowUpdateTrigger struct { @@ -491,6 +495,17 @@ const ( type V2WorkflowRunResultArtifactManagerMetadata map[string]string +func (m *V2WorkflowRunResultArtifactManagerMetadata) Set(k, v string) { + (*m)[k] = v +} + +func (m *V2WorkflowRunResultArtifactManagerMetadata) Get(k string) string { + if m == nil { + return "" + } + return (*m)[k] +} + func V2WorkflowRunResultArtifactManagerMetadataFromCDNItemLink(i CDNItemLink) *V2WorkflowRunResultArtifactManagerMetadata { return &V2WorkflowRunResultArtifactManagerMetadata{ "cdn_http_url": i.CDNHttpURL, @@ -613,6 +628,8 @@ const ( V2WorkflowRunResultTypeRelease = "release" V2WorkflowRunResultTypeGeneric = "generic" V2WorkflowRunResultTypeVariable = "variable" + V2WorkflowRunResultTypeDocker = "docker" + V2WorkflowRunResultTypeHelm = "helm" // Other values may be instantiated from Artifactory Manager repository type ) From 811104021269435f6b2e6690c6d1ca356183895f Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Wed, 3 Jan 2024 16:37:12 +0100 Subject: [PATCH 02/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- engine/api/v2_workflow_run_craft.go | 32 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/engine/api/v2_workflow_run_craft.go b/engine/api/v2_workflow_run_craft.go index 5f6f4de9b3..a3ac461106 100644 --- a/engine/api/v2_workflow_run_craft.go +++ b/engine/api/v2_workflow_run_craft.go @@ -590,18 +590,28 @@ func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p s // Compute current semver version with CDS metadata currentVersion, _ := semver.NewVersion(wr.RunEvent.GitTrigger.SemverCurrent) if currentVersion != nil { - suffix := currentVersion.Metadata() - splittedSuffix := strings.Split(suffix, ".") - metadataStr := strconv.FormatInt(wr.RunNumber, 10) - if len(splittedSuffix) >= 2 { - metadataStr += "." + splittedSuffix[0] + ".sha." + splittedSuffix[1] - } - for i := 2; i < len(splittedSuffix); i++ { - metadataStr += "." + splittedSuffix[i] + if currentVersion.Metadata() == "" { // Tags doesn't have metadata + semverCurrent = wr.Event.GitTrigger.SemverCurrent + } else { + suffix := currentVersion.Metadata() + splittedSuffix := strings.Split(suffix, ".") + var metadataStr string + if len(splittedSuffix) >= 2 { + metadataStr += splittedSuffix[0] + ".sha." + splittedSuffix[1] + } + for i := 2; i < len(splittedSuffix); i++ { + metadataStr += "." + splittedSuffix[i] + } + preRelease := currentVersion.Prerelease() + var v = *currentVersion + if preRelease != "" { + v, _ = currentVersion.SetPrerelease(preRelease + "-" + strconv.FormatInt(wr.RunNumber, 10)) + } else { + v, _ = currentVersion.SetPrerelease(strconv.FormatInt(wr.RunNumber, 10)) + } + v, _ = v.SetMetadata(metadataStr) + semverCurrent = v.String() } - - v, _ := currentVersion.SetMetadata(metadataStr) - semverCurrent = v.String() } else { // If no semver found, compute it from 0.1.0 semverCurrent = "0.1.0+" + strconv.FormatInt(wr.RunNumber, 10) + ".sha." + commit From d897b2d7fc9a6fe7d381376cc84b0ceb5a639354 Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Wed, 3 Jan 2024 18:29:14 +0100 Subject: [PATCH 03/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- tests/lib/v2_create_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/v2_create_project.yml b/tests/lib/v2_create_project.yml index ad687647e8..a4e63311f9 100644 --- a/tests/lib/v2_create_project.yml +++ b/tests/lib/v2_create_project.yml @@ -21,7 +21,7 @@ steps: {{.input.cdsctl_command}} project create {{.input.cds_project}} "Test Project {{.input.cds_project}}" {{.input.cds_project}} # Get SSH Key info from project - script: > - {{.input.cdsctl_command}} project keys list {{.input.cds_project}} --format json + {{.input.cdsctl_command}} project keys list {{.input.cds_project}} --filter type=ssh --format json vars: sshKeyPub: from: result.systemoutjson.systemoutjson0.publickey From a5632c5da81cdd07da52810679a8145d75e1f759 Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Thu, 4 Jan 2024 13:51:57 +0100 Subject: [PATCH 04/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- tests/08_v2_checkout.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/08_v2_checkout.yml b/tests/08_v2_checkout.yml index 4105707d17..649805ef62 100644 --- a/tests/08_v2_checkout.yml +++ b/tests/08_v2_checkout.yml @@ -63,8 +63,12 @@ testcases: init: runs-on: {{.cds_project}}/my_vcs_server/{{.git.user}}/{{.git_repo}}/docker-debian steps: - - uses: actions/checkout - id: checkout + - id: checkout + uses: actions/checkout + - id: git-variables + run: |- + #!/bin/bash -x + env | grep GIT - name: Check CDS project analyses status script: "{{.cdsctl}} -f {{.cdsctl.config}} experimental project analysis list {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} --format json" @@ -86,7 +90,10 @@ testcases: - name: Download logs script: {{.cdsctl}} -f {{.cdsctl.config}} experimental workflow logs download {{.cds_project}} my_vcs_server {{.git.user}}/{{.git_repo}} {{.cds_workflow}} 1 - - name: Check log content + - name: Check checkout log content script: "cat {{.cds_workflow}}-1-1-init-checkout | grep \"Checkout completed\"" + - name: Check git-variables log content to check SEMVER variables are available + script: "cat {{.cds_workflow}}-1-1-init-git-variables | grep SEMVER" + From ae2e48831bf1bdc23f1bb7a4f4bab5e658c09871 Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Thu, 4 Jan 2024 14:25:43 +0100 Subject: [PATCH 05/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- engine/api/v2_workflow_run_craft.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/api/v2_workflow_run_craft.go b/engine/api/v2_workflow_run_craft.go index a3ac461106..a4216e0a70 100644 --- a/engine/api/v2_workflow_run_craft.go +++ b/engine/api/v2_workflow_run_craft.go @@ -591,7 +591,7 @@ func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p s currentVersion, _ := semver.NewVersion(wr.RunEvent.GitTrigger.SemverCurrent) if currentVersion != nil { if currentVersion.Metadata() == "" { // Tags doesn't have metadata - semverCurrent = wr.Event.GitTrigger.SemverCurrent + semverCurrent = wr.RunEvent.GitTrigger.SemverCurrent } else { suffix := currentVersion.Metadata() splittedSuffix := strings.Split(suffix, ".") From c6224944373816138eea5d93b12046efc90de9ee Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Thu, 4 Jan 2024 15:59:11 +0100 Subject: [PATCH 06/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- engine/api/v2_workflow_run_craft.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/engine/api/v2_workflow_run_craft.go b/engine/api/v2_workflow_run_craft.go index a4216e0a70..98a111dc27 100644 --- a/engine/api/v2_workflow_run_craft.go +++ b/engine/api/v2_workflow_run_craft.go @@ -647,8 +647,6 @@ func buildRunContext(ctx context.Context, db *gorp.DbMap, store cache.Store, p s SemverNext: semverNext, } - log.Warn(ctx, "git context: %+v", gitContext) - if gitContext.SSHKey != "" { gitContext.Connection = "ssh" } else { From 77a8396437e8f9b13c511609d50d8043469864f0 Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Thu, 4 Jan 2024 16:40:43 +0100 Subject: [PATCH 07/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- tests/08_v2_checkout.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/08_v2_checkout.yml b/tests/08_v2_checkout.yml index 649805ef62..c3f2dfddcd 100644 --- a/tests/08_v2_checkout.yml +++ b/tests/08_v2_checkout.yml @@ -65,7 +65,7 @@ testcases: steps: - id: checkout uses: actions/checkout - - id: git-variables + - id: gitvariables run: |- #!/bin/bash -x env | grep GIT @@ -93,7 +93,7 @@ testcases: - name: Check checkout log content script: "cat {{.cds_workflow}}-1-1-init-checkout | grep \"Checkout completed\"" - - name: Check git-variables log content to check SEMVER variables are available - script: "cat {{.cds_workflow}}-1-1-init-git-variables | grep SEMVER" + - name: Check gitvariables log content to check SEMVER variables are available + script: "cat {{.cds_workflow}}-1-1-init-gitvariables | grep SEMVER" From 89c4b4994da04e28a72f0742eabadf94ae300450 Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Thu, 4 Jan 2024 17:08:23 +0100 Subject: [PATCH 08/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- tests/08_v2_checkout.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/08_v2_checkout.yml b/tests/08_v2_checkout.yml index c3f2dfddcd..a8e4f1326f 100644 --- a/tests/08_v2_checkout.yml +++ b/tests/08_v2_checkout.yml @@ -3,6 +3,8 @@ vars: cds_project: "ITV2CHECKOUT" git_repo: "it_v2_checkout" cds_workflow: "WorkflowCheckout" + worker_model: "{{.cds_project}}/my_vcs_server/{{.git.user}}/{{.git_repo}}/docker-debian" + testcases: - name: Prepare test steps: @@ -61,7 +63,7 @@ testcases: on: [push] jobs: init: - runs-on: {{.cds_project}}/my_vcs_server/{{.git.user}}/{{.git_repo}}/docker-debian + runs-on: "{{.worker_model}}" steps: - id: checkout uses: actions/checkout From 0c88517739a67dcd2832b73dcac182725787cc74 Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Thu, 4 Jan 2024 17:30:30 +0100 Subject: [PATCH 09/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- tests/08_v2_checkout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/08_v2_checkout.yml b/tests/08_v2_checkout.yml index a8e4f1326f..f46d9b3686 100644 --- a/tests/08_v2_checkout.yml +++ b/tests/08_v2_checkout.yml @@ -86,7 +86,7 @@ testcases: assertions: - result.systemoutjson ShouldHaveLength 1 - result.systemoutjson.systemoutjson0.status ShouldEqual "Success" - retry: 2 + retry: 5 delay: 30 - name: Download logs From 901417a01db3d0a856f170de0ec07beb9691800c Mon Sep 17 00:00:00 2001 From: "francois.samin" Date: Fri, 5 Jan 2024 09:33:22 +0100 Subject: [PATCH 10/10] feat(api, repositories, hooks): compute semver for workflow run git context Signed-off-by: francois.samin --- engine/hooks/trigger_signingkey.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/hooks/trigger_signingkey.go b/engine/hooks/trigger_signingkey.go index ddb9f72250..5adfdd95ec 100644 --- a/engine/hooks/trigger_signingkey.go +++ b/engine/hooks/trigger_signingkey.go @@ -3,9 +3,10 @@ package hooks import ( "context" "fmt" - "github.com/rockbears/log" "time" + "github.com/rockbears/log" + "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/telemetry" ) @@ -60,6 +61,8 @@ func (s *Service) triggerGetSigningKey(ctx context.Context, hre *sdk.HookReposit } if ope.Status == sdk.OperationStatusDone { hre.SignKey = ope.Setup.Checkout.Result.SignKeyID + hre.SemverCurrent = ope.Setup.Checkout.Result.Semver.Current + hre.SemverNext = ope.Setup.Checkout.Result.Semver.Next if !ope.Setup.Checkout.Result.CommitVerified { hre.Status = sdk.HookEventStatusSkipped hre.LastError = fmt.Sprintf("User with key '%s' not found in CDS", hre.SignKey)