Replay pipeline using cli exec by downloading metadata

cli/exec/exec.go

@@ -37,6 +37,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local"
 	backend_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/compiler"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/linter"
@@ -76,6 +77,7 @@ func execDir(ctx context.Context, c *cli.Command, dir string) error {
 	if runtime.GOOS == "windows" {
 		repoPath = convertPathForWindows(repoPath)
 	}
+	// TODO: respect depends_on and do parallel runs with output to multiple _windows_ e.g. tmux like
 	return filepath.Walk(dir, func(path string, info os.FileInfo, e error) error {
 		if e != nil {
 			return e
@@ -84,7 +86,7 @@ func execDir(ctx context.Context, c *cli.Command, dir string) error {
 		// check if it is a regular file (not dir)
 		if info.Mode().IsRegular() && (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) {
 			fmt.Println("#", info.Name())
-			_ = runExec(ctx, c, path, repoPath) // TODO: should we drop errors or store them and report back?
+			_ = runExec(ctx, c, path, repoPath, false) // TODO: should we drop errors or store them and report back?
 			fmt.Println("")
 			return nil
 		}
@@ -103,10 +105,10 @@ func execFile(ctx context.Context, c *cli.Command, file string) error {
 	if runtime.GOOS == "windows" {
 		repoPath = convertPathForWindows(repoPath)
 	}
-	return runExec(ctx, c, file, repoPath)
+	return runExec(ctx, c, file, repoPath, true)
 }
 
-func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error {
+func runExec(ctx context.Context, c *cli.Command, file, repoPath string, singleExec bool) error {
 	dat, err := os.ReadFile(file)
 	if err != nil {
 		return err
@@ -121,19 +123,28 @@ func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error {
 		axes = append(axes, matrix.Axis{})
 	}
 	for _, axis := range axes {
-		err := execWithAxis(ctx, c, file, repoPath, axis)
+		err := execWithAxis(ctx, c, file, repoPath, axis, singleExec)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis) error {
-	metadata, err := metadataFromContext(ctx, c, axis)
+func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis, singleExec bool) error {
+	var metadataWorkflow *metadata.Workflow
+	if !singleExec {
+		// TODO: proper try to use the engine to generate the same metadata for workflows
+		// https://github.com/woodpecker-ci/woodpecker/pull/3967
+		metadataWorkflow.Name = strings.TrimSuffix(strings.TrimSuffix(file, ".yaml"), ".yml")
+	}
+	metadata, err := metadataFromContext(ctx, c, axis, metadataWorkflow)
 	if err != nil {
 		return fmt.Errorf("could not create metadata: %w", err)
+	} else if metadata == nil {
+		return fmt.Errorf("metadata is nil")
 	}
+
 	environ := metadata.Environ()
 	var secrets []compiler.Secret
 	for key, val := range metadata.Workflow.Matrix {
@@ -239,7 +250,7 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
 			c.String("netrc-password"),
 			c.String("netrc-machine"),
 		),
-		compiler.WithMetadata(metadata),
+		compiler.WithMetadata(*metadata),
 		compiler.WithSecret(secrets...),
 		compiler.WithEnviron(pipelineEnv),
 	).Compile(conf)

cli/exec/flags.go

@@ -32,6 +32,11 @@ var flags = []cli.Flag{
 		Name:    "repo-path",
 		Usage:   "path to local repository",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("WOODPECKER_METADATA_FILE"),
+		Name:    "metadata-file",
+		Usage:   "path to pipeline metadata file (normally downloaded from UI). Parameters can be adjusted by applying additional cli flags",
+	},
 	&cli.DurationFlag{
 		Sources: cli.EnvVars("WOODPECKER_TIMEOUT"),
 		Name:    "timeout",

cli/exec/metadata.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"runtime"
 	"strings"
 
@@ -29,108 +30,131 @@ import (
 )
 
 // return the metadata from the cli context.
-func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (metadata.Metadata, error) {
+func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis, w *metadata.Workflow) (*metadata.Metadata, error) {
+	m := &metadata.Metadata{}
+
+	if c.IsSet("metadata-file") {
+		metadataFile, err := os.Open(c.String("metadata-file"))
+		if err != nil {
+			return nil, err
+		}
+		defer metadataFile.Close()
+
+		if err := json.NewDecoder(metadataFile).Decode(m); err != nil {
+			return nil, err
+		}
+	}
+
 	platform := c.String("system-platform")
 	if platform == "" {
 		platform = runtime.GOOS + "/" + runtime.GOARCH
 	}
 
-	fullRepoName := c.String("repo-name")
-	repoOwner := ""
-	repoName := ""
-	if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
-		repoOwner = fullRepoName[:idx]
-		repoName = fullRepoName[idx+1:]
-	}
-
-	var changedFiles []string
-	changedFilesRaw := c.String("pipeline-changed-files")
-	if len(changedFilesRaw) != 0 && changedFilesRaw[0] == '[' {
-		if err := json.Unmarshal([]byte(changedFilesRaw), &changedFiles); err != nil {
-			return metadata.Metadata{}, fmt.Errorf("pipeline-changed-files detected json but could not parse it: %w", err)
+	metadataFileAndOverrideOrDefault(c, "repo-name", func(fullRepoName string) {
+		if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
+			m.Repo.Owner = fullRepoName[:idx]
+			m.Repo.Name = fullRepoName[idx+1:]
 		}
-	} else {
-		for _, file := range strings.Split(changedFilesRaw, ",") {
-			changedFiles = append(changedFiles, strings.TrimSpace(file))
+	}, c.String)
+
+	var err error
+	metadataFileAndOverrideOrDefault(c, "pipeline-changed-files", func(changedFilesRaw string) {
+		var changedFiles []string
+		if len(changedFilesRaw) != 0 && changedFilesRaw[0] == '[' {
+			if jsonErr := json.Unmarshal([]byte(changedFilesRaw), &changedFiles); jsonErr != nil {
+				err = fmt.Errorf("pipeline-changed-files detected json but could not parse it: %w", jsonErr)
+			}
+		} else {
+			for _, file := range strings.Split(changedFilesRaw, ",") {
+				changedFiles = append(changedFiles, strings.TrimSpace(file))
+			}
 		}
+		m.Curr.Commit.ChangedFiles = changedFiles
+	}, c.String)
+	if err != nil {
+		return nil, err
 	}
 
-	return metadata.Metadata{
-		Repo: metadata.Repo{
-			Name:        repoName,
-			Owner:       repoOwner,
-			RemoteID:    c.String("repo-remote-id"),
-			ForgeURL:    c.String("repo-url"),
-			SCM:         c.String("repo-scm"),
-			Branch:      c.String("repo-default-branch"),
-			CloneURL:    c.String("repo-clone-url"),
-			CloneSSHURL: c.String("repo-clone-ssh-url"),
-			Private:     c.Bool("repo-private"),
-			Trusted:     c.Bool("repo-trusted"),
-		},
-		Curr: metadata.Pipeline{
-			Number:     c.Int("pipeline-number"),
-			Parent:     c.Int("pipeline-parent"),
-			Created:    c.Int("pipeline-created"),
-			Started:    c.Int("pipeline-started"),
-			Finished:   c.Int("pipeline-finished"),
-			Status:     c.String("pipeline-status"),
-			Event:      c.String("pipeline-event"),
-			ForgeURL:   c.String("pipeline-url"),
-			DeployTo:   c.String("pipeline-deploy-to"),
-			DeployTask: c.String("pipeline-deploy-task"),
-			Commit: metadata.Commit{
-				Sha:     c.String("commit-sha"),
-				Ref:     c.String("commit-ref"),
-				Refspec: c.String("commit-refspec"),
-				Branch:  c.String("commit-branch"),
-				Message: c.String("commit-message"),
-				Author: metadata.Author{
-					Name:   c.String("commit-author-name"),
-					Email:  c.String("commit-author-email"),
-					Avatar: c.String("commit-author-avatar"),
-				},
-				PullRequestLabels: c.StringSlice("commit-pull-labels"),
-				IsPrerelease:      c.Bool("commit-release-is-pre"),
-				ChangedFiles:      changedFiles,
-			},
-		},
-		Prev: metadata.Pipeline{
-			Number:   c.Int("prev-pipeline-number"),
-			Created:  c.Int("prev-pipeline-created"),
-			Started:  c.Int("prev-pipeline-started"),
-			Finished: c.Int("prev-pipeline-finished"),
-			Status:   c.String("prev-pipeline-status"),
-			Event:    c.String("prev-pipeline-event"),
-			ForgeURL: c.String("prev-pipeline-url"),
-			Commit: metadata.Commit{
-				Sha:     c.String("prev-commit-sha"),
-				Ref:     c.String("prev-commit-ref"),
-				Refspec: c.String("prev-commit-refspec"),
-				Branch:  c.String("prev-commit-branch"),
-				Message: c.String("prev-commit-message"),
-				Author: metadata.Author{
-					Name:   c.String("prev-commit-author-name"),
-					Email:  c.String("prev-commit-author-email"),
-					Avatar: c.String("prev-commit-author-avatar"),
-				},
-			},
-		},
-		Workflow: metadata.Workflow{
-			Name:   c.String("workflow-name"),
-			Number: int(c.Int("workflow-number")),
-			Matrix: axis,
-		},
-		Sys: metadata.System{
-			Name:     c.String("system-name"),
-			URL:      c.String("system-url"),
-			Host:     c.String("system-host"),
-			Platform: platform,
-			Version:  version.Version,
-		},
-		Forge: metadata.Forge{
-			Type: c.String("forge-type"),
-			URL:  c.String("forge-url"),
-		},
-	}, nil
+	// Repo
+	metadataFileAndOverrideOrDefault(c, "repo-remote-id", func(s string) { m.Repo.RemoteID = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-url", func(s string) { m.Repo.ForgeURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-scm", func(s string) { m.Repo.SCM = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-default-branch", func(s string) { m.Repo.Branch = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-clone-url", func(s string) { m.Repo.CloneURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-clone-ssh-url", func(s string) { m.Repo.CloneSSHURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-private", func(b bool) { m.Repo.Private = b }, c.Bool)
+	metadataFileAndOverrideOrDefault(c, "repo-trusted", func(b bool) { m.Repo.Trusted = b }, c.Bool)
+
+	// Current Pipeline
+	metadataFileAndOverrideOrDefault(c, "pipeline-number", func(i int64) { m.Curr.Number = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-parent", func(i int64) { m.Curr.Parent = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-created", func(i int64) { m.Curr.Created = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-started", func(i int64) { m.Curr.Started = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-finished", func(i int64) { m.Curr.Finished = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-status", func(s string) { m.Curr.Status = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-event", func(s string) { m.Curr.Event = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-url", func(s string) { m.Curr.ForgeURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-deploy-to", func(s string) { m.Curr.DeployTo = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-deploy-task", func(s string) { m.Curr.DeployTask = s }, c.String)
+
+	// Current Pipeline Commit
+	metadataFileAndOverrideOrDefault(c, "commit-sha", func(s string) { m.Curr.Commit.Sha = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-ref", func(s string) { m.Curr.Commit.Ref = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-refspec", func(s string) { m.Curr.Commit.Refspec = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-branch", func(s string) { m.Curr.Commit.Branch = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-message", func(s string) { m.Curr.Commit.Message = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-author-name", func(s string) { m.Curr.Commit.Author.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-author-email", func(s string) { m.Curr.Commit.Author.Email = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-author-avatar", func(s string) { m.Curr.Commit.Author.Avatar = s }, c.String)
+
+	metadataFileAndOverrideOrDefault(c, "commit-pull-labels", func(sl []string) { m.Curr.Commit.PullRequestLabels = sl }, c.StringSlice)
+	metadataFileAndOverrideOrDefault(c, "commit-release-is-pre", func(b bool) { m.Curr.Commit.IsPrerelease = b }, c.Bool)
+
+	// Previous Pipeline
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-number", func(i int64) { m.Prev.Number = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-created", func(i int64) { m.Prev.Created = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-started", func(i int64) { m.Prev.Started = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-finished", func(i int64) { m.Prev.Finished = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-status", func(s string) { m.Prev.Status = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-event", func(s string) { m.Prev.Event = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-url", func(s string) { m.Prev.ForgeURL = s }, c.String)
+
+	// Previous Pipeline Commit
+	metadataFileAndOverrideOrDefault(c, "prev-commit-sha", func(s string) { m.Prev.Commit.Sha = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-ref", func(s string) { m.Prev.Commit.Ref = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-refspec", func(s string) { m.Prev.Commit.Refspec = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-branch", func(s string) { m.Prev.Commit.Branch = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-message", func(s string) { m.Prev.Commit.Message = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-author-name", func(s string) { m.Prev.Commit.Author.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-author-email", func(s string) { m.Prev.Commit.Author.Email = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-author-avatar", func(s string) { m.Prev.Commit.Author.Avatar = s }, c.String)
+
+	// Workflow
+	metadataFileAndOverrideOrDefault(c, "workflow-name", func(s string) { m.Workflow.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "workflow-number", func(i int64) { m.Workflow.Number = int(i) }, c.Int)
+	m.Workflow.Matrix = axis
+
+	// System
+	metadataFileAndOverrideOrDefault(c, "system-name", func(s string) { m.Sys.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "system-url", func(s string) { m.Sys.URL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "system-host", func(s string) { m.Sys.Host = s }, c.String)
+	m.Sys.Platform = platform
+	m.Sys.Version = version.Version
+
+	// Forge
+	metadataFileAndOverrideOrDefault(c, "forge-type", func(s string) { m.Forge.Type = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "forge-url", func(s string) { m.Forge.URL = s }, c.String)
+
+	if w != nil {
+		m.Workflow = *w
+	}
+
+	return m, nil
+}
+
+// metadataFileAndOverrideOrDefault will either use the flag default or if metadata file is set only overload if explicit set.
+func metadataFileAndOverrideOrDefault[T any](c *cli.Command, flag string, setter func(T), getter func(string) T) {
+	if !c.IsSet("metadata-file") || c.IsSet(flag) {
+		setter(getter(flag))
+	}
 }