Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replay pipeline using cli exec by downloading metadata #4103

Merged
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
b9a7d6c
Add option to use WOODPECKER_METADATA_FILE to set the metadata for th…
6543 Sep 11, 2024
7b092b3
add check aganist nil and use it
6543 Sep 11, 2024
7976901
rename StepBuilder.Last to StepBuilder.Prev to be consisten with meta…
6543 Sep 11, 2024
b11e882
First draft of metadata API
6543 Sep 11, 2024
8b00bd0
first draft of debug pipeline tab
6543 Sep 11, 2024
1d0384b
Add dropdown to download workflow specific metadata
6543 Sep 11, 2024
4fd307b
Add PipelineMetadata to golang api-client
6543 Sep 11, 2024
480fc10
more test coverage
6543 Sep 12, 2024
eb2b4df
tests for cli
6543 Sep 12, 2024
fcc8688
fix lint
6543 Sep 12, 2024
055c717
fix lint
6543 Sep 12, 2024
2e4e953
fix lint
6543 Sep 12, 2024
44bfe63
any
6543 Sep 12, 2024
96d252d
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 13, 2024
60252ef
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 16, 2024
bd5571d
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 16, 2024
4f9b1d4
generate
6543 Sep 16, 2024
8cf8aff
remove unused secret struct
6543 Sep 16, 2024
0d50bac
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 16, 2024
ea24133
harden test
6543 Sep 16, 2024
09ecd9b
fix bug
6543 Sep 16, 2024
82ba011
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 17, 2024
4fc9e33
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 20, 2024
0e8285a
well good the linter catched it
6543 Sep 20, 2024
9d31b98
use SelectField
6543 Sep 20, 2024
040b773
add todo notes
6543 Sep 20, 2024
c9815d4
fmt
6543 Sep 20, 2024
6148a26
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 22, 2024
991aced
rm workflows param: go sdk
6543 Sep 22, 2024
8e42642
rm workflows param: webui
6543 Sep 22, 2024
3554a7b
rm workflows param: server
6543 Sep 22, 2024
5f2a06b
rm workflows param: test
6543 Sep 22, 2024
c8a787c
Panel
6543 Sep 23, 2024
bfed706
fmt
6543 Sep 23, 2024
a0dcd37
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 23, 2024
3eae1d2
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 24, 2024
f62f0da
show cli example in webui
6543 Sep 24, 2024
8918d91
update file name dynamic in example
6543 Sep 24, 2024
c3fd642
wow
6543 Sep 24, 2024
915fbc1
docu
6543 Sep 24, 2024
ba00568
Merge remote-tracking branch 'upstream/main' into feat/download-and-u…
6543 Sep 24, 2024
e99c837
reapply
6543 Sep 24, 2024
7b26fda
Apply suggestions from code review
6543 Sep 24, 2024
3fe3eb4
fix
6543 Sep 24, 2024
e75c8f1
Merge branch 'main' into feat/download-and-use-metadata-in-exec
6543 Sep 24, 2024
67bb314
Update cli/exec/flags.go
anbraten Sep 25, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions cli/exec/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,13 @@ func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error {
}

func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis) error {
metadata := metadataFromContext(ctx, c, axis)
metadata, err := metadataFromContext(ctx, c, axis)
if err != nil {
return err
} else if metadata == nil {
return fmt.Errorf("metadata is nil")
}

environ := metadata.Environ()
var secrets []compiler.Secret
for key, val := range metadata.Workflow.Matrix {
Expand Down Expand Up @@ -234,7 +240,7 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
c.String("netrc-password"),
c.String("netrc-machine"),
),
compiler.WithMetadata(metadata),
compiler.WithMetadata(*metadata),
compiler.WithSecret(secrets...),
compiler.WithEnviron(pipelineEnv),
).Compile(conf)
Expand Down
5 changes: 5 additions & 0 deletions cli/exec/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ var flags = []cli.Flag{
Name: "repo-path",
Usage: "path to local repository",
},
&cli.StringFlag{
Sources: cli.EnvVars("WOODPECKER_METADATA_FILE"),
Name: "metadata-file",
Usage: "path to metadata file who has stored the pipeline environment to emulate (can be downloaded from an existing pipeline), infos can be overwritten.",
anbraten marked this conversation as resolved.
Show resolved Hide resolved
},
&cli.DurationFlag{
Sources: cli.EnvVars("WOODPECKER_TIMEOUT"),
Name: "timeout",
Expand Down
181 changes: 98 additions & 83 deletions cli/exec/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ package exec

import (
"context"
"encoding/json"
"os"
"runtime"
"strings"

Expand All @@ -27,94 +29,107 @@ import (
)

// return the metadata from the cli context.
func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) metadata.Metadata {
func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (*metadata.Metadata, error) {
m := &metadata.Metadata{}

if c.IsSet("metadata-file") {
metadataFile, err := os.Open(c.String("metadata-file"))
if err != nil {
return nil, err
}
defer metadataFile.Close()

if err := json.NewDecoder(metadataFile).Decode(m); err != nil {
return nil, err
}
}

platform := c.String("system-platform")
if platform == "" {
platform = runtime.GOOS + "/" + runtime.GOARCH
}

fullRepoName := c.String("repo-name")
repoOwner := ""
repoName := ""
if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
repoOwner = fullRepoName[:idx]
repoName = fullRepoName[idx+1:]
}
metadataFileAndOverrideOrDefault(c, "repo-name", func(fullRepoName string) {
if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
m.Repo.Owner = fullRepoName[:idx]
m.Repo.Name = fullRepoName[idx+1:]
}
}, c.String)

// Repo
metadataFileAndOverrideOrDefault(c, "repo-remote-id", func(s string) { m.Repo.RemoteID = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-url", func(s string) { m.Repo.ForgeURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-clone-url", func(s string) { m.Repo.CloneURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-clone-ssh-url", func(s string) { m.Repo.CloneSSHURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "repo-private", func(b bool) { m.Repo.Private = b }, c.Bool)
metadataFileAndOverrideOrDefault(c, "repo-trusted", func(b bool) { m.Repo.Trusted = b }, c.Bool)

// Current Pipeline
metadataFileAndOverrideOrDefault(c, "pipeline-number", func(i int64) { m.Curr.Number = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-parent", func(i int64) { m.Curr.Parent = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-created", func(i int64) { m.Curr.Created = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-started", func(i int64) { m.Curr.Started = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-finished", func(i int64) { m.Curr.Finished = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "pipeline-status", func(s string) { m.Curr.Status = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-event", func(s string) { m.Curr.Event = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-url", func(s string) { m.Curr.ForgeURL = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-deploy-to", func(s string) { m.Curr.DeployTo = s }, c.String)
metadataFileAndOverrideOrDefault(c, "pipeline-deploy-task", func(s string) { m.Curr.DeployTask = s }, c.String)

// Current Pipeline Commit
metadataFileAndOverrideOrDefault(c, "commit-sha", func(s string) { m.Curr.Commit.Sha = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-ref", func(s string) { m.Curr.Commit.Ref = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-refspec", func(s string) { m.Curr.Commit.Refspec = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-branch", func(s string) { m.Curr.Commit.Branch = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-message", func(s string) { m.Curr.Commit.Message = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-author-name", func(s string) { m.Curr.Commit.Author.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-author-email", func(s string) { m.Curr.Commit.Author.Email = s }, c.String)
metadataFileAndOverrideOrDefault(c, "commit-author-avatar", func(s string) { m.Curr.Commit.Author.Avatar = s }, c.String)

// Previous Pipeline
metadataFileAndOverrideOrDefault(c, "prev-pipeline-number", func(i int64) { m.Prev.Number = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-created", func(i int64) { m.Prev.Created = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-started", func(i int64) { m.Prev.Started = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-finished", func(i int64) { m.Prev.Finished = i }, c.Int)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-status", func(s string) { m.Prev.Status = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-event", func(s string) { m.Prev.Event = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-pipeline-url", func(s string) { m.Prev.ForgeURL = s }, c.String)

// Previous Pipeline Commit
metadataFileAndOverrideOrDefault(c, "prev-commit-sha", func(s string) { m.Prev.Commit.Sha = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-ref", func(s string) { m.Prev.Commit.Ref = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-refspec", func(s string) { m.Prev.Commit.Refspec = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-branch", func(s string) { m.Prev.Commit.Branch = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-message", func(s string) { m.Prev.Commit.Message = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-author-name", func(s string) { m.Prev.Commit.Author.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-author-email", func(s string) { m.Prev.Commit.Author.Email = s }, c.String)
metadataFileAndOverrideOrDefault(c, "prev-commit-author-avatar", func(s string) { m.Prev.Commit.Author.Avatar = s }, c.String)

// Workflow
metadataFileAndOverrideOrDefault(c, "workflow-name", func(s string) { m.Workflow.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "workflow-number", func(i int64) { m.Workflow.Number = int(i) }, c.Int)
m.Workflow.Matrix = axis

// Step
metadataFileAndOverrideOrDefault(c, "step-name", func(s string) { m.Step.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "step-number", func(i int64) { m.Step.Number = int(i) }, c.Int)

// System
metadataFileAndOverrideOrDefault(c, "system-name", func(s string) { m.Sys.Name = s }, c.String)
metadataFileAndOverrideOrDefault(c, "system-url", func(s string) { m.Sys.URL = s }, c.String)
m.Sys.Platform = platform
m.Sys.Version = version.Version

// Forge
metadataFileAndOverrideOrDefault(c, "forge-type", func(s string) { m.Forge.Type = s }, c.String)
metadataFileAndOverrideOrDefault(c, "forge-url", func(s string) { m.Forge.URL = s }, c.String)

return m, nil
}

return metadata.Metadata{
Repo: metadata.Repo{
Name: repoName,
Owner: repoOwner,
RemoteID: c.String("repo-remote-id"),
ForgeURL: c.String("repo-url"),
CloneURL: c.String("repo-clone-url"),
CloneSSHURL: c.String("repo-clone-ssh-url"),
Private: c.Bool("repo-private"),
Trusted: c.Bool("repo-trusted"),
},
Curr: metadata.Pipeline{
Number: c.Int("pipeline-number"),
Parent: c.Int("pipeline-parent"),
Created: c.Int("pipeline-created"),
Started: c.Int("pipeline-started"),
Finished: c.Int("pipeline-finished"),
Status: c.String("pipeline-status"),
Event: c.String("pipeline-event"),
ForgeURL: c.String("pipeline-url"),
DeployTo: c.String("pipeline-deploy-to"),
DeployTask: c.String("pipeline-deploy-task"),
Commit: metadata.Commit{
Sha: c.String("commit-sha"),
Ref: c.String("commit-ref"),
Refspec: c.String("commit-refspec"),
Branch: c.String("commit-branch"),
Message: c.String("commit-message"),
Author: metadata.Author{
Name: c.String("commit-author-name"),
Email: c.String("commit-author-email"),
Avatar: c.String("commit-author-avatar"),
},
},
},
Prev: metadata.Pipeline{
Number: c.Int("prev-pipeline-number"),
Created: c.Int("prev-pipeline-created"),
Started: c.Int("prev-pipeline-started"),
Finished: c.Int("prev-pipeline-finished"),
Status: c.String("prev-pipeline-status"),
Event: c.String("prev-pipeline-event"),
ForgeURL: c.String("prev-pipeline-url"),
Commit: metadata.Commit{
Sha: c.String("prev-commit-sha"),
Ref: c.String("prev-commit-ref"),
Refspec: c.String("prev-commit-refspec"),
Branch: c.String("prev-commit-branch"),
Message: c.String("prev-commit-message"),
Author: metadata.Author{
Name: c.String("prev-commit-author-name"),
Email: c.String("prev-commit-author-email"),
Avatar: c.String("prev-commit-author-avatar"),
},
},
},
Workflow: metadata.Workflow{
Name: c.String("workflow-name"),
Number: int(c.Int("workflow-number")),
Matrix: axis,
},
Step: metadata.Step{
Name: c.String("step-name"),
Number: int(c.Int("step-number")),
},
Sys: metadata.System{
Name: c.String("system-name"),
URL: c.String("system-url"),
Platform: platform,
Version: version.Version,
},
Forge: metadata.Forge{
Type: c.String("forge-type"),
URL: c.String("forge-url"),
},
// metadataFileAndOverrideOrDefault will either use the flag default or if metadata file is set only overload if explicit set.
func metadataFileAndOverrideOrDefault[T any](c *cli.Command, flag string, setter func(T), getter func(string) T) {
if !c.IsSet("metadata-file") || c.IsSet(flag) {
setter(getter(flag))
}
}
140 changes: 140 additions & 0 deletions cli/exec/metadata_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// Copyright 2024 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
"context"
"encoding/json"
"os"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v3"

"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/matrix"
)

func TestMetadataFromContext(t *testing.T) {
sampleMetadata := &metadata.Metadata{
Repo: metadata.Repo{Owner: "test-user", Name: "test-repo"},
Curr: metadata.Pipeline{Number: 5},
}

runCommand := func(flags []cli.Flag, fn func(c *cli.Command)) {
c := &cli.Command{
Flags: flags,
Action: func(_ context.Context, c *cli.Command) error {
fn(c)
return nil
},
}
assert.NoError(t, c.Run(context.Background(), []string{"woodpecker-cli"}))
}

t.Run("LoadFromFile", func(t *testing.T) {
tempFileName := createTempFile(t, sampleMetadata)

flags := []cli.Flag{
&cli.StringFlag{Name: "metadata-file"},
}

runCommand(flags, func(c *cli.Command) {
_ = c.Set("metadata-file", tempFileName)

m, err := metadataFromContext(context.Background(), c, nil)
require.NoError(t, err)
assert.Equal(t, "test-repo", m.Repo.Name)
assert.Equal(t, int64(5), m.Curr.Number)
})
})

t.Run("OverrideFromFlags", func(t *testing.T) {
tempFileName := createTempFile(t, sampleMetadata)

flags := []cli.Flag{
&cli.StringFlag{Name: "metadata-file"},
&cli.StringFlag{Name: "repo-name"},
&cli.IntFlag{Name: "pipeline-number"},
}

runCommand(flags, func(c *cli.Command) {
_ = c.Set("metadata-file", tempFileName)
_ = c.Set("repo-name", "aUser/override-repo")
_ = c.Set("pipeline-number", "10")

m, err := metadataFromContext(context.Background(), c, nil)
require.NoError(t, err)
assert.Equal(t, "override-repo", m.Repo.Name)
assert.Equal(t, int64(10), m.Curr.Number)
})
})

t.Run("InvalidFile", func(t *testing.T) {
tempFile, err := os.CreateTemp("", "invalid.json")
require.NoError(t, err)
t.Cleanup(func() { os.Remove(tempFile.Name()) })

_, err = tempFile.Write([]byte("invalid json"))
require.NoError(t, err)

flags := []cli.Flag{
&cli.StringFlag{Name: "metadata-file"},
}

runCommand(flags, func(c *cli.Command) {
_ = c.Set("metadata-file", tempFile.Name())

_, err = metadataFromContext(context.Background(), c, nil)
assert.Error(t, err)
})
})

t.Run("DefaultValues", func(t *testing.T) {
flags := []cli.Flag{
&cli.StringFlag{Name: "repo-name", Value: "test/default-repo"},
&cli.IntFlag{Name: "pipeline-number", Value: 1},
}

runCommand(flags, func(c *cli.Command) {
m, err := metadataFromContext(context.Background(), c, nil)
require.NoError(t, err)
assert.Equal(t, "test", m.Repo.Owner)
assert.Equal(t, "default-repo", m.Repo.Name)
assert.Equal(t, int64(1), m.Curr.Number)
})
})

t.Run("MatrixAxis", func(t *testing.T) {
runCommand([]cli.Flag{}, func(c *cli.Command) {
axis := matrix.Axis{"go": "1.16", "os": "linux"}
m, err := metadataFromContext(context.Background(), c, axis)
require.NoError(t, err)
assert.EqualValues(t, map[string]string{"go": "1.16", "os": "linux"}, m.Workflow.Matrix)
})
})
}

func createTempFile(t *testing.T, content interface{}) string {
t.Helper()
tempFile, err := os.CreateTemp("", "metadata.json")
require.NoError(t, err)
t.Cleanup(func() { os.Remove(tempFile.Name()) })

err = json.NewEncoder(tempFile).Encode(content)
require.NoError(t, err)
return tempFile.Name()
}
Loading