Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

git-init: add support for fetching submodules 🚝 #1531

Merged
merged 2 commits into from
Nov 7, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmd/git-init/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ var (
revision = flag.String("revision", "", "The Git revision to make the repository HEAD")
path = flag.String("path", "", "Path of directory under which git repository will be copied")
terminationMessagePath = flag.String("terminationMessagePath", "/dev/termination-log", "Location of file containing termination message")
submodules = flag.Bool("submodules", true, "Initialize and fetch git submodules")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WDYT about surfacing this option all the way up into the Git resource, so users can turn it off if they'd prefer?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@imjasonh I was waiting for someone to comment this to do it 😛

)

func main() {
Expand All @@ -39,6 +40,11 @@ func main() {
if err := git.Fetch(logger, *revision, *path, *url); err != nil {
logger.Fatalf("Error fetching git repository: %s", err)
}
if *submodules {
if err := git.SubmoduleFetch(logger, *path); err != nil {
logger.Fatalf("Error initalizing or fetching the git submodules")
}
}

commit, err := git.Commit(logger, *revision, *path)
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions docs/resources.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,9 @@ Params that can be added are the following:
(branch, tag, commit SHA or ref) to clone. You can use this to control what
commit [or branch](#using-a-branch) is used. _If no revision is specified,
the resource will default to `latest` from `master`._
1. `submodules`: defines if the resource should initialize and
fetch the submodules, value is either `true` or `false`. _If not
specified, this will default to true_

When used as an input, the Git resource includes the exact commit fetched in the `resourceResults`
section of the `taskRun`'s status object:
Expand Down
23 changes: 19 additions & 4 deletions pkg/apis/pipeline/v1alpha1/git_resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ type GitResource struct {
// Git revision (branch, tag, commit SHA or ref) to clone. See
// https://git-scm.com/docs/gitrevisions#_specifying_revisions for more
// information.
Revision string `json:"revision"`
Revision string `json:"revision"`
Submodules bool `json:"submodules"`

GitImage string `json:"-"`
}
Expand All @@ -50,16 +51,19 @@ func NewGitResource(gitImage string, r *PipelineResource) (*GitResource, error)
return nil, xerrors.Errorf("GitResource: Cannot create a Git resource from a %s Pipeline Resource", r.Spec.Type)
}
gitResource := GitResource{
Name: r.Name,
Type: r.Spec.Type,
GitImage: gitImage,
Name: r.Name,
Type: r.Spec.Type,
GitImage: gitImage,
Submodules: true,
}
for _, param := range r.Spec.Params {
switch {
case strings.EqualFold(param.Name, "URL"):
gitResource.URL = param.Value
case strings.EqualFold(param.Name, "Revision"):
gitResource.Revision = param.Value
case strings.EqualFold(param.Name, "Submodules"):
gitResource.Submodules = toBool(param.Value, true)
}
}
// default revision to master if nothing is provided
Expand All @@ -69,6 +73,17 @@ func NewGitResource(gitImage string, r *PipelineResource) (*GitResource, error)
return &gitResource, nil
}

func toBool(s string, d bool) bool {
switch s {
case "true":
return true
case "false":
return false
default:
return d
}
}

// GetName returns the name of the resource
func (s GitResource) GetName() string {
return s.Name
Expand Down
55 changes: 45 additions & 10 deletions pkg/apis/pipeline/v1alpha1/git_resource_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,12 @@ func Test_Valid_NewGitResource(t *testing.T) {
),
),
want: &v1alpha1.GitResource{
Name: "git-resource",
Type: v1alpha1.PipelineResourceTypeGit,
URL: "git@github.com:test/test.git",
Revision: "test",
GitImage: "override-with-git:latest",
Name: "git-resource",
Type: v1alpha1.PipelineResourceTypeGit,
URL: "git@github.com:test/test.git",
Revision: "test",
GitImage: "override-with-git:latest",
Submodules: true,
},
}, {
desc: "Without Revision",
Expand All @@ -60,11 +61,45 @@ func Test_Valid_NewGitResource(t *testing.T) {
),
),
want: &v1alpha1.GitResource{
Name: "git-resource",
Type: v1alpha1.PipelineResourceTypeGit,
URL: "git@github.com:test/test.git",
Revision: "master",
GitImage: "override-with-git:latest",
Name: "git-resource",
Type: v1alpha1.PipelineResourceTypeGit,
URL: "git@github.com:test/test.git",
Revision: "master",
GitImage: "override-with-git:latest",
Submodules: true,
},
}, {
desc: "With Submodules",
pipelineResource: tb.PipelineResource("git-resource", "default",
tb.PipelineResourceSpec(v1alpha1.PipelineResourceTypeGit,
tb.PipelineResourceSpecParam("URL", "git@github.com:test/test.git"),
tb.PipelineResourceSpecParam("Revision", "test"),
tb.PipelineResourceSpecParam("Submodules", "false"),
),
),
want: &v1alpha1.GitResource{
Name: "git-resource",
Type: v1alpha1.PipelineResourceTypeGit,
URL: "git@github.com:test/test.git",
Revision: "test",
GitImage: "override-with-git:latest",
Submodules: false,
},
}, {
desc: "Without Submodules",
pipelineResource: tb.PipelineResource("git-resource", "default",
tb.PipelineResourceSpec(v1alpha1.PipelineResourceTypeGit,
tb.PipelineResourceSpecParam("URL", "git@github.com:test/test.git"),
tb.PipelineResourceSpecParam("Revision", "test"),
),
),
want: &v1alpha1.GitResource{
Name: "git-resource",
Type: v1alpha1.PipelineResourceTypeGit,
URL: "git@github.com:test/test.git",
Revision: "test",
GitImage: "override-with-git:latest",
Submodules: true,
},
}} {
t.Run(tc.desc, func(t *testing.T) {
Expand Down
73 changes: 50 additions & 23 deletions pkg/git/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,31 +45,9 @@ func run(logger *zap.SugaredLogger, dir string, args ...string) (string, error)

// Fetch fetches the specified git repository at the revision into path.
func Fetch(logger *zap.SugaredLogger, revision, path, url string) error {
// HACK: This is to get git+ssh to work since ssh doesn't respect the HOME
// env variable.
homepath, err := homedir.Dir()
if err != nil {
logger.Errorf("Unexpected error: getting the user home directory: %v", err)
if err := ensureHomeEnv(logger); err != nil {
return err
}
homeenv := os.Getenv("HOME")
euid := os.Geteuid()
// Special case the root user/directory
if euid == 0 {
if err := os.Symlink(homeenv+"/.ssh", "/root/.ssh"); err != nil {
// Only do a warning, in case we don't have a real home
// directory writable in our image
logger.Warnf("Unexpected error: creating symlink: %v", err)
}
} else if homeenv != "" && homeenv != homepath {
if _, err := os.Stat(homepath + "/.ssh"); os.IsNotExist(err) {
if err := os.Symlink(homeenv+"/.ssh", homepath+"/.ssh"); err != nil {
// Only do a warning, in case we don't have a real home
// directory writable in our image
logger.Warnf("Unexpected error: creating symlink: %v", err)
}
}
}

if revision == "" {
revision = "master"
Expand Down Expand Up @@ -111,3 +89,52 @@ func Commit(logger *zap.SugaredLogger, revision, path string) (string, error) {
}
return strings.TrimSuffix(output, "\n"), nil
}

func SubmoduleFetch(logger *zap.SugaredLogger, path string) error {
if err := ensureHomeEnv(logger); err != nil {
return err
}

if path != "" {
if err := os.Chdir(path); err != nil {
return xerrors.Errorf("Failed to change directory with path %s; err: %w", path, err)
}
}
if _, err := run(logger, "", "submodule", "init"); err != nil {
return err
}
if _, err := run(logger, "", "submodule", "update", "--recursive"); err != nil {
return err
}
logger.Infof("Successfully initialized and updated submodules in path %s", path)
return nil
}

func ensureHomeEnv(logger *zap.SugaredLogger) error {
// HACK: This is to get git+ssh to work since ssh doesn't respect the HOME
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like this hack is copied from Fetch(). Opportunity to de-dup by creating a helper func?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sbwsg indeed 😛 I was a bit lazy 🤫, I need to enhance that 😉

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice 👍

// env variable.
homepath, err := homedir.Dir()
if err != nil {
logger.Errorf("Unexpected error: getting the user home directory: %v", err)
return err
}
homeenv := os.Getenv("HOME")
euid := os.Geteuid()
// Special case the root user/directory
if euid == 0 {
if err := os.Symlink(homeenv+"/.ssh", "/root/.ssh"); err != nil {
// Only do a warning, in case we don't have a real home
// directory writable in our image
logger.Warnf("Unexpected error: creating symlink: %v", err)
}
} else if homeenv != "" && homeenv != homepath {
if _, err := os.Stat(homepath + "/.ssh"); os.IsNotExist(err) {
if err := os.Symlink(homeenv+"/.ssh", homepath+"/.ssh"); err != nil {
// Only do a warning, in case we don't have a real home
// directory writable in our image
logger.Warnf("Unexpected error: creating symlink: %v", err)
}
}
}
return nil
}