Skip to content

Commit

Permalink
Fix .dockerignore for build context copies in later stages (GoogleCon…
Browse files Browse the repository at this point in the history 
…tainerTools#1447)

* Extend .dockerignore integration test with copies in later stages

.dockerignore should continue to apply when copying from the build context in later stages, but it currently doesn't

* Replace excluded global with passed along FileContext struct

This new FileContext struct allows much cleaner handling of context specific file exclusions.
The global excluded file state is no longer needed.

Additionally this also fixes the issue where excluded files aren't being applied for build context copies in later build stages.
  • Loading branch information
@tscni
tscni authored Oct 8, 2020
1 parent 0ef3a6b commit 5f4e2f1
Show file tree
Hide file tree
Showing 14 changed files with 150 additions and 126 deletions.
2 changes: 1 addition & 1 deletion cmd/executor/cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ func resolveEnvironmentBuildArgs(arguments []string, resolver func(string) strin
// copy Dockerfile to /kaniko/Dockerfile so that if it's specified in the .dockerignore
// it won't be copied into the image
func copyDockerfile() error {
if _, err := util.CopyFile(opts.DockerfilePath, constants.DockerfilePath, "", util.DoNotChangeUID, util.DoNotChangeGID); err != nil {
if _, err := util.CopyFile(opts.DockerfilePath, constants.DockerfilePath, util.FileContext{}, util.DoNotChangeUID, util.DoNotChangeGID); err != nil {
return errors.Wrap(err, "copying dockerfile")
}
opts.DockerfilePath = constants.DockerfilePath
Expand Down
12 changes: 10 additions & 2 deletions integration/dockerfiles/Dockerfile_test_dockerignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,13 @@ COPY ignore/* /foo
From base as first
COPY --from=base /foo ignore/bar

FROM first
COPY --from=first ignore/* /fooAnother/
# Make sure that .dockerignore also applies for later stages
FROM scratch as base2
COPY ignore/* /foo

From base2 as second
COPY --from=base2 /foo ignore/bar

FROM scratch
COPY --from=first ignore/* /fooAnother/
COPY --from=second ignore/* /fooAnother2/
12 changes: 6 additions & 6 deletions pkg/commands/add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
type AddCommand struct {
BaseCommand
cmd *instructions.AddCommand
buildcontext string
fileContext util.FileContext
snapshotFiles []string
}

Expand All @@ -52,7 +52,7 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
return errors.Wrap(err, "getting user group from chown")
}

srcs, dest, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
srcs, dest, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.fileContext, replacementEnvs)
if err != nil {
return err
}
Expand All @@ -64,7 +64,7 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
// 1. Download and copy it to the specified dest
// Else, add to the list of unresolved sources
for _, src := range srcs {
fullPath := filepath.Join(a.buildcontext, src)
fullPath := filepath.Join(a.fileContext.Root, src)
if util.IsSrcRemoteFileURL(src) {
urlDest, err := util.URLDestinationFilepath(src, dest, config.WorkingDir, replacementEnvs)
if err != nil {
Expand Down Expand Up @@ -101,7 +101,7 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
SourcesAndDest: append(unresolvedSrcs, dest),
Chown: a.cmd.Chown,
},
buildcontext: a.buildcontext,
fileContext: a.fileContext,
}

if err := copyCmd.ExecuteCommand(config, buildArgs); err != nil {
Expand All @@ -124,7 +124,7 @@ func (a *AddCommand) String() string {
func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)

srcs, _, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs)
srcs, _, err := util.ResolveEnvAndWildcards(a.cmd.SourcesAndDest, a.fileContext, replacementEnvs)
if err != nil {
return nil, err
}
Expand All @@ -137,7 +137,7 @@ func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfi
if util.IsFileLocalTarArchive(src) {
continue
}
fullPath := filepath.Join(a.buildcontext, src)
fullPath := filepath.Join(a.fileContext.Root, src)
files = append(files, fullPath)
}

Expand Down
7 changes: 4 additions & 3 deletions pkg/commands/commands.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package commands

import (
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/util"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/pkg/errors"
Expand Down Expand Up @@ -59,23 +60,23 @@ type DockerCommand interface {
ShouldDetectDeletedFiles() bool
}

func GetCommand(cmd instructions.Command, buildcontext string, useNewRun bool) (DockerCommand, error) {
func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool) (DockerCommand, error) {
switch c := cmd.(type) {
case *instructions.RunCommand:
if useNewRun {
return &RunMarkerCommand{cmd: c}, nil
}
return &RunCommand{cmd: c}, nil
case *instructions.CopyCommand:
return &CopyCommand{cmd: c, buildcontext: buildcontext}, nil
return &CopyCommand{cmd: c, fileContext: fileContext}, nil
case *instructions.ExposeCommand:
return &ExposeCommand{cmd: c}, nil
case *instructions.EnvCommand:
return &EnvCommand{cmd: c}, nil
case *instructions.WorkdirCommand:
return &WorkdirCommand{cmd: c}, nil
case *instructions.AddCommand:
return &AddCommand{cmd: c, buildcontext: buildcontext}, nil
return &AddCommand{cmd: c, fileContext: fileContext}, nil
case *instructions.CmdCommand:
return &CmdCommand{cmd: c}, nil
case *instructions.EntrypointCommand:
Expand Down
22 changes: 11 additions & 11 deletions pkg/commands/copy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,14 @@ var (
type CopyCommand struct {
BaseCommand
cmd *instructions.CopyCommand
buildcontext string
fileContext util.FileContext
snapshotFiles []string
}

func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
// Resolve from
if c.cmd.From != "" {
c.buildcontext = filepath.Join(kConfig.KanikoDir, c.cmd.From)
c.fileContext = util.FileContext{Root: filepath.Join(kConfig.KanikoDir, c.cmd.From)}
}

replacementEnvs := buildArgs.ReplacementEnvs(config.Env)
Expand All @@ -55,14 +55,14 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
return errors.Wrap(err, "getting user group from chown")
}

srcs, dest, err := util.ResolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs)
srcs, dest, err := util.ResolveEnvAndWildcards(c.cmd.SourcesAndDest, c.fileContext, replacementEnvs)
if err != nil {
return errors.Wrap(err, "resolving src")
}

// For each source, iterate through and copy it over
for _, src := range srcs {
fullPath := filepath.Join(c.buildcontext, src)
fullPath := filepath.Join(c.fileContext.Root, src)

fi, err := os.Lstat(fullPath)
if err != nil {
Expand All @@ -89,14 +89,14 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
}

if fi.IsDir() {
copiedFiles, err := util.CopyDir(fullPath, destPath, c.buildcontext, uid, gid)
copiedFiles, err := util.CopyDir(fullPath, destPath, c.fileContext, uid, gid)
if err != nil {
return errors.Wrap(err, "copying dir")
}
c.snapshotFiles = append(c.snapshotFiles, copiedFiles...)
} else if util.IsSymlink(fi) {
// If file is a symlink, we want to copy the target file to destPath
exclude, err := util.CopySymlink(fullPath, destPath, c.buildcontext)
exclude, err := util.CopySymlink(fullPath, destPath, c.fileContext)
if err != nil {
return errors.Wrap(err, "copying symlink")
}
Expand All @@ -106,7 +106,7 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu
c.snapshotFiles = append(c.snapshotFiles, destPath)
} else {
// ... Else, we want to copy over a file
exclude, err := util.CopyFile(fullPath, destPath, c.buildcontext, uid, gid)
exclude, err := util.CopyFile(fullPath, destPath, c.fileContext, uid, gid)
if err != nil {
return errors.Wrap(err, "copying file")
}
Expand All @@ -130,7 +130,7 @@ func (c *CopyCommand) String() string {
}

func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
return copyCmdFilesUsedFromContext(config, buildArgs, c.cmd, c.buildcontext)
return copyCmdFilesUsedFromContext(config, buildArgs, c.cmd, c.fileContext)
}

func (c *CopyCommand) MetadataOnly() bool {
Expand Down Expand Up @@ -186,7 +186,7 @@ func resolveIfSymlink(destPath string) (string, error) {

func copyCmdFilesUsedFromContext(
config *v1.Config, buildArgs *dockerfile.BuildArgs, cmd *instructions.CopyCommand,
buildcontext string,
fileContext util.FileContext,
) ([]string, error) {
// We don't use the context if we're performing a copy --from.
if cmd.From != "" {
Expand All @@ -196,15 +196,15 @@ func copyCmdFilesUsedFromContext(
replacementEnvs := buildArgs.ReplacementEnvs(config.Env)

srcs, _, err := util.ResolveEnvAndWildcards(
cmd.SourcesAndDest, buildcontext, replacementEnvs,
cmd.SourcesAndDest, fileContext, replacementEnvs,
)
if err != nil {
return nil, err
}

files := []string{}
for _, src := range srcs {
fullPath := filepath.Join(buildcontext, src)
fullPath := filepath.Join(fileContext.Root, src)
files = append(files, fullPath)
}

Expand Down
32 changes: 17 additions & 15 deletions pkg/commands/copy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"testing"

"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/GoogleContainerTools/kaniko/testutil"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
Expand Down Expand Up @@ -113,6 +114,7 @@ func TestCopyExecuteCmd(t *testing.T) {
Env: []string{},
WorkingDir: tempDir,
}
fileContext := util.FileContext{Root: tempDir}

for _, test := range copyTests {
t.Run(test.name, func(t *testing.T) {
Expand All @@ -122,7 +124,7 @@ func TestCopyExecuteCmd(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: test.sourcesAndDest,
},
buildcontext: tempDir,
fileContext: fileContext,
}

buildArgs := copySetUpBuildArgs()
Expand Down Expand Up @@ -275,7 +277,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, "dest"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -307,7 +309,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "bam.txt"), "dest/"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand All @@ -334,7 +336,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "bam.txt"), "dest"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -363,7 +365,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "bam.txt"), "dest"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -392,7 +394,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "sym.link"), "dest/"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -437,7 +439,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "dead.link"), "dest/"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -478,7 +480,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{"another", "dest"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -524,7 +526,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, "dest"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -568,7 +570,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{"another", "dest"},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -611,7 +613,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, linkedDest},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -656,7 +658,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{fmt.Sprintf("%s/bam.txt", srcDir), linkedDest},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -705,7 +707,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
SourcesAndDest: []string{fmt.Sprintf("%s/bam.txt", srcDir), testDir},
Chown: "alice:group",
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -750,7 +752,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
SourcesAndDest: []string{fmt.Sprintf("%s/bam.txt", srcDir), testDir},
Chown: "missing:missing",
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down Expand Up @@ -781,7 +783,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, dest},
},
buildcontext: testDir,
fileContext: util.FileContext{Root: testDir},
}

cfg := &v1.Config{
Expand Down
Loading

0 comments on commit 5f4e2f1

Please sign in to comment.