Skip to content

Commit

Permalink
feat: Remove binaries from argoexec image. Fixes #7486 (#8292)
Browse files Browse the repository at this point in the history 
Signed-off-by: Alex Collins <alex_collins@intuit.com>
  • Loading branch information
@alexec
alexec authored Apr 5, 2022
1 parent af80774 commit 153540f
Show file tree
Hide file tree
Showing 6 changed files with 246 additions and 275 deletions.
54 changes: 16 additions & 38 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
#syntax=docker/dockerfile:1.2

ARG DOCKER_CHANNEL=stable
ARG DOCKER_VERSION=20.10.14
# NOTE: kubectl version should be one minor version less than https://storage.googleapis.com/kubernetes-release/release/stable.txt
ARG KUBECTL_VERSION=1.22.3
ARG JQ_VERSION=1.6

FROM golang:1.17 as builder

RUN apt-get update && apt-get --no-install-recommends install -y \
Expand All @@ -15,9 +9,7 @@ RUN apt-get update && apt-get --no-install-recommends install -y \
apt-transport-https \
ca-certificates \
wget \
gcc \
libcap2-bin \
zip && \
gcc && \
apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
Expand All @@ -37,33 +29,6 @@ RUN go mod download

COPY . .

####################################################################################################

FROM alpine:3 as argoexec-base

ARG DOCKER_CHANNEL
ARG DOCKER_VERSION
ARG KUBECTL_VERSION

RUN apk --no-cache add curl git tar jq

COPY hack/arch.sh hack/os.sh /bin/

RUN if [ $(arch.sh) = ppc64le ] || [ $(arch.sh) = s390x ]; then \
curl -o docker.tgz https://download.docker.com/$(os.sh)/static/${DOCKER_CHANNEL}/$(uname -m)/docker-18.06.3-ce.tgz; \
else \
curl -o docker.tgz https://download.docker.com/$(os.sh)/static/${DOCKER_CHANNEL}/$(uname -m)/docker-${DOCKER_VERSION}.tgz; \
fi && \
tar --extract --file docker.tgz --strip-components 1 --directory /usr/local/bin/ && \
rm docker.tgz
RUN curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/$(os.sh)/$(arch.sh)/kubectl && \
chmod +x /usr/local/bin/kubectl
RUN rm /bin/arch.sh /bin/os.sh

COPY hack/ssh_known_hosts /etc/ssh/
COPY hack/nsswitch.conf /etc/


####################################################################################################

FROM node:16 as argo-ui
Expand All @@ -81,6 +46,15 @@ RUN NODE_OPTIONS="--max-old-space-size=2048" JOBS=max yarn --cwd ui build

FROM builder as argoexec-build

COPY hack/arch.sh hack/os.sh /bin/

# NOTE: kubectl version should be one minor version less than https://storage.googleapis.com/kubernetes-release/release/stable.txt
RUN curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.22.3/bin/$(os.sh)/$(arch.sh)/kubectl && \
chmod +x /usr/local/bin/kubectl

RUN curl -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \
chmod +x /usr/local/bin/jq

# Tell git to forget about all of the files that were not included because of .dockerignore in order to ensure that
# the git state is "clean" even though said .dockerignore files are not present
RUN cat .dockerignore >> .gitignore
Expand Down Expand Up @@ -118,10 +92,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build make dist/argo

####################################################################################################

FROM argoexec-base as argoexec
FROM gcr.io/distroless/static as argoexec

COPY --from=argoexec-build /go/src/github.com/argoproj/argo-workflows/dist/argoexec /usr/local/bin/
COPY --from=argoexec-build /usr/local/bin/kubectl /bin/
COPY --from=argoexec-build /usr/local/bin/jq /bin/
COPY --from=argoexec-build /go/src/github.com/argoproj/argo-workflows/dist/argoexec /bin/
COPY --from=argoexec-build /etc/mime.types /etc/mime.types
COPY hack/ssh_known_hosts /etc/ssh/
COPY hack/nsswitch.conf /etc/

ENTRYPOINT [ "argoexec" ]

Expand Down
165 changes: 53 additions & 112 deletions workflow/artifacts/git/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,7 @@ import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"

"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
Expand Down Expand Up @@ -45,65 +42,33 @@ func GetUser(url string) string {
return "git"
}

func (g *ArtifactDriver) auth(sshUser string) (func(), transport.AuthMethod, []string, error) {
func (g *ArtifactDriver) auth(sshUser string) (func(), transport.AuthMethod, error) {
if g.SSHPrivateKey != "" {
signer, err := ssh.ParsePrivateKey([]byte(g.SSHPrivateKey))
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
privateKeyFile, err := ioutil.TempFile("", "id_rsa.")
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
err = ioutil.WriteFile(privateKeyFile.Name(), []byte(g.SSHPrivateKey), 0o600)
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
auth := &ssh2.PublicKeys{User: sshUser, Signer: signer}
if g.InsecureIgnoreHostKey {
auth.HostKeyCallback = ssh.InsecureIgnoreHostKey()
}
args := []string{"ssh", "-i", privateKeyFile.Name()}
if g.InsecureIgnoreHostKey {
args = append(args, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null")
} else {
args = append(args, "-o", "StrictHostKeyChecking=yes", "-o")
}
env := []string{"GIT_SSH_COMMAND=" + strings.Join(args, " ")}
if g.InsecureIgnoreHostKey {
auth.HostKeyCallback = ssh.InsecureIgnoreHostKey()
env = append(env, "GIT_SSL_NO_VERIFY=true")
}
return func() { _ = os.Remove(privateKeyFile.Name()) },
auth,
env,
nil
return func() { _ = os.Remove(privateKeyFile.Name()) }, auth, nil
}
if g.Username != "" || g.Password != "" {
filename := filepath.Join(os.TempDir(), "git-ask-pass.sh")
_, err := os.Stat(filename)
if os.IsNotExist(err) {
//nolint:gosec
err := ioutil.WriteFile(filename, []byte(`#!/bin/sh
case "$1" in
Username*) echo "${GIT_USERNAME}" ;;
Password*) echo "${GIT_PASSWORD}" ;;
esac
`), 0o755)
if err != nil {
return nil, nil, nil, err
}
}
return func() {},
&http.BasicAuth{Username: g.Username, Password: g.Password},
[]string{
"GIT_ASKPASS=" + filename,
"GIT_USERNAME=" + g.Username,
"GIT_PASSWORD=" + g.Password,
},
nil
return func() {}, &http.BasicAuth{Username: g.Username, Password: g.Password}, nil
}
return func() {}, nil, nil, nil
return func() {}, nil, nil
}

// Save is unsupported for git output artifacts
Expand All @@ -112,111 +77,87 @@ func (g *ArtifactDriver) Save(string, *wfv1.Artifact) error {
}

func (g *ArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) error {
sshUser := GetUser(inputArtifact.Git.Repo)
closer, auth, env, err := g.auth(sshUser)
a := inputArtifact.Git
sshUser := GetUser(a.Repo)
closer, auth, err := g.auth(sshUser)
if err != nil {
return err
}
defer closer()

var recurseSubmodules = git.DefaultSubmoduleRecursionDepth
if inputArtifact.Git.DisableSubmodules {
log.Info("Recursive cloning of submodules is disabled")
recurseSubmodules = git.NoRecurseSubmodules
}
repo, err := git.PlainClone(path, false, &git.CloneOptions{
URL: inputArtifact.Git.Repo,
RecurseSubmodules: recurseSubmodules,
Auth: auth,
Depth: inputArtifact.Git.GetDepth(),
})
depth := a.GetDepth()
r, err := git.PlainClone(path, false, &git.CloneOptions{URL: a.Repo, Auth: auth, Depth: depth})
switch err {
case transport.ErrEmptyRemoteRepository:
log.Info("Cloned an empty repository ")
log.Info("Cloned an empty repository")
r, err := git.PlainInit(path, false)
if err != nil {
return err
return fmt.Errorf("failed to plain init: %w", err)
}
if _, err := r.CreateRemote(&config.RemoteConfig{Name: git.DefaultRemoteName, URLs: []string{inputArtifact.Git.Repo}}); err != nil {
return err
if _, err := r.CreateRemote(&config.RemoteConfig{Name: git.DefaultRemoteName, URLs: []string{a.Repo}}); err != nil {
return fmt.Errorf("failed to create remote %q: %w", a.Repo, err)
}
branchName := inputArtifact.Git.Revision
branchName := a.Revision
if branchName == "" {
branchName = "master"
}
if err = r.CreateBranch(&config.Branch{Name: branchName, Remote: git.DefaultRemoteName, Merge: plumbing.Master}); err != nil {
return err
return fmt.Errorf("failed to create branch %q: %w", branchName, err)
}
return nil
default:
return err
case nil:
// fallthrough ...
default:
return fmt.Errorf("failed to clone %q: %w", a.Repo, err)
}
if inputArtifact.Git.Fetch != nil {
refSpecs := make([]config.RefSpec, len(inputArtifact.Git.Fetch))
for i, spec := range inputArtifact.Git.Fetch {
if len(a.Fetch) > 0 {
refSpecs := make([]config.RefSpec, len(a.Fetch))
for i, spec := range a.Fetch {
refSpecs[i] = config.RefSpec(spec)
}
fetchOptions := git.FetchOptions{
Auth: auth,
RefSpecs: refSpecs,
Depth: inputArtifact.Git.GetDepth(),
opts := &git.FetchOptions{Auth: auth, RefSpecs: refSpecs, Depth: depth}
if err := opts.Validate(); err != nil {
return fmt.Errorf("failed to validate fetch %v: %w", refSpecs, err)
}
if err = r.Fetch(opts); isFetchErr(err) {
return fmt.Errorf("failed to fetch %v: %w", refSpecs, err)
}
}
w, err := r.Worktree()
if err != nil {
return fmt.Errorf("failed to get work tree: %w", err)
}
if a.Revision != "" {
if err := r.Fetch(&git.FetchOptions{RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*"}}); isFetchErr(err) {
return fmt.Errorf("failed to fatch refs: %w", err)
}
err = fetchOptions.Validate()
h, err := r.ResolveRevision(plumbing.Revision(a.Revision))
if err != nil {
return err
return fmt.Errorf("failed to get resolve revision: %w", err)
}
err = repo.Fetch(&fetchOptions)
if isAlreadyUpToDateErr(err) {
return err
if err := w.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(h.String())}); err != nil {
return fmt.Errorf("failed to checkout %q: %w", h, err)
}
}
if inputArtifact.Git.Revision != "" {
// We still rely on forking git for checkout, since go-git does not have a reliable
// way of resolving revisions (e.g. mybranch, HEAD^, v1.2.3)
rev := getRevisionForCheckout(inputArtifact.Git.Revision)
log.Info("Checking out revision ", rev)
cmd := exec.Command("git", "checkout", rev, "--")
cmd.Dir = path
cmd.Env = env
output, err := cmd.Output()
if !a.DisableSubmodules {
s, err := w.Submodules()
if err != nil {
return g.error(err, cmd)
}
log.Infof("`%s` stdout:\n%s", cmd.Args, string(output))
if !inputArtifact.Git.DisableSubmodules {
submodulesCmd := exec.Command("git", "submodule", "update", "--init", "--recursive", "--force")
submodulesCmd.Dir = path
submodulesCmd.Env = env
submoduleOutput, err := submodulesCmd.Output()
if err != nil {
return g.error(err, cmd)
}
log.Infof("`%s` stdout:\n%s", cmd.Args, string(submoduleOutput))
return fmt.Errorf("failed to get submodules: %w", err)
}
if err := s.Update(&git.SubmoduleUpdateOptions{
Init: true,
RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
Auth: auth,
}); err != nil {
return fmt.Errorf("failed to update submodules: %w", err)
}
}
return nil
}

// getRevisionForCheckout trims "refs/heads/" from the revision name (if present)
// so that `git checkout` will succeed.
func getRevisionForCheckout(revision string) string {
return strings.TrimPrefix(revision, "refs/heads/")
}

func isAlreadyUpToDateErr(err error) bool {
func isFetchErr(err error) bool {
return err != nil && err.Error() != "already up-to-date"
}

func (g *ArtifactDriver) error(err error, cmd *exec.Cmd) error {
if exErr, ok := err.(*exec.ExitError); ok {
log.Errorf("`%s` stderr:\n%s", cmd.Args, string(exErr.Stderr))
return errors.New(strings.Split(string(exErr.Stderr), "\n")[0])
}
return err
}

func (g *ArtifactDriver) ListObjects(artifact *wfv1.Artifact) ([]string, error) {
return nil, fmt.Errorf("ListObjects is currently not supported for this artifact type, but it will be in a future version")
}
Loading

0 comments on commit 153540f

Please sign in to comment.