Submodules are diry after checkout -- need to update

workflow/artifacts/git/git.go

@@ -1,7 +1,11 @@
 package git
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"os/user"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -31,21 +35,47 @@ func (g *GitArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) erro
 		}
 		auth := &ssh2.PublicKeys{User: "git", Signer: signer}
 		auth.HostKeyCallback = ssh.InsecureIgnoreHostKey()
-		return gitClone(path, inputArtifact, auth)
+		return gitClone(path, inputArtifact, auth, g.SSHPrivateKey)
 	}
 	if g.Username != "" || g.Password != "" {
 		auth := &http.BasicAuth{Username: g.Username, Password: g.Password}
-		return gitClone(path, inputArtifact, auth)
+		return gitClone(path, inputArtifact, auth, "")
 	}
-	return gitClone(path, inputArtifact, nil)
+	return gitClone(path, inputArtifact, nil, "")
 }
 
 // Save is unsupported for git output artifacts
 func (g *GitArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) error {
 	return errors.Errorf(errors.CodeBadRequest, "Git output artifacts unsupported")
 }
 
-func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMethod) error {
+func writePrivateKey(key string) error {
+	usr, err := user.Current()
+	if err != nil {
+		return errors.InternalWrapError(err)
+	}
+	sshDir := fmt.Sprintf("%s/.ssh", usr.HomeDir)
+	err = os.Mkdir(sshDir, 0700)
+	if err != nil {
+		return errors.InternalWrapError(err)
+	}
+
+	sshConfig := `Host *
+	StrictHostKeyChecking no
+	UserKnownHostsFile /dev/null`
+	err = ioutil.WriteFile(fmt.Sprintf("%s/config", sshDir), []byte(sshConfig), 0644)
+	if err != nil {
+		return errors.InternalWrapError(err)
+	}
+	err = ioutil.WriteFile(fmt.Sprintf("%s/id_rsa", sshDir), []byte(key), 0600)
+	if err != nil {
+		return errors.InternalWrapError(err)
+	}
+
+	return nil
+}
+
+func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMethod, privateKey string) error {
 	cloneOptions := git.CloneOptions{
 		URL:               inputArtifact.Git.Repo,
 		RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
@@ -70,6 +100,23 @@ func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMeth
 			return errors.InternalWrapError(err)
 		}
 		log.Errorf("`%s` stdout:\n%s", cmd.Args, string(output))
+		if privateKey != "" {
+			err := writePrivateKey(privateKey)
+			if err != nil {
+				return errors.InternalWrapError(err)
+			}
+		}
+		submodulesCmd := exec.Command("git", "submodule", "update", "--init", "--recursive", "--force")
+		submodulesCmd.Dir = path
+		submoduleOutput, err := submodulesCmd.Output()
+		if err != nil {
+			if exErr, ok := err.(*exec.ExitError); ok {
+				log.Errorf("`%s` stderr:\n%s", submodulesCmd.Args, string(exErr.Stderr))
+				return errors.InternalError(strings.Split(string(exErr.Stderr), "\n")[0])
+			}
+			return errors.InternalWrapError(err)
+		}
+		log.Errorf("`%s` stdout:\n%s", submodulesCmd.Args, string(submoduleOutput))
 	}
 	return nil
 }

workflow/controller/operator.go

@@ -734,9 +734,10 @@ func inferFailedReason(pod *apiv1.Pod) (wfv1.NodePhase, string) {
 		}
 		errMsg := fmt.Sprintf("failed with exit code %d", ctr.State.Terminated.ExitCode)
 		if ctr.Name != common.MainContainerName {
-			if ctr.State.Terminated.ExitCode == 137 {
+			if ctr.State.Terminated.ExitCode == 137 || ctr.State.Terminated.ExitCode == 143 {
 				// if the sidecar was SIGKILL'd (exit code 137) assume it was because argoexec
 				// forcibly killed the container, which we ignore the error for.
+				// Java code 143 is a normal exit 128 + 15 https://github.com/elastic/elasticsearch/issues/31847
 				log.Infof("Ignoring %d exit code of sidecar '%s'", ctr.State.Terminated.ExitCode, ctr.Name)
 				continue
 			}