[CSE-13] extended error handling for workflows (#16)

* wip

* mixed case imports cause all sorts of problems, switch to lowercase

* fixing build issu

* fixing error deserialization

* fixing error deserialization

* unmatched string logic

* make workflows fail on error trigger

* properly evaluate workflow failures

* dev version bump

* serialize errors and warnings into wf crd

* ugh go types

* rewriting error handling to support file sources

* temporarily commenting out a test

* fixing warning handler

* fixing error handling

fixing executor

fixing executor

Fixing executor

fixing executor

fixing executor

fixing executor

asdf

fixing executor

fixing operator

operator

fixing executor

cleanup

* cleaning up types

* updating codegen

* fixing version

* updating codegen

* add podname and stage name to error result

* version 2.5.0->2.4.0

* ErrorCondition->ExceptionCondition

* codegen

.argo-ci/ci.yaml

@@ -9,7 +9,7 @@ spec:
     - name: revision
       value: master
     - name: repo
-      value: https://github.com/CyrusBiotechnology/argo.git
+      value: https://github.com/cyrusbiotechnology/argo.git
 
   templates:
   - name: argo-ci
@@ -39,31 +39,31 @@ spec:
       - name: cmd
       artifacts:
       - name: code
-        path: /go/src/github.com/CyrusBiotechnology/argo
+        path: /go/src/github.com/cyrusbiotechnology/argo
         git:
           repo: "{{workflow.parameters.repo}}"
           revision: "{{workflow.parameters.revision}}"
     container:
       image: argoproj/argo-ci-builder:latest
       command: [sh, -c]
       args: ["{{inputs.parameters.cmd}}"]
-      workingDir: /go/src/github.com/CyrusBiotechnology/argo
+      workingDir: /go/src/github.com/cyrusbiotechnology/argo
 
   - name: ci-dind
     inputs:
       parameters:
       - name: cmd
       artifacts:
       - name: code
-        path: /go/src/github.com/CyrusBiotechnology/argo
+        path: /go/src/github.com/cyrusbiotechnology/argo
         git:
           repo: "{{workflow.parameters.repo}}"
           revision: "{{workflow.parameters.revision}}"
     container:
       image: argoproj/argo-ci-builder:latest
       command: [sh, -c]
       args: ["until docker ps; do sleep 3; done && {{inputs.parameters.cmd}}"]
-      workingDir: /go/src/github.com/CyrusBiotechnology/argo
+      workingDir: /go/src/github.com/cyrusbiotechnology/argo
       env:
       - name: DOCKER_HOST
         value: 127.0.0.1

Branding Assets.md

@@ -1,3 +1,3 @@
 # Argo Branding Assets
 ## Logo
-![Argo Logo](https://github.com/CyrusBiotechnology/argo/blob/master/argo-logo600.png "Argo Logo")
+![Argo Logo](https://github.com/cyrusbiotechnology/argo/blob/master/argo-logo600.png "Argo Logo")

CONTRIBUTING.md

@@ -29,8 +29,8 @@ Go to https://groups.google.com/forum/#!forum/argoproj
 
 ### Quickstart
 ```
-$ go get github.com/CyrusBiotechnology/argo
-$ cd $(go env GOPATH)/src/github.com/CyrusBiotechnology/argo
+$ go get github.com/cyrusbiotechnology/argo
+$ cd $(go env GOPATH)/src/github.com/cyrusbiotechnology/argo
 $ dep ensure -vendor-only
 $ make
 ```

Jenkinsfile

@@ -11,8 +11,8 @@ def runUtilityCommand(buildCommand) {
     // Run an arbitrary command inside the docker builder image
     sh "docker run --rm " +
        "-v ${pwd()}/dist/pkg:/root/go/pkg " +
-       "-v ${pwd()}:/root/go/src/github.com/CyrusBiotechnology/argo " +
-       "-w /root/go/src/github.com/CyrusBiotechnology/argo argo-builder ${buildCommand}"
+       "-v ${pwd()}:/root/go/src/github.com/cyrusbiotechnology/argo " +
+       "-w /root/go/src/github.com/cyrusbiotechnology/argo argo-builder ${buildCommand}"
 }
 
 pipeline {
@@ -85,8 +85,8 @@ pipeline {
         stage('push CLI to artifactory') {
             steps {
                 withCredentials([usernamePassword(credentialsId: 'Artifactory', usernameVariable: 'ARTI_NAME', passwordVariable: 'ARTI_PASS')]) {
-                    runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/CyrusBiotechnology/argo/dist/argo-darwin-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-mac-${VERSION}")
-                    runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/CyrusBiotechnology/argo/dist/argo-linux-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-linux-${VERSION}")
+                    runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/cyrusbiotechnology/argo/dist/argo-darwin-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-mac-${VERSION}")
+                    runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/cyrusbiotechnology/argo/dist/argo-linux-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-linux-${VERSION}")
                 }
             }
         }

Makefile

@@ -1,4 +1,4 @@
-PACKAGE=github.com/CyrusBiotechnology/argo
+PACKAGE=github.com/cyrusbiotechnology/argo
 CURRENT_DIR=$(shell pwd)
 DIST_DIR=${CURRENT_DIR}/dist
 ARGO_CLI_NAME=argo

README.md

@@ -22,9 +22,9 @@ Argo is an open source container-native workflow engine for getting work done on
 * Argo with Kubernetes puts a cloud-scale supercomputer at your fingertips.
 
 ## Documentation
-* [Get started here](https://github.com/CyrusBiotechnology/argo/blob/master/demo.md)
-* [How to write Argo workflow specs](https://github.com/CyrusBiotechnology/argo/blob/master/examples/README.md)
-* [How to configure your artifact repository](https://github.com/CyrusBiotechnology/argo/blob/master/ARTIFACT_REPO.md)
+* [Get started here](https://github.com/cyrusbiotechnology/argo/blob/master/demo.md)
+* [How to write Argo workflow specs](https://github.com/cyrusbiotechnology/argo/blob/master/examples/README.md)
+* [How to configure your artifact repository](https://github.com/cyrusbiotechnology/argo/blob/master/ARTIFACT_REPO.md)
 
 ## Features
 * DAG or Steps based declaration of workflows

VERSION

@@ -1 +1 @@
-2.3.0
+2.4.0

api/openapi-spec/swagger.json

@@ -2,7 +2,7 @@
   "swagger": "2.0",
   "info": {
     "title": "Argo",
-    "version": "v2.2.1"
+    "version": "v2.4.0"
   },
   "paths": {},
   "definitions": {
@@ -229,6 +229,52 @@
         }
       }
     },
+    "io.argoproj.workflow.v1alpha1.ExceptionCondition": {
+      "description": "ExceptionCondition is a container for defining an error or warning rule",
+      "required": [
+        "name"
+      ],
+      "properties": {
+        "message": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "patternMatched": {
+          "type": "string"
+        },
+        "patternUnmatched": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string"
+        }
+      }
+    },
+    "io.argoproj.workflow.v1alpha1.ExceptionResult": {
+      "description": "ExceptionResult contains the results on an extended error or warning condition evaluation",
+      "required": [
+        "name",
+        "message",
+        "podName",
+        "stepName"
+      ],
+      "properties": {
+        "message": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "podName": {
+          "type": "string"
+        },
+        "stepName": {
+          "type": "string"
+        }
+      }
+    },
     "io.argoproj.workflow.v1alpha1.GCSArtifact": {
       "description": "GCSArtifact is the location of a GCS artifact",
       "required": [
@@ -821,6 +867,12 @@
           "description": "DAG template subtype which runs a DAG",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.DAGTemplate"
         },
+        "errors": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition"
+          }
+        },
         "inputs": {
           "description": "Inputs describe what inputs parameters and artifacts are supplied to this template",
           "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Inputs"
@@ -888,6 +940,12 @@
           "items": {
             "$ref": "#/definitions/io.k8s.api.core.v1.Toleration"
           }
+        },
+        "warnings": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition"
+          }
         }
       }
     },

cmd/argo/commands/common.go

@@ -7,9 +7,9 @@ import (
 	"strconv"
 	"strings"
 
-	wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
-	wfclientset "github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned"
-	"github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
+	wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+	wfclientset "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned"
+	"github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"

cmd/argo/commands/delete.go

@@ -10,7 +10,7 @@ import (
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"github.com/CyrusBiotechnology/argo/workflow/common"
+	"github.com/cyrusbiotechnology/argo/workflow/common"
 )
 
 var (

cmd/argo/commands/get.go

@@ -13,7 +13,7 @@ import (
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
+	wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
 )
 
 const onExitSuffix = "onExit"

cmd/argo/commands/lint.go

@@ -7,8 +7,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	cmdutil "github.com/CyrusBiotechnology/argo/util/cmd"
-	"github.com/CyrusBiotechnology/argo/workflow/validate"
+	cmdutil "github.com/cyrusbiotechnology/argo/util/cmd"
+	"github.com/cyrusbiotechnology/argo/workflow/validate"
 )
 
 func NewLintCommand() *cobra.Command {

cmd/argo/commands/list.go

@@ -17,10 +17,10 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
 
-	wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
-	"github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
-	"github.com/CyrusBiotechnology/argo/workflow/common"
-	"github.com/CyrusBiotechnology/argo/workflow/util"
+	wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+	"github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
+	"github.com/cyrusbiotechnology/argo/workflow/common"
+	"github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 type listFlags struct {

cmd/argo/commands/logs.go

@@ -11,10 +11,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
-	wfclientset "github.com/CyrusBiotechnology/argo/pkg/client/clientset/versioned"
-	wfinformers "github.com/CyrusBiotechnology/argo/pkg/client/informers/externalversions"
 	"github.com/argoproj/pkg/errors"
+	"github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+	wfclientset "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned"
+	wfinformers "github.com/cyrusbiotechnology/argo/pkg/client/informers/externalversions"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"k8s.io/api/core/v1"

cmd/argo/commands/resubmit.go

@@ -3,8 +3,8 @@ package commands
 import (
 	"os"
 
-	"github.com/CyrusBiotechnology/argo/workflow/util"
 	"github.com/argoproj/pkg/errors"
+	"github.com/cyrusbiotechnology/argo/workflow/util"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )

cmd/argo/commands/resume.go

@@ -5,7 +5,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/CyrusBiotechnology/argo/workflow/util"
+	"github.com/cyrusbiotechnology/argo/workflow/util"
 	"github.com/spf13/cobra"
 )
 

cmd/argo/commands/retry.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"github.com/CyrusBiotechnology/argo/workflow/util"
+	"github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 func NewRetryCommand() *cobra.Command {

cmd/argo/commands/root.go

@@ -3,7 +3,7 @@ package commands
 import (
 	"os"
 
-	"github.com/CyrusBiotechnology/argo/util/cmd"
+	"github.com/cyrusbiotechnology/argo/util/cmd"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/tools/clientcmd"
 )

cmd/argo/commands/submit.go

@@ -10,10 +10,10 @@ import (
 	"github.com/argoproj/pkg/json"
 	"github.com/spf13/cobra"
 
-	wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
-	cmdutil "github.com/CyrusBiotechnology/argo/util/cmd"
-	"github.com/CyrusBiotechnology/argo/workflow/common"
-	"github.com/CyrusBiotechnology/argo/workflow/util"
+	wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
+	cmdutil "github.com/cyrusbiotechnology/argo/util/cmd"
+	"github.com/cyrusbiotechnology/argo/workflow/common"
+	"github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 // cliSubmitOpts holds submition options specific to CLI submission (e.g. controlling output)

cmd/argo/commands/suspend.go

@@ -5,7 +5,7 @@ import (
 	"log"
 	"os"
 
-	"github.com/CyrusBiotechnology/argo/workflow/util"
+	"github.com/cyrusbiotechnology/argo/workflow/util"
 	"github.com/spf13/cobra"
 )
 

cmd/argo/commands/terminate.go

@@ -7,7 +7,7 @@ import (
 	"github.com/argoproj/pkg/errors"
 	"github.com/spf13/cobra"
 
-	"github.com/CyrusBiotechnology/argo/workflow/util"
+	"github.com/cyrusbiotechnology/argo/workflow/util"
 )
 
 func NewTerminateCommand() *cobra.Command {

cmd/argo/commands/wait.go

@@ -5,8 +5,8 @@ import (
 	"os"
 	"sync"
 
-	wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
 	"github.com/argoproj/pkg/errors"
+	wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
 	"github.com/spf13/cobra"
 	apierr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

cmd/argo/commands/watch.go

@@ -10,7 +10,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 
-	wfv1 "github.com/CyrusBiotechnology/argo/pkg/apis/workflow/v1alpha1"
+	wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1"
 )
 
 func NewWatchCommand() *cobra.Command {

cmd/argo/main.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/CyrusBiotechnology/argo/cmd/argo/commands"
+	"github.com/cyrusbiotechnology/argo/cmd/argo/commands"
 	// load the gcp plugin (required to authenticate against GKE clusters).
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	// load the oidc plugin (required to authenticate with OpenID Connect).

cmd/argoexec/commands/resource.go

@@ -3,7 +3,7 @@ package commands
 import (
 	"os"
 
-	"github.com/CyrusBiotechnology/argo/workflow/common"
+	"github.com/cyrusbiotechnology/argo/workflow/common"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )

cmd/argoexec/commands/root.go

@@ -10,12 +10,12 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 
-	"github.com/CyrusBiotechnology/argo"
-	"github.com/CyrusBiotechnology/argo/util/cmd"
-	"github.com/CyrusBiotechnology/argo/workflow/common"
-	"github.com/CyrusBiotechnology/argo/workflow/executor"
-	"github.com/CyrusBiotechnology/argo/workflow/executor/docker"
-	"github.com/CyrusBiotechnology/argo/workflow/executor/kubelet"
+	"github.com/cyrusbiotechnology/argo"
+	"github.com/cyrusbiotechnology/argo/util/cmd"
+	"github.com/cyrusbiotechnology/argo/workflow/common"
+	"github.com/cyrusbiotechnology/argo/workflow/executor"
+	"github.com/cyrusbiotechnology/argo/workflow/executor/docker"
+	"github.com/cyrusbiotechnology/argo/workflow/executor/kubelet"
 )
 
 const (