diff --git a/.argo-ci/ci.yaml b/.argo-ci/ci.yaml index 4d44bb462f80..84443799a8b3 100644 --- a/.argo-ci/ci.yaml +++ b/.argo-ci/ci.yaml @@ -22,16 +22,13 @@ spec: value: "{{item}}" withItems: - make controller-image executor-image - - make cli-linux - - make cli-darwin + - make release-clis - name: test template: ci-builder arguments: parameters: - name: cmd - value: "{{item}}" - withItems: - - dep ensure && make lint test verify-codegen + value: dep ensure && make lint test verify-codegen - name: ci-builder inputs: @@ -67,10 +64,11 @@ spec: env: - name: DOCKER_HOST value: 127.0.0.1 + - name: DOCKER_BUILDKIT + value: "1" sidecars: - name: dind - image: docker:17.10-dind + image: docker:18.09-dind securityContext: privileged: true mirrorVolumeMounts: true - diff --git a/.dockerignore b/.dockerignore index 848b59e797ff..f515f4519087 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ -* -!dist -dist/pkg -!Gopkg.* \ No newline at end of file +# Prevent vendor directory from being copied to ensure we are not not pulling unexpected cruft from +# a user's workspace, and are only building off of what is locked by dep. +vendor +dist \ No newline at end of file diff --git a/ARTIFACT_REPO.md b/ARTIFACT_REPO.md index d0b1ef7b0f30..c126fa5ace86 100644 --- a/ARTIFACT_REPO.md +++ b/ARTIFACT_REPO.md @@ -14,12 +14,12 @@ $ helm install stable/minio --name argo-artifacts --set service.type=LoadBalance Login to the Minio UI using a web browser (port 9000) after obtaining the external IP using `kubectl`. ``` -$ kubectl get service argo-artifacts-minio +$ kubectl get service argo-artifacts ``` On Minikube: ``` -$ minikube service --url argo-artifacts-minio +$ minikube service --url argo-artifacts ``` NOTE: When minio is installed via Helm, it uses the following hard-wired default credentials, @@ -106,7 +106,7 @@ For Minio, the `accessKeySecret` and `secretKeySecret` naturally correspond the Example: ``` -$ kubectl edit configmap workflow-controller-configmap -n kube-system +$ kubectl edit configmap workflow-controller-configmap -n argo # assumes argo was installed in the argo namespace ... data: config: | diff --git a/CHANGELOG.md b/CHANGELOG.md index db4a421e05e1..bd85bfee8910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,94 @@ # Changelog +## 2.3.0-rc2 (2019-04-21) + +### Changes since 2.3.0-rc1 ++ Support parameter substitution in the volumes attribute (#1238) +- Fix regression where argoexec wait would not return when podname was too long +- wait will conditionally become privileged if main/sidecar privileged (issue #1323) +- `argo list` was not displaying non-zero priorities correctly +- Pod creation with secret volumemount (#1318) +- Export the methods of `KubernetesClientInterface` (#1294) + + +## 2.3.0-rc1 (2019-04-10) + +### Notes about upgrading from v2.2 + +* Secrets are passed to the wait sidecar using volumeMounts instead of performing K8s API calls + performed by the. This is much more secure since it limits the privileges of the workflow pod + to no longer require namespace level secret access. However, as a consequence, workflow pods which + reference a secret that does not exist, will now indefinitely stay in a Pending state, as opposed + to the previous behavior of failing during runtime. + + +### Deprecation Notice +The workflow-controller-configmap introduces a new config field, `executor`, which is a container +spec and provides controls over the executor sidecar container (i.e. `init`/`wait`). The fields +`executorImage`, `executorResources`, and `executorImagePullPolicy` are deprecated and will be +removed in a future release. + +### New Features: ++ Support for PNS (Process Namespace Sharing) executor (#1214) ++ Support for K8s API based Executor (#1010) (@dtaniwaki) ++ Adds limited support for Kubelet/K8s API artifact collection by mirroring volume mounts to wait sidecar ++ Support HDFS Artifact (#1159) (@dtaniwaki) ++ System level workflow parallelism limits & priorities (#1065) ++ Support larger workflows through node status compression (#1264) ++ Support nested steps workflow parallelism (#1046) (@WeiTang114) ++ Add feature to continue workflow on failed/error steps/tasks (#1205) (@schrodit) ++ Parameter and Argument names should support snake case (#1048) (@bbc88ks) ++ Add support for ppc64le and s390x (#1102) (@chenzhiwei) ++ Install mime-support in argoexec to set proper mime types for S3 artifacts ++ Allow owner reference to be set in submit util (#1120) (@nareshku) ++ add support for hostNetwork & dnsPolicy config (#1161) (@Dreamheart) ++ Add schedulerName to workflow and template spec (#1184) (@houz42) ++ Executor can access the k8s apiserver with a out-of-cluster config file (@houz42) ++ Proxy Priority and PriorityClassName to pods (#1179) (@dtaniwaki) ++ Add the `mergeStrategy` option to resource patching (#1269) (@ian-howell) ++ Add workflow labels and annotations global vars (#1280) (@discordianfish) ++ Support for optional input/output artifacts (#1277) ++ Add dns config support (#1301) (@xianlubird) ++ Added support for artifact path references (#1300) (@Ark-kun) ++ Add support for init containers (#1183) (@dtaniwaki) ++ Secrets should be passed to pods using volumes instead of API calls (#1302) ++ Azure AKS authentication issues #1079 (@gerardaus) + +### New Features: +* Update dependencies to K8s v1.12 and client-go 9.0 +* Add namespace explicitly to pod metadata (#1059) (@dvavili) +* Raise not implemented error when artifact saving is unsupported (#1062) (@dtaniwaki) +* Retry logic to s3 load and save function (#1082) (@kshamajain99) +* Remove docker_lib mount volume which is not needed anymore (#1115) (@ywskycn) +* Documentation improvements and fixes (@protochron, @jmcarp, @locona, @kivio, @fischerjulian, @annawinkler, @jdfalko, @groodt, @migggy, @nstott, @adrienjt) +* Validate ArchiveLocation artifacts (#1167) (@dtaniwaki) +* Git cloning via SSH was not verifying host public key (#1261) +* Speed up podReconciliation using parallel goroutine (#1286) (@xianlubird) + + +- Initialize child node before marking phase. Fixes panic on invalid `When` (#1075) (@jmcarp) +- Submodules are dirty after checkout -- need to update (#1052) (@andreimc) +- Fix output artifact and parameter conflict (#1125) (@Ark-kun) +- Remove container wait timeout from 'argo logs --follow' (#1142) +- Fix panic in ttl controller (#1143) +- Kill daemoned step if workflow consist of single daemoned step (#1144) +- Fix global artifact overwriting in nested workflow (#1086) (@WeiTang114) +- Fix issue where steps with exhausted retires would not complete (#1148) +- Fix metadata for DAG with loops (#1149) +- Replace exponential retry with poll (#1166) (@kzadorozhny) +- Dockerfile: argoexec base image correction (#1213) (@elikatsis) +- Set executor image pull policy for resource template (#1174) (@dtaniwaki) +- fix dag retries (#1221) (@houz42) +- Remove extra quotes around output parameter value (#1232) (@elikatsis) +- Include stderr when retrieving docker logs (#1225) (@shahin) +- Fix the Prometheus address references (#1237) (@spacez320) +- Kubernetes Resource action: patch is not supported (#1245) +- Fake outputs don't notify and task completes successfully (#1247) +- Reduce redundancy pod label action (#1271) (@xianlubird) +- Fix bug with DockerExecutor's CopyFile (#1275) +- Fix for Resource creation where template has same parameter templating (#1283) +- Fixes an issue where daemon steps were not getting terminated properly + ## 2.2.1 (2018-10-18) ### Changelog since v2.2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1491117a1cb1..033ea2360a24 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,17 +15,17 @@ Go to https://github.com/argoproj/ ## How to suggest a new feature -Go to https://groups.google.com/forum/#!forum/argoproj -* Create a new topic to discuss your feature. +Go to https://github.com/argoproj/ +* Open an issue and discuss it. ## How to setup your dev environment ### Requirements -* Golang 1.10 +* Golang 1.11 * Docker * dep v0.5 * Mac Install: `brew install dep` -* gometalinter v2.0.5 +* gometalinter v2.0.12 ### Quickstart ``` @@ -36,9 +36,16 @@ $ make ``` ### Build workflow-controller and executor images -The following will build the workflow-controller and executor images tagged with the `latest` tag, then push to a personal dockerhub repository: +The following will build the release versions of workflow-controller and executor images tagged +with the `latest` tag, then push to a personal dockerhub repository, `mydockerrepo`: +``` +$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=mydockerrepo DOCKER_PUSH=true +``` +Building release versions of the images will be slow during development, since the build happens +inside a docker build context, which cannot re-use the golang build cache between builds. To build +images quicker (for development purposes), images can be built by adding DEV_IMAGE=true. ``` -$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=jessesuen DOCKER_PUSH=true +$ make controller-image executor-image IMAGE_TAG=latest IMAGE_NAMESPACE=mydockerrepo DOCKER_PUSH=true DEV_IMAGE=true ``` ### Build argo cli @@ -49,5 +56,6 @@ $ ./dist/argo version ### Deploying controller with alternative controller/executor images ``` -$ argo install --controller-image jessesuen/workflow-controller:latest --executor-image jessesuen/argoexec:latest +$ helm install argo/argo --set images.namespace=mydockerrepo --set +images.controller workflow-controller:latest ``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000000..a0464961ef65 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,99 @@ +#################################################################################################### +# Builder image +# Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image +# Also used as the image in CI jobs so needs all dependencies +#################################################################################################### +FROM golang:1.11.5 as builder + +RUN apt-get update && apt-get install -y \ + git \ + make \ + wget \ + gcc \ + zip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +WORKDIR /tmp + +# Install docker +ENV DOCKER_CHANNEL stable +ENV DOCKER_VERSION 18.09.1 +RUN wget -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz" && \ + tar --extract --file docker.tgz --strip-components 1 --directory /usr/local/bin/ && \ + rm docker.tgz + +# Install dep +ENV DEP_VERSION=0.5.0 +RUN wget https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -O /usr/local/bin/dep && \ + chmod +x /usr/local/bin/dep + +# Install gometalinter +ENV GOMETALINTER_VERSION=2.0.12 +RUN curl -sLo- https://github.com/alecthomas/gometalinter/releases/download/v${GOMETALINTER_VERSION}/gometalinter-${GOMETALINTER_VERSION}-linux-amd64.tar.gz | \ + tar -xzC "$GOPATH/bin" --exclude COPYING --exclude README.md --strip-components 1 -f- && \ + ln -s $GOPATH/bin/gometalinter $GOPATH/bin/gometalinter.v2 + + +#################################################################################################### +# argoexec-base +# Used as the base for both the release and development version of argoexec +#################################################################################################### +FROM debian:9.6-slim as argoexec-base +# NOTE: keep the version synced with https://storage.googleapis.com/kubernetes-release/release/stable.txt +ENV KUBECTL_VERSION=1.13.4 +RUN apt-get update && \ + apt-get install -y curl jq procps git tar mime-support && \ + rm -rf /var/lib/apt/lists/* && \ + curl -L -o /usr/local/bin/kubectl -LO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ + chmod +x /usr/local/bin/kubectl +COPY hack/ssh_known_hosts /etc/ssh/ssh_known_hosts +COPY --from=builder /usr/local/bin/docker /usr/local/bin/ + + +#################################################################################################### +# Argo Build stage which performs the actual build of Argo binaries +#################################################################################################### +FROM builder as builder-base + +# A dummy directory is created under $GOPATH/src/dummy so we are able to use dep +# to install all the packages of our dep lock file +COPY Gopkg.toml ${GOPATH}/src/dummy/Gopkg.toml +COPY Gopkg.lock ${GOPATH}/src/dummy/Gopkg.lock + +RUN cd ${GOPATH}/src/dummy && \ + dep ensure -vendor-only && \ + mv vendor/* ${GOPATH}/src/ && \ + rmdir vendor + +WORKDIR /go/src/github.com/cyrusbiotechnology/argo +COPY . . + +FROM builder-base as argo-build +# Perform the build + +ARG MAKE_TARGET="controller executor cli-linux-amd64" +RUN make $MAKE_TARGET + + +#################################################################################################### +# argoexec +#################################################################################################### +FROM argoexec-base as argoexec +COPY --from=argo-build /go/src/github.com/cyrusbiotechnology/argo/dist/argoexec /usr/local/bin/ + + +#################################################################################################### +# workflow-controller +#################################################################################################### +FROM scratch as workflow-controller +COPY --from=argo-build /go/src/github.com/cyrusbiotechnology/argo/dist/workflow-controller /bin/ +ENTRYPOINT [ "workflow-controller" ] + + +#################################################################################################### +# argocli +#################################################################################################### +FROM scratch as argocli +COPY --from=argo-build /go/src/github.com/cyrusbiotechnology/argo/dist/argo-linux-amd64 /bin/argo +ENTRYPOINT [ "argo" ] diff --git a/Dockerfile-argoexec b/Dockerfile-argoexec deleted file mode 100644 index 2461140bbc25..000000000000 --- a/Dockerfile-argoexec +++ /dev/null @@ -1,16 +0,0 @@ -FROM debian:9.5-slim - -RUN apt-get update && \ - apt-get install -y curl jq procps git tar && \ - rm -rf /var/lib/apt/lists/* && \ - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ - chmod +x ./kubectl && \ - mv ./kubectl /bin/ - -ENV DOCKER_VERSION=18.06.0 -RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}-ce.tgz && \ - tar -xzf docker-${DOCKER_VERSION}-ce.tgz && \ - mv docker/docker /usr/local/bin/docker && \ - rm -rf ./docker - -COPY dist/argoexec /bin/ diff --git a/Dockerfile-builder b/Dockerfile-builder deleted file mode 100644 index 8cb721fcd932..000000000000 --- a/Dockerfile-builder +++ /dev/null @@ -1,32 +0,0 @@ -FROM debian:9.5-slim - -RUN apt-get update && apt-get install -y \ - git \ - make \ - curl \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Install go -ENV GO_VERSION 1.10.3 -ENV GO_ARCH amd64 -ENV GOPATH /root/go -ENV PATH ${GOPATH}/bin:/usr/local/go/bin:${PATH} -RUN wget https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \ - tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \ - rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \ - wget https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 -O /usr/local/bin/dep && \ - chmod +x /usr/local/bin/dep && \ - mkdir -p ${GOPATH}/bin && \ - curl -sLo- https://github.com/alecthomas/gometalinter/releases/download/v2.0.5/gometalinter-2.0.5-linux-amd64.tar.gz | \ - tar -xzC "$GOPATH/bin" --exclude COPYING --exclude README.md --strip-components 1 -f- - -# A dummy directory is created under $GOPATH/src/dummy so we are able to use dep -# to install all the packages of our dep lock file -COPY Gopkg.toml ${GOPATH}/src/dummy/Gopkg.toml -COPY Gopkg.lock ${GOPATH}/src/dummy/Gopkg.lock -RUN cd ${GOPATH}/src/dummy && \ - dep ensure -vendor-only && \ - mv vendor/* ${GOPATH}/src/ && \ - rmdir vendor diff --git a/Dockerfile-ci-builder b/Dockerfile-ci-builder deleted file mode 100644 index 943176928518..000000000000 --- a/Dockerfile-ci-builder +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang:1.10.3 - -WORKDIR /tmp - -RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-18.06.0-ce.tgz && \ - tar -xzf docker-18.06.0-ce.tgz && \ - mv docker/docker /usr/local/bin/docker && \ - rm -rf ./docker && \ - wget https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 -O /usr/local/bin/dep && \ - chmod +x /usr/local/bin/dep && \ - curl -sLo- https://github.com/alecthomas/gometalinter/releases/download/v2.0.5/gometalinter-2.0.5-linux-amd64.tar.gz | \ - tar -xzC "$GOPATH/bin" --exclude COPYING --exclude README.md --strip-components 1 -f- diff --git a/Dockerfile-cli b/Dockerfile-cli deleted file mode 100644 index 39f6c45b9523..000000000000 --- a/Dockerfile-cli +++ /dev/null @@ -1,4 +0,0 @@ -FROM alpine:3.7 - -COPY dist/argo-linux-amd64 /bin/argo -ENTRYPOINT [ "/bin/argo" ] diff --git a/Dockerfile-workflow-controller b/Dockerfile-workflow-controller deleted file mode 100644 index b7694f7d0dc6..000000000000 --- a/Dockerfile-workflow-controller +++ /dev/null @@ -1,5 +0,0 @@ -FROM debian:9.4 - -COPY dist/workflow-controller /bin/ - -ENTRYPOINT [ "/bin/workflow-controller" ] diff --git a/Dockerfile.argoexec-dev b/Dockerfile.argoexec-dev new file mode 100644 index 000000000000..e1437f7be80b --- /dev/null +++ b/Dockerfile.argoexec-dev @@ -0,0 +1,5 @@ +#################################################################################################### +# argoexec-dev +#################################################################################################### +FROM argoexec-base +COPY argoexec /usr/local/bin/ diff --git a/Dockerfile.workflow-controller-dev b/Dockerfile.workflow-controller-dev new file mode 100644 index 000000000000..f2132614c852 --- /dev/null +++ b/Dockerfile.workflow-controller-dev @@ -0,0 +1,6 @@ +#################################################################################################### +# workflow-controller-dev +#################################################################################################### +FROM scratch +COPY workflow-controller /bin/ +ENTRYPOINT [ "workflow-controller" ] diff --git a/Gopkg.lock b/Gopkg.lock index 6b124fc64a6e..e276f7df7c25 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -15,6 +15,22 @@ revision = "64a2037ec6be8a4b0c1d1f706ed35b428b989239" version = "v0.26.0" +[[projects]] + name = "github.com/Azure/go-autorest" + packages = [ + "autorest", + "autorest/adal", + "autorest/azure", + "autorest/date" + ] + revision = "1ff28809256a84bb6966640ff3d0371af82ccba4" + +[[projects]] + name = "github.com/BurntSushi/toml" + packages = ["."] + revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" + version = "v0.3.1" + [[projects]] name = "github.com/Knetic/govaluate" packages = ["."] @@ -23,8 +39,8 @@ [[projects]] name = "github.com/PuerkitoBio/purell" packages = ["."] - revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" - version = "v1.1.0" + revision = "44968752391892e1b0d0b821ee79e9a85fa13049" + version = "v1.1.1" [[projects]] branch = "master" @@ -36,7 +52,9 @@ branch = "master" name = "github.com/argoproj/pkg" packages = [ + "cli", "errors", + "exec", "file", "humanize", "json", @@ -47,19 +65,35 @@ "strftime", "time" ] - revision = "1aa3e0c55668da17703adba5c534fff6930db589" + revision = "7e3ef65c8d44303738c7e815bd9b1b297b39f5c8" [[projects]] - branch = "master" name = "github.com/beorn7/perks" packages = ["quantile"] - revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + revision = "4b2b341e8d7715fae06375aa633dbb6e91b3fb46" + version = "v1.0.0" + +[[projects]] + name = "github.com/colinmarc/hdfs" + packages = [ + ".", + "protocol/hadoop_common", + "protocol/hadoop_hdfs", + "rpc" + ] + revision = "48eb8d6c34a97ffc73b406356f0f2e1c569b42a5" [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + +[[projects]] + name = "github.com/dgrijalva/jwt-go" + packages = ["."] + revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e" + version = "v3.2.0" [[projects]] branch = "master" @@ -68,13 +102,13 @@ ".", "spdy" ] - revision = "bc6354cbbc295e925e4c611ffe90c1f287ee54db" + revision = "6480d4af844c189cf5dd913db24ddd339d3a4f85" [[projects]] - branch = "master" name = "github.com/dustin/go-humanize" packages = ["."] revision = "9f541cc9db5d55bce703bd99987c9d5cb8eea45e" + version = "v1.0.0" [[projects]] name = "github.com/emicklei/go-restful" @@ -82,8 +116,8 @@ ".", "log" ] - revision = "3eb9738c1697594ea6e71a7156a9bb32ed216cf0" - version = "v2.8.0" + revision = "b9bbc5664f49b6deec52393bd68f39830687a347" + version = "v2.9.3" [[projects]] name = "github.com/emirpasic/gods" @@ -95,56 +129,44 @@ "trees/binaryheap", "utils" ] - revision = "f6c17b524822278a87e3b3bd809fec33b51f5b46" - version = "v1.9.0" + revision = "1615341f118ae12f353cc8a983f35b584342c9b3" + version = "v1.12.0" [[projects]] name = "github.com/evanphx/json-patch" packages = ["."] - revision = "afac545df32f2287a079e2dfb7ba2745a643747e" - version = "v3.0.0" - -[[projects]] - name = "github.com/fsnotify/fsnotify" - packages = ["."] - revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" - version = "v1.4.7" + revision = "72bf35d0ff611848c1dc9df0f976c81192392fa5" + version = "v4.1.0" [[projects]] branch = "master" name = "github.com/ghodss/yaml" packages = ["."] - revision = "c7ce16629ff4cd059ed96ed06419dd3856fd3577" - -[[projects]] - name = "github.com/go-ini/ini" - packages = ["."] - revision = "358ee7663966325963d4e8b2e1fbd570c5195153" - version = "v1.38.1" + revision = "25d852aebe32c875e9c044af3eef9c7dc6bc777f" [[projects]] name = "github.com/go-openapi/jsonpointer" packages = ["."] - revision = "3a0015ad55fa9873f41605d3e8f28cd279c32ab2" - version = "0.15.0" + revision = "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004" + version = "v0.19.0" [[projects]] name = "github.com/go-openapi/jsonreference" packages = ["."] - revision = "3fb327e6747da3043567ee86abd02bb6376b6be2" - version = "0.15.0" + revision = "8483a886a90412cd6858df4ea3483dce9c8e35a3" + version = "v0.19.0" [[projects]] name = "github.com/go-openapi/spec" packages = ["."] - revision = "bce47c9386f9ecd6b86f450478a80103c3fe1402" - version = "0.15.0" + revision = "53d776530bf78a11b03a7b52dd8a083086b045e5" + version = "v0.19.0" [[projects]] name = "github.com/go-openapi/swag" packages = ["."] - revision = "2b0bd4f193d011c203529df626a65d63cb8a79e8" - version = "0.15.0" + revision = "b3e2804c8535ee0d1b89320afd98474d5b8e9e3b" + version = "v0.19.0" [[projects]] name = "github.com/gogo/protobuf" @@ -152,8 +174,8 @@ "proto", "sortkeys" ] - revision = "636bf0302bc95575d69441b25a2603156ffdddf1" - version = "v1.1.1" + revision = "ba06b47c162d49f2af050fb4c75bcbc86a159d5c" + version = "v1.2.1" [[projects]] branch = "master" @@ -165,26 +187,40 @@ name = "github.com/golang/protobuf" packages = [ "proto", + "protoc-gen-go", "protoc-gen-go/descriptor", + "protoc-gen-go/generator", + "protoc-gen-go/generator/internal/remap", + "protoc-gen-go/grpc", + "protoc-gen-go/plugin", "ptypes", "ptypes/any", "ptypes/duration", "ptypes/timestamp" ] - revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" - version = "v1.1.0" + revision = "b5d812f8a3706043e23a9cd5babf2e5423744d30" + version = "v1.3.1" + +[[projects]] + name = "github.com/google/btree" + packages = ["."] + revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306" + version = "v1.0.0" [[projects]] - branch = "master" name = "github.com/google/gofuzz" packages = ["."] - revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" + revision = "f140a6486e521aad38f5917de355cbf147cc0496" + version = "v1.0.0" [[projects]] name = "github.com/googleapis/gax-go" - packages = ["."] - revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f" - version = "v2.0.0" + packages = [ + ".", + "v2" + ] + revision = "beaecbbdd8af86aa3acf14180d53828ce69400b2" + version = "v2.0.4" [[projects]] name = "github.com/googleapis/gnostic" @@ -199,29 +235,38 @@ [[projects]] name = "github.com/gorilla/websocket" packages = ["."] - revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" - version = "v1.2.0" + revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d" + version = "v1.4.0" [[projects]] branch = "master" - name = "github.com/hashicorp/golang-lru" + name = "github.com/gregjones/httpcache" packages = [ ".", - "simplelru" + "diskcache" ] - revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" + revision = "3befbb6ad0cc97d4c25d851e9528915809e1a22f" [[projects]] - branch = "master" - name = "github.com/howeyc/gopass" + name = "github.com/hashicorp/go-uuid" packages = ["."] - revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8" + revision = "4f571afc59f3043a65f8fe6bf46d887b10a01d43" + version = "v1.0.1" + +[[projects]] + name = "github.com/hashicorp/golang-lru" + packages = [ + ".", + "simplelru" + ] + revision = "7087cb70de9f7a8bc0a10c375cb0d2280a8edf9c" + version = "v0.5.1" [[projects]] name = "github.com/imdario/mergo" packages = ["."] - revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4" - version = "v0.3.6" + revision = "7c29201646fa3de8506f701213473dd407f19646" + version = "v0.3.7" [[projects]] name = "github.com/inconshreveable/mousetrap" @@ -235,17 +280,32 @@ packages = ["io"] revision = "d14ea06fba99483203c19d92cfcd13ebe73135f4" +[[projects]] + branch = "master" + name = "github.com/jcmturner/gofork" + packages = [ + "encoding/asn1", + "x/crypto/pbkdf2" + ] + revision = "dc7c13fece037a4a36e2b3c69db4991498d30692" + [[projects]] name = "github.com/json-iterator/go" packages = ["."] - revision = "1624edc4454b8682399def8740d46db5e4362ba4" - version = "1.1.5" + revision = "0ff49de124c6f76f8494e194af75bde0f1a49a29" + version = "v1.1.6" [[projects]] name = "github.com/kevinburke/ssh_config" packages = ["."] - revision = "9fc7bb800b555d63157c65a904c86a2cc7b4e795" - version = "0.4" + revision = "81db2a75821ed34e682567d48be488a1c3121088" + version = "0.5" + +[[projects]] + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e" + version = "v1.0.2" [[projects]] branch = "master" @@ -255,7 +315,7 @@ "jlexer", "jwriter" ] - revision = "03f2033d19d5860aef995fe360ac7d395cd8ce65" + revision = "1ea4449da9834f4d333f1cc461c374aea217d249" [[projects]] name = "github.com/matttproud/golang_protobuf_extensions" @@ -273,14 +333,20 @@ "pkg/s3utils", "pkg/set" ] - revision = "70799fe8dae6ecfb6c7d7e9e048fce27f23a1992" - version = "v6.0.5" + revision = "a8704b60278f98501c10f694a9c4df8bdd1fac56" + version = "v6.0.23" [[projects]] - branch = "master" name = "github.com/mitchellh/go-homedir" packages = ["."] - revision = "58046073cbffe2f25d425fe1331102f55cf719de" + revision = "af06845cf3004701891bf4fdb884bfe4920b3727" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/go-ps" + packages = ["."] + revision = "4fdf99ab29366514c69ccccddab5dc58b8d84062" [[projects]] name = "github.com/modern-go/concurrent" @@ -300,11 +366,23 @@ revision = "c37440a7cf42ac63b919c752ca73a85067e05992" version = "v0.2.0" +[[projects]] + branch = "master" + name = "github.com/petar/GoLLRB" + packages = ["llrb"] + revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" + +[[projects]] + name = "github.com/peterbourgon/diskv" + packages = ["."] + revision = "0be1b92a6df0e4f5cb0a5d15fb7f643d0ad93ce6" + version = "v3.0.0" + [[projects]] name = "github.com/pkg/errors" packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" + revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" + version = "v0.8.1" [[projects]] name = "github.com/pmezard/go-difflib" @@ -325,28 +403,23 @@ branch = "master" name = "github.com/prometheus/client_model" packages = ["go"] - revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" + revision = "fd36f4220a901265f90734c3183c5f0c91daa0b8" [[projects]] - branch = "master" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", "model" ] - revision = "c7de2306084e37d54b8be01f3541a8464345e9a5" + revision = "a82f4c12f983cc2649298185f296632953e50d3e" + version = "v0.3.0" [[projects]] branch = "master" name = "github.com/prometheus/procfs" - packages = [ - ".", - "internal/util", - "nfs", - "xfs" - ] - revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" + packages = ["."] + revision = "87a4384529e0652f5035fb5cc8095faf73ea9b0b" [[projects]] name = "github.com/sergi/go-diff" @@ -357,20 +430,19 @@ [[projects]] name = "github.com/sirupsen/logrus" packages = ["."] - revision = "3e01752db0189b9157070a0e1668a620f9a85da2" - version = "v1.0.6" + revision = "8bdbc7bcc01dcbb8ec23dc8a28e332258d25251f" + version = "v1.4.1" [[projects]] - branch = "master" name = "github.com/spf13/cobra" packages = ["."] - revision = "7c4570c3ebeb8129a1f7456d0908a8b676b6f9f1" + revision = "fe5e611709b0c57fa4a89136deaa8e1d4004d053" [[projects]] name = "github.com/spf13/pflag" packages = ["."] - revision = "583c0c0531f06d5278b7d917446061adc344b5cd" - version = "v1.0.1" + revision = "298182f68c66c05229eb03ac171abe6e309ee79a" + version = "v1.0.3" [[projects]] name = "github.com/src-d/gcfg" @@ -380,8 +452,8 @@ "token", "types" ] - revision = "f187355171c936ac84a82793659ebb4936bc1c23" - version = "v1.3.0" + revision = "1ac3a1ac202429a54835fe8408a92880156b489d" + version = "v1.4.0" [[projects]] name = "github.com/stretchr/objx" @@ -397,38 +469,44 @@ "require", "suite" ] - revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" - version = "v1.2.2" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" [[projects]] name = "github.com/tidwall/gjson" packages = ["."] - revision = "1e3f6aeaa5bad08d777ea7807b279a07885dd8b2" - version = "v1.1.3" + revision = "eee0b6226f0d1db2675a176fdfaa8419bcad4ca8" + version = "v1.2.1" [[projects]] - branch = "master" name = "github.com/tidwall/match" packages = ["."] - revision = "1731857f09b1f38450e2c12409748407822dc6be" + revision = "33827db735fff6510490d69a8622612558a557ed" + version = "v1.0.1" [[projects]] branch = "master" + name = "github.com/tidwall/pretty" + packages = ["."] + revision = "1166b9ac2b65e46a43d8618d30d1554f4652d49b" + +[[projects]] name = "github.com/valyala/bytebufferpool" packages = ["."] revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" + version = "v1.0.0" [[projects]] - branch = "master" name = "github.com/valyala/fasttemplate" packages = ["."] - revision = "dcecefd839c4193db0d35b88ec65b4c12d360ab0" + revision = "8b5e4e491ab636663841c42ea3c5a9adebabaf36" + version = "v1.0.1" [[projects]] name = "github.com/xanzy/ssh-agent" packages = ["."] - revision = "640f0ab560aeb89d523bb6ac322b1244d5c3796c" - version = "v0.2.0" + revision = "6a3e2ff9e7c564f36873c2e36413f634534f1c44" + version = "v0.2.1" [[projects]] name = "go.opencensus.io" @@ -436,8 +514,11 @@ ".", "internal", "internal/tagencoding", + "metric/metricdata", + "metric/metricproducer", "plugin/ochttp", "plugin/ochttp/propagation/b3", + "resource", "stats", "stats/internal", "stats/view", @@ -447,8 +528,8 @@ "trace/propagation", "trace/tracestate" ] - revision = "79993219becaa7e29e3b60cb67f5b8e82dee11d6" - version = "v0.17.0" + revision = "df6e2001952312404b06f5f6f03fcb4aec1648e5" + version = "v0.21.0" [[projects]] branch = "master" @@ -462,19 +543,39 @@ "ed25519/internal/edwards25519", "internal/chacha20", "internal/subtle", + "md4", "openpgp", "openpgp/armor", "openpgp/elgamal", "openpgp/errors", "openpgp/packet", "openpgp/s2k", + "pbkdf2", "poly1305", "ssh", "ssh/agent", "ssh/knownhosts", "ssh/terminal" ] - revision = "f027049dab0ad238e394a753dba2d14753473a04" + revision = "c05e17bb3b2dca130fc919668a96b4bec9eb9442" + +[[projects]] + branch = "master" + name = "golang.org/x/exp" + packages = [ + "apidiff", + "cmd/apidiff" + ] + revision = "8c7d1c524af6eaf18eadc4f57955a748e7001194" + +[[projects]] + branch = "master" + name = "golang.org/x/lint" + packages = [ + ".", + "golint" + ] + revision = "959b441ac422379a43da2230f62be024250818b0" [[projects]] branch = "master" @@ -487,9 +588,10 @@ "http2/hpack", "idna", "internal/timeseries", + "publicsuffix", "trace" ] - revision = "f9ce57c11b242f0f1599cf25c89d8cb02c45295a" + revision = "4829fb13d2c62012c17688fa7f629f371014946d" [[projects]] branch = "master" @@ -501,7 +603,7 @@ "jws", "jwt" ] - revision = "3d292e4d0cdc3a0113e6d207bb137145ef1de42f" + revision = "9f3314589c9a9136388751d9adae6b0ed400978a" [[projects]] branch = "master" @@ -511,7 +613,7 @@ "unix", "windows" ] - revision = "904bdc257025c7b3f43c19360ad3ab85783fad78" + revision = "16072639606ea9e22c7d86e4cbd6af6314f4193c" [[projects]] name = "golang.org/x/text" @@ -520,6 +622,8 @@ "collate/build", "internal/colltab", "internal/gen", + "internal/language", + "internal/language/compact", "internal/tag", "internal/triegen", "internal/ucd", @@ -532,27 +636,38 @@ "unicode/rangetable", "width" ] - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" + revision = "c942b20a5d85b458c4dce1589326051d85e25d6d" + version = "v0.3.1" [[projects]] branch = "master" name = "golang.org/x/time" packages = ["rate"] - revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" + revision = "9d24e82272b4f38b78bc8cff74fa936d31ccd8ef" [[projects]] branch = "master" name = "golang.org/x/tools" packages = [ + "cmd/goimports", "go/ast/astutil", + "go/buildutil", + "go/gcexportdata", + "go/internal/cgo", + "go/internal/gcimporter", + "go/internal/packagesdriver", + "go/loader", + "go/packages", + "go/types/typeutil", "imports", - "internal/fastwalk" + "internal/fastwalk", + "internal/gopathwalk", + "internal/module", + "internal/semver" ] - revision = "ca6481ae56504398949d597084558e50ad07117a" + revision = "36563e24a2627da92566d43aa1c7a2dd895fc60d" [[projects]] - branch = "master" name = "google.golang.org/api" packages = [ "gensupport", @@ -566,7 +681,8 @@ "transport/http", "transport/http/internal/propagation" ] - revision = "44c6748ece026e0fe668793d8f92e521356400a3" + revision = "0cbcb99a9ea0c8023c794b2693cbe1def82ed4d7" + version = "v0.3.2" [[projects]] name = "google.golang.org/appengine" @@ -582,8 +698,8 @@ "internal/urlfetch", "urlfetch" ] - revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" - version = "v1.1.0" + revision = "54a98f90d1c46b7731eb8fb305d2a321c30ef610" + version = "v1.5.0" [[projects]] branch = "master" @@ -594,7 +710,7 @@ "googleapis/rpc/code", "googleapis/rpc/status" ] - revision = "0e822944c569bf5c9afd034adaa56208bd2906ac" + revision = "e7d98fc518a78c9f8b5ee77be7b0b317475d89e1" [[projects]] name = "google.golang.org/grpc" @@ -603,17 +719,23 @@ "balancer", "balancer/base", "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", "codes", "connectivity", "credentials", + "credentials/internal", "encoding", "encoding/proto", "grpclog", "internal", "internal/backoff", + "internal/balancerload", + "internal/binarylog", "internal/channelz", "internal/envconfig", "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", "internal/transport", "keepalive", "metadata", @@ -626,8 +748,8 @@ "status", "tap" ] - revision = "8dea3dc473e90c8179e519d91302d0597c0ca1d1" - version = "v1.15.0" + revision = "25c4f928eaa6d96443009bd842389fb4fa48664e" + version = "v1.20.1" [[projects]] name = "gopkg.in/inf.v0" @@ -635,6 +757,68 @@ revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" version = "v0.9.1" +[[projects]] + name = "gopkg.in/ini.v1" + packages = ["."] + revision = "c85607071cf08ca1adaf48319cd1aa322e81d8c1" + version = "v1.42.0" + +[[projects]] + name = "gopkg.in/jcmturner/aescts.v1" + packages = ["."] + revision = "f6abebb3171c4c1b1fea279cb7c7325020a26290" + version = "v1.0.1" + +[[projects]] + name = "gopkg.in/jcmturner/dnsutils.v1" + packages = ["."] + revision = "13eeb8d49ffb74d7a75784c35e4d900607a3943c" + version = "v1.0.1" + +[[projects]] + name = "gopkg.in/jcmturner/gokrb5.v5" + packages = [ + "asn1tools", + "client", + "config", + "credentials", + "crypto", + "crypto/common", + "crypto/etype", + "crypto/rfc3961", + "crypto/rfc3962", + "crypto/rfc4757", + "crypto/rfc8009", + "gssapi", + "iana", + "iana/addrtype", + "iana/adtype", + "iana/asnAppTag", + "iana/chksumtype", + "iana/errorcode", + "iana/etypeID", + "iana/flags", + "iana/keyusage", + "iana/msgtype", + "iana/nametype", + "iana/patype", + "kadmin", + "keytab", + "krberror", + "messages", + "mstypes", + "pac", + "types" + ] + revision = "32ba44ca5b42f17a4a9f33ff4305e70665a1bc0f" + version = "v5.3.0" + +[[projects]] + name = "gopkg.in/jcmturner/rpc.v0" + packages = ["ndr"] + revision = "4480c480c9cd343b54b0acb5b62261cbd33d7adf" + version = "v0.0.2" + [[projects]] name = "gopkg.in/src-d/go-billy.v4" packages = [ @@ -644,8 +828,8 @@ "osfs", "util" ] - revision = "83cf655d40b15b427014d7875d10850f96edba14" - version = "v4.2.0" + revision = "982626487c60a5252e7d0b695ca23fb0fa2fd670" + version = "v4.3.0" [[projects]] name = "gopkg.in/src-d/go-git.v4" @@ -653,6 +837,7 @@ ".", "config", "internal/revision", + "internal/url", "plumbing", "plumbing/cache", "plumbing/filemode", @@ -691,8 +876,8 @@ "utils/merkletrie/internal/frame", "utils/merkletrie/noder" ] - revision = "3bd5e82b2512d85becae9677fa06b5a973fd4cfb" - version = "v4.5.0" + revision = "aa6f288c256ff8baf8a7745546a9752323dc0d89" + version = "v4.11.0" [[projects]] name = "gopkg.in/warnings.v0" @@ -703,11 +888,39 @@ [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" - version = "v2.2.1" + revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" + version = "v2.2.2" [[projects]] - branch = "release-1.10" + name = "honnef.co/go/tools" + packages = [ + "arg", + "callgraph", + "callgraph/static", + "cmd/staticcheck", + "config", + "deprecated", + "functions", + "internal/sharedcheck", + "lint", + "lint/lintdsl", + "lint/lintutil", + "lint/lintutil/format", + "simple", + "ssa", + "ssa/ssautil", + "ssautil", + "staticcheck", + "staticcheck/vrp", + "stylecheck", + "unused", + "version" + ] + revision = "95959eaf5e3c41c66151dcfd91779616b84077a8" + version = "2019.1.1" + +[[projects]] + branch = "release-1.12" name = "k8s.io/api" packages = [ "admissionregistration/v1alpha1", @@ -721,10 +934,12 @@ "authorization/v1beta1", "autoscaling/v1", "autoscaling/v2beta1", + "autoscaling/v2beta2", "batch/v1", "batch/v1beta1", "batch/v2alpha1", "certificates/v1beta1", + "coordination/v1beta1", "core/v1", "events/v1beta1", "extensions/v1beta1", @@ -734,15 +949,16 @@ "rbac/v1alpha1", "rbac/v1beta1", "scheduling/v1alpha1", + "scheduling/v1beta1", "settings/v1alpha1", "storage/v1", "storage/v1alpha1", "storage/v1beta1" ] - revision = "0f11257a8a25954878633ebdc9841c67d8f83bdb" + revision = "6db15a15d2d3874a6c3ddb2140ac9f3bc7058428" [[projects]] - branch = "release-1.10" + branch = "release-1.12" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -775,23 +991,27 @@ "pkg/util/httpstream/spdy", "pkg/util/intstr", "pkg/util/json", + "pkg/util/mergepatch", + "pkg/util/naming", "pkg/util/net", "pkg/util/remotecommand", "pkg/util/runtime", "pkg/util/sets", + "pkg/util/strategicpatch", "pkg/util/validation", "pkg/util/validation/field", "pkg/util/wait", "pkg/util/yaml", "pkg/version", "pkg/watch", + "third_party/forked/golang/json", "third_party/forked/golang/netutil", "third_party/forked/golang/reflect" ] - revision = "e386b2658ed20923da8cc9250e552f082899a1ee" + revision = "01f179d85dbce0f2e0e4351a92394b38694b7cae" [[projects]] - branch = "release-7.0" + branch = "release-9.0" name = "k8s.io/client-go" packages = [ "discovery", @@ -823,6 +1043,8 @@ "kubernetes/typed/autoscaling/v1/fake", "kubernetes/typed/autoscaling/v2beta1", "kubernetes/typed/autoscaling/v2beta1/fake", + "kubernetes/typed/autoscaling/v2beta2", + "kubernetes/typed/autoscaling/v2beta2/fake", "kubernetes/typed/batch/v1", "kubernetes/typed/batch/v1/fake", "kubernetes/typed/batch/v1beta1", @@ -831,6 +1053,8 @@ "kubernetes/typed/batch/v2alpha1/fake", "kubernetes/typed/certificates/v1beta1", "kubernetes/typed/certificates/v1beta1/fake", + "kubernetes/typed/coordination/v1beta1", + "kubernetes/typed/coordination/v1beta1/fake", "kubernetes/typed/core/v1", "kubernetes/typed/core/v1/fake", "kubernetes/typed/events/v1beta1", @@ -849,6 +1073,8 @@ "kubernetes/typed/rbac/v1beta1/fake", "kubernetes/typed/scheduling/v1alpha1", "kubernetes/typed/scheduling/v1alpha1/fake", + "kubernetes/typed/scheduling/v1beta1", + "kubernetes/typed/scheduling/v1beta1/fake", "kubernetes/typed/settings/v1alpha1", "kubernetes/typed/settings/v1alpha1/fake", "kubernetes/typed/storage/v1", @@ -859,7 +1085,9 @@ "kubernetes/typed/storage/v1beta1/fake", "pkg/apis/clientauthentication", "pkg/apis/clientauthentication/v1alpha1", + "pkg/apis/clientauthentication/v1beta1", "pkg/version", + "plugin/pkg/client/auth/azure", "plugin/pkg/client/auth/exec", "plugin/pkg/client/auth/gcp", "plugin/pkg/client/auth/oidc", @@ -877,10 +1105,12 @@ "tools/pager", "tools/reference", "tools/remotecommand", + "tools/watch", "transport", "transport/spdy", "util/buffer", "util/cert", + "util/connrotation", "util/exec", "util/flowcontrol", "util/homedir", @@ -889,10 +1119,10 @@ "util/retry", "util/workqueue" ] - revision = "a312bfe35c401f70e5ea0add48b50da283031dc3" + revision = "77e032213d34c856222b4d4647c1c175ba8d22b9" [[projects]] - branch = "release-1.10" + branch = "release-1.12" name = "k8s.io/code-generator" packages = [ "cmd/client-gen", @@ -913,7 +1143,7 @@ "cmd/lister-gen/generators", "pkg/util" ] - revision = "9de8e796a74d16d2a285165727d04c185ebca6dc" + revision = "b1289fc74931d4b6b04bd1a259acfc88a2cb0a66" [[projects]] branch = "master" @@ -927,17 +1157,32 @@ "parser", "types" ] - revision = "c42f3cdacc394f43077ff17e327d1b351c0304e4" + revision = "e17681d19d3ac4837a019ece36c2a0ec31ffe985" + +[[projects]] + name = "k8s.io/klog" + packages = ["."] + revision = "e531227889390a39d9533dde61f590fe9f4b0035" + version = "v0.3.0" [[projects]] branch = "master" name = "k8s.io/kube-openapi" - packages = ["pkg/common"] - revision = "e3762e86a74c878ffed47484592986685639c2cd" + packages = [ + "pkg/common", + "pkg/util/proto" + ] + revision = "6b3d3b2d5666c5912bab8b7bf26bf50f75a8f887" + +[[projects]] + branch = "master" + name = "k8s.io/utils" + packages = ["pointer"] + revision = "21c4ce38f2a793ec01e925ddc31216500183b773" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "aa791f727ca547798fb61f90a8c3800c55674ecf2daf0d1818e82a7ed0db9d71" + inputs-digest = "f32bcd98041871575601108af8703d15a31ac8b6c27338818fd2cb0033d9b01c" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 2a5f36b93050..be42e0bd7974 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -7,15 +7,15 @@ required = [ [[constraint]] name = "k8s.io/api" - branch = "release-1.10" + branch = "release-1.12" [[constraint]] name = "k8s.io/apimachinery" - branch = "release-1.10" + branch = "release-1.12" [[constraint]] name = "k8s.io/code-generator" - branch = "release-1.10" + branch = "release-1.12" [[constraint]] name = "k8s.io/kube-openapi" @@ -23,11 +23,7 @@ required = [ [[constraint]] name = "k8s.io/client-go" - branch = "release-7.0" - -[[override]] - name = "k8s.io/kubernetes" - version = "~1.10.0" + branch = "release-9.0" [[constraint]] name = "github.com/stretchr/testify" @@ -35,7 +31,7 @@ required = [ [[constraint]] name = "github.com/spf13/cobra" - branch = "master" + revision = "fe5e611709b0c57fa4a89136deaa8e1d4004d053" [[constraint]] name = "gopkg.in/src-d/go-git.v4" @@ -52,3 +48,25 @@ required = [ [[constraint]] name = "github.com/ghodss/yaml" branch = "master" + +# vendor/k8s.io/client-go/plugin/pkg/client/auth/azure/azure.go:300:25: +# cannot call non-function spt.Token (type adal.Token) +[[override]] + name = "github.com/Azure/go-autorest" + revision = "1ff28809256a84bb6966640ff3d0371af82ccba4" + +[[constraint]] + name = "github.com/colinmarc/hdfs" + revision = "48eb8d6c34a97ffc73b406356f0f2e1c569b42a5" + +[[constraint]] + name = "gopkg.in/jcmturner/gokrb5.v5" + version = "5.3.0" + +[[constraint]] + name = "cloud.google.com/go" + version = "0.26.0" + +[[constraint]] + name = "google.golang.org/api" + version = "0.3.2" diff --git a/Jenkinsfile b/Jenkinsfile index 38a1de3db6ad..543ce07ae7a5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,10 +9,7 @@ def NAMESPACE = '' def runUtilityCommand(buildCommand) { // Run an arbitrary command inside the docker builder image - sh "docker run --rm " + - "-v ${pwd()}/dist/pkg:/root/go/pkg " + - "-v ${pwd()}:/root/go/src/github.com/cyrusbiotechnology/argo " + - "-w /root/go/src/github.com/cyrusbiotechnology/argo argo-builder ${buildCommand}" + sh "docker run -v ${pwd()}/dist:/go/src/github.com/cyrusbiotechnology/argo/dist --rm builder-base:latest ${buildCommand}" } pipeline { @@ -41,30 +38,33 @@ pipeline { stage('build utility container') { steps { - sh "docker build -t argo-builder -f Dockerfile-builder ." + sh "docker build -t builder-base --target builder-base ." } } + stage('run tests') { steps { runUtilityCommand("go test ./...") } } + stage('build controller') { steps { - runUtilityCommand("make controller") - sh "docker build -t workflow-controller:${VERSION} -f Dockerfile-workflow-controller ." + sh "docker build -t workflow-controller:${VERSION} --target workflow-controller ." } } stage('build executor') { steps { - runUtilityCommand("make executor") - sh "docker build -t argoexec:${VERSION} -f Dockerfile-argoexec ." + sh "docker build -t argoexec:${VERSION} --target argoexec ." } } + + + stage('build Linux and MacOS CLIs') { steps { runUtilityCommand("make cli CGO_ENABLED=0 LDFLAGS='-extldflags \"-static\"' ARGO_CLI_NAME=argo-linux-amd64") @@ -85,8 +85,8 @@ pipeline { stage('push CLI to artifactory') { steps { withCredentials([usernamePassword(credentialsId: 'Artifactory', usernameVariable: 'ARTI_NAME', passwordVariable: 'ARTI_PASS')]) { - runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/cyrusbiotechnology/argo/dist/argo-darwin-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-mac-${VERSION}") - runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /root/go/src/github.com/cyrusbiotechnology/argo/dist/argo-linux-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-linux-${VERSION}") + runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /go/src/github.com/cyrusbiotechnology/argo/dist/argo-darwin-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-mac-${VERSION}") + runUtilityCommand("curl -u ${ARTI_NAME}:${ARTI_PASS} -T /go/src/github.com/cyrusbiotechnology/argo/dist/argo-linux-amd64 https://cyrusbio.jfrog.io/cyrusbio/argo-cli/argo-linux-${VERSION}") } } } diff --git a/Makefile b/Makefile index 161c3daf826b..db568e218a91 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,13 @@ GIT_COMMIT=$(shell git rev-parse HEAD) GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi) GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi) -BUILDER_IMAGE=argo-builder -# NOTE: the volume mount of ${DIST_DIR}/pkg below is optional and serves only -# to speed up subsequent builds by caching ${GOPATH}/pkg between builds. -BUILDER_CMD=docker run --rm \ - -v ${CURRENT_DIR}:/root/go/src/${PACKAGE} \ - -v ${DIST_DIR}/pkg:/root/go/pkg \ - -w /root/go/src/${PACKAGE} ${BUILDER_IMAGE} +# docker image publishing options +DOCKER_PUSH=false +IMAGE_TAG=latest +# perform static compilation +STATIC_BUILD=true +# build development images +DEV_IMAGE=false override LDFLAGS += \ -X ${PACKAGE}.version=${VERSION} \ @@ -23,22 +23,16 @@ override LDFLAGS += \ -X ${PACKAGE}.gitCommit=${GIT_COMMIT} \ -X ${PACKAGE}.gitTreeState=${GIT_TREE_STATE} -# docker image publishing options -DOCKER_PUSH=false -IMAGE_TAG=latest +ifeq (${STATIC_BUILD}, true) +override LDFLAGS += -extldflags "-static" +endif ifneq (${GIT_TAG},) IMAGE_TAG=${GIT_TAG} override LDFLAGS += -X ${PACKAGE}.gitTag=${GIT_TAG} endif -ifneq (${IMAGE_NAMESPACE},) -override LDFLAGS += -X ${PACKAGE}/cmd/argo/commands.imageNamespace=${IMAGE_NAMESPACE} -endif -ifneq (${IMAGE_TAG},) -override LDFLAGS += -X ${PACKAGE}/cmd/argo/commands.imageTag=${IMAGE_TAG} -endif -ifeq (${DOCKER_PUSH},true) +ifeq (${DOCKER_PUSH}, true) ifndef IMAGE_NAMESPACE $(error IMAGE_NAMESPACE must be set to push images (e.g. IMAGE_NAMESPACE=argoproj)) endif @@ -50,72 +44,85 @@ endif # Build the project .PHONY: all -all: cli cli-image controller-image executor-image +all: cli controller-image executor-image -.PHONY: builder -builder: - docker build -t ${BUILDER_IMAGE} -f Dockerfile-builder . +.PHONY: builder-image +builder-image: + docker build -t $(IMAGE_PREFIX)argo-ci-builder:$(IMAGE_TAG) --target builder . .PHONY: cli cli: - CGO_ENABLED=0 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${ARGO_CLI_NAME} ./cmd/argo + go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${ARGO_CLI_NAME} ./cmd/argo + +.PHONY: cli-linux-amd64 +cli-linux-amd64: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/argo-linux-amd64 ./cmd/argo + +.PHONY: cli-linux-ppc64le +cli-linux-ppc64le: + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/argo-linux-ppc64le ./cmd/argo + +.PHONY: cli-linux-s390x +cli-linux-s390x: + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/argo-linux-s390x ./cmd/argo .PHONY: cli-linux -cli-linux: builder - ${BUILDER_CMD} make cli \ - CGO_ENABLED=0 \ - IMAGE_TAG=$(IMAGE_TAG) \ - IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) \ - LDFLAGS='-extldflags "-static"' \ - ARGO_CLI_NAME=argo-linux-amd64 +cli-linux: cli-linux-amd64 cli-linux-ppc64le cli-linux-s390x .PHONY: cli-darwin -cli-darwin: builder - ${BUILDER_CMD} make cli \ - GOOS=darwin \ - IMAGE_TAG=$(IMAGE_TAG) \ - IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) \ - ARGO_CLI_NAME=argo-darwin-amd64 +cli-darwin: + CGO_ENABLED=0 GOOS=darwin go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/argo-darwin-amd64 ./cmd/argo .PHONY: cli-windows -cli-windows: builder - ${BUILDER_CMD} make cli \ - GOARCH=amd64 \ - GOOS=windows \ - IMAGE_TAG=$(IMAGE_TAG) \ - IMAGE_NAMESPACE=$(IMAGE_NAMESPACE) \ - LDFLAGS='-extldflags "-static"' \ - ARGO_CLI_NAME=argo-windows-amd64 - -.PHONY: controller -controller: - go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/workflow-controller ./cmd/workflow-controller +cli-windows: + CGO_ENABLED=0 GOARCH=amd64 GOOS=windows go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/argo-windows-amd64 ./cmd/argo .PHONY: cli-image -cli-image: cli-linux - docker build -t $(IMAGE_PREFIX)argocli:$(IMAGE_TAG) -f Dockerfile-cli . +cli-image: + docker build -t $(IMAGE_PREFIX)argocli:$(IMAGE_TAG) --target argocli . @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)argocli:$(IMAGE_TAG) ; fi -.PHONY: controller-linux -controller-linux: builder - ${BUILDER_CMD} make controller +.PHONY: controller +controller: + CGO_ENABLED=0 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/workflow-controller ./cmd/workflow-controller .PHONY: controller-image -controller-image: controller-linux - docker build -t $(IMAGE_PREFIX)workflow-controller:$(IMAGE_TAG) -f Dockerfile-workflow-controller . +controller-image: +ifeq ($(DEV_IMAGE), true) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o workflow-controller ./cmd/workflow-controller + docker build -t $(IMAGE_PREFIX)workflow-controller:$(IMAGE_TAG) -f Dockerfile.workflow-controller-dev . + rm -f workflow-controller +else + docker build -t $(IMAGE_PREFIX)workflow-controller:$(IMAGE_TAG) --target workflow-controller . +endif @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)workflow-controller:$(IMAGE_TAG) ; fi .PHONY: executor executor: go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/argoexec ./cmd/argoexec -.PHONY: executor-linux -executor-linux: builder - ${BUILDER_CMD} make executor - +.PHONY: executor-base-image +executor-base-image: + docker build -t argoexec-base --target argoexec-base . + +# The DEV_IMAGE versions of controller-image and executor-image are speed optimized development +# builds of workflow-controller and argoexec images respectively. It allows for faster image builds +# by re-using the golang build cache of the desktop environment. Ideally, we would not need extra +# Dockerfiles for these, and the targets would be defined as new targets in the main Dockerfile, but +# intelligent skipping of docker build stages requires DOCKER_BUILDKIT=1 enabled, which not all +# docker daemons support (including the daemon currently used by minikube). +# TODO: move these targets to the main Dockerfile once DOCKER_BUILDKIT=1 is more pervasive. +# NOTE: have to output ouside of dist directory since dist is under .dockerignore .PHONY: executor-image -executor-image: executor-linux - docker build -t $(IMAGE_PREFIX)argoexec:$(IMAGE_TAG) -f Dockerfile-argoexec . +ifeq ($(DEV_IMAGE), true) +executor-image: executor-base-image + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o argoexec ./cmd/argoexec + docker build -t $(IMAGE_PREFIX)argoexec:$(IMAGE_TAG) -f Dockerfile.argoexec-dev . + rm -f argoexec +else +executor-image: + docker build -t $(IMAGE_PREFIX)argoexec:$(IMAGE_TAG) --target argoexec . +endif @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)argoexec:$(IMAGE_TAG) ; fi .PHONY: lint @@ -126,8 +133,8 @@ lint: test: go test ./... -.PHONY: update-codegen -update-codegen: +.PHONY: codegen +codegen: ./hack/update-codegen.sh ./hack/update-openapigen.sh go run ./hack/gen-openapi-spec/main.go ${VERSION} > ${CURRENT_DIR}/api/openapi-spec/swagger.json @@ -140,8 +147,8 @@ verify-codegen: go run ./hack/gen-openapi-spec/main.go ${VERSION} > ${CURRENT_DIR}/dist/swagger.json diff ${CURRENT_DIR}/dist/swagger.json ${CURRENT_DIR}/api/openapi-spec/swagger.json -.PHONY: update-manifests -update-manifests: +.PHONY: manifests +manifests: ./hack/update-manifests.sh .PHONY: clean @@ -152,10 +159,22 @@ clean: precheckin: test lint verify-codegen .PHONY: release-precheck -release-precheck: precheckin +release-precheck: manifests codegen precheckin @if [ "$(GIT_TREE_STATE)" != "clean" ]; then echo 'git tree state is $(GIT_TREE_STATE)' ; exit 1; fi @if [ -z "$(GIT_TAG)" ]; then echo 'commit must be tagged to perform release' ; exit 1; fi @if [ "$(GIT_TAG)" != "v$(VERSION)" ]; then echo 'git tag ($(GIT_TAG)) does not match VERSION (v$(VERSION))'; exit 1; fi +.PHONY: release-clis +release-clis: cli-image + docker build --iidfile /tmp/argo-cli-build --target argo-build --build-arg MAKE_TARGET="cli-darwin cli-windows" . + docker create --name tmp-cli `cat /tmp/argo-cli-build` + mkdir -p ${DIST_DIR} + docker cp tmp-cli:/go/src/github.com/cyrusbiotechnology/argo/dist/argo-darwin-amd64 ${DIST_DIR}/argo-darwin-amd64 + docker cp tmp-cli:/go/src/github.com/cyrusbiotechnology/argo/dist/argo-windows-amd64 ${DIST_DIR}/argo-windows-amd64 + docker rm tmp-cli + docker create --name tmp-cli $(IMAGE_PREFIX)argocli:$(IMAGE_TAG) + docker cp tmp-cli:/bin/argo ${DIST_DIR}/argo-linux-amd64 + docker rm tmp-cli + .PHONY: release -release: release-precheck controller-image cli-darwin cli-linux cli-windows executor-image cli-image +release: release-precheck controller-image executor-image cli-image release-clis diff --git a/OWNERS b/OWNERS index 244615e3a069..585b9d1aa85c 100644 --- a/OWNERS +++ b/OWNERS @@ -5,3 +5,6 @@ approvers: - alexmt - edlee2121 - jessesuen + +reviewers: +- dtaniwaki diff --git a/README.md b/README.md index 5281d9787254..1f3e8197e9aa 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,48 @@ -# Argo - The Workflow Engine for Kubernetes +[![slack](https://img.shields.io/badge/slack-argoproj-brightgreen.svg?logo=slack)](https://argoproj.github.io/community/join-slack) + +# Argoproj - Get stuff done with Kubernetes ![Argo Image](argo.png) ## News -We are excited to welcome [Adobe](https://www.adobe.com/) and [BlackRock](https://www.blackrock.com/) as the latest corporate members of the Argo Community! We are also thrilled that BlackRock has developed an eventing framework for Argo and has decided to contribute it to the Argo Community. Please check out the new repo and try [Argo Events](https://github.com/argoproj/argo-events)! +KubeCon 2018 in Seattle was the biggest KubeCon yet with 8000 developers attending. We connected with many existing and new Argoproj users and contributions, and gave away a lot of Argo T-shirts at our booth sponsored by Intuit! + +We were also super excited to see KubeCon presentations about Argo by Argo developers, users and partners. +* [CI/CD in Light Speed with K8s and Argo CD](https://www.youtube.com/watch?v=OdzH82VpMwI&feature=youtu.be) + * How Intuit uses Argo CD. +* [Automating Research Workflows at BlackRock](https://www.youtube.com/watch?v=ZK510prml8o&t=0s&index=169&list=PLj6h78yzYM2PZf9eA7bhWnIh_mK1vyOfU) + * Why BlackRock created Argo Events and how they use it. +* [Machine Learning as Code](https://www.youtube.com/watch?v=VXrGp5er1ZE&t=0s&index=135&list=PLj6h78yzYM2PZf9eA7bhWnIh_mK1vyOfU) + * How Kubeflow uses Argo Workflows as its core workflow engine and Argo CD to declaratively deploy ML pipelines and models. + +If you actively use Argo in your organization and your organization would be interested in participating in the Argo Community, please ask a representative to contact saradhi_sreegiriraju@intuit.com for additional information. + +## What is Argoproj? -If you actively use Argo in your organization and believe that your organization may be interested in actively participating in the Argo Community, please ask a representative to contact saradhi_sreegiriraju@intuit.com for additional information. +Argoproj is a collection of tools for getting work done with Kubernetes. +* [Argo Workflows](https://github.com/cyrusbiotechnology/argo) - Container-native Workflow Engine +* [Argo CD](https://github.com/cyrusbiotechnology/argo-cd) - Declarative GitOps Continuous Delivery +* [Argo Events](https://github.com/cyrusbiotechnology/argo-events) - Event-based Dependency Manager -## What is Argo? -Argo is an open source container-native workflow engine for getting work done on Kubernetes. Argo is implemented as a Kubernetes CRD (Custom Resource Definition). +## What is Argo Workflows? +Argo Workflows is an open source container-native workflow engine for orchestrating parallel jobs on Kubernetes. Argo Workflows is implemented as a Kubernetes CRD (Custom Resource Definition). * Define workflows where each step in the workflow is a container. * Model multi-step workflows as a sequence of tasks or capture the dependencies between tasks using a graph (DAG). -* Easily run compute intensive jobs for machine learning or data processing in a fraction of the time using Argo workflows on Kubernetes. +* Easily run compute intensive jobs for machine learning or data processing in a fraction of the time using Argo Workflows on Kubernetes. * Run CI/CD pipelines natively on Kubernetes without configuring complex software development products. -## Why Argo? -* Argo is designed from the ground up for containers without the overhead and limitations of legacy VM and server-based environments. -* Argo is cloud agnostic and can run on any kubernetes cluster. -* Argo with Kubernetes puts a cloud-scale supercomputer at your fingertips. +## Why Argo Workflows? +* Designed from the ground up for containers without the overhead and limitations of legacy VM and server-based environments. +* Cloud agnostic and can run on any Kubernetes cluster. +* Easily orchestrate highly parallel jobs on Kubernetes. +* Argo Workflows puts a cloud-scale supercomputer at your fingertips! ## Documentation -* [Get started here](https://github.com/cyrusbiotechnology/argo/blob/master/demo.md) -* [How to write Argo workflow specs](https://github.com/cyrusbiotechnology/argo/blob/master/examples/README.md) -* [How to configure your artifact repository](https://github.com/cyrusbiotechnology/argo/blob/master/ARTIFACT_REPO.md) +* [Get started here](demo.md) +* [How to write Argo Workflow specs](examples/README.md) +* [How to configure your artifact repository](ARTIFACT_REPO.md) ## Features * DAG or Steps based declaration of workflows @@ -53,21 +71,34 @@ As the Argo Community grows, we'd like to keep track of our users. Please send a Currently **officially** using Argo: +1. [Admiralty](https://admiralty.io/) 1. [Adobe](https://www.adobe.com/) +1. [Alibaba Cloud](https://www.alibabacloud.com/about) 1. [BlackRock](https://www.blackrock.com/) +1. [Canva](https://www.canva.com/) 1. [CoreFiling](https://www.corefiling.com/) +1. [Cratejoy](https://www.cratejoy.com/) 1. [Cyrus Biotechnology](https://cyrusbio.com/) 1. [Datadog](https://www.datadoghq.com/) +1. [Equinor](https://www.equinor.com/) +1. [Gardener](https://gardener.cloud/) 1. [Gladly](https://gladly.com/) +1. [GitHub](https://github.com/) 1. [Google](https://www.google.com/intl/en/about/our-company/) 1. [Interline Technologies](https://www.interline.io/blog/scaling-openstreetmap-data-workflows/) 1. [Intuit](https://www.intuit.com/) +1. [Karius](https://www.kariusdx.com/) +1. [KintoHub](https://www.kintohub.com/) 1. [Localytics](https://www.localytics.com/) 1. [NVIDIA](https://www.nvidia.com/) -1. [KintoHub](https://www.kintohub.com/) +1. [Preferred Networks](https://www.preferred-networks.jp/en/) +1. [Quantibio](http://quantibio.com/us/en/) +1. [SAP Hybris](https://cx.sap.com/) 1. [Styra](https://www.styra.com/) ## Community Blogs and Presentations +* [Running Argo Workflows Across Multiple Kubernetes Clusters](https://admiralty.io/blog/running-argo-workflows-across-multiple-kubernetes-clusters/) +* [Open Source Model Management Roundup: Polyaxon, Argo, and Seldon](https://www.anaconda.com/blog/developer-blog/open-source-model-management-roundup-polyaxon-argo-and-seldon/) * [Producing 200 OpenStreetMap extracts in 35 minutes using a scalable data workflow](https://www.interline.io/blog/scaling-openstreetmap-data-workflows/) * [Argo integration review](http://dev.matt.hillsdon.net/2018/03/24/argo-integration-review.html) * TGI Kubernetes with Joe Beda: [Argo workflow system](https://www.youtube.com/watch?v=M_rxPPLG8pU&start=859) @@ -75,6 +106,5 @@ Currently **officially** using Argo: ## Project Resources * Argo GitHub: https://github.com/argoproj -* Argo Slack: [click here to join](https://join.slack.com/t/argoproj/shared_invite/enQtMzExODU3MzIyNjYzLTA5MTFjNjI0Nzg3NzNiMDZiNmRiODM4Y2M1NWQxOGYzMzZkNTc1YWVkYTZkNzdlNmYyZjMxNWI3NjY2MDc1MzI) * Argo website: https://argoproj.github.io/ -* Argo forum: https://groups.google.com/forum/#!forum/argoproj +* Argo Slack: [click here to join](https://join.slack.com/t/argoproj/shared_invite/enQtMzExODU3MzIyNjYzLTA5MTFjNjI0Nzg3NzNiMDZiNmRiODM4Y2M1NWQxOGYzMzZkNTc1YWVkYTZkNzdlNmYyZjMxNWI3NjY2MDc1MzI) diff --git a/ROADMAP.md b/ROADMAP.md index fbc7e0ddf529..cf31798b54a4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,14 +1,13 @@ # Roadmap -## v2.2 - -### Proposed Items - -The following are candidate items for v2.2 release - -* Workflow composability - support for Jsonnet in CLI -* Queuing / Admission control - ability to limit number of concurrent workflows -* Scheduling - investigate k8s PriorityClasses and re-use in workflows -* Persistence - workflow history/state -* `argo run` to run workflows against clusters without a controller - #794 -* UI – filtering to improve performance +## v2.4 +* Persistence - support offloading of workflow state into database layer +* Large workflow support (enabled by persistence feature) +* Backlog and bug fixes + +## Proposed Items +* Argo API server +* Best effort workflow steps +* Template level finalizers +* Artifact loop aggregation +* Pod reclamation controls diff --git a/VERSION b/VERSION index 437459cd94c9..914ec967116c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5.0 +2.6.0 \ No newline at end of file diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index de7c63472616..258274021262 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "info": { "title": "Argo", - "version": "v2.4.0" + "version": "v2.6.0" }, "paths": {}, "definitions": { @@ -70,6 +70,10 @@ "description": "GlobalName exports an output artifact to the global scope, making it available as '{{workflow.outputs.artifacts.XXXX}} and in workflow.status.outputs.artifacts", "type": "string" }, + "hdfs": { + "description": "HDFS contains HDFS artifact location details", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HDFSArtifact" + }, "http": { "description": "HTTP contains HTTP artifact location details", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HTTPArtifact" @@ -83,6 +87,10 @@ "description": "name of the artifact. must be unique within a template's inputs/outputs.", "type": "string" }, + "optional": { + "description": "Make Artifacts optional, if Artifacts doesn't generate or exist", + "type": "boolean" + }, "path": { "description": "Path is the container path to the artifact", "type": "string" @@ -116,6 +124,10 @@ "description": "Git contains git artifact location details", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GitArtifact" }, + "hdfs": { + "description": "HDFS contains HDFS artifact location details", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HDFSArtifact" + }, "http": { "description": "HTTP contains HTTP artifact location details", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HTTPArtifact" @@ -163,6 +175,17 @@ } } }, + "io.argoproj.workflow.v1alpha1.ContinueOn": { + "description": "ContinueOn defines if a workflow should continue even if a task or step fails/errors. It can be specified if the workflow should continue when the pod errors, fails or both.", + "properties": { + "error": { + "type": "boolean" + }, + "failed": { + "type": "boolean" + } + } + }, "io.argoproj.workflow.v1alpha1.DAGTask": { "description": "DAGTask represents a node in the graph during DAG execution", "required": [ @@ -174,6 +197,10 @@ "description": "Arguments are the parameter and artifact arguments to the template", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Arguments" }, + "continueOn": { + "description": "ContinueOn makes argo to proceed with the following step even if this step fails. Errors and Failed states can be specified", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ContinueOn" + }, "dependencies": { "description": "Dependencies are name of other targets which this depends on", "type": "array", @@ -279,12 +306,16 @@ "description": "GCSArtifact is the location of a GCS artifact", "required": [ "bucket", + "credentialsSecret", "key" ], "properties": { "bucket": { "type": "string" }, + "credentialsSecret": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, "key": { "type": "string" } @@ -293,11 +324,15 @@ "io.argoproj.workflow.v1alpha1.GCSBucket": { "description": "GCSBucket contains the access information required for acting with a GCS bucket", "required": [ - "bucket" + "bucket", + "credentialsSecret" ], "properties": { "bucket": { "type": "string" + }, + "credentialsSecret": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" } } }, @@ -307,6 +342,10 @@ "repo" ], "properties": { + "insecureIgnoreHostKey": { + "description": "InsecureIgnoreHostKey disables SSH strict host key checking during git clone", + "type": "boolean" + }, "passwordSecret": { "description": "PasswordSecret is the secret selector to the repository password", "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" @@ -329,6 +368,130 @@ } } }, + "io.argoproj.workflow.v1alpha1.HDFSArtifact": { + "description": "HDFSArtifact is the location of an HDFS artifact", + "required": [ + "addresses", + "path" + ], + "properties": { + "addresses": { + "description": "Addresses is accessible addresses of HDFS name nodes", + "type": "array", + "items": { + "type": "string" + } + }, + "force": { + "description": "Force copies a file forcibly even if it exists (default: false)", + "type": "boolean" + }, + "hdfsUser": { + "description": "HDFSUser is the user to access HDFS file system. It is ignored if either ccache or keytab is used.", + "type": "string" + }, + "krbCCacheSecret": { + "description": "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, + "krbConfigConfigMap": { + "description": "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector" + }, + "krbKeytabSecret": { + "description": "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, + "krbRealm": { + "description": "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.", + "type": "string" + }, + "krbServicePrincipalName": { + "description": "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.", + "type": "string" + }, + "krbUsername": { + "description": "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.", + "type": "string" + }, + "path": { + "description": "Path is a file path in HDFS", + "type": "string" + } + } + }, + "io.argoproj.workflow.v1alpha1.HDFSConfig": { + "description": "HDFSConfig is configurations for HDFS", + "required": [ + "addresses" + ], + "properties": { + "addresses": { + "description": "Addresses is accessible addresses of HDFS name nodes", + "type": "array", + "items": { + "type": "string" + } + }, + "hdfsUser": { + "description": "HDFSUser is the user to access HDFS file system. It is ignored if either ccache or keytab is used.", + "type": "string" + }, + "krbCCacheSecret": { + "description": "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, + "krbConfigConfigMap": { + "description": "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector" + }, + "krbKeytabSecret": { + "description": "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, + "krbRealm": { + "description": "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.", + "type": "string" + }, + "krbServicePrincipalName": { + "description": "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.", + "type": "string" + }, + "krbUsername": { + "description": "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.", + "type": "string" + } + } + }, + "io.argoproj.workflow.v1alpha1.HDFSKrbConfig": { + "description": "HDFSKrbConfig is auth configurations for Kerberos", + "properties": { + "krbCCacheSecret": { + "description": "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, + "krbConfigConfigMap": { + "description": "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.", + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector" + }, + "krbKeytabSecret": { + "description": "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.", + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector" + }, + "krbRealm": { + "description": "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.", + "type": "string" + }, + "krbServicePrincipalName": { + "description": "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.", + "type": "string" + }, + "krbUsername": { + "description": "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.", + "type": "string" + } + } + }, "io.argoproj.workflow.v1alpha1.HTTPArtifact": { "description": "HTTPArtifact allows an file served on HTTP to be placed as an input artifact in a container", "required": [ @@ -467,6 +630,10 @@ "description": "Manifest contains the kubernetes manifest", "type": "string" }, + "mergeStrategy": { + "description": "MergeStrategy is the strategy used to merge a patch. It defaults to \"strategic\" Must be one of: strategic, merge, json", + "type": "string" + }, "successCondition": { "description": "SuccessCondition is a label selector expression which describes the conditions of the k8s resource in which it is acceptable to proceed to the following step", "type": "string" @@ -705,8 +872,154 @@ } } }, - "io.argoproj.workflow.v1alpha1.Sidecar": { - "description": "Sidecar is a container which runs alongside the main container", + "io.argoproj.workflow.v1alpha1.SuspendTemplate": { + "description": "SuspendTemplate is a template subtype to suspend a workflow at a predetermined point in time" + }, + "io.argoproj.workflow.v1alpha1.TarStrategy": { + "description": "TarStrategy will tar and gzip the file or directory when saving" + }, + "io.argoproj.workflow.v1alpha1.Template": { + "description": "Template is a reusable and composable unit of execution in a workflow", + "required": [ + "name" + ], + "properties": { + "activeDeadlineSeconds": { + "description": "Optional duration in seconds relative to the StartTime that the pod may be active on a node before the system actively tries to terminate the pod; value must be positive integer This field is only applicable to container and script templates.", + "type": "integer", + "format": "int64" + }, + "affinity": { + "description": "Affinity sets the pod's scheduling constraints Overrides the affinity set at the workflow level (if any)", + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "archiveLocation": { + "description": "Location in which all files related to the step will be stored (logs, artifacts, etc...). Can be overridden by individual items in Outputs. If omitted, will use the default artifact repository location configured in the controller, appended with the \u003cworkflowname\u003e/\u003cnodename\u003e in the key.", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactLocation" + }, + "container": { + "description": "Container is the main container image to run in the pod", + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + }, + "daemon": { + "description": "Deamon will allow a workflow to proceed to the next step so long as the container reaches readiness", + "type": "boolean" + }, + "dag": { + "description": "DAG template subtype which runs a DAG", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.DAGTemplate" + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition" + } + }, + "initContainers": { + "description": "InitContainers is a list of containers which run before the main container.", + "type": "array", + "items": { + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.UserContainer" + } + }, + "inputs": { + "description": "Inputs describe what inputs parameters and artifacts are supplied to this template", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Inputs" + }, + "metadata": { + "description": "Metdata sets the pods's metadata, i.e. annotations and labels", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Metadata" + }, + "name": { + "description": "Name is the name of the template", + "type": "string" + }, + "nodeSelector": { + "description": "NodeSelector is a selector to schedule this step of the workflow to be run on the selected node(s). Overrides the selector set at the workflow level.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "outputs": { + "description": "Outputs describe the parameters and artifacts that this template produces", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Outputs" + }, + "parallelism": { + "description": "Parallelism limits the max total parallel pods that can execute at the same time within the boundaries of this template invocation. If additional steps/dag templates are invoked, the pods created by those templates will not be counted towards this total.", + "type": "integer", + "format": "int64" + }, + "priority": { + "description": "Priority to apply to workflow pods.", + "type": "integer", + "format": "int32" + }, + "priorityClassName": { + "description": "PriorityClassName to apply to workflow pods.", + "type": "string" + }, + "resource": { + "description": "Resource template subtype which can run k8s resources", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ResourceTemplate" + }, + "retryStrategy": { + "description": "RetryStrategy describes how to retry a template when it fails", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.RetryStrategy" + }, + "schedulerName": { + "description": "If specified, the pod will be dispatched by specified scheduler. Or it will be dispatched by workflow scope scheduler if specified. If neither specified, the pod will be dispatched by default scheduler.", + "type": "string" + }, + "script": { + "description": "Script runs a portion of code against an interpreter", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ScriptTemplate" + }, + "sidecars": { + "description": "Sidecars is a list of containers which run alongside the main container Sidecars are automatically killed when the main container completes", + "type": "array", + "items": { + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.UserContainer" + } + }, + "steps": { + "description": "Steps define a series of sequential/parallel workflow steps", + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.WorkflowStep" + } + } + }, + "suspend": { + "description": "Suspend template subtype which can suspend a workflow when reaching the step", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.SuspendTemplate" + }, + "tolerations": { + "description": "Tolerations to apply to workflow pods.", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "volumes": { + "description": "Volumes is a list of volumes that can be mounted by containers in a template.", + "type": "array", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "warnings": { + "type": "array", + "items": { + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition" + } + } + } + }, + "io.argoproj.workflow.v1alpha1.UserContainer": { + "description": "UserContainer is a container specified by a user.", "required": [ "name" ], @@ -758,7 +1071,7 @@ "$ref": "#/definitions/io.k8s.api.core.v1.Probe" }, "mirrorVolumeMounts": { - "description": "MirrorVolumeMounts will mount the same volumes specified in the main container to the sidecar (including artifacts), at the same mountPaths. This enables dind daemon to partially see the same filesystem as the main container in order to use features such as docker volume binding", + "description": "MirrorVolumeMounts will mount the same volumes specified in the main container to the container (including artifacts), at the same mountPaths. This enables dind daemon to partially see the same filesystem as the main container in order to use features such as docker volume binding", "type": "boolean" }, "name": { @@ -830,125 +1143,6 @@ } } }, - "io.argoproj.workflow.v1alpha1.SuspendTemplate": { - "description": "SuspendTemplate is a template subtype to suspend a workflow at a predetermined point in time" - }, - "io.argoproj.workflow.v1alpha1.TarStrategy": { - "description": "TarStrategy will tar and gzip the file or directory when saving" - }, - "io.argoproj.workflow.v1alpha1.Template": { - "description": "Template is a reusable and composable unit of execution in a workflow", - "required": [ - "name" - ], - "properties": { - "activeDeadlineSeconds": { - "description": "Optional duration in seconds relative to the StartTime that the pod may be active on a node before the system actively tries to terminate the pod; value must be positive integer This field is only applicable to container and script templates.", - "type": "integer", - "format": "int64" - }, - "affinity": { - "description": "Affinity sets the pod's scheduling constraints Overrides the affinity set at the workflow level (if any)", - "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" - }, - "archiveLocation": { - "description": "Location in which all files related to the step will be stored (logs, artifacts, etc...). Can be overridden by individual items in Outputs. If omitted, will use the default artifact repository location configured in the controller, appended with the \u003cworkflowname\u003e/\u003cnodename\u003e in the key.", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactLocation" - }, - "container": { - "description": "Container is the main container image to run in the pod", - "$ref": "#/definitions/io.k8s.api.core.v1.Container" - }, - "daemon": { - "description": "Deamon will allow a workflow to proceed to the next step so long as the container reaches readiness", - "type": "boolean" - }, - "dag": { - "description": "DAG template subtype which runs a DAG", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.DAGTemplate" - }, - "errors": { - "type": "array", - "items": { - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition" - } - }, - "inputs": { - "description": "Inputs describe what inputs parameters and artifacts are supplied to this template", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Inputs" - }, - "metadata": { - "description": "Metdata sets the pods's metadata, i.e. annotations and labels", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Metadata" - }, - "name": { - "description": "Name is the name of the template", - "type": "string" - }, - "nodeSelector": { - "description": "NodeSelector is a selector to schedule this step of the workflow to be run on the selected node(s). Overrides the selector set at the workflow level.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "outputs": { - "description": "Outputs describe the parameters and artifacts that this template produces", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Outputs" - }, - "parallelism": { - "description": "Parallelism limits the max total parallel pods that can execute at the same time within the boundaries of this template invocation. If additional steps/dag templates are invoked, the pods created by those templates will not be counted towards this total.", - "type": "integer", - "format": "int64" - }, - "resource": { - "description": "Resource template subtype which can run k8s resources", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ResourceTemplate" - }, - "retryStrategy": { - "description": "RetryStrategy describes how to retry a template when it fails", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.RetryStrategy" - }, - "script": { - "description": "Script runs a portion of code against an interpreter", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ScriptTemplate" - }, - "sidecars": { - "description": "Sidecars is a list of containers which run alongside the main container Sidecars are automatically killed when the main container completes", - "type": "array", - "items": { - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Sidecar" - } - }, - "steps": { - "description": "Steps define a series of sequential/parallel workflow steps", - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.WorkflowStep" - } - } - }, - "suspend": { - "description": "Suspend template subtype which can suspend a workflow when reaching the step", - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.SuspendTemplate" - }, - "tolerations": { - "description": "Tolerations to apply to workflow pods.", - "type": "array", - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" - } - }, - "warnings": { - "type": "array", - "items": { - "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ExceptionCondition" - } - } - } - }, "io.argoproj.workflow.v1alpha1.ValueFrom": { "description": "ValueFrom describes a location in which to obtain the value to a parameter", "properties": { @@ -1043,10 +1237,22 @@ "description": "Arguments contain the parameters and artifacts sent to the workflow entrypoint Parameters are referencable globally using the 'workflow' variable prefix. e.g. {{workflow.parameters.myparam}}", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Arguments" }, + "dnsConfig": { + "description": "PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.", + "$ref": "#/definitions/io.k8s.api.core.v1.PodDNSConfig" + }, + "dnsPolicy": { + "description": "Set DNS policy for the pod. Defaults to \"ClusterFirst\". Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", + "type": "string" + }, "entrypoint": { "description": "Entrypoint is a template reference to the starting point of the workflow", "type": "string" }, + "hostNetwork": { + "description": "Host networking requested for this workflow pod. Default to false.", + "type": "boolean" + }, "imagePullSecrets": { "description": "ImagePullSecrets is a list of references to secrets in the same namespace to use for pulling any images in pods that reference this ServiceAccount. ImagePullSecrets are distinct from Secrets because Secrets can be mounted in the pod, but ImagePullSecrets are only accessed by the kubelet. More info: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod", "type": "array", @@ -1070,6 +1276,24 @@ "type": "integer", "format": "int64" }, + "podPriority": { + "description": "Priority to apply to workflow pods.", + "type": "integer", + "format": "int32" + }, + "podPriorityClassName": { + "description": "PriorityClassName to apply to workflow pods.", + "type": "string" + }, + "priority": { + "description": "Priority is used if controller is configured to process limited number of workflows in parallel. Workflows with higher priority are processed first.", + "type": "integer", + "format": "int32" + }, + "schedulerName": { + "description": "Set scheduler name for all pods. Will be overridden if container/script template's scheduler name is set. Default scheduler will be used if neither specified.", + "type": "string" + }, "serviceAccountName": { "description": "ServiceAccountName is the name of the ServiceAccount to run all pods of the workflow as.", "type": "string" @@ -1120,6 +1344,10 @@ "description": "Arguments hold arguments to the template", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.Arguments" }, + "continueOn": { + "description": "ContinueOn makes argo to proceed with the following step even if this step fails. Errors and Failed states can be specified", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ContinueOn" + }, "name": { "description": "Name of the step", "type": "string" diff --git a/cmd/argo/commands/completion.go b/cmd/argo/commands/completion.go index 53f5bf8fc9f7..d6cba691292a 100644 --- a/cmd/argo/commands/completion.go +++ b/cmd/argo/commands/completion.go @@ -9,6 +9,30 @@ import ( "github.com/spf13/cobra" ) +const ( + bashCompletionFunc = ` +__argo_get_workflow() { + local argo_out + if argo_out=$(argo list --output name 2>/dev/null); then + COMPREPLY+=( $( compgen -W "${argo_out[*]}" -- "$cur" ) ) + fi +} + +__argo_custom_func() { + case ${last_command} in + argo_delete | argo_get | argo_logs |\ + argo_resubmit | argo_resume | argo_retry | argo_suspend |\ + argo_terminate | argo_wait | argo_watch) + __argo_get_workflow + return + ;; + *) + ;; + esac +} + ` +) + func NewCompletionCommand() *cobra.Command { var command = &cobra.Command{ Use: "completion SHELL", @@ -30,6 +54,7 @@ variable. } shell := args[0] rootCommand := NewCommand() + rootCommand.BashCompletionFunction = bashCompletionFunc availableCompletions := map[string]func(io.Writer) error{ "bash": rootCommand.GenBashCompletion, "zsh": rootCommand.GenZshCompletion, diff --git a/cmd/argo/commands/get.go b/cmd/argo/commands/get.go index 2ae34ac72b9d..358273edba9b 100644 --- a/cmd/argo/commands/get.go +++ b/cmd/argo/commands/get.go @@ -9,11 +9,11 @@ import ( "text/tabwriter" "github.com/argoproj/pkg/humanize" + wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/workflow/util" "github.com/ghodss/yaml" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" ) const onExitSuffix = "onExit" @@ -36,6 +36,10 @@ func NewGetCommand() *cobra.Command { if err != nil { log.Fatal(err) } + err = util.DecompressWorkflow(wf) + if err != nil { + log.Fatal(err) + } printWorkflow(wf, output) }, } @@ -71,7 +75,7 @@ func printWorkflowHelper(wf *wfv1.Workflow, outFmt string) { serviceAccount = "default" } fmt.Printf(fmtStr, "ServiceAccount:", serviceAccount) - fmt.Printf(fmtStr, "Status:", worklowStatus(wf)) + fmt.Printf(fmtStr, "Status:", workflowStatus(wf)) if wf.Status.Message != "" { fmt.Printf(fmtStr, "Message:", wf.Status.Message) } diff --git a/cmd/argo/commands/list.go b/cmd/argo/commands/list.go index adc9829c9bf9..c1a17eb0ec24 100644 --- a/cmd/argo/commands/list.go +++ b/cmd/argo/commands/list.go @@ -108,7 +108,7 @@ func printTable(wfList []wfv1.Workflow, listArgs *listFlags) { if listArgs.allNamespaces { fmt.Fprint(w, "NAMESPACE\t") } - fmt.Fprint(w, "NAME\tSTATUS\tAGE\tDURATION") + fmt.Fprint(w, "NAME\tSTATUS\tAGE\tDURATION\tPRIORITY") if listArgs.output == "wide" { fmt.Fprint(w, "\tP/R/C\tPARAMETERS") } @@ -119,7 +119,11 @@ func printTable(wfList []wfv1.Workflow, listArgs *listFlags) { if listArgs.allNamespaces { fmt.Fprintf(w, "%s\t", wf.ObjectMeta.Namespace) } - fmt.Fprintf(w, "%s\t%s\t%s\t%s", wf.ObjectMeta.Name, worklowStatus(&wf), ageStr, durationStr) + var priority int + if wf.Spec.Priority != nil { + priority = int(*wf.Spec.Priority) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d", wf.ObjectMeta.Name, workflowStatus(&wf), ageStr, durationStr, priority) if listArgs.output == "wide" { pending, running, completed := countPendingRunningCompleted(&wf) fmt.Fprintf(w, "\t%d/%d/%d", pending, running, completed) @@ -134,6 +138,10 @@ func countPendingRunningCompleted(wf *wfv1.Workflow) (int, int, int) { pending := 0 running := 0 completed := 0 + err := util.DecompressWorkflow(wf) + if err != nil { + log.Fatal(err) + } for _, node := range wf.Status.Nodes { tmpl := wf.GetTemplate(node.TemplateName) if tmpl == nil || !tmpl.IsPodType() { @@ -196,7 +204,7 @@ func (f ByFinishedAt) Less(i, j int) bool { } // workflowStatus returns a human readable inferred workflow status based on workflow phase and conditions -func worklowStatus(wf *wfv1.Workflow) wfv1.NodePhase { +func workflowStatus(wf *wfv1.Workflow) wfv1.NodePhase { switch wf.Status.Phase { case wfv1.NodeRunning: if util.IsWorkflowSuspended(wf) { diff --git a/cmd/argo/commands/logs.go b/cmd/argo/commands/logs.go index c8180b9542ea..b8d64ef0e725 100644 --- a/cmd/argo/commands/logs.go +++ b/cmd/argo/commands/logs.go @@ -2,6 +2,7 @@ package commands import ( "bufio" + "context" "fmt" "hash/fnv" "math" @@ -11,16 +12,21 @@ import ( "sync" "time" - "github.com/argoproj/pkg/errors" - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" - wfclientset "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned" - wfinformers "github.com/cyrusbiotechnology/argo/pkg/client/informers/externalversions" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + pkgwatch "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/watch" + + "github.com/argoproj/pkg/errors" + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + workflowv1 "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/workflow/util" ) type logEntry struct { @@ -101,8 +107,8 @@ func (p *logPrinter) PrintWorkflowLogs(workflow string) error { return err } timeByPod := p.printRecentWorkflowLogs(wf) - if p.follow && wf.Status.Phase == v1alpha1.NodeRunning { - p.printLiveWorkflowLogs(wf, timeByPod) + if p.follow { + p.printLiveWorkflowLogs(wf.Name, wfClient, timeByPod) } return nil } @@ -114,7 +120,7 @@ func (p *logPrinter) PrintPodLogs(podName string) error { return err } var logs []logEntry - err = p.getPodLogs("", podName, namespace, p.follow, p.tail, p.sinceSeconds, p.sinceTime, func(entry logEntry) { + err = p.getPodLogs(context.Background(), "", podName, namespace, p.follow, p.tail, p.sinceSeconds, p.sinceTime, func(entry logEntry) { logs = append(logs, entry) }) if err != nil { @@ -129,6 +135,11 @@ func (p *logPrinter) PrintPodLogs(podName string) error { // Prints logs for workflow pod steps and return most recent log timestamp per pod name func (p *logPrinter) printRecentWorkflowLogs(wf *v1alpha1.Workflow) map[string]*time.Time { var podNodes []v1alpha1.NodeStatus + err := util.DecompressWorkflow(wf) + if err != nil { + log.Warn(err) + return nil + } for _, node := range wf.Status.Nodes { if node.Type == v1alpha1.NodeTypePod && node.Phase != v1alpha1.NodeError { podNodes = append(podNodes, node) @@ -144,7 +155,7 @@ func (p *logPrinter) printRecentWorkflowLogs(wf *v1alpha1.Workflow) map[string]* go func() { defer wg.Done() var podLogs []logEntry - err := p.getPodLogs(getDisplayName(node), node.ID, wf.Namespace, false, p.tail, p.sinceSeconds, p.sinceTime, func(entry logEntry) { + err := p.getPodLogs(context.Background(), getDisplayName(node), node.ID, wf.Namespace, false, p.tail, p.sinceSeconds, p.sinceTime, func(entry logEntry) { podLogs = append(podLogs, entry) }) @@ -178,35 +189,19 @@ func (p *logPrinter) printRecentWorkflowLogs(wf *v1alpha1.Workflow) map[string]* return timeByPod } -func (p *logPrinter) setupWorkflowInformer(namespace string, name string, callback func(wf *v1alpha1.Workflow, done bool)) cache.SharedIndexInformer { - wfcClientset := wfclientset.NewForConfigOrDie(restConfig) - wfInformerFactory := wfinformers.NewFilteredSharedInformerFactory(wfcClientset, 20*time.Minute, namespace, nil) - informer := wfInformerFactory.Argoproj().V1alpha1().Workflows().Informer() - informer.AddEventHandler( - cache.ResourceEventHandlerFuncs{ - UpdateFunc: func(old, new interface{}) { - updatedWf := new.(*v1alpha1.Workflow) - if updatedWf.Name == name { - callback(updatedWf, updatedWf.Status.Phase != v1alpha1.NodeRunning) - } - }, - DeleteFunc: func(obj interface{}) { - deletedWf := obj.(*v1alpha1.Workflow) - if deletedWf.Name == name { - callback(deletedWf, true) - } - }, - }, - ) - return informer -} - // Prints live logs for workflow pods, starting from time specified in timeByPod name. -func (p *logPrinter) printLiveWorkflowLogs(workflow *v1alpha1.Workflow, timeByPod map[string]*time.Time) { +func (p *logPrinter) printLiveWorkflowLogs(workflowName string, wfClient workflowv1.WorkflowInterface, timeByPod map[string]*time.Time) { logs := make(chan logEntry) streamedPods := make(map[string]bool) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() processPods := func(wf *v1alpha1.Workflow) { + err := util.DecompressWorkflow(wf) + if err != nil { + log.Warn(err) + return + } for id := range wf.Status.Nodes { node := wf.Status.Nodes[id] if node.Type == v1alpha1.NodeTypePod && node.Phase != v1alpha1.NodeError && streamedPods[node.ID] == false { @@ -218,7 +213,7 @@ func (p *logPrinter) printLiveWorkflowLogs(workflow *v1alpha1.Workflow, timeByPo sinceTime := metav1.NewTime(podTime.Add(time.Second)) sinceTimePtr = &sinceTime } - err := p.getPodLogs(getDisplayName(node), node.ID, wf.Namespace, true, nil, nil, sinceTimePtr, func(entry logEntry) { + err := p.getPodLogs(ctx, getDisplayName(node), node.ID, wf.Namespace, true, nil, nil, sinceTimePtr, func(entry logEntry) { logs <- entry }) if err != nil { @@ -229,20 +224,31 @@ func (p *logPrinter) printLiveWorkflowLogs(workflow *v1alpha1.Workflow, timeByPo } } - processPods(workflow) - informer := p.setupWorkflowInformer(workflow.Namespace, workflow.Name, func(wf *v1alpha1.Workflow, done bool) { - if done { - close(logs) - } else { - processPods(wf) - } - }) - - stopChannel := make(chan struct{}) go func() { - informer.Run(stopChannel) + defer close(logs) + fieldSelector := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", workflowName)) + listOpts := metav1.ListOptions{FieldSelector: fieldSelector.String()} + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return wfClient.List(listOpts) + }, + WatchFunc: func(options metav1.ListOptions) (pkgwatch.Interface, error) { + return wfClient.Watch(listOpts) + }, + } + _, err := watch.UntilWithSync(ctx, lw, &v1alpha1.Workflow{}, nil, func(event pkgwatch.Event) (b bool, e error) { + if wf, ok := event.Object.(*v1alpha1.Workflow); ok { + if !wf.Status.Completed() { + processPods(wf) + } + return wf.Status.Completed(), nil + } + return true, nil + }) + if err != nil { + log.Fatal(err) + } }() - defer close(stopChannel) for entry := range logs { p.printLogEntry(entry) @@ -273,35 +279,56 @@ func (p *logPrinter) printLogEntry(entry logEntry) { fmt.Println(line) } -func (p *logPrinter) ensureContainerStarted(podName string, podNamespace string, container string, retryCnt int, retryTimeout time.Duration) error { - for retryCnt > 0 { - pod, err := p.kubeClient.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{}) +func (p *logPrinter) hasContainerStarted(podName string, podNamespace string, container string) (bool, error) { + pod, err := p.kubeClient.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{}) + if err != nil { + return false, err + } + var containerStatus *v1.ContainerStatus + for _, status := range pod.Status.ContainerStatuses { + if status.Name == container { + containerStatus = &status + break + } + } + if containerStatus == nil { + return false, nil + } + + if containerStatus.State.Waiting != nil { + return false, nil + } + return true, nil +} + +func (p *logPrinter) getPodLogs( + ctx context.Context, + displayName string, + podName string, + podNamespace string, + follow bool, + tail *int64, + sinceSeconds *int64, + sinceTime *metav1.Time, + callback func(entry logEntry)) error { + + for ctx.Err() == nil { + hasStarted, err := p.hasContainerStarted(podName, podNamespace, p.container) + if err != nil { return err } - var containerStatus *v1.ContainerStatus - for _, status := range pod.Status.ContainerStatuses { - if status.Name == container { - containerStatus = &status - break + if !hasStarted { + if follow { + time.Sleep(1 * time.Second) + } else { + return nil } - } - if containerStatus == nil || containerStatus.State.Waiting != nil { - time.Sleep(retryTimeout) - retryCnt-- } else { - return nil + break } } - return fmt.Errorf("container '%s' of pod '%s' has not started within expected timeout", container, podName) -} -func (p *logPrinter) getPodLogs( - displayName string, podName string, podNamespace string, follow bool, tail *int64, sinceSeconds *int64, sinceTime *metav1.Time, callback func(entry logEntry)) error { - err := p.ensureContainerStarted(podName, podNamespace, p.container, 10, time.Second) - if err != nil { - return err - } stream, err := p.kubeClient.CoreV1().Pods(podNamespace).GetLogs(podName, &v1.PodLogOptions{ Container: p.container, Follow: follow, diff --git a/cmd/argo/commands/submit.go b/cmd/argo/commands/submit.go index b71f09696a8a..1d9dc5a837e7 100644 --- a/cmd/argo/commands/submit.go +++ b/cmd/argo/commands/submit.go @@ -18,16 +18,18 @@ import ( // cliSubmitOpts holds submition options specific to CLI submission (e.g. controlling output) type cliSubmitOpts struct { - output string // --output - wait bool // --wait - watch bool // --watch - strict bool // --strict + output string // --output + wait bool // --wait + watch bool // --watch + strict bool // --strict + priority *int32 // --priority } func NewSubmitCommand() *cobra.Command { var ( submitOpts util.SubmitOpts cliSubmitOpts cliSubmitOpts + priority int32 ) var command = &cobra.Command{ Use: "submit FILE1 FILE2...", @@ -37,6 +39,10 @@ func NewSubmitCommand() *cobra.Command { cmd.HelpFunc()(cmd, args) os.Exit(1) } + if cmd.Flag("priority").Changed { + cliSubmitOpts.priority = &priority + } + SubmitWorkflows(args, &submitOpts, &cliSubmitOpts) }, } @@ -51,6 +57,7 @@ func NewSubmitCommand() *cobra.Command { command.Flags().BoolVarP(&cliSubmitOpts.wait, "wait", "w", false, "wait for the workflow to complete") command.Flags().BoolVar(&cliSubmitOpts.watch, "watch", false, "watch the workflow until it completes") command.Flags().BoolVar(&cliSubmitOpts.strict, "strict", true, "perform strict workflow validation") + command.Flags().Int32Var(&priority, "priority", 0, "workflow priority") return command } @@ -106,6 +113,7 @@ func SubmitWorkflows(filePaths []string, submitOpts *util.SubmitOpts, cliOpts *c var workflowNames []string for _, wf := range workflows { + wf.Spec.Priority = cliOpts.priority created, err := util.SubmitWorkflow(wfClient, &wf, submitOpts) if err != nil { log.Fatalf("Failed to submit workflow: %v", err) @@ -137,7 +145,7 @@ func unmarshalWorkflows(wfBytes []byte, strict bool) []wfv1.Workflow { func waitOrWatch(workflowNames []string, cliSubmitOpts cliSubmitOpts) { if cliSubmitOpts.wait { - WaitWorkflows(workflowNames, false, cliSubmitOpts.output == "json") + WaitWorkflows(workflowNames, false, !(cliSubmitOpts.output == "" || cliSubmitOpts.output == "wide")) } else if cliSubmitOpts.watch { watchWorkflow(workflowNames[0]) } diff --git a/cmd/argo/commands/watch.go b/cmd/argo/commands/watch.go index a57bc4e3f238..9f07a88ea39c 100644 --- a/cmd/argo/commands/watch.go +++ b/cmd/argo/commands/watch.go @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/fields" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/workflow/util" ) func NewWatchCommand() *cobra.Command { @@ -53,6 +54,8 @@ func watchWorkflow(name string) { errors.CheckError(err) continue } + err := util.DecompressWorkflow(wf) + errors.CheckError(err) print("\033[H\033[2J") print("\033[0;0H") printWorkflowHelper(wf, "") diff --git a/cmd/argo/main.go b/cmd/argo/main.go index 208a1391ac3d..2af1fc903aca 100644 --- a/cmd/argo/main.go +++ b/cmd/argo/main.go @@ -5,6 +5,8 @@ import ( "os" "github.com/cyrusbiotechnology/argo/cmd/argo/commands" + // load the azure plugin (required to authenticate against AKS clusters). + _ "k8s.io/client-go/plugin/pkg/client/auth/azure" // load the gcp plugin (required to authenticate against GKE clusters). _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // load the oidc plugin (required to authenticate with OpenID Connect). diff --git a/cmd/argoexec/commands/init.go b/cmd/argoexec/commands/init.go index c581a240496c..270cada7e52f 100644 --- a/cmd/argoexec/commands/init.go +++ b/cmd/argoexec/commands/init.go @@ -6,19 +6,18 @@ import ( "github.com/spf13/cobra" ) -func init() { - RootCmd.AddCommand(initCmd) -} - -var initCmd = &cobra.Command{ - Use: "init", - Short: "Load artifacts", - Run: func(cmd *cobra.Command, args []string) { - err := loadArtifacts() - if err != nil { - log.Fatalf("%+v", err) - } - }, +func NewInitCommand() *cobra.Command { + var command = cobra.Command{ + Use: "init", + Short: "Load artifacts", + Run: func(cmd *cobra.Command, args []string) { + err := loadArtifacts() + if err != nil { + log.Fatalf("%+v", err) + } + }, + } + return &command } func loadArtifacts() error { diff --git a/cmd/argoexec/commands/resource.go b/cmd/argoexec/commands/resource.go index d894de5c471c..35266c21aedc 100644 --- a/cmd/argoexec/commands/resource.go +++ b/cmd/argoexec/commands/resource.go @@ -1,6 +1,7 @@ package commands import ( + "fmt" "os" "github.com/cyrusbiotechnology/argo/workflow/common" @@ -8,23 +9,22 @@ import ( "github.com/spf13/cobra" ) -func init() { - RootCmd.AddCommand(resourceCmd) -} - -var resourceCmd = &cobra.Command{ - Use: "resource (get|create|apply|delete) MANIFEST", - Short: "update a resource and wait for resource conditions", - Run: func(cmd *cobra.Command, args []string) { - if len(args) != 1 { - cmd.HelpFunc()(cmd, args) - os.Exit(1) - } - err := execResource(args[0]) - if err != nil { - log.Fatalf("%+v", err) - } - }, +func NewResourceCommand() *cobra.Command { + var command = cobra.Command{ + Use: "resource (get|create|apply|delete) MANIFEST", + Short: "update a resource and wait for resource conditions", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 1 { + cmd.HelpFunc()(cmd, args) + os.Exit(1) + } + err := execResource(args[0]) + if err != nil { + log.Fatalf("%+v", err) + } + }, + } + return &command } func execResource(action string) error { @@ -35,20 +35,28 @@ func execResource(action string) error { wfExecutor.AddError(err) return err } - resourceName, err := wfExecutor.ExecResource(action, common.ExecutorResourceManifestPath) - if err != nil { + isDelete := action == "delete" + if isDelete && (wfExecutor.Template.Resource.SuccessCondition != "" || wfExecutor.Template.Resource.FailureCondition != "" || len(wfExecutor.Template.Outputs.Parameters) > 0) { + err = fmt.Errorf("successCondition, failureCondition and outputs are not supported for delete action") wfExecutor.AddError(err) return err } - err = wfExecutor.WaitResource(resourceName) + resourceNamespace, resourceName, err := wfExecutor.ExecResource(action, common.ExecutorResourceManifestPath, isDelete) if err != nil { wfExecutor.AddError(err) return err } - err = wfExecutor.SaveResourceParameters(resourceName) - if err != nil { - wfExecutor.AddError(err) - return err + if !isDelete { + err = wfExecutor.WaitResource(resourceNamespace, resourceName) + if err != nil { + wfExecutor.AddError(err) + return err + } + err = wfExecutor.SaveResourceParameters(resourceNamespace, resourceName) + if err != nil { + wfExecutor.AddError(err) + return err + } } return nil } diff --git a/cmd/argoexec/commands/root.go b/cmd/argoexec/commands/root.go index f1607885567f..af81cfe6f073 100644 --- a/cmd/argoexec/commands/root.go +++ b/cmd/argoexec/commands/root.go @@ -1,10 +1,11 @@ package commands import ( + "encoding/json" "os" - "github.com/argoproj/pkg/kube/cli" - "github.com/ghodss/yaml" + "github.com/argoproj/pkg/cli" + kubecli "github.com/argoproj/pkg/kube/cli" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" @@ -15,7 +16,9 @@ import ( "github.com/cyrusbiotechnology/argo/workflow/common" "github.com/cyrusbiotechnology/argo/workflow/executor" "github.com/cyrusbiotechnology/argo/workflow/executor/docker" + "github.com/cyrusbiotechnology/argo/workflow/executor/k8sapi" "github.com/cyrusbiotechnology/argo/workflow/executor/kubelet" + "github.com/cyrusbiotechnology/argo/workflow/executor/pns" ) const ( @@ -24,78 +27,84 @@ const ( ) var ( - // GlobalArgs hold global CLI flags - GlobalArgs globalFlags - - clientConfig clientcmd.ClientConfig -) - -type globalFlags struct { + clientConfig clientcmd.ClientConfig + logLevel string // --loglevel + glogLevel int // --gloglevel podAnnotationsPath string // --pod-annotations -} +) func init() { - clientConfig = cli.AddKubectlFlagsToCmd(RootCmd) - RootCmd.PersistentFlags().StringVar(&GlobalArgs.podAnnotationsPath, "pod-annotations", common.PodMetadataAnnotationsPath, "Pod annotations file from k8s downward API") - RootCmd.AddCommand(cmd.NewVersionCmd(CLIName)) + cobra.OnInitialize(initConfig) } -// RootCmd is the argo root level command -var RootCmd = &cobra.Command{ - Use: CLIName, - Short: "argoexec is the executor sidecar to workflow containers", - Run: func(cmd *cobra.Command, args []string) { - cmd.HelpFunc()(cmd, args) - }, +func initConfig() { + cli.SetLogLevel(logLevel) + cli.SetGLogLevel(glogLevel) } -func initExecutor() *executor.WorkflowExecutor { - podAnnotationsPath := common.PodMetadataAnnotationsPath - - // Use the path specified from the flag - if GlobalArgs.podAnnotationsPath != "" { - podAnnotationsPath = GlobalArgs.podAnnotationsPath +func NewRootCommand() *cobra.Command { + var command = cobra.Command{ + Use: CLIName, + Short: "argoexec is the executor sidecar to workflow containers", + Run: func(cmd *cobra.Command, args []string) { + cmd.HelpFunc()(cmd, args) + }, } + command.AddCommand(NewInitCommand()) + command.AddCommand(NewResourceCommand()) + command.AddCommand(NewWaitCommand()) + command.AddCommand(cmd.NewVersionCmd(CLIName)) + + clientConfig = kubecli.AddKubectlFlagsToCmd(&command) + command.PersistentFlags().StringVar(&podAnnotationsPath, "pod-annotations", common.PodMetadataAnnotationsPath, "Pod annotations file from k8s downward API") + command.PersistentFlags().StringVar(&logLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error") + command.PersistentFlags().IntVar(&glogLevel, "gloglevel", 0, "Set the glog logging level") + + return &command +} + +func initExecutor() *executor.WorkflowExecutor { config, err := clientConfig.ClientConfig() - if err != nil { - panic(err.Error()) - } + checkErr(err) + namespace, _, err := clientConfig.Namespace() - if err != nil { - panic(err.Error()) - } + checkErr(err) clientset, err := kubernetes.NewForConfig(config) - if err != nil { - panic(err.Error()) - } + checkErr(err) + podName, ok := os.LookupEnv(common.EnvVarPodName) if !ok { log.Fatalf("Unable to determine pod name from environment variable %s", common.EnvVarPodName) } + tmpl, err := executor.LoadTemplate(podAnnotationsPath) + checkErr(err) + var cre executor.ContainerRuntimeExecutor switch os.Getenv(common.EnvVarContainerRuntimeExecutor) { + case common.ContainerRuntimeExecutorK8sAPI: + cre, err = k8sapi.NewK8sAPIExecutor(clientset, config, podName, namespace) case common.ContainerRuntimeExecutorKubelet: cre, err = kubelet.NewKubeletExecutor() - if err != nil { - panic(err.Error()) - } + case common.ContainerRuntimeExecutorPNS: + cre, err = pns.NewPNSExecutor(clientset, podName, namespace, tmpl.Outputs.HasOutputs()) default: cre, err = docker.NewDockerExecutor() - if err != nil { - panic(err.Error()) - } - } - wfExecutor := executor.NewExecutor(clientset, podName, namespace, podAnnotationsPath, cre) - err = wfExecutor.LoadTemplate() - if err != nil { - panic(err.Error()) } + checkErr(err) - yamlBytes, _ := yaml.Marshal(&wfExecutor.Template) + wfExecutor := executor.NewExecutor(clientset, podName, namespace, podAnnotationsPath, cre, *tmpl) + yamlBytes, _ := json.Marshal(&wfExecutor.Template) vers := argo.GetVersion() - log.Infof("Executor (version: %s, build_date: %s) initialized with template:\n%s", vers, vers.BuildDate, string(yamlBytes)) + log.Infof("Executor (version: %s, build_date: %s) initialized (pod: %s/%s) with template:\n%s", vers, vers.BuildDate, namespace, podName, string(yamlBytes)) return &wfExecutor } + +// checkErr is a convenience function to panic upon error +func checkErr(err error) { + if err != nil { + panic(err.Error()) + } +} diff --git a/cmd/argoexec/commands/wait.go b/cmd/argoexec/commands/wait.go index 6b5c305f08cf..b49aa7edd7f2 100644 --- a/cmd/argoexec/commands/wait.go +++ b/cmd/argoexec/commands/wait.go @@ -9,19 +9,18 @@ import ( "github.com/spf13/cobra" ) -func init() { - RootCmd.AddCommand(waitCmd) -} - -var waitCmd = &cobra.Command{ - Use: "wait", - Short: "wait for main container to finish and save artifacts", - Run: func(cmd *cobra.Command, args []string) { - err := waitContainer() - if err != nil { - log.Fatalf("%+v", err) - } - }, +func NewWaitCommand() *cobra.Command { + var command = cobra.Command{ + Use: "wait", + Short: "wait for main container to finish and save artifacts", + Run: func(cmd *cobra.Command, args []string) { + err := waitContainer() + if err != nil { + log.Fatalf("%+v", err) + } + }, + } + return &command } func waitContainer() error { @@ -30,18 +29,18 @@ func waitContainer() error { defer stats.LogStats() stats.StartStatsTicker(5 * time.Minute) - // Wait for main container to complete and kill sidecars + // Wait for main container to complete err := wfExecutor.Wait() if err != nil { wfExecutor.AddError(err) - // do not return here so we can still try to save outputs + // do not return here so we can still try to kill sidecars & save outputs } - logArt, err := wfExecutor.SaveLogs() + err = wfExecutor.KillSidecars() if err != nil { wfExecutor.AddError(err) - return err + // do not return here so we can still try save outputs } - err = wfExecutor.SaveArtifacts() + logArt, err := wfExecutor.SaveLogs() if err != nil { wfExecutor.AddError(err) return err @@ -52,6 +51,12 @@ func waitContainer() error { wfExecutor.AddError(err) return err } + // Saving output artifacts + err = wfExecutor.SaveArtifacts() + if err != nil { + wfExecutor.AddError(err) + return err + } // Capture output script result err = wfExecutor.CaptureScriptResult() if err != nil { diff --git a/cmd/argoexec/main.go b/cmd/argoexec/main.go index 74a28f6bc92b..5a73e18c68d7 100644 --- a/cmd/argoexec/main.go +++ b/cmd/argoexec/main.go @@ -5,6 +5,8 @@ import ( "os" "github.com/cyrusbiotechnology/argo/cmd/argoexec/commands" + // load the azure plugin (required to authenticate against AKS clusters). + _ "k8s.io/client-go/plugin/pkg/client/auth/azure" // load the gcp plugin (required to authenticate against GKE clusters). _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // load the oidc plugin (required to authenticate with OpenID Connect). @@ -12,7 +14,7 @@ import ( ) func main() { - if err := commands.RootCmd.Execute(); err != nil { + if err := commands.NewRootCommand().Execute(); err != nil { fmt.Println(err) os.Exit(1) } diff --git a/cmd/workflow-controller/main.go b/cmd/workflow-controller/main.go index e3d56a9218b4..ab972505951d 100644 --- a/cmd/workflow-controller/main.go +++ b/cmd/workflow-controller/main.go @@ -2,16 +2,16 @@ package main import ( "context" - "flag" "fmt" "os" - "strconv" "time" - "github.com/argoproj/pkg/kube/cli" + "github.com/argoproj/pkg/cli" + kubecli "github.com/argoproj/pkg/kube/cli" "github.com/argoproj/pkg/stats" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth/azure" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" "k8s.io/client-go/tools/clientcmd" @@ -44,16 +44,11 @@ func NewRootCommand() *cobra.Command { Use: CLIName, Short: "workflow-controller is the controller to operate on workflows", RunE: func(c *cobra.Command, args []string) error { - - cmdutil.SetLogLevel(logLevel) + cli.SetLogLevel(logLevel) + cli.SetGLogLevel(glogLevel) stats.RegisterStackDumper() stats.StartStatsTicker(5 * time.Minute) - // Set the glog level for the k8s go-client - _ = flag.CommandLine.Parse([]string{}) - _ = flag.Lookup("logtostderr").Value.Set("true") - _ = flag.Lookup("v").Value.Set(strconv.Itoa(glogLevel)) - config, err := clientConfig.ClientConfig() if err != nil { return err @@ -90,7 +85,7 @@ func NewRootCommand() *cobra.Command { }, } - clientConfig = cli.AddKubectlFlagsToCmd(&command) + clientConfig = kubecli.AddKubectlFlagsToCmd(&command) command.AddCommand(cmdutil.NewVersionCmd(CLIName)) command.Flags().StringVar(&configMap, "configmap", "workflow-controller-configmap", "Name of K8s configmap to retrieve workflow controller configuration") command.Flags().StringVar(&configFile, "config-file", "", "Path to a yaml config file. Cannot be specified at the same time as --configmap") diff --git a/community/Argo Individual CLA.pdf b/community/Argo Individual CLA.pdf index f91c4a5a3048..e25d08bc4738 100644 Binary files a/community/Argo Individual CLA.pdf and b/community/Argo Individual CLA.pdf differ diff --git a/community/README.md b/community/README.md index 18a92952ad7d..4ee176d838f7 100644 --- a/community/README.md +++ b/community/README.md @@ -4,14 +4,7 @@ Welcome to the Argo community! Argo is an open, community driven project to make it easy to use Kubernetes for getting useful work done. This document describes the organizational structure of the Argo Community including the roles, responsibilities and processes that govern Argo projects and community. - -We will have our **first community meeting on May 9th 10 am PST**. - -Map that to your local time with this [timezone table](https://www.google.com/search?q=1800+in+utc) - -See it on the web as [Google Calendar](https://calendar.google.com/calendar/embed?src=argoproj%40gmail.com&ctz=America%2FLos_Angeles) , or paste this [iCal url](https://calendar.google.com/calendar/ical/argoproj%40gmail.com/private-52229421c00ee71c176517df5bf1941e/basic.ics) into any iCal client. - -Meeting notes is available [here](https://docs.google.com/document/d/16aWGQ1Te5IRptFuAIFtg3rONRQqHC1Z3X9rdDHYhYfE/edit?usp=sharing). +Community meeting notes is available [here](https://docs.google.com/document/d/16aWGQ1Te5IRptFuAIFtg3rONRQqHC1Z3X9rdDHYhYfE/edit?usp=sharing). ## Projects @@ -20,9 +13,9 @@ Argo is organized into a set of projects. Each project has at least one owner. T The projects are: * Argo Workflows -* Argo CI * Argo CD * Argo Events +* Argo CI ## Community Roles and Responsibilities @@ -42,9 +35,11 @@ manner. ## Contributing to Argo -Read and abide by the [Argo Code of Conduct](https://github.com/cyrusbiotechnology/argo/blob/master/CODE_OF_CONDUCT.md) : +Read and abide by the [Argo Code of Conduct](https://github.com/cyrusbiotechnology/argo/blob/master/CODE_OF_CONDUCT.md). + Before submitting a pull request, please sign the [CLA](https://github.com/cyrusbiotechnology/argo/blob/master/community/Argo%20Individual%20CLA.pdf). This agreement gives us permission to use and redistribute your contributions as part of the project. +Contributors will be asked to read and sign a [CLA](https://github.com/cyrusbiotechnology/argo/blob/master/community/Argo%20Individual%20CLA.pdf). This agreement gives us permission to use and redistribute your contributions as part of the Argo Project and protects the users and contributors of the project. ## Community Meetings diff --git a/demo.md b/demo.md index 99f79228b824..65f4749fbdbd 100644 --- a/demo.md +++ b/demo.md @@ -24,7 +24,7 @@ chmod +x /usr/local/bin/argo ## 2. Install the Controller and UI ``` kubectl create ns argo -kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.2.1/manifests/install.yaml +kubectl apply -n argo -f https://raw.githubusercontent.com/cyrusbiotechnology/argo/v2.2.1/manifests/install.yaml ``` NOTE: On GKE, you may need to grant your account the ability to create new clusterroles ``` @@ -32,22 +32,25 @@ kubectl create clusterrolebinding YOURNAME-cluster-admin-binding --clusterrole=c ``` ## 3. Configure the service account to run workflows -For clusters with RBAC enabled, the 'default' service account is too limited to support features -like artifacts, outputs, access to secrets, etc... Run the following command to grant admin -privileges to the 'default' service account in the namespace 'default': + +To run all of the examples in this guide, the 'default' service account is too limited to support +features such as artifacts, outputs, access to secrets, etc... For demo purposes, run the following +command to grant admin privileges to the 'default' service account in the namespace 'default': ``` kubectl create rolebinding default-admin --clusterrole=admin --serviceaccount=default:default ``` -NOTE: You can also submit workflows which run with a different service account using: +For the bare minimum set of privileges which a workflow needs to function, see +[Workflow RBAC](docs/workflow-rbac.md). You can also submit workflows which run with a different +service account using: ``` argo submit --serviceaccount ``` ## 4. Run Simple Example Workflows ``` -argo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml -argo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/coinflip.yaml -argo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/loops-maps.yaml +argo submit --watch https://raw.githubusercontent.com/cyrusbiotechnology/argo/master/examples/hello-world.yaml +argo submit --watch https://raw.githubusercontent.com/cyrusbiotechnology/argo/master/examples/coinflip.yaml +argo submit --watch https://raw.githubusercontent.com/cyrusbiotechnology/argo/master/examples/loops-maps.yaml argo list argo get xxx-workflow-name-xxx argo logs xxx-pod-name-xxx #from get command above @@ -57,14 +60,14 @@ You can also create workflows directly with kubectl. However, the Argo CLI offer that kubectl does not, such as YAML validation, workflow visualization, parameter passing, retries and resubmits, suspend and resume, and more. ``` -kubectl create -f https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml +kubectl create -f https://raw.githubusercontent.com/cyrusbiotechnology/argo/master/examples/hello-world.yaml kubectl get wf kubectl get wf hello-world-xxx kubectl get po --selector=workflows.argoproj.io/workflow=hello-world-xxx --show-all kubectl logs hello-world-yyy -c main ``` -Additional examples are availabe [here](https://github.com/cyrusbiotechnology/argo/blob/master/examples/README.md). +Additional examples are available [here](https://github.com/cyrusbiotechnology/argo/blob/master/examples/README.md). ## 5. Install an Artifact Repository @@ -72,18 +75,21 @@ Argo supports S3 (AWS, GCS, Minio) as well as Artifactory as artifact repositori uses Minio for the sake of portability. Instructions on how to configure other artifact repositories are [here](https://github.com/cyrusbiotechnology/argo/blob/master/ARTIFACT_REPO.md). ``` -brew install kubernetes-helm # mac -helm init -helm install stable/minio --name argo-artifacts --set service.type=LoadBalancer --set persistence.enabled=false +helm install stable/minio \ + --name argo-artifacts \ + --set service.type=LoadBalancer \ + --set defaultBucket.enabled=true \ + --set defaultBucket.name=my-bucket \ + --set persistence.enabled=false ``` Login to the Minio UI using a web browser (port 9000) after exposing obtaining the external IP using `kubectl`. ``` -kubectl get service argo-artifacts-minio -o wide +kubectl get service argo-artifacts -o wide ``` On Minikube: ``` -minikube service --url argo-artifacts-minio +minikube service --url argo-artifacts ``` NOTE: When minio is installed via Helm, it uses the following hard-wired default credentials, @@ -95,8 +101,8 @@ Create a bucket named `my-bucket` from the Minio UI. ## 6. Reconfigure the workflow controller to use the Minio artifact repository -Edit the workflow-controller config map to reference the service name (argo-artifacts-minio) and -secret (argo-artifacts-minio) created by the helm install: +Edit the workflow-controller config map to reference the service name (argo-artifacts) and +secret (argo-artifacts) created by the helm install: ``` kubectl edit cm -n argo workflow-controller-configmap ... @@ -105,18 +111,18 @@ data: artifactRepository: s3: bucket: my-bucket - endpoint: argo-artifacts-minio.default:9000 + endpoint: argo-artifacts.default:9000 insecure: true # accessKeySecret and secretKeySecret are secret selectors. - # It references the k8s secret named 'argo-artifacts-minio' + # It references the k8s secret named 'argo-artifacts' # which was created during the minio helm install. The keys, # 'accesskey' and 'secretkey', inside that secret are where the # actual minio credentials are stored. accessKeySecret: - name: argo-artifacts-minio + name: argo-artifacts key: accesskey secretKeySecret: - name: argo-artifacts-minio + name: argo-artifacts key: secretkey ``` @@ -126,7 +132,7 @@ namespace you use for workflows. ## 7. Run a workflow which uses artifacts ``` -argo submit https://raw.githubusercontent.com/argoproj/argo/master/examples/artifact-passing.yaml +argo submit https://raw.githubusercontent.com/cyrusbiotechnology/argo/master/examples/artifact-passing.yaml ``` ## 8. Access the Argo UI diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000000..d0f3d87ad6ed --- /dev/null +++ b/docs/README.md @@ -0,0 +1,9 @@ +# Argo Documentation + +## [Getting Started](../demo.md) + +## Features +* [Controller Configuration](workflow-controller-configmap.yaml) +* [RBAC](workflow-rbac.md) +* [REST API](rest-api.md) +* [Workflow Variables](variables.md) diff --git a/docs/example-golang/main.go b/docs/example-golang/main.go new file mode 100644 index 000000000000..fb790e3076c0 --- /dev/null +++ b/docs/example-golang/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/argoproj/pkg/errors" + wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + wfclientset "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/tools/clientcmd" +) + +var ( + helloWorldWorkflow = wfv1.Workflow{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "hello-world-", + }, + Spec: wfv1.WorkflowSpec{ + Entrypoint: "whalesay", + Templates: []wfv1.Template{ + { + Name: "whalesay", + Container: &corev1.Container{ + Image: "docker/whalesay:latest", + Command: []string{"cowsay", "hello world"}, + }, + }, + }, + }, + } +) + +func main() { + // use the current context in kubeconfig + kubeconfig := flag.String("kubeconfig", filepath.Join(os.Getenv("HOME"), ".kube", "config"), "(optional) absolute path to the kubeconfig file") + flag.Parse() + + // use the current context in kubeconfig + config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) + checkErr(err) + namespace := "default" + + // create the workflow client + wfClient := wfclientset.NewForConfigOrDie(config).ArgoprojV1alpha1().Workflows(namespace) + + // submit the hello world workflow + createdWf, err := wfClient.Create(&helloWorldWorkflow) + checkErr(err) + fmt.Printf("Workflow %s submitted\n", createdWf.Name) + + // wait for the workflow to complete + fieldSelector := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", createdWf.Name)) + watchIf, err := wfClient.Watch(metav1.ListOptions{FieldSelector: fieldSelector.String()}) + errors.CheckError(err) + defer watchIf.Stop() + for next := range watchIf.ResultChan() { + wf, ok := next.Object.(*wfv1.Workflow) + if !ok { + continue + } + if !wf.Status.FinishedAt.IsZero() { + fmt.Printf("Workflow %s %s at %v\n", wf.Name, wf.Status.Phase, wf.Status.FinishedAt) + break + } + } +} + +func checkErr(err error) { + if err != nil { + panic(err.Error()) + } +} diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 000000000000..4a71834f56de --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,38 @@ +# Release Instructions + +1. Update CHANGELOG.md with changes in the release + +2. Update VERSION with new tag + +3. Update codegen, manifests with new tag + +``` +make codegen manifests IMAGE_NAMESPACE=argoproj IMAGE_TAG=vX.Y.Z +``` + +4. Commit VERSION and manifest changes + +5. git tag the release + +``` +git tag vX.Y.Z +``` + +6. Build the release + +``` +make release IMAGE_NAMESPACE=argoproj IMAGE_TAG=vX.Y.Z +``` + +7. If successful, publish the release: +``` +export ARGO_RELEASE=vX.Y.Z +docker push argoproj/workflow-controller:${ARGO_RELEASE} +docker push argoproj/argoexec:${ARGO_RELEASE} +docker push argoproj/argocli:${ARGO_RELEASE} +git push upstream ${ARGO_RELEASE} +``` + +8. Draft GitHub release with the content from CHANGELOG.md, and CLI binaries produced in the `dist` directory + +* https://github.com/argoproj/argo/releases/new diff --git a/docs/rest-api.md b/docs/rest-api.md new file mode 100644 index 000000000000..c88a5698de7c --- /dev/null +++ b/docs/rest-api.md @@ -0,0 +1,40 @@ +# REST API + +Argo is implemented as a kubernetes controller and Workflow [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). +Argo itself does not run an API server, and with all CRDs, it extends the Kubernetes API server by +introducing a new API Group/Version (argorproj.io/v1alpha1) and Kind (Workflow). When CRDs are +registered in a cluster, access to those resources are made available by exposing new endpoints in +the kubernetes API server. For example, to list workflows in the default namespace, a client would +make an HTTP GET request to: `https:///apis/argoproj.io/v1alpha1/namespaces/default/workflows` + +> NOTE: the optional argo-ui does run a thin API layer to power the UI, but is not intended for + programatic interaction. + +A common scenario is to programatically submit and retrieve workflows. To do this, you would use the +existing Kubernetes REST client in the language of preference, which often libraries for performing +CRUD operation on custom resource objects. + +## Examples + +### Golang + +A kubernetes Workflow clientset library is auto-generated under [argoproj/argo/pkg/client](https://github.com/argoproj/argo/tree/master/pkg/client) and can be imported by golang +applications. See the [golang code example](example-golang/main.go) on how to make use of this client. + +### Python +The python kubernetes client has libraries for interacting with custom objects. See: https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/CustomObjectsApi.md + + +### Java +The Java kubernetes client has libraries for interacting with custom objects. See: +https://github.com/kubernetes-client/java/blob/master/kubernetes/docs/CustomObjectsApi.md + +### Ruby +The Ruby kubernetes client has libraries for interacting with custom objects. See: +https://github.com/kubernetes-client/ruby/tree/master/kubernetes +See this [external Ruby example](https://github.com/fischerjulian/argo_workflows_ruby_example) on how to make use of this client. + +## OpenAPI + +An OpenAPI Spec is generated under [argoproj/argo/api/openapi-spec](https://github.com/argoproj/argo/blob/master/api/openapi-spec/swagger.json). This spec may be +used to auto-generate concrete datastructures in other languages. diff --git a/docs/variables.md b/docs/variables.md index bb08d36426c8..95e078135e17 100644 --- a/docs/variables.md +++ b/docs/variables.md @@ -28,6 +28,9 @@ The following variables are made available to reference various metadata of a wo | Variable | Description| |----------|------------| | `pod.name` | Pod name of the container/script | +| `inputs.artifacts..path` | Local path of the input artifact | +| `outputs.artifacts..path` | Local path of the output artifact | +| `outputs.parameters..path` | Local path of the output parameter | ## Loops (withItems / withParam) | Variable | Description| @@ -43,16 +46,13 @@ The following variables are made available to reference various metadata of a wo | `workflow.uid` | Workflow UID. Useful for setting ownership reference to a resource, or a unique artifact location | | `workflow.parameters.` | Input parameter to the workflow | | `workflow.outputs.parameters.` | Input artifact to the workflow | +| `workflow.annotations.` | Workflow annotations | +| `workflow.labels.` | Workflow labels | +| `workflow.creationTimestamp` | Workflow creation timestamp formatted in RFC 3339 (e.g. `2018-08-23T05:42:49Z`) | +| `workflow.creationTimestamp.` | Creation timestamp formatted with a [strftime](http://strftime.org) format character | + ## Exit Handler: | Variable | Description| |----------|------------| | `workflow.status` | Workflow status. One of: `Succeeded`, `Failed`, `Error` | - -## Coming in v2.2: -| Variable | Description| -|----------|------------| -| `workflow.artifacts.` | Input artifact to the workflow | -| `workflow.outputs.artifacts.` | Output artifact to the workflow | -| `workflow.creationTimestamp` | Workflow creation timestamp formatted in RFC 3339 (e.g. `2018-08-23T05:42:49Z`) | -| `workflow.creationTimestamp.` | Creation timestamp formatted with a [strftime](http://strftime.org) format character | diff --git a/docs/workflow-controller-configmap.yaml b/docs/workflow-controller-configmap.yaml index 8bd773d6e60e..5f2e6bd2ba52 100644 --- a/docs/workflow-controller-configmap.yaml +++ b/docs/workflow-controller-configmap.yaml @@ -15,9 +15,26 @@ data: instanceID: my-ci-controller # namespace limits the controller's watch/queries to a specific namespace. This allows the - # controller to run with namespace scope (role), instead of cluster scope (clusterrole). + # controller to run with namespace scope (Role), instead of cluster scope (ClusterRole). namespace: argo + # Parallelism limits the max total parallel workflows that can execute at the same time + # (available since Argo v2.3) + parallelism: 10 + + # uncomment flowing lines if workflow controller runs in a different k8s cluster with the + # workflow workloads, or needs to communicate with the k8s apiserver using an out-of-cluster + # kubeconfig secret + # kubeConfig: + # # name of the kubeconfig secret, may not be empty when kubeConfig specified + # secretName: kubeconfig-secret + # # key of the kubeconfig secret, may not be empty when kubeConfig specified + # secretKey: kubeconfig + # # mounting path of the kubeconfig secret, default to /kube/config + # mountPath: /kubeconfig/mount/path + # # volume name when mounting the secret, default to kubeconfig + # volumeName: kube-config-volume + # artifactRepository defines the default location to be used as the artifact repository for # container artifacts. artifactRepository: @@ -32,8 +49,10 @@ data: endpoint: s3.amazonaws.com bucket: my-bucket region: us-west-2 + # insecure will disable TLS. Primarily used for minio installs not configured with TLS + insecure: false # keyFormat is a format pattern to define how artifacts will be organized in a bucket. - # It can reference workflow metadata variables such as workflow.namespace, workflow.name, + # It can reference workflow metadata variables such as workflow.namespace, workflow.name, # pod.name. Can also use strftime formating of workflow.creationTimestamp so that workflow # artifacts can be organized by date. If omitted, will use `{{workflow.name}}/{{pod.name}}`, # which has potential for have collisions. @@ -46,10 +65,8 @@ data: /{{workflow.creationTimestamp.d}}\ /{{workflow.name}}\ /{{pod.name}}" - # insecure will disable TLS. used for minio installs not configured with TLS - insecure: false # The actual secret object (in this example my-s3-credentials), should be created in every - # namespace which a workflow wants which wants to store its artifacts to S3. If omitted, + # namespace where a workflow needs to store its artifacts to S3. If omitted, # attempts to use IAM role to access the bucket (instead of accessKey/secretKey). accessKeySecret: name: my-s3-credentials @@ -59,24 +76,36 @@ data: key: secretKey # Specifies the container runtime interface to use (default: docker) + # must be one of: docker, kubelet, k8sapi, pns containerRuntimeExecutor: docker # kubelet port when using kubelet executor (default: 10250) kubeletPort: 10250 - # disable the TLS verification of the kubelet executo (default: false) + # disable the TLS verification of the kubelet executor (default: false) kubeletInsecure: false - # executorResources specifies the resource requirements that will be used for the executor - # sidecar/init container. This is useful in clusters which require resources to be specified as - # part of admission control. - executorResources: - requests: - cpu: 0.1 - memory: 64Mi - limits: - cpu: 0.5 - memory: 512Mi + # executor controls how the init and wait container should be customized + # (available since Argo v2.3) + executor: + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 0.1 + memory: 64Mi + limits: + cpu: 0.5 + memory: 512Mi + # args & env allows command line arguments and environment variables to be appended to the + # executor container and is mainly used for development/debugging purposes. + args: + - --loglevel + - debug + - --gloglevel + - "6" + env: + - name: SOME_ENV_VAR + value: "1" # metricsConfig controls the path and port for prometheus metrics metricsConfig: diff --git a/docs/workflow-rbac.md b/docs/workflow-rbac.md new file mode 100644 index 000000000000..18ecc54aacb9 --- /dev/null +++ b/docs/workflow-rbac.md @@ -0,0 +1,44 @@ +# Workfow RBAC + +All pods in a workflow run with the service account specified in `workflow.spec.serviceAccountName`, +or if omitted, the `default` service account of the workflow's namespace. The amount of access which +a workflow needs is dependent on what the workflow needs to do. For example, if your workflow needs +to deploy a resource, then the workflow's service account will require 'create' privileges on that +resource. + +The bare minimum for a workflow to function is outlined below: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: workflow-role +rules: +# pod get/watch is used to identify the container IDs of the current pod +# pod patch is used to annotate the step's outputs back to controller (e.g. artifact location) +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - patch +# logs get/watch are used to get the pods logs for script outputs, and for log archival +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get + - watch +# secrets get is used to retrieve credentials to artifact repository. NOTE: starting n Argo v2.3, +# the API secret access will be removed in favor of volume mounting the secrets to the workflow pod +# (issue #1072) +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +``` diff --git a/errors/errors.go b/errors/errors.go index c21119bb0b10..1fbe662557a7 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -10,12 +10,13 @@ import ( // Externally visible error codes const ( - CodeUnauthorized = "ERR_UNAUTHORIZED" - CodeBadRequest = "ERR_BAD_REQUEST" - CodeForbidden = "ERR_FORBIDDEN" - CodeNotFound = "ERR_NOT_FOUND" - CodeTimeout = "ERR_TIMEOUT" - CodeInternal = "ERR_INTERNAL" + CodeUnauthorized = "ERR_UNAUTHORIZED" + CodeBadRequest = "ERR_BAD_REQUEST" + CodeForbidden = "ERR_FORBIDDEN" + CodeNotFound = "ERR_NOT_FOUND" + CodeNotImplemented = "ERR_NOT_IMPLEMENTED" + CodeTimeout = "ERR_TIMEOUT" + CodeInternal = "ERR_INTERNAL" ) // ArgoError is an error interface that additionally adds support for diff --git a/examples/README.md b/examples/README.md index cd8009f34684..302adeb0839e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,11 +4,13 @@ Argo is an open source project that provides container-native workflows for Kubernetes. Each step in an Argo workflow is defined as a container. -Argo is implemented as a Kubernetes CRD (Custom Resource Definition). As a result, Argo workflows can be managed using kubectl and natively integrates with other Kubernetes services such as volumes, secrets, and RBAC. The new Argo software is lightweight and installs in under a minute but provides complete workflow features including parameter substitution, artifacts, fixtures, loops and recursive workflows. +Argo is implemented as a Kubernetes CRD (Custom Resource Definition). As a result, Argo workflows can be managed using `kubectl` and natively integrates with other Kubernetes services such as volumes, secrets, and RBAC. The new Argo software is light-weight and installs in under a minute, and provides complete workflow features including parameter substitution, artifacts, fixtures, loops and recursive workflows. Many of the Argo examples used in this walkthrough are available at https://github.com/cyrusbiotechnology/argo/tree/master/examples. If you like this project, please give us a star! -## Table of Content +For a complete description of the Argo workflow spec, please refer to https://github.com/argoproj/argo/blob/master/pkg/apis/workflow/v1alpha1/types.go + +## Table of Contents - [Argo CLI](#argo-cli) - [Hello World!](#hello-world) @@ -30,12 +32,13 @@ Many of the Argo examples used in this walkthrough are available at https://gith - [Sidecars](#sidecars) - [Hardwired Artifacts](#hardwired-artifacts) - [Kubernetes Resources](#kubernetes-resources) -- [Docker-in-Docker (aka. DinD) Using Sidecars](#docker-in-docker-aka-dind-using-sidecars) -- [Continuous integration example](#continuous-integration-example) +- [Docker-in-Docker Using Sidecars](#docker-in-docker-using-sidecars) +- [Custom Template Variable Reference](#custom-template-variable-reference) +- [Continuous Integration Example](#continuous-integration-example) ## Argo CLI -In case you want to follow along with this walkthrough, here's a quick overview of the most useful argo CLI commands. +In case you want to follow along with this walkthrough, here's a quick overview of the most useful argo command line interface (CLI) commands. [Install Argo here](https://github.com/cyrusbiotechnology/argo/blob/master/demo.md) @@ -48,24 +51,26 @@ argo logs hello-world-xxx-yyy # get logs from a specific step in a workflow argo delete hello-world-xxx # delete workflow ``` -You can also run workflow specs directly using kubectl but the argo CLI provides syntax checking, nicer output, and requires less typing. +You can also run workflow specs directly using `kubectl` but the Argo CLI provides syntax checking, nicer output, and requires less typing. + ```sh kubectl create -f hello-world.yaml kubectl get wf kubectl get wf hello-world-xxx -kubectl get po --selector=workflows.argoproj.io/workflow=hello-world-xxx --show-all #similar to argo +kubectl get po --selector=workflows.argoproj.io/workflow=hello-world-xxx --show-all # similar to argo kubectl logs hello-world-xxx-yyy -c main kubectl delete wf hello-world-xxx ``` ## Hello World! -Let's start by creating a very simple workflow template to echo "hello world" using the docker/whalesay container image from DockerHub. +Let's start by creating a very simple workflow template to echo "hello world" using the docker/whalesay container image from DockerHub. -You can run this directly from your shell with a simple docker command. -``` +You can run this directly from your shell with a simple docker command: + +```sh bash% docker run docker/whalesay cowsay "hello world" _____________ < hello world > @@ -88,32 +93,33 @@ This message shows that your installation appears to be working correctly. ``` Below, we run the same container on a Kubernetes cluster using an Argo workflow template. -Be sure to read the comments. They provide useful explanations. +Be sure to read the comments as they provide useful explanations. + ```yaml apiVersion: argoproj.io/v1alpha1 -kind: Workflow #new type of k8s spec +kind: Workflow # new type of k8s spec metadata: - generateName: hello-world- #name of workflow spec + generateName: hello-world- # name of the workflow spec spec: - entrypoint: whalesay #invoke the whalesay template + entrypoint: whalesay # invoke the whalesay template templates: - - name: whalesay #name of template + - name: whalesay # name of the template container: image: docker/whalesay command: [cowsay] args: ["hello world"] - resources: #don't use too much resources + resources: # limit the resources limits: memory: 32Mi cpu: 100m ``` -Argo adds a new `kind` of Kubernetes spec called a `Workflow`. -The above spec contains a single `template` called `whalesay` which runs the `docker/whalesay` container and invokes `cowsay "hello world"`. -The `whalesay` template is denoted as the `entrypoint` for the spec. The entrypoint specifies the initial template that should be invoked when the workflow spec is executed by Kubernetes. Being able to specify the entrypoint is more useful when there are more than one template defined in the Kubernetes workflow spec :-) + +Argo adds a new `kind` of Kubernetes spec called a `Workflow`. The above spec contains a single `template` called `whalesay` which runs the `docker/whalesay` container and invokes `cowsay "hello world"`. The `whalesay` template is the `entrypoint` for the spec. The entrypoint specifies the initial template that should be invoked when the workflow spec is executed by Kubernetes. Being able to specify the entrypoint is more useful when there is more than one template defined in the Kubernetes workflow spec. :-) ## Parameters Let's look at a slightly more complex workflow spec with parameters. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -133,28 +139,43 @@ spec: - name: whalesay inputs: parameters: - - name: message #parameter declaration + - name: message # parameter declaration container: # run cowsay with that message input parameter as args image: docker/whalesay command: [cowsay] args: ["{{inputs.parameters.message}}"] ``` -This time, the `whalesay` template takes an input parameter named `message` which is passed as the `args` to the `cowsay` command. In order to reference parameters (e.g. "{{inputs.parameters.message}}"), the parameters must be enclosed in double quotes to escape the curly braces in YAML. + +This time, the `whalesay` template takes an input parameter named `message` that is passed as the `args` to the `cowsay` command. In order to reference parameters (e.g., ``"{{inputs.parameters.message}}"``), the parameters must be enclosed in double quotes to escape the curly braces in YAML. The argo CLI provides a convenient way to override parameters used to invoke the entrypoint. For example, the following command would bind the `message` parameter to "goodbye world" instead of the default "hello world". + ```sh argo submit arguments-parameters.yaml -p message="goodbye world" ``` -Command line parameters can also be used to override the default entrypoint and invoke any template in the workflow spec. For example, if you add a new version of the `whalesay` template called `whalesay-caps` but you don't want to change the default entrypoint, you can invoke this from the command line as follows. +In case of multiple parameters that can be overriten, the argo CLI provides a command to load parameters files in YAML or JSON format. Here is an example of that kind of parameter file: + +```yaml +message: goodbye world +``` + +To run use following command: + +```sh +argo submit arguments-parameters.yaml --parameter-file params.yaml +``` + +Command-line parameters can also be used to override the default entrypoint and invoke any template in the workflow spec. For example, if you add a new version of the `whalesay` template called `whalesay-caps` but you don't want to change the default entrypoint, you can invoke this from the command line as follows: + ```sh argo submit arguments-parameters.yaml --entrypoint whalesay-caps ``` -By using a combination of the `--entrypoint` and `-p` parameters, you can invoke any template in the workflow spec with any parameter that you like. +By using a combination of the `--entrypoint` and `-p` parameters, you can call any template in the workflow spec with any parameter that you like. -The values set in the `spec.arguments.parameters` are globally scoped and can be accessed via `{{workflow.parameters.parameter_name}}`. This can be useful to pass information to multiple steps in a workflow. For example, if you wanted to run your workflows with different logging levels, set in environment of each container, you could have a set up similar to this: +The values set in the `spec.arguments.parameters` are globally scoped and can be accessed via `{{workflow.parameters.parameter_name}}`. This can be useful to pass information to multiple steps in a workflow. For example, if you wanted to run your workflows with different logging levels that are set in the environment of each container, you could have a YAML file similar to this one: ```yaml apiVersion: argoproj.io/v1alpha1 @@ -165,7 +186,7 @@ spec: entrypoint: A arguments: parameters: - - name: log_level + - name: log-level value: INFO templates: @@ -174,22 +195,23 @@ spec: image: containerA env: - name: LOG_LEVEL - value: "{{workflow.parameters.log_level}}" + value: "{{workflow.parameters.log-level}}" command: [runA] - - - name: B - container: - image: containerB - env: - - name: LOG_LEVEL - value: "{{workflow.parameters.log_level}}" - command: [runB] + - name: B + container: + image: containerB + env: + - name: LOG_LEVEL + value: "{{workflow.parameters.log-level}}" + command: [runB] ``` -In this workflow, both steps `A` and `B` would have the same log level set to `INFO` and can easily be changed between workflow submissions using the `-p` flag. +In this workflow, both steps `A` and `B` would have the same log-level set to `INFO` and can easily be changed between workflow submissions using the `-p` flag. ## Steps -In this example, we'll see how to create multi-step workflows as well as how to define more than one template in a workflow spec and how to create nested workflows. Be sure to read the comments. They provide useful explanations. +In this example, we'll see how to create multi-step workflows, how to define more than one template in a workflow spec, and how to create nested workflows. Be sure to read the comments as they provide useful explanations. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -204,19 +226,19 @@ spec: # Instead of just running a container # This template has a sequence of steps steps: - - - name: hello1 #hello1 is run before the following steps + - - name: hello1 # hello1 is run before the following steps template: whalesay arguments: parameters: - name: message value: "hello1" - - - name: hello2a #double dash => run after previous step + - - name: hello2a # double dash => run after previous step template: whalesay arguments: parameters: - name: message value: "hello2a" - - name: hello2b #single dash => run in parallel with previous step + - name: hello2b # single dash => run in parallel with previous step template: whalesay arguments: parameters: @@ -233,11 +255,10 @@ spec: command: [cowsay] args: ["{{inputs.parameters.message}}"] ``` -The above workflow spec prints three different flavors of "hello". -The `hello-hello-hello` template consists of three `steps`. -The first step named `hello1` will be run in sequence whereas the next two steps named `hello2a` and `hello2b` will be run in parallel with each other. -Using the argo CLI command, we can graphically display the execution history of this workflow spec, which shows that the steps named `hello2a` and `hello2b` ran in parallel with each other. -``` + +The above workflow spec prints three different flavors of "hello". The `hello-hello-hello` template consists of three `steps`. The first step named `hello1` will be run in sequence whereas the next two steps named `hello2a` and `hello2b` will be run in parallel with each other. Using the argo CLI command, we can graphically display the execution history of this workflow spec, which shows that the steps named `hello2a` and `hello2b` ran in parallel with each other. + +```sh STEP PODNAME ✔ arguments-parameters-rbm92 ├---✔ hello1 steps-rbm92-2023062412 @@ -247,12 +268,9 @@ STEP PODNAME ## DAG -As an alternative to specifying sequences of steps, you can define the workflow as a graph by specifying the dependencies of each task. -This can be simpler to maintain for complex workflows and allows for maximum parallelism when running tasks. +As an alternative to specifying sequences of steps, you can define the workflow as a directed-acyclic graph (DAG) by specifying the dependencies of each task. This can be simpler to maintain for complex workflows and allows for maximum parallelism when running tasks. -In the following workflow, step `A` runs first, as it has no dependencies. -Once `A` has finished, steps `B` and `C` run in parallel. -Finally, once `B` and `C` have completed, step `D` can run. +In the following workflow, step `A` runs first, as it has no dependencies. Once `A` has finished, steps `B` and `C` run in parallel. Finally, once `B` and `C` have completed, step `D` can run. ```yaml apiVersion: argoproj.io/v1alpha1 @@ -293,8 +311,7 @@ spec: parameters: [{name: message, value: D}] ``` -The dependency graph may have [multiple roots](./dag-multiroot.yaml). -The templates called from a dag or steps template can themselves be dag or steps templates. This can allow for complex workflows to be split into manageable pieces. +The dependency graph may have [multiple roots](./dag-multiroot.yaml). The templates called from a DAG or steps template can themselves be DAG or steps templates. This can allow for complex workflows to be split into manageable pieces. ## Artifacts @@ -304,7 +321,8 @@ You will need to have configured an artifact repository to run this example. When running workflows, it is very common to have steps that generate or consume artifacts. Often, the output artifacts of one step may be used as input artifacts to a subsequent step. -The below workflow spec consists of two steps that run in sequence. The first step named `generate-artifact` will generate an artifact using the `whalesay` template which will be consumed by the second step named `print-message` that consumes the generated artifact. +The below workflow spec consists of two steps that run in sequence. The first step named `generate-artifact` will generate an artifact using the `whalesay` template that will be consumed by the second step named `print-message` that then consumes the generated artifact. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -350,14 +368,14 @@ spec: command: [sh, -c] args: ["cat /tmp/message"] ``` -The `whalesay` template uses the `cowsay` command to generate a file named `/tmp/hello-world.txt`. It then `outputs` this file as an artifact named `hello-art`. In general, the artifact's `path` may be a directory rather than just a file. -The `print-message` template takes an input artifact named `message`, unpacks it at the `path` named `/tmp/message` and then prints the contents of `/tmp/message` using the `cat` command. -The `artifact-example` template passes the `hello-art` artifact generated as an output of the `generate-artifact` step as the `message` input artifact to the `print-message` step. -DAG templates use the tasks prefix to refer to another task, for example `{{tasks.generate-artifact.outputs.artifacts.hello-art}}`. + +The `whalesay` template uses the `cowsay` command to generate a file named `/tmp/hello-world.txt`. It then `outputs` this file as an artifact named `hello-art`. In general, the artifact's `path` may be a directory rather than just a file. The `print-message` template takes an input artifact named `message`, unpacks it at the `path` named `/tmp/message` and then prints the contents of `/tmp/message` using the `cat` command. +The `artifact-example` template passes the `hello-art` artifact generated as an output of the `generate-artifact` step as the `message` input artifact to the `print-message` step. DAG templates use the tasks prefix to refer to another task, for example `{{tasks.generate-artifact.outputs.artifacts.hello-art}}`. ## The Structure of Workflow Specs -We now know enough about the basic components of a workflow spec to review its basic structure. +We now know enough about the basic components of a workflow spec to review its basic structure: + - Kubernetes header including metadata - Spec body - Entrypoint invocation with optionally arguments @@ -372,11 +390,11 @@ We now know enough about the basic components of a workflow spec to review its b To summarize, workflow specs are composed of a set of Argo templates where each template consists of an optional input section, an optional output section and either a container invocation or a list of steps where each step invokes another template. -Note that the controller section of the workflow spec will accept the same options as the controller section of a pod spec, including but not limited to env vars, secrets, and volume mounts. Similarly, for volume claims and volumes. +Note that the controller section of the workflow spec will accept the same options as the controller section of a pod spec, including but not limited to environment variables, secrets, and volume mounts. Similarly, for volume claims and volumes. ## Secrets -Argo supports the same secrets syntax and mechanisms as Kubernetes Pod specs, which allows access to secrets as environment variables or volume mounts. -- https://kubernetes.io/docs/concepts/configuration/secret/ + +Argo supports the same secrets syntax and mechanisms as Kubernetes Pod specs, which allows access to secrets as environment variables or volume mounts. See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/secret/) for more information. ```yaml # To run this example, first create the secret by running: @@ -416,7 +434,9 @@ spec: ``` ## Scripts & Results -Often times, we just want a template that executes a script specified as a here-script (aka. here document) in the workflow spec. + +Often, we just want a template that executes a script specified as a here-script (also known as a `here document`) in the workflow spec. This example shows how to do that: + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -469,13 +489,15 @@ spec: command: [sh, -c] args: ["echo result was: {{inputs.parameters.message}}"] ``` -The `script` keyword allows the specification of the script body using the `source` tag. This creates a temporary file containing the script body and then passes the name of the temporary file as the final parameter to `command`, which should be an interpreter that executes the script body.. + +The `script` keyword allows the specification of the script body using the `source` tag. This creates a temporary file containing the script body and then passes the name of the temporary file as the final parameter to `command`, which should be an interpreter that executes the script body. The use of the `script` feature also assigns the standard output of running the script to a special output parameter named `result`. This allows you to use the result of running the script itself in the rest of the workflow spec. In this example, the result is simply echoed by the print-message template. ## Output Parameters Output parameters provide a general mechanism to use the result of a step as a parameter rather than as an artifact. This allows you to use the result from any type of step, not just a `script`, for conditional tests, loops, and arguments. Output parameters work similarly to `script result` except that the value of the output parameter is set to the contents of a generated file rather than the contents of `stdout`. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -500,12 +522,12 @@ spec: container: image: docker/whalesay:latest command: [sh, -c] - args: ["echo -n hello world > /tmp/hello_world.txt"] #generate the content of hello_world.txt + args: ["echo -n hello world > /tmp/hello_world.txt"] # generate the content of hello_world.txt outputs: parameters: - - name: hello-param #name of output parameter + - name: hello-param # name of output parameter valueFrom: - path: /tmp/hello_world.txt #set the value of hello-param to the contents of this hello-world.txt + path: /tmp/hello_world.txt # set the value of hello-param to the contents of this hello-world.txt - name: print-message inputs: @@ -521,7 +543,8 @@ DAG templates use the tasks prefix to refer to another task, for example `{{task ## Loops -When writing workflows, it is often very useful to be able to iterate over a set of inputs. +When writing workflows, it is often very useful to be able to iterate over a set of inputs as shown in this example: + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -538,9 +561,9 @@ spec: parameters: - name: message value: "{{item}}" - withItems: #invoke whalesay once for each item in parallel - - hello world #item 1 - - goodbye world #item 2 + withItems: # invoke whalesay once for each item in parallel + - hello world # item 1 + - goodbye world # item 2 - name: whalesay inputs: @@ -552,7 +575,8 @@ spec: args: ["{{inputs.parameters.message}}"] ``` -We can also iterate over a sets of items. +We can also iterate over sets of items: + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -588,7 +612,8 @@ spec: args: [/etc/os-release] ``` -We can pass lists of items as parameters. +We can pass lists of items as parameters: + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -598,7 +623,7 @@ spec: entrypoint: loop-param-arg-example arguments: parameters: - - name: os-list #a list of items + - name: os-list # a list of items value: | [ { "image": "debian", "tag": "9.1" }, @@ -621,7 +646,7 @@ spec: value: "{{item.image}}" - name: tag value: "{{item.tag}}" - withParam: "{{inputs.parameters.os-list}}" #parameter specifies the list to iterate over + withParam: "{{inputs.parameters.os-list}}" # parameter specifies the list to iterate over # This template is the same as in the previous example - name: cat-os-release @@ -636,6 +661,7 @@ spec: ``` We can even dynamically generate the list of items to iterate over! + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -678,7 +704,9 @@ spec: ``` ## Conditionals -We also support conditional execution. + +We also support conditional execution as shown in this example: + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -694,10 +722,10 @@ spec: template: flip-coin # evaluate the result in parallel - - name: heads - template: heads #invoke heads template if "heads" + template: heads # call heads template if "heads" when: "{{steps.flip-coin.outputs.result}} == heads" - name: tails - template: tails #invoke tails template if "tails" + template: tails # call tails template if "tails" when: "{{steps.flip-coin.outputs.result}} == tails" # Return heads or tails based on a random number @@ -724,7 +752,9 @@ spec: ``` ## Recursion + Templates can recursively invoke each other! In this variation of the above coin-flip template, we continue to flip coins until it comes up heads. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -740,9 +770,9 @@ spec: template: flip-coin # evaluate the result in parallel - - name: heads - template: heads #invoke heads template if "heads" + template: heads # call heads template if "heads" when: "{{steps.flip-coin.outputs.result}} == heads" - - name: tails #keep flipping coins if "tails" + - name: tails # keep flipping coins if "tails" template: coinflip when: "{{steps.flip-coin.outputs.result}} == tails" @@ -763,7 +793,8 @@ spec: ``` Here's the result of a couple of runs of coinflip for comparison. -``` + +```sh argo get coinflip-recursive-tzcb5 STEP PODNAME MESSAGE @@ -787,17 +818,18 @@ STEP PODNAME MESSAGE └-·-✔ heads coinflip-recursive-tzcb5-4080323273 └-○ tails ``` -In the first run, the coin immediately comes up heads and we stop. -In the second run, the coin comes up tail three times before it finally comes up heads and we stop. + +In the first run, the coin immediately comes up heads and we stop. In the second run, the coin comes up tail three times before it finally comes up heads and we stop. ## Exit handlers -An exit handler is a template that always executes, irrespective of success or failure, at the end of the workflow. +An exit handler is a template that *always* executes, irrespective of success or failure, at the end of the workflow. Some common use cases of exit handlers are: + - cleaning up after a workflow runs -- sending notifications of workflow status (e.g. e-mail/slack) -- posting the pass/fail status to a webhook result (e.g. github build result) +- sending notifications of workflow status (e.g., e-mail/Slack) +- posting the pass/fail status to a webhook result (e.g. GitHub build result) - resubmitting or submitting another workflow ```yaml @@ -807,7 +839,7 @@ metadata: generateName: exit-handlers- spec: entrypoint: intentional-fail - onExit: exit-handler #invoke exit-hander template at end of the workflow + onExit: exit-handler # invoke exit-hander template at end of the workflow templates: # primary workflow template - name: intentional-fail @@ -848,7 +880,9 @@ spec: ``` ## Timeouts -To limit the elapsed time for a workflow, you can set `activeDeadlineSeconds`. + +To limit the elapsed time for a workflow, you can set the variable `activeDeadlineSeconds`. + ```yaml # To enforce a timeout for a container template, specify a value for activeDeadlineSeconds. apiVersion: argoproj.io/v1alpha1 @@ -863,11 +897,13 @@ spec: image: alpine:latest command: [sh, -c] args: ["echo sleeping for 1m; sleep 60; echo done"] - activeDeadlineSeconds: 10 #terminate container template after 10 seconds + activeDeadlineSeconds: 10 # terminate container template after 10 seconds ``` ## Volumes + The following example dynamically creates a volume and then uses the volume in a two step workflow. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -875,14 +911,14 @@ metadata: generateName: volumes-pvc- spec: entrypoint: volumes-pvc-example - volumeClaimTemplates: #define volume, same syntax as k8s Pod spec + volumeClaimTemplates: # define volume, same syntax as k8s Pod spec - metadata: - name: workdir #name of volume claim + name: workdir # name of volume claim spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 1Gi #Gi => 1024 * 1024 * 1024 + storage: 1Gi # Gi => 1024 * 1024 * 1024 templates: - name: volumes-pvc-example @@ -898,7 +934,7 @@ spec: command: [sh, -c] args: ["echo generating message in volume; cowsay hello world | tee /mnt/vol/hello_world.txt"] # Mount workdir volume at /mnt/vol before invoking docker/whalesay - volumeMounts: #same syntax as k8s Pod spec + volumeMounts: # same syntax as k8s Pod spec - name: workdir mountPath: /mnt/vol @@ -908,15 +944,16 @@ spec: command: [sh, -c] args: ["echo getting message from volume; find /mnt/vol; cat /mnt/vol/hello_world.txt"] # Mount workdir volume at /mnt/vol before invoking docker/whalesay - volumeMounts: #same syntax as k8s Pod spec + volumeMounts: # same syntax as k8s Pod spec - name: workdir mountPath: /mnt/vol ``` -Volumes are a very useful way to move large amounts of data from one step in a workflow to another. -Depending on the system, some volumes may be accessible concurrently from multiple steps. + +Volumes are a very useful way to move large amounts of data from one step in a workflow to another. Depending on the system, some volumes may be accessible concurrently from multiple steps. In some cases, you want to access an already existing volume rather than creating/destroying one dynamically. + ```yaml # Define Kubernetes PVC kind: PersistentVolumeClaim @@ -971,7 +1008,9 @@ spec: ``` ## Daemon Containers -Argo workflows can start containers that run in the background (aka. daemon containers) while the workflow itself continues execution. The daemons will be automatically destroyed when the workflow exits the template scope in which the daemon was invoked. Deamons containers are useful for starting up services to be tested or to be used in testing (aka. fixtures). We also find it very useful when running large simulations to spin up a database as a daemon for collecting and organizing the results. The big advantage of daemons compared with sidecars is that their existence can persist across multiple steps or even the entire workflow. + +Argo workflows can start containers that run in the background (also known as `daemon containers`) while the workflow itself continues execution. Note that the daemons will be *automatically destroyed* when the workflow exits the template scope in which the daemon was invoked. Daemon containers are useful for starting up services to be tested or to be used in testing (e.g., fixtures). We also find it very useful when running large simulations to spin up a database as a daemon for collecting and organizing the results. The big advantage of daemons compared with sidecars is that their existence can persist across multiple steps or even the entire workflow. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -983,35 +1022,35 @@ spec: - name: daemon-example steps: - - name: influx - template: influxdb #start an influxdb as a daemon (see the influxdb template spec below) + template: influxdb # start an influxdb as a daemon (see the influxdb template spec below) - - - name: init-database #initialize influxdb + - - name: init-database # initialize influxdb template: influxdb-client arguments: parameters: - name: cmd value: curl -XPOST 'http://{{steps.influx.ip}}:8086/query' --data-urlencode "q=CREATE DATABASE mydb" - - - name: producer-1 #add entries to influxdb + - - name: producer-1 # add entries to influxdb template: influxdb-client arguments: parameters: - name: cmd value: for i in $(seq 1 20); do curl -XPOST 'http://{{steps.influx.ip}}:8086/write?db=mydb' -d "cpu,host=server01,region=uswest load=$i" ; sleep .5 ; done - - name: producer-2 #add entries to influxdb + - name: producer-2 # add entries to influxdb template: influxdb-client arguments: parameters: - name: cmd value: for i in $(seq 1 20); do curl -XPOST 'http://{{steps.influx.ip}}:8086/write?db=mydb' -d "cpu,host=server02,region=uswest load=$((RANDOM % 100))" ; sleep .5 ; done - - name: producer-3 #add entries to influxdb + - name: producer-3 # add entries to influxdb template: influxdb-client arguments: parameters: - name: cmd value: curl -XPOST 'http://{{steps.influx.ip}}:8086/write?db=mydb' -d 'cpu,host=server03,region=useast load=15.4' - - - name: consumer #consume intries from influxdb + - - name: consumer # consume intries from influxdb template: influxdb-client arguments: parameters: @@ -1019,11 +1058,11 @@ spec: value: curl --silent -G http://{{steps.influx.ip}}:8086/query?pretty=true --data-urlencode "db=mydb" --data-urlencode "q=SELECT * FROM cpu" - name: influxdb - daemon: true #start influxdb as a daemon + daemon: true # start influxdb as a daemon container: image: influxdb:1.2 - restartPolicy: Always #restart container if it fails - readinessProbe: #wait for readinessProbe to succeed + restartPolicy: Always # restart container if it fails + readinessProbe: # wait for readinessProbe to succeed httpGet: path: /ping port: 8086 @@ -1045,8 +1084,9 @@ spec: DAG templates use the tasks prefix to refer to another task, for example `{{tasks.influx.ip}}`. ## Sidecars -A sidecar is another container that executes concurrently in the same pod as the "main" container and is useful -in creating multi-container pods. + +A sidecar is another container that executes concurrently in the same pod as the main container and is useful in creating multi-container pods. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -1066,10 +1106,13 @@ spec: - name: nginx image: nginx:1.13 ``` -In the above example, we create a sidecar container that runs nginx as a simple web server. The order in which containers may come up is random. This is why the 'main' container polls the nginx container until it is ready to service requests. This is a good design pattern when designing multi-container systems. Always wait for any services you need to come up before running your main code. + +In the above example, we create a sidecar container that runs nginx as a simple web server. The order in which containers come up is random, so in this example the main container polls the nginx container until it is ready to service requests. This is a good design pattern when designing multi-container systems: always wait for any services you need to come up before running your main code. ## Hardwired Artifacts -With Argo, you can use any container image that you like to generate any kind of artifact. In practice, however, we find certain types of artifacts are very common and provide a more convenient way to generate and use these artifacts. In particular, we have "hardwired" support for git, http and s3 artifacts. + +With Argo, you can use any container image that you like to generate any kind of artifact. In practice, however, we find certain types of artifacts are very common, so there is built-in support for git, http, and s3 artifacts. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -1113,10 +1156,10 @@ spec: args: ["ls -l /src /bin/kubectl /s3"] ``` - ## Kubernetes Resources In many cases, you will want to manage Kubernetes resources from Argo workflows. The resource template allows you to create, delete or updated any type of Kubernetes resource. + ```yaml # in a workflow. The resource template type accepts any k8s manifest # (including CRDs) and can perform any kubectl action against it (e.g. create, @@ -1129,8 +1172,8 @@ spec: entrypoint: pi-tmpl templates: - name: pi-tmpl - resource: #indicates that this is a resource template - action: create #can be any kubectl action (e.g. create, delete, apply, patch) + resource: # indicates that this is a resource template + action: create # can be any kubectl action (e.g. create, delete, apply, patch) # The successCondition and failureCondition are optional expressions. # If failureCondition is true, the step is considered failed. # If successCondition is true, the step is considered successful. @@ -1160,9 +1203,43 @@ spec: Resources created in this way are independent of the workflow. If you want the resource to be deleted when the workflow is deleted then you can use [Kubernetes garbage collection](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/) with the workflow resource as an owner reference ([example](./k8s-owner-reference.yaml)). -## Docker-in-Docker (aka. DinD) Using Sidecars -An application of sidecars is to implement DinD (Docker-in-Docker). -DinD is useful when you want to run Docker commands from inside a container. For example, you may want to build and push a container image from inside your build container. In the following example, we use the docker:dind container to run a Docker daemon in a sidecar and give the main container access to the daemon. +**Note:** +When patching, the resource will accept another attribute, `mergeStrategy`, which can either be `strategic`, `merge`, or `json`. If this attribute is not supplied, it will default to `strategic`. Keep in mind that Custom Resources cannot be patched with `strategic`, so a different strategy must be chosen. For example, suppose you have the [CronTab CustomResourceDefinition](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#create-a-customresourcedefinition) defined, and the following instance of a CronTab: + +```yaml +apiVersion: "stable.example.com/v1" +kind: CronTab +spec: + cronSpec: "* * * * */5" + image: my-awesome-cron-image +``` + +This Crontab can be modified using the following Argo Workflow: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: k8s-patch- +spec: + entrypoint: cront-tmpl + templates: + - name: cront-tmpl + resource: + action: patch + mergeStrategy: merge # Must be one of [strategic merge json] + manifest: | + apiVersion: "stable.example.com/v1" + kind: CronTab + spec: + cronSpec: "* * * * */10" + image: my-awesome-cron-image +``` + +## Docker-in-Docker Using Sidecars + +An application of sidecars is to implement Docker-in-Docker (DinD). DinD is useful when you want to run Docker commands from inside a container. For example, you may want to build and push a container image from inside your build container. In the following example, we use the docker:dind container to run a Docker daemon in a sidecar and give the main container access to the daemon. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -1177,13 +1254,13 @@ spec: command: [sh, -c] args: ["until docker ps; do sleep 3; done; docker run --rm debian:latest cat /etc/os-release"] env: - - name: DOCKER_HOST #the docker daemon can be access on the standard port on localhost + - name: DOCKER_HOST # the docker daemon can be access on the standard port on localhost value: 127.0.0.1 sidecars: - name: dind - image: docker:17.10-dind #Docker already provides an image for running a Docker daemon + image: docker:17.10-dind # Docker already provides an image for running a Docker daemon securityContext: - privileged: true #the Docker daemon can only run in a privileged container + privileged: true # the Docker daemon can only run in a privileged container # mirrorVolumeMounts will mount the same volumes specified in the main container # to the sidecar (including artifacts), at the same mountPaths. This enables # dind daemon to (partially) see the same filesystem as the main container in @@ -1191,7 +1268,49 @@ spec: mirrorVolumeMounts: true ``` -## Continuous integration example +## Custom Template Variable Reference + +In this example, we can see how we can use the other template language variable reference (E.g: Jinja) in Argo workflow template. +Argo will validate and resolve only the variable that starts with Argo allowed prefix +{***"item", "steps", "inputs", "outputs", "workflow", "tasks"***} + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: custom-template-variable- +spec: + entrypoint: hello-hello-hello + + templates: + - name: hello-hello-hello + steps: + - - name: hello1 + template: whalesay + arguments: + parameters: [{name: message, value: "hello1"}] + - - name: hello2a + template: whalesay + arguments: + parameters: [{name: message, value: "hello2a"}] + - name: hello2b + template: whalesay + arguments: + parameters: [{name: message, value: "hello2b"}] + + - name: whalesay + inputs: + parameters: + - name: message + container: + image: docker/whalesay + command: [cowsay] + args: ["{{user.username}}"] + +``` + +## Continuous Integration Example + Continuous integration is a popular application for workflows. Currently, Argo does not provide event triggers for automatically kicking off your CI jobs, but we plan to do so in the near future. Until then, you can easily write a cron job that checks for new commits and kicks off the needed workflow, or use your existing Jenkins server to kick off the workflow. A good example of a CI workflow spec is provided at https://github.com/cyrusbiotechnology/argo/tree/master/examples/influxdb-ci.yaml. Because it just uses the concepts that we've already covered and is somewhat long, we don't go into details here. diff --git a/examples/artifact-disable-archive.yaml b/examples/artifact-disable-archive.yaml new file mode 100644 index 000000000000..444b01ac5b53 --- /dev/null +++ b/examples/artifact-disable-archive.yaml @@ -0,0 +1,51 @@ +# This example demonstrates the ability to disable the default behavior of archiving (tar.gz) +# when saving output artifacts. For directories, when archive is set to none, files in directory +# will be copied recursively in the case of S3. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: artifact-disable-archive- +spec: + entrypoint: artifact-disable-archive + templates: + - name: artifact-disable-archive + steps: + - - name: generate-artifact + template: whalesay + - - name: consume-artifact + template: print-message + arguments: + artifacts: + - name: etc + from: "{{steps.generate-artifact.outputs.artifacts.etc}}" + - name: hello-txt + from: "{{steps.generate-artifact.outputs.artifacts.hello-txt}}" + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay hello world | tee /tmp/hello_world.txt ; sleep 1"] + outputs: + artifacts: + - name: etc + path: /etc + archive: + none: {} + - name: hello-txt + path: /tmp/hello_world.txt + archive: + none: {} + + - name: print-message + inputs: + artifacts: + - name: etc + path: /tmp/etc + - name: hello-txt + path: /tmp/hello.txt + container: + image: alpine:latest + command: [sh, -c] + args: + - cat /tmp/hello.txt && cd /tmp/etc && find . diff --git a/examples/artifact-passing.yaml b/examples/artifact-passing.yaml index dd301b9ac116..90fdeacd3728 100644 --- a/examples/artifact-passing.yaml +++ b/examples/artifact-passing.yaml @@ -22,7 +22,7 @@ spec: container: image: docker/whalesay:latest command: [sh, -c] - args: ["cowsay hello world | tee /tmp/hello_world.txt"] + args: ["sleep 1; cowsay hello world | tee /tmp/hello_world.txt"] outputs: artifacts: - name: hello-art diff --git a/examples/artifact-path-placeholders.yaml b/examples/artifact-path-placeholders.yaml new file mode 100644 index 000000000000..3371b5e893c5 --- /dev/null +++ b/examples/artifact-path-placeholders.yaml @@ -0,0 +1,40 @@ +# This example demonstrates the how to refer to input and output artifact paths. +# Referring to the path instead of copy/pasting it prevents errors when paths change. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: artifact-path-placeholders- +spec: + entrypoint: head-lines + arguments: + parameters: + - name: lines-count + value: 3 + artifacts: + - name: text + raw: + data: | + 1 + 2 + 3 + 4 + 5 + templates: + - name: head-lines + inputs: + parameters: + - name: lines-count + artifacts: + - name: text + path: /inputs/text/data + outputs: + parameters: + - name: actual-lines-count + valueFrom: + path: /outputs/actual-lines-count/data + artifacts: + - name: text + path: /outputs/text/data + container: + image: busybox + command: [sh, -c, 'head -n {{inputs.parameters.lines-count}} <"{{inputs.artifacts.text.path}}" | tee "{{outputs.artifacts.text.path}}" | wc -l > "{{outputs.parameters.actual-lines-count.path}}"'] diff --git a/examples/ci-output-artifact.yaml b/examples/ci-output-artifact.yaml index fababef1bb13..591fec3cea39 100644 --- a/examples/ci-output-artifact.yaml +++ b/examples/ci-output-artifact.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - generateName: ci-example- + generateName: ci-output-artifact- spec: entrypoint: ci-example # a temporary volume, named workdir, will be used as a working @@ -75,7 +75,10 @@ spec: - name: release-artifact container: - image: debian:9.4 + image: alpine:3.8 + volumeMounts: + - name: workdir + mountPath: /go outputs: artifacts: - name: release diff --git a/examples/continue-on-fail.yaml b/examples/continue-on-fail.yaml new file mode 100644 index 000000000000..7681e99c597f --- /dev/null +++ b/examples/continue-on-fail.yaml @@ -0,0 +1,36 @@ +# Example on specifying parallelism on the outer workflow and limiting the number of its +# children workflowss to be run at the same time. +# +# If the parallelism of A is 1, the four steps of seq-step will run sequentially. + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: continue-on-fail- +spec: + entrypoint: workflow-ignore + templates: + - name: workflow-ignore + steps: + - - name: A + template: whalesay + - - name: B + template: whalesay + - name: C + template: intentional-fail + continueOn: + failed: true + - - name: D + template: whalesay + + - name: whalesay + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["hello world"] + + - name: intentional-fail + container: + image: alpine:latest + command: [sh, -c] + args: ["echo intentional failure; exit 1"] diff --git a/examples/dag-continue-on-fail.yaml b/examples/dag-continue-on-fail.yaml new file mode 100644 index 000000000000..dc9600babb52 --- /dev/null +++ b/examples/dag-continue-on-fail.yaml @@ -0,0 +1,44 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: dag-contiue-on-fail- +spec: + entrypoint: workflow + templates: + - name: workflow + dag: + tasks: + - name: A + template: whalesay + - name: B + dependencies: [A] + template: intentional-fail + continueOn: + failed: true + - name: C + dependencies: [A] + template: whalesay + - name: D + dependencies: [B, C] + template: whalesay + - name: E + dependencies: [A] + template: intentional-fail + - name: F + dependencies: [A] + template: whalesay + - name: G + dependencies: [E, F] + template: whalesay + + - name: whalesay + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["hello world"] + + - name: intentional-fail + container: + image: alpine:latest + command: [sh, -c] + args: ["echo intentional failure; exit 1"] \ No newline at end of file diff --git a/examples/dns-config.yaml b/examples/dns-config.yaml new file mode 100644 index 000000000000..35a621864827 --- /dev/null +++ b/examples/dns-config.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow # new type of k8s spec +metadata: + generateName: test-dns-config- # name of the workflow spec +spec: + entrypoint: whalesay # invoke the whalesay template + templates: + - name: whalesay # name of the template + container: + image: docker/whalesay + command: [cowsay] + args: ["hello world"] + resources: # limit the resources + limits: + memory: 32Mi + cpu: 100m + dnsConfig: + nameservers: + - 1.2.3.4 + options: + - name: ndots + value: "2" \ No newline at end of file diff --git a/examples/global-outputs.yaml b/examples/global-outputs.yaml index e621b7c1fb5f..f2a270aa6141 100644 --- a/examples/global-outputs.yaml +++ b/examples/global-outputs.yaml @@ -19,7 +19,7 @@ spec: container: image: alpine:3.7 command: [sh, -c] - args: ["echo -n hello world > /tmp/hello_world.txt"] + args: ["sleep 1; echo -n hello world > /tmp/hello_world.txt"] outputs: parameters: # export a global parameter. The parameter will be programatically available in the completed diff --git a/examples/hdfs-artifact.yaml b/examples/hdfs-artifact.yaml new file mode 100644 index 000000000000..0031b756387f --- /dev/null +++ b/examples/hdfs-artifact.yaml @@ -0,0 +1,81 @@ +# This example demonstrates the use of hdfs as the store for artifacts. This example assumes the following: +# 1. you have hdfs running in the same namespace as where this workflow will be run and you have created a repo with the name "generic-local" +# 2. you have created a kubernetes secret for storing hdfs username/password. To create kubernetes secret required for this example, +# run the following command: +# $ kubectl create secret generic my-hdfs-credentials --from-literal=username= --from-literal=password= + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hdfs-artifact- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + - - name: consume-artifact + template: print-message + arguments: + artifacts: + - name: message + from: "{{steps.generate-artifact.outputs.artifacts.hello-art}}" + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay hello world | tee /tmp/hello_world.txt"] + outputs: + artifacts: + - name: hello-art + path: /tmp/hello_world.txt + hdfs: + addresses: + - my-hdfs-namenode-0.my-hdfs-namenode.default.svc.cluster.local:8020 + - my-hdfs-namenode-1.my-hdfs-namenode.default.svc.cluster.local:8020 + path: "/tmp/argo/foo" + hdfsUser: root + force: true + # krbCCacheSecret: + # name: krb + # key: krb5cc_0 + # krbKeytabSecret: + # name: krb + # key: user1.keytab + # krbUsername: "user1" + # krbRealm: "MYCOMPANY.COM" + # krbConfigConfigMap: + # name: my-hdfs-krb5-config + # key: krb5.conf + # krbServicePrincipalName: hdfs/_HOST + + - name: print-message + inputs: + artifacts: + - name: message + path: /tmp/message + hdfs: + addresses: + - my-hdfs-namenode-0.my-hdfs-namenode.default.svc.cluster.local:8020 + - my-hdfs-namenode-1.my-hdfs-namenode.default.svc.cluster.local:8020 + path: "/tmp/argo/foo" + hdfsUser: root + force: true + # krbCCacheSecret: + # name: krb + # key: krb5cc_0 + # krbKeytabSecret: + # name: krb + # key: user1.keytab + # krbUsername: "user1" + # krbRealm: "MYCOMPANY.COM" + # krbConfigConfigMap: + # name: my-hdfs-krb5-config + # key: krb5.conf + # krbServicePrincipalName: hdfs/_HOST + container: + image: alpine:latest + command: [sh, -c] + args: ["cat /tmp/message"] diff --git a/examples/influxdb-ci.yaml b/examples/influxdb-ci.yaml index 121a6fd2d8a0..d26c765fdd78 100644 --- a/examples/influxdb-ci.yaml +++ b/examples/influxdb-ci.yaml @@ -194,6 +194,10 @@ spec: - name: influxd path: /app daemon: true + outputs: + artifacts: + - name: data + path: /var/lib/influxdb/data container: image: debian:9.4 readinessProbe: diff --git a/examples/init-container.yaml b/examples/init-container.yaml new file mode 100644 index 000000000000..a113fce55f18 --- /dev/null +++ b/examples/init-container.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: init-container- +spec: + entrypoint: init-container-example + templates: + - name: init-container-example + container: + image: alpine:latest + command: ["echo", "bye"] + volumeMounts: + - name: foo + mountPath: /foo + initContainers: + - name: hello + image: alpine:latest + command: ["echo", "hello"] + mirrorVolumeMounts: true + volumes: + - name: foo + emptyDir: diff --git a/examples/output-parameter.yaml b/examples/output-parameter.yaml index c9ccf686955f..d15f30f466ce 100644 --- a/examples/output-parameter.yaml +++ b/examples/output-parameter.yaml @@ -32,7 +32,7 @@ spec: container: image: docker/whalesay:latest command: [sh, -c] - args: ["echo -n hello world > /tmp/hello_world.txt"] + args: ["sleep 1; echo -n hello world > /tmp/hello_world.txt"] outputs: parameters: - name: hello-param diff --git a/examples/parallelism-nested-dag.yaml b/examples/parallelism-nested-dag.yaml new file mode 100644 index 000000000000..bcc7bd6ca064 --- /dev/null +++ b/examples/parallelism-nested-dag.yaml @@ -0,0 +1,87 @@ +# Example on specifying parallelism on the outer DAG and limiting the number of its +# children DAGs to be run at the same time. +# +# As the parallelism of A is 2, only two of the three DAGs (b2, b3, b4) will start +# running after b1 is finished, and the left DAG will run after either one is finished. + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: parallelism-nested-dag- +spec: + entrypoint: A + templates: + - name: A + parallelism: 2 + dag: + tasks: + - name: b1 + template: B + arguments: + parameters: + - name: msg + value: "1" + - name: b2 + template: B + dependencies: [b1] + arguments: + parameters: + - name: msg + value: "2" + - name: b3 + template: B + dependencies: [b1] + arguments: + parameters: + - name: msg + value: "3" + - name: b4 + template: B + dependencies: [b1] + arguments: + parameters: + - name: msg + value: "4" + - name: b5 + template: B + dependencies: [b2, b3, b4] + arguments: + parameters: + - name: msg + value: "5" + + - name: B + inputs: + parameters: + - name: msg + dag: + tasks: + - name: c1 + template: one-job + arguments: + parameters: + - name: msg + value: "{{inputs.parameters.msg}} c1" + - name: c2 + template: one-job + dependencies: [c1] + arguments: + parameters: + - name: msg + value: "{{inputs.parameters.msg}} c2" + - name: c3 + template: one-job + dependencies: [c1] + arguments: + parameters: + - name: msg + value: "{{inputs.parameters.msg}} c3" + + - name: one-job + inputs: + parameters: + - name: msg + container: + image: alpine + command: ['/bin/sh', '-c'] + args: ["echo {{inputs.parameters.msg}}; sleep 10"] diff --git a/examples/parallelism-nested-workflow.yaml b/examples/parallelism-nested-workflow.yaml new file mode 100644 index 000000000000..5cba4de3391b --- /dev/null +++ b/examples/parallelism-nested-workflow.yaml @@ -0,0 +1,52 @@ +# Example on specifying parallelism on the outer workflow and limiting the number of its +# children workflowss to be run at the same time. +# +# As the parallelism of A is 1, the four steps of seq-step will run sequentially. + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: parallelism-nested-workflow- +spec: + arguments: + parameters: + - name: seq-list + value: | + ["a","b","c","d"] + entrypoint: A + templates: + - name: A + parallelism: 1 + inputs: + parameters: + - name: seq-list + steps: + - - name: seq-step + template: B + arguments: + parameters: + - name: seq-id + value: "{{item}}" + withParam: "{{inputs.parameters.seq-list}}" + + - name: B + inputs: + parameters: + - name: seq-id + steps: + - - name: jobs + template: one-job + arguments: + parameters: + - name: seq-id + value: "{{inputs.parameters.seq-id}}" + withParam: "[1, 2]" + + - name: one-job + inputs: + parameters: + - name: seq-id + container: + image: alpine + command: ['/bin/sh', '-c'] + args: ["echo {{inputs.parameters.seq-id}}; sleep 30"] diff --git a/examples/parameter-aggregation-dag.yaml b/examples/parameter-aggregation-dag.yaml index 49bc3bc6a24f..3a534e153bf0 100644 --- a/examples/parameter-aggregation-dag.yaml +++ b/examples/parameter-aggregation-dag.yaml @@ -49,6 +49,7 @@ spec: command: [sh, -xc] args: - | + sleep 1 && echo {{inputs.parameters.num}} > /tmp/num && if [ $(({{inputs.parameters.num}}%2)) -eq 0 ]; then echo "even" > /tmp/even; diff --git a/examples/parameter-aggregation.yaml b/examples/parameter-aggregation.yaml index f7df1f7f053a..4baec52e8af1 100644 --- a/examples/parameter-aggregation.yaml +++ b/examples/parameter-aggregation.yaml @@ -46,6 +46,7 @@ spec: command: [sh, -xc] args: - | + sleep 1 && echo {{inputs.parameters.num}} > /tmp/num && if [ $(({{inputs.parameters.num}}%2)) -eq 0 ]; then echo "even" > /tmp/even; diff --git a/examples/sidecar-dind.yaml b/examples/sidecar-dind.yaml index 467bb101dade..7bf8b67998c6 100644 --- a/examples/sidecar-dind.yaml +++ b/examples/sidecar-dind.yaml @@ -19,7 +19,7 @@ spec: value: 127.0.0.1 sidecars: - name: dind - image: docker:17.10-dind + image: docker:18.09.4-dind securityContext: privileged: true # mirrorVolumeMounts will mount the same volumes specified in the main container diff --git a/gometalinter.json b/gometalinter.json index 408aa6a9c98a..42b62bf2758f 100644 --- a/gometalinter.json +++ b/gometalinter.json @@ -19,6 +19,7 @@ ], "Exclude": [ "pkg/client", - "vendor/" + "vendor/", + ".*warning.*fmt.Fprint" ] } diff --git a/hack/ssh_known_hosts b/hack/ssh_known_hosts new file mode 100644 index 000000000000..31a7bae3fce5 --- /dev/null +++ b/hack/ssh_known_hosts @@ -0,0 +1,8 @@ +# This file was automatically generated. DO NOT EDIT +bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== +github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== +gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY= +gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf +gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9 +ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H +vs-ssh.visualstudio.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H diff --git a/hack/update-manifests.sh b/hack/update-manifests.sh index b24787e3f489..e73c111ddc61 100755 --- a/hack/update-manifests.sh +++ b/hack/update-manifests.sh @@ -1,12 +1,21 @@ -#!/bin/sh +#!/bin/sh -x -e -IMAGE_NAMESPACE=${IMAGE_NAMESPACE:='argoproj'} -IMAGE_TAG=${IMAGE_TAG:='latest'} +SRCROOT="$( CDPATH='' cd -- "$(dirname "$0")/.." && pwd -P )" +AUTOGENMSG="# This is an auto-generated file. DO NOT EDIT" -autogen_warning="# This is an auto-generated file. DO NOT EDIT" +IMAGE_NAMESPACE="${IMAGE_NAMESPACE:-argoproj}" +IMAGE_TAG="${IMAGE_TAG:-latest}" -echo $autogen_warning > manifests/install.yaml -kustomize build manifests/cluster-install >> manifests/install.yaml +cd ${SRCROOT}/manifests/base && kustomize edit set image \ + argoproj/workflow-controller=${IMAGE_NAMESPACE}/workflow-controller:${IMAGE_TAG} \ + argoproj/argoui=${IMAGE_NAMESPACE}/argoui:${IMAGE_TAG} -echo $autogen_warning > manifests/namespace-install.yaml -kustomize build manifests/namespace-install >> manifests/namespace-install.yaml +echo "${AUTOGENMSG}" > "${SRCROOT}/manifests/install.yaml" +kustomize build "${SRCROOT}/manifests/cluster-install" >> "${SRCROOT}/manifests/install.yaml" +sed -i.bak "s@- .*/argoexec:.*@- ${IMAGE_NAMESPACE}/argoexec:${IMAGE_TAG}@" "${SRCROOT}/manifests/install.yaml" +rm -f "${SRCROOT}/manifests/install.yaml.bak" + +echo "${AUTOGENMSG}" > "${SRCROOT}/manifests/namespace-install.yaml" +kustomize build "${SRCROOT}/manifests/namespace-install" >> "${SRCROOT}/manifests/namespace-install.yaml" +sed -i.bak "s@- .*/argoexec:.*@- ${IMAGE_NAMESPACE}/argoexec:${IMAGE_TAG}@" "${SRCROOT}/manifests/namespace-install.yaml" +rm -f "${SRCROOT}/manifests/namespace-install.yaml.bak" diff --git a/hack/update-ssh-known-hosts.sh b/hack/update-ssh-known-hosts.sh new file mode 100755 index 000000000000..aa74c6489add --- /dev/null +++ b/hack/update-ssh-known-hosts.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +KNOWN_HOSTS_FILE=$(dirname "$0")/ssh_known_hosts +HEADER="# This file was automatically generated. DO NOT EDIT" +echo "$HEADER" > $KNOWN_HOSTS_FILE +ssh-keyscan github.com gitlab.com bitbucket.org ssh.dev.azure.com vs-ssh.visualstudio.com | sort -u >> $KNOWN_HOSTS_FILE +chmod 0644 $KNOWN_HOSTS_FILE + +# Public SSH keys can be verified at the following URLs: +# - github.com: https://help.github.com/articles/github-s-ssh-key-fingerprints/ +# - gitlab.com: https://docs.gitlab.com/ee/user/gitlab_com/#ssh-host-keys-fingerprints +# - bitbucket.org: https://confluence.atlassian.com/bitbucket/ssh-keys-935365775.html +# - ssh.dev.azure.com, vs-ssh.visualstudio.com: https://docs.microsoft.com/en-us/azure/devops/repos/git/use-ssh-keys-to-authenticate?view=azure-devops +diff - <(ssh-keygen -l -f $KNOWN_HOSTS_FILE | sort -k 3) < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pkg/apis/workflow/v1alpha1/openapi_generated.go b/pkg/apis/workflow/v1alpha1/openapi_generated.go index 7d4bf39a4c4e..48e4013a7fe9 100644 --- a/pkg/apis/workflow/v1alpha1/openapi_generated.go +++ b/pkg/apis/workflow/v1alpha1/openapi_generated.go @@ -19,6 +19,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactLocation": schema_pkg_apis_workflow_v1alpha1_ArtifactLocation(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact": schema_pkg_apis_workflow_v1alpha1_ArtifactoryArtifact(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryAuth": schema_pkg_apis_workflow_v1alpha1_ArtifactoryAuth(ref), + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ContinueOn": schema_pkg_apis_workflow_v1alpha1_ContinueOn(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.DAGTask": schema_pkg_apis_workflow_v1alpha1_DAGTask(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.DAGTemplate": schema_pkg_apis_workflow_v1alpha1_DAGTemplate(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ExceptionCondition": schema_pkg_apis_workflow_v1alpha1_ExceptionCondition(ref), @@ -26,6 +27,9 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GCSArtifact": schema_pkg_apis_workflow_v1alpha1_GCSArtifact(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GCSBucket": schema_pkg_apis_workflow_v1alpha1_GCSBucket(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GitArtifact": schema_pkg_apis_workflow_v1alpha1_GitArtifact(ref), + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HDFSArtifact": schema_pkg_apis_workflow_v1alpha1_HDFSArtifact(ref), + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HDFSConfig": schema_pkg_apis_workflow_v1alpha1_HDFSConfig(ref), + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HDFSKrbConfig": schema_pkg_apis_workflow_v1alpha1_HDFSKrbConfig(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact": schema_pkg_apis_workflow_v1alpha1_HTTPArtifact(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Inputs": schema_pkg_apis_workflow_v1alpha1_Inputs(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Item": schema_pkg_apis_workflow_v1alpha1_Item(ref), @@ -40,10 +44,10 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.S3Bucket": schema_pkg_apis_workflow_v1alpha1_S3Bucket(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ScriptTemplate": schema_pkg_apis_workflow_v1alpha1_ScriptTemplate(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sequence": schema_pkg_apis_workflow_v1alpha1_Sequence(ref), - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sidecar": schema_pkg_apis_workflow_v1alpha1_Sidecar(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.SuspendTemplate": schema_pkg_apis_workflow_v1alpha1_SuspendTemplate(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.TarStrategy": schema_pkg_apis_workflow_v1alpha1_TarStrategy(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Template": schema_pkg_apis_workflow_v1alpha1_Template(ref), + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.UserContainer": schema_pkg_apis_workflow_v1alpha1_UserContainer(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ValueFrom": schema_pkg_apis_workflow_v1alpha1_ValueFrom(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Workflow": schema_pkg_apis_workflow_v1alpha1_Workflow(ref), "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.WorkflowList": schema_pkg_apis_workflow_v1alpha1_WorkflowList(ref), @@ -181,6 +185,12 @@ func schema_pkg_apis_workflow_v1alpha1_Artifact(ref common.ReferenceCallback) co Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact"), }, }, + "hdfs": { + SchemaProps: spec.SchemaProps{ + Description: "HDFS contains HDFS artifact location details", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HDFSArtifact"), + }, + }, "raw": { SchemaProps: spec.SchemaProps{ Description: "Raw contains raw artifact location details", @@ -206,12 +216,19 @@ func schema_pkg_apis_workflow_v1alpha1_Artifact(ref common.ReferenceCallback) co Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArchiveStrategy"), }, }, + "optional": { + SchemaProps: spec.SchemaProps{ + Description: "Make Artifacts optional, if Artifacts doesn't generate or exist", + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"name"}, }, }, Dependencies: []string{ - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArchiveStrategy", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GCSArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArchiveStrategy", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GCSArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HDFSArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, } } @@ -252,6 +269,12 @@ func schema_pkg_apis_workflow_v1alpha1_ArtifactLocation(ref common.ReferenceCall Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact"), }, }, + "hdfs": { + SchemaProps: spec.SchemaProps{ + Description: "HDFS contains HDFS artifact location details", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HDFSArtifact"), + }, + }, "raw": { SchemaProps: spec.SchemaProps{ Description: "Raw contains raw artifact location details", @@ -268,7 +291,7 @@ func schema_pkg_apis_workflow_v1alpha1_ArtifactLocation(ref common.ReferenceCall }, }, Dependencies: []string{ - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GCSArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GCSArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HDFSArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, } } @@ -332,6 +355,31 @@ func schema_pkg_apis_workflow_v1alpha1_ArtifactoryAuth(ref common.ReferenceCallb } } +func schema_pkg_apis_workflow_v1alpha1_ContinueOn(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ContinueOn defines if a workflow should continue even if a task or step fails/errors. It can be specified if the workflow should continue when the pod errors, fails or both.", + Properties: map[string]spec.Schema{ + "error": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + "failed": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + }, + }, + Dependencies: []string{}, + } +} + func schema_pkg_apis_workflow_v1alpha1_DAGTask(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -405,12 +453,18 @@ func schema_pkg_apis_workflow_v1alpha1_DAGTask(ref common.ReferenceCallback) com Format: "", }, }, + "continueOn": { + SchemaProps: spec.SchemaProps{ + Description: "ContinueOn makes argo to proceed with the following step even if this step fails. Errors and Failed states can be specified", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ContinueOn"), + }, + }, }, Required: []string{"name", "template"}, }, }, Dependencies: []string{ - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Arguments", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Item", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sequence"}, + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Arguments", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ContinueOn", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Item", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sequence"}, } } @@ -543,6 +597,11 @@ func schema_pkg_apis_workflow_v1alpha1_GCSArtifact(ref common.ReferenceCallback) Format: "", }, }, + "credentialsSecret": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, "key": { SchemaProps: spec.SchemaProps{ Type: []string{"string"}, @@ -550,10 +609,11 @@ func schema_pkg_apis_workflow_v1alpha1_GCSArtifact(ref common.ReferenceCallback) }, }, }, - Required: []string{"bucket", "key"}, + Required: []string{"bucket", "credentialsSecret", "key"}, }, }, - Dependencies: []string{}, + Dependencies: []string{ + "k8s.io/api/core/v1.SecretKeySelector"}, } } @@ -569,11 +629,17 @@ func schema_pkg_apis_workflow_v1alpha1_GCSBucket(ref common.ReferenceCallback) c Format: "", }, }, + "credentialsSecret": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, }, - Required: []string{"bucket"}, + Required: []string{"bucket", "credentialsSecret"}, }, }, - Dependencies: []string{}, + Dependencies: []string{ + "k8s.io/api/core/v1.SecretKeySelector"}, } } @@ -615,6 +681,13 @@ func schema_pkg_apis_workflow_v1alpha1_GitArtifact(ref common.ReferenceCallback) Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), }, }, + "insecureIgnoreHostKey": { + SchemaProps: spec.SchemaProps{ + Description: "InsecureIgnoreHostKey disables SSH strict host key checking during git clone", + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"repo"}, }, @@ -624,6 +697,223 @@ func schema_pkg_apis_workflow_v1alpha1_GitArtifact(ref common.ReferenceCallback) } } +func schema_pkg_apis_workflow_v1alpha1_HDFSArtifact(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "HDFSArtifact is the location of an HDFS artifact", + Properties: map[string]spec.Schema{ + "krbCCacheSecret": { + SchemaProps: spec.SchemaProps{ + Description: "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.", + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, + "krbKeytabSecret": { + SchemaProps: spec.SchemaProps{ + Description: "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.", + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, + "krbUsername": { + SchemaProps: spec.SchemaProps{ + Description: "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "krbRealm": { + SchemaProps: spec.SchemaProps{ + Description: "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "krbConfigConfigMap": { + SchemaProps: spec.SchemaProps{ + Description: "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.", + Ref: ref("k8s.io/api/core/v1.ConfigMapKeySelector"), + }, + }, + "krbServicePrincipalName": { + SchemaProps: spec.SchemaProps{ + Description: "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "addresses": { + SchemaProps: spec.SchemaProps{ + Description: "Addresses is accessible addresses of HDFS name nodes", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "hdfsUser": { + SchemaProps: spec.SchemaProps{ + Description: "HDFSUser is the user to access HDFS file system. It is ignored if either ccache or keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "path": { + SchemaProps: spec.SchemaProps{ + Description: "Path is a file path in HDFS", + Type: []string{"string"}, + Format: "", + }, + }, + "force": { + SchemaProps: spec.SchemaProps{ + Description: "Force copies a file forcibly even if it exists (default: false)", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"addresses", "path"}, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.ConfigMapKeySelector", "k8s.io/api/core/v1.SecretKeySelector"}, + } +} + +func schema_pkg_apis_workflow_v1alpha1_HDFSConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "HDFSConfig is configurations for HDFS", + Properties: map[string]spec.Schema{ + "krbCCacheSecret": { + SchemaProps: spec.SchemaProps{ + Description: "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.", + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, + "krbKeytabSecret": { + SchemaProps: spec.SchemaProps{ + Description: "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.", + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, + "krbUsername": { + SchemaProps: spec.SchemaProps{ + Description: "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "krbRealm": { + SchemaProps: spec.SchemaProps{ + Description: "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "krbConfigConfigMap": { + SchemaProps: spec.SchemaProps{ + Description: "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.", + Ref: ref("k8s.io/api/core/v1.ConfigMapKeySelector"), + }, + }, + "krbServicePrincipalName": { + SchemaProps: spec.SchemaProps{ + Description: "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "addresses": { + SchemaProps: spec.SchemaProps{ + Description: "Addresses is accessible addresses of HDFS name nodes", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "hdfsUser": { + SchemaProps: spec.SchemaProps{ + Description: "HDFSUser is the user to access HDFS file system. It is ignored if either ccache or keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"addresses"}, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.ConfigMapKeySelector", "k8s.io/api/core/v1.SecretKeySelector"}, + } +} + +func schema_pkg_apis_workflow_v1alpha1_HDFSKrbConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "HDFSKrbConfig is auth configurations for Kerberos", + Properties: map[string]spec.Schema{ + "krbCCacheSecret": { + SchemaProps: spec.SchemaProps{ + Description: "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.", + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, + "krbKeytabSecret": { + SchemaProps: spec.SchemaProps{ + Description: "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.", + Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), + }, + }, + "krbUsername": { + SchemaProps: spec.SchemaProps{ + Description: "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "krbRealm": { + SchemaProps: spec.SchemaProps{ + Description: "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + "krbConfigConfigMap": { + SchemaProps: spec.SchemaProps{ + Description: "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.", + Ref: ref("k8s.io/api/core/v1.ConfigMapKeySelector"), + }, + }, + "krbServicePrincipalName": { + SchemaProps: spec.SchemaProps{ + Description: "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.ConfigMapKeySelector", "k8s.io/api/core/v1.SecretKeySelector"}, + } +} + func schema_pkg_apis_workflow_v1alpha1_HTTPArtifact(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -878,6 +1168,13 @@ func schema_pkg_apis_workflow_v1alpha1_ResourceTemplate(ref common.ReferenceCall Format: "", }, }, + "mergeStrategy": { + SchemaProps: spec.SchemaProps{ + Description: "MergeStrategy is the strategy used to merge a patch. It defaults to \"strategic\" Must be one of: strategic, merge, json", + Type: []string{"string"}, + Format: "", + }, + }, "manifest": { SchemaProps: spec.SchemaProps{ Description: "Manifest contains the kubernetes manifest", @@ -1317,36 +1614,300 @@ func schema_pkg_apis_workflow_v1alpha1_Sequence(ref common.ReferenceCallback) co } } -func schema_pkg_apis_workflow_v1alpha1_Sidecar(ref common.ReferenceCallback) common.OpenAPIDefinition { +func schema_pkg_apis_workflow_v1alpha1_SuspendTemplate(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "SuspendTemplate is a template subtype to suspend a workflow at a predetermined point in time", + Properties: map[string]spec.Schema{}, + }, + }, + Dependencies: []string{}, + } +} + +func schema_pkg_apis_workflow_v1alpha1_TarStrategy(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TarStrategy will tar and gzip the file or directory when saving", + Properties: map[string]spec.Schema{}, + }, + }, + Dependencies: []string{}, + } +} + +func schema_pkg_apis_workflow_v1alpha1_Template(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "Sidecar is a container which runs alongside the main container", + Description: "Template is a reusable and composable unit of execution in a workflow", Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", + Description: "Name is the name of the template", Type: []string{"string"}, Format: "", }, }, - "image": { + "inputs": { SchemaProps: spec.SchemaProps{ - Description: "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", - Type: []string{"string"}, - Format: "", + Description: "Inputs describe what inputs parameters and artifacts are supplied to this template", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Inputs"), }, }, - "command": { + "outputs": { SchemaProps: spec.SchemaProps{ - Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", - }, + Description: "Outputs describe the parameters and artifacts that this template produces", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Outputs"), + }, + }, + "nodeSelector": { + SchemaProps: spec.SchemaProps{ + Description: "NodeSelector is a selector to schedule this step of the workflow to be run on the selected node(s). Overrides the selector set at the workflow level.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "affinity": { + SchemaProps: spec.SchemaProps{ + Description: "Affinity sets the pod's scheduling constraints Overrides the affinity set at the workflow level (if any)", + Ref: ref("k8s.io/api/core/v1.Affinity"), + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Metdata sets the pods's metadata, i.e. annotations and labels", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Metadata"), + }, + }, + "daemon": { + SchemaProps: spec.SchemaProps{ + Description: "Deamon will allow a workflow to proceed to the next step so long as the container reaches readiness", + Type: []string{"boolean"}, + Format: "", + }, + }, + "steps": { + SchemaProps: spec.SchemaProps{ + Description: "Steps define a series of sequential/parallel workflow steps", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.WorkflowStep"), + }, + }, + }, + }, + }, + }, + }, + }, + "container": { + SchemaProps: spec.SchemaProps{ + Description: "Container is the main container image to run in the pod", + Ref: ref("k8s.io/api/core/v1.Container"), + }, + }, + "script": { + SchemaProps: spec.SchemaProps{ + Description: "Script runs a portion of code against an interpreter", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ScriptTemplate"), + }, + }, + "resource": { + SchemaProps: spec.SchemaProps{ + Description: "Resource template subtype which can run k8s resources", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ResourceTemplate"), + }, + }, + "dag": { + SchemaProps: spec.SchemaProps{ + Description: "DAG template subtype which runs a DAG", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.DAGTemplate"), + }, + }, + "suspend": { + SchemaProps: spec.SchemaProps{ + Description: "Suspend template subtype which can suspend a workflow when reaching the step", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.SuspendTemplate"), + }, + }, + "volumes": { + SchemaProps: spec.SchemaProps{ + Description: "Volumes is a list of volumes that can be mounted by containers in a template.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.Volume"), + }, + }, + }, + }, + }, + "initContainers": { + SchemaProps: spec.SchemaProps{ + Description: "InitContainers is a list of containers which run before the main container.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.UserContainer"), + }, + }, + }, + }, + }, + "sidecars": { + SchemaProps: spec.SchemaProps{ + Description: "Sidecars is a list of containers which run alongside the main container Sidecars are automatically killed when the main container completes", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.UserContainer"), + }, + }, + }, + }, + }, + "archiveLocation": { + SchemaProps: spec.SchemaProps{ + Description: "Location in which all files related to the step will be stored (logs, artifacts, etc...). Can be overridden by individual items in Outputs. If omitted, will use the default artifact repository location configured in the controller, appended with the / in the key.", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactLocation"), + }, + }, + "activeDeadlineSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "Optional duration in seconds relative to the StartTime that the pod may be active on a node before the system actively tries to terminate the pod; value must be positive integer This field is only applicable to container and script templates.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "retryStrategy": { + SchemaProps: spec.SchemaProps{ + Description: "RetryStrategy describes how to retry a template when it fails", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RetryStrategy"), + }, + }, + "parallelism": { + SchemaProps: spec.SchemaProps{ + Description: "Parallelism limits the max total parallel pods that can execute at the same time within the boundaries of this template invocation. If additional steps/dag templates are invoked, the pods created by those templates will not be counted towards this total.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "tolerations": { + SchemaProps: spec.SchemaProps{ + Description: "Tolerations to apply to workflow pods.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.Toleration"), + }, + }, + }, + }, + }, + "schedulerName": { + SchemaProps: spec.SchemaProps{ + Description: "If specified, the pod will be dispatched by specified scheduler. Or it will be dispatched by workflow scope scheduler if specified. If neither specified, the pod will be dispatched by default scheduler.", + Type: []string{"string"}, + Format: "", + }, + }, + "priorityClassName": { + SchemaProps: spec.SchemaProps{ + Description: "PriorityClassName to apply to workflow pods.", + Type: []string{"string"}, + Format: "", + }, + }, + "priority": { + SchemaProps: spec.SchemaProps{ + Description: "Priority to apply to workflow pods.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "errors": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ExceptionCondition"), + }, + }, + }, + }, + }, + "warnings": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ExceptionCondition"), + }, + }, + }, + }, + }, + }, + Required: []string{"name"}, + }, + }, + Dependencies: []string{ + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactLocation", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.DAGTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ExceptionCondition", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Inputs", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Metadata", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Outputs", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ResourceTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RetryStrategy", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ScriptTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.SuspendTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.UserContainer", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.WorkflowStep", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Container", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume"}, + } +} + +func schema_pkg_apis_workflow_v1alpha1_UserContainer(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "UserContainer is a container specified by a user.", + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", + Type: []string{"string"}, + Format: "", + }, + }, + "image": { + SchemaProps: spec.SchemaProps{ + Description: "Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", + Type: []string{"string"}, + Format: "", + }, + }, + "command": { + SchemaProps: spec.SchemaProps{ + Description: "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, }, }, }, @@ -1535,7 +2096,7 @@ func schema_pkg_apis_workflow_v1alpha1_Sidecar(ref common.ReferenceCallback) com }, "mirrorVolumeMounts": { SchemaProps: spec.SchemaProps{ - Description: "MirrorVolumeMounts will mount the same volumes specified in the main container to the sidecar (including artifacts), at the same mountPaths. This enables dind daemon to partially see the same filesystem as the main container in order to use features such as docker volume binding", + Description: "MirrorVolumeMounts will mount the same volumes specified in the main container to the container (including artifacts), at the same mountPaths. This enables dind daemon to partially see the same filesystem as the main container in order to use features such as docker volume binding", Type: []string{"boolean"}, Format: "", }, @@ -1549,223 +2110,6 @@ func schema_pkg_apis_workflow_v1alpha1_Sidecar(ref common.ReferenceCallback) com } } -func schema_pkg_apis_workflow_v1alpha1_SuspendTemplate(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "SuspendTemplate is a template subtype to suspend a workflow at a predetermined point in time", - Properties: map[string]spec.Schema{}, - }, - }, - Dependencies: []string{}, - } -} - -func schema_pkg_apis_workflow_v1alpha1_TarStrategy(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "TarStrategy will tar and gzip the file or directory when saving", - Properties: map[string]spec.Schema{}, - }, - }, - Dependencies: []string{}, - } -} - -func schema_pkg_apis_workflow_v1alpha1_Template(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "Template is a reusable and composable unit of execution in a workflow", - Properties: map[string]spec.Schema{ - "name": { - SchemaProps: spec.SchemaProps{ - Description: "Name is the name of the template", - Type: []string{"string"}, - Format: "", - }, - }, - "inputs": { - SchemaProps: spec.SchemaProps{ - Description: "Inputs describe what inputs parameters and artifacts are supplied to this template", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Inputs"), - }, - }, - "outputs": { - SchemaProps: spec.SchemaProps{ - Description: "Outputs describe the parameters and artifacts that this template produces", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Outputs"), - }, - }, - "nodeSelector": { - SchemaProps: spec.SchemaProps{ - Description: "NodeSelector is a selector to schedule this step of the workflow to be run on the selected node(s). Overrides the selector set at the workflow level.", - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, - "affinity": { - SchemaProps: spec.SchemaProps{ - Description: "Affinity sets the pod's scheduling constraints Overrides the affinity set at the workflow level (if any)", - Ref: ref("k8s.io/api/core/v1.Affinity"), - }, - }, - "metadata": { - SchemaProps: spec.SchemaProps{ - Description: "Metdata sets the pods's metadata, i.e. annotations and labels", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Metadata"), - }, - }, - "daemon": { - SchemaProps: spec.SchemaProps{ - Description: "Deamon will allow a workflow to proceed to the next step so long as the container reaches readiness", - Type: []string{"boolean"}, - Format: "", - }, - }, - "steps": { - SchemaProps: spec.SchemaProps{ - Description: "Steps define a series of sequential/parallel workflow steps", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.WorkflowStep"), - }, - }, - }, - }, - }, - }, - }, - }, - "container": { - SchemaProps: spec.SchemaProps{ - Description: "Container is the main container image to run in the pod", - Ref: ref("k8s.io/api/core/v1.Container"), - }, - }, - "script": { - SchemaProps: spec.SchemaProps{ - Description: "Script runs a portion of code against an interpreter", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ScriptTemplate"), - }, - }, - "resource": { - SchemaProps: spec.SchemaProps{ - Description: "Resource template subtype which can run k8s resources", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ResourceTemplate"), - }, - }, - "dag": { - SchemaProps: spec.SchemaProps{ - Description: "DAG template subtype which runs a DAG", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.DAGTemplate"), - }, - }, - "suspend": { - SchemaProps: spec.SchemaProps{ - Description: "Suspend template subtype which can suspend a workflow when reaching the step", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.SuspendTemplate"), - }, - }, - "sidecars": { - SchemaProps: spec.SchemaProps{ - Description: "Sidecars is a list of containers which run alongside the main container Sidecars are automatically killed when the main container completes", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sidecar"), - }, - }, - }, - }, - }, - "archiveLocation": { - SchemaProps: spec.SchemaProps{ - Description: "Location in which all files related to the step will be stored (logs, artifacts, etc...). Can be overridden by individual items in Outputs. If omitted, will use the default artifact repository location configured in the controller, appended with the / in the key.", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactLocation"), - }, - }, - "activeDeadlineSeconds": { - SchemaProps: spec.SchemaProps{ - Description: "Optional duration in seconds relative to the StartTime that the pod may be active on a node before the system actively tries to terminate the pod; value must be positive integer This field is only applicable to container and script templates.", - Type: []string{"integer"}, - Format: "int64", - }, - }, - "retryStrategy": { - SchemaProps: spec.SchemaProps{ - Description: "RetryStrategy describes how to retry a template when it fails", - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RetryStrategy"), - }, - }, - "parallelism": { - SchemaProps: spec.SchemaProps{ - Description: "Parallelism limits the max total parallel pods that can execute at the same time within the boundaries of this template invocation. If additional steps/dag templates are invoked, the pods created by those templates will not be counted towards this total.", - Type: []string{"integer"}, - Format: "int64", - }, - }, - "tolerations": { - SchemaProps: spec.SchemaProps{ - Description: "Tolerations to apply to workflow pods.", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/api/core/v1.Toleration"), - }, - }, - }, - }, - }, - "errors": { - SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ExceptionCondition"), - }, - }, - }, - }, - }, - "warnings": { - SchemaProps: spec.SchemaProps{ - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ExceptionCondition"), - }, - }, - }, - }, - }, - }, - Required: []string{"name"}, - }, - }, - Dependencies: []string{ - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ArtifactLocation", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.DAGTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ExceptionCondition", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Inputs", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Metadata", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Outputs", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ResourceTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.RetryStrategy", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ScriptTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sidecar", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.SuspendTemplate", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.WorkflowStep", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Container", "k8s.io/api/core/v1.Toleration"}, - } -} - func schema_pkg_apis_workflow_v1alpha1_ValueFrom(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -2022,6 +2366,26 @@ func schema_pkg_apis_workflow_v1alpha1_WorkflowSpec(ref common.ReferenceCallback }, }, }, + "hostNetwork": { + SchemaProps: spec.SchemaProps{ + Description: "Host networking requested for this workflow pod. Default to false.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "dnsPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "Set DNS policy for the pod. Defaults to \"ClusterFirst\". Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", + Type: []string{"string"}, + Format: "", + }, + }, + "dnsConfig": { + SchemaProps: spec.SchemaProps{ + Description: "PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.", + Ref: ref("k8s.io/api/core/v1.PodDNSConfig"), + }, + }, "onExit": { SchemaProps: spec.SchemaProps{ Description: "OnExit is a template reference which is invoked at the end of the workflow, irrespective of the success, failure, or error of the primary workflow.", @@ -2043,12 +2407,40 @@ func schema_pkg_apis_workflow_v1alpha1_WorkflowSpec(ref common.ReferenceCallback Format: "int64", }, }, + "priority": { + SchemaProps: spec.SchemaProps{ + Description: "Priority is used if controller is configured to process limited number of workflows in parallel. Workflows with higher priority are processed first.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "schedulerName": { + SchemaProps: spec.SchemaProps{ + Description: "Set scheduler name for all pods. Will be overridden if container/script template's scheduler name is set. Default scheduler will be used if neither specified.", + Type: []string{"string"}, + Format: "", + }, + }, + "podPriorityClassName": { + SchemaProps: spec.SchemaProps{ + Description: "PriorityClassName to apply to workflow pods.", + Type: []string{"string"}, + Format: "", + }, + }, + "podPriority": { + SchemaProps: spec.SchemaProps{ + Description: "Priority to apply to workflow pods.", + Type: []string{"integer"}, + Format: "int32", + }, + }, }, Required: []string{"templates", "entrypoint"}, }, }, Dependencies: []string{ - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Arguments", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Template", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaim", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume"}, + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Arguments", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Template", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaim", "k8s.io/api/core/v1.PodDNSConfig", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume"}, } } @@ -2111,10 +2503,16 @@ func schema_pkg_apis_workflow_v1alpha1_WorkflowStep(ref common.ReferenceCallback Format: "", }, }, + "continueOn": { + SchemaProps: spec.SchemaProps{ + Description: "ContinueOn makes argo to proceed with the following step even if this step fails. Errors and Failed states can be specified", + Ref: ref("github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ContinueOn"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Arguments", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Item", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sequence"}, + "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Arguments", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.ContinueOn", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Item", "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1.Sequence"}, } } diff --git a/pkg/apis/workflow/v1alpha1/types.go b/pkg/apis/workflow/v1alpha1/types.go index 00ffa597edf5..3d0ff8cffed4 100644 --- a/pkg/apis/workflow/v1alpha1/types.go +++ b/pkg/apis/workflow/v1alpha1/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "hash/fnv" + "strings" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -117,6 +118,21 @@ type WorkflowSpec struct { // More info: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod ImagePullSecrets []apiv1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + // Host networking requested for this workflow pod. Default to false. + HostNetwork *bool `json:"hostNetwork,omitempty"` + + // Set DNS policy for the pod. + // Defaults to "ClusterFirst". + // Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + // DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + // To have DNS options set along with hostNetwork, you have to specify DNS policy + // explicitly to 'ClusterFirstWithHostNet'. + DNSPolicy *apiv1.DNSPolicy `json:"dnsPolicy,omitempty"` + + // PodDNSConfig defines the DNS parameters of a pod in addition to + // those generated from DNSPolicy. + DNSConfig *apiv1.PodDNSConfig `json:"dnsConfig,omitempty"` + // OnExit is a template reference which is invoked at the end of the // workflow, irrespective of the success, failure, or error of the // primary workflow. @@ -133,6 +149,21 @@ type WorkflowSpec struct { // allowed to run before the controller terminates the workflow. A value of zero is used to // terminate a Running workflow ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"` + + // Priority is used if controller is configured to process limited number of workflows in parallel. Workflows with higher priority are processed first. + Priority *int32 `json:"priority,omitempty"` + + // Set scheduler name for all pods. + // Will be overridden if container/script template's scheduler name is set. + // Default scheduler will be used if neither specified. + // +optional + SchedulerName string `json:"schedulerName,omitempty"` + + // PriorityClassName to apply to workflow pods. + PodPriorityClassName string `json:"podPriorityClassName,omitempty"` + + // Priority to apply to workflow pods. + PodPriority *int32 `json:"podPriority,omitempty"` } // Template is a reusable and composable unit of execution in a workflow @@ -178,9 +209,15 @@ type Template struct { // Suspend template subtype which can suspend a workflow when reaching the step Suspend *SuspendTemplate `json:"suspend,omitempty"` + // Volumes is a list of volumes that can be mounted by containers in a template. + Volumes []apiv1.Volume `json:"volumes,omitempty"` + + // InitContainers is a list of containers which run before the main container. + InitContainers []UserContainer `json:"initContainers,omitempty"` + // Sidecars is a list of containers which run alongside the main container // Sidecars are automatically killed when the main container completes - Sidecars []Sidecar `json:"sidecars,omitempty"` + Sidecars []UserContainer `json:"sidecars,omitempty"` // Location in which all files related to the step will be stored (logs, artifacts, etc...). // Can be overridden by individual items in Outputs. If omitted, will use the default @@ -204,6 +241,18 @@ type Template struct { // Tolerations to apply to workflow pods. Tolerations []apiv1.Toleration `json:"tolerations,omitempty"` + // If specified, the pod will be dispatched by specified scheduler. + // Or it will be dispatched by workflow scope scheduler if specified. + // If neither specified, the pod will be dispatched by default scheduler. + // +optional + SchedulerName string `json:"schedulerName,omitempty"` + + // PriorityClassName to apply to workflow pods. + PriorityClassName string `json:"priorityClassName,omitempty"` + + // Priority to apply to workflow pods. + Priority *int32 `json:"priority,omitempty"` + Errors []ExceptionCondition `json:"errors,omitempty"` Warnings []ExceptionCondition `json:"warnings,omitempty"` } @@ -283,6 +332,9 @@ type Artifact struct { // Archive controls how the artifact will be saved to the artifact repository. Archive *ArchiveStrategy `json:"archive,omitempty"` + + // Make Artifacts optional, if Artifacts doesn't generate or exist + Optional bool `json:"optional,omitempty"` } // ArchiveStrategy describes how to archive files/directory when saving artifacts @@ -319,6 +371,9 @@ type ArtifactLocation struct { // Artifactory contains artifactory artifact location details Artifactory *ArtifactoryArtifact `json:"artifactory,omitempty"` + // HDFS contains HDFS artifact location details + HDFS *HDFSArtifact `json:"hdfs,omitempty"` + // Raw contains raw artifact location details Raw *RawArtifact `json:"raw,omitempty"` @@ -361,6 +416,10 @@ type WorkflowStep struct { // When is an expression in which the step should conditionally execute When string `json:"when,omitempty"` + + // ContinueOn makes argo to proceed with the following step even if this step fails. + // Errors and Failed states can be specified + ContinueOn *ContinueOn `json:"continueOn,omitempty"` } // Item expands a single workflow step into multiple parallel steps @@ -424,12 +483,12 @@ type Arguments struct { Artifacts []Artifact `json:"artifacts,omitempty"` } -// Sidecar is a container which runs alongside the main container -type Sidecar struct { +// UserContainer is a container specified by a user. +type UserContainer struct { apiv1.Container `json:",inline"` // MirrorVolumeMounts will mount the same volumes specified in the main container - // to the sidecar (including artifacts), at the same mountPaths. This enables + // to the container (including artifacts), at the same mountPaths. This enables // dind daemon to partially see the same filesystem as the main container in // order to use features such as docker volume binding MirrorVolumeMounts *bool `json:"mirrorVolumeMounts,omitempty"` @@ -450,6 +509,9 @@ type WorkflowStatus struct { // A human readable message indicating details about why the workflow is in this condition. Message string `json:"message,omitempty"` + // Compressed and base64 decoded Nodes map + CompressedNodes string `json:"compressedNodes,omitempty"` + // Nodes is a mapping between a node ID and the node's status. Nodes map[string]NodeStatus `json:"nodes,omitempty"` @@ -539,12 +601,21 @@ func (n NodeStatus) String() string { return fmt.Sprintf("%s (%s)", n.Name, n.ID) } -// Completed returns whether or not the node has completed execution +func isCompletedPhase(phase NodePhase) bool { + return phase == NodeSucceeded || + phase == NodeFailed || + phase == NodeError || + phase == NodeSkipped +} + +// Remove returns whether or not the workflow has completed execution +func (ws *WorkflowStatus) Completed() bool { + return isCompletedPhase(ws.Phase) +} + +// Remove returns whether or not the node has completed execution func (n NodeStatus) Completed() bool { - return n.Phase == NodeSucceeded || - n.Phase == NodeFailed || - n.Phase == NodeError || - n.Phase == NodeSkipped + return isCompletedPhase(n.Phase) || n.IsDaemoned() && n.Phase != NodePending } // IsDaemoned returns whether or not the node is deamoned @@ -557,7 +628,7 @@ func (n NodeStatus) IsDaemoned() bool { // Successful returns whether or not this node completed successfully func (n NodeStatus) Successful() bool { - return n.Phase == NodeSucceeded || n.Phase == NodeSkipped + return n.Phase == NodeSucceeded || n.Phase == NodeSkipped || n.IsDaemoned() && n.Phase != NodePending } // CanRetry returns whether the node should be retried or not. @@ -603,9 +674,14 @@ func (s *S3Artifact) String() string { return fmt.Sprintf("%s://%s/%s/%s", protocol, s.Endpoint, s.Bucket, s.Key) } +func (s *S3Artifact) HasLocation() bool { + return s != nil && s.Bucket != "" +} + // GCSBucket contains the access information required for acting with a GCS bucket type GCSBucket struct { - Bucket string `json:"bucket"` + Bucket string `json:"bucket"` + CredentialsSecret apiv1.SecretKeySelector `json:"credentialsSecret"` } // GCSArtifact is the location of a GCS artifact @@ -618,6 +694,10 @@ func (s *GCSArtifact) String() string { return fmt.Sprintf("gs://%s/%s", s.Bucket, s.Key) } +func (s *GCSArtifact) HasLocation() bool { + return s != nil && s.Bucket != "" +} + // GitArtifact is the location of an git artifact type GitArtifact struct { // Repo is the git repository @@ -631,8 +711,16 @@ type GitArtifact struct { // PasswordSecret is the secret selector to the repository password PasswordSecret *apiv1.SecretKeySelector `json:"passwordSecret,omitempty"` + // SSHPrivateKeySecret is the secret selector to the repository ssh private key SSHPrivateKeySecret *apiv1.SecretKeySelector `json:"sshPrivateKeySecret,omitempty"` + + // InsecureIgnoreHostKey disables SSH strict host key checking during git clone + InsecureIgnoreHostKey bool `json:"insecureIgnoreHostKey,omitempty"` +} + +func (g *GitArtifact) HasLocation() bool { + return g != nil && g.Repo != "" } // ArtifactoryAuth describes the secret selectors required for authenticating to artifactory @@ -655,18 +743,96 @@ func (a *ArtifactoryArtifact) String() string { return a.URL } +func (a *ArtifactoryArtifact) HasLocation() bool { + return a != nil && a.URL != "" +} + +// HDFSArtifact is the location of an HDFS artifact +type HDFSArtifact struct { + HDFSConfig `json:",inline"` + + // Path is a file path in HDFS + Path string `json:"path"` + + // Force copies a file forcibly even if it exists (default: false) + Force bool `json:"force,omitempty"` +} + +func (h *HDFSArtifact) HasLocation() bool { + return h != nil && len(h.Addresses) > 0 +} + +// HDFSConfig is configurations for HDFS +type HDFSConfig struct { + HDFSKrbConfig `json:",inline"` + + // Addresses is accessible addresses of HDFS name nodes + Addresses []string `json:"addresses"` + + // HDFSUser is the user to access HDFS file system. + // It is ignored if either ccache or keytab is used. + HDFSUser string `json:"hdfsUser,omitempty"` +} + +// HDFSKrbConfig is auth configurations for Kerberos +type HDFSKrbConfig struct { + // KrbCCacheSecret is the secret selector for Kerberos ccache + // Either ccache or keytab can be set to use Kerberos. + KrbCCacheSecret *apiv1.SecretKeySelector `json:"krbCCacheSecret,omitempty"` + + // KrbKeytabSecret is the secret selector for Kerberos keytab + // Either ccache or keytab can be set to use Kerberos. + KrbKeytabSecret *apiv1.SecretKeySelector `json:"krbKeytabSecret,omitempty"` + + // KrbUsername is the Kerberos username used with Kerberos keytab + // It must be set if keytab is used. + KrbUsername string `json:"krbUsername,omitempty"` + + // KrbRealm is the Kerberos realm used with Kerberos keytab + // It must be set if keytab is used. + KrbRealm string `json:"krbRealm,omitempty"` + + // KrbConfig is the configmap selector for Kerberos config as string + // It must be set if either ccache or keytab is used. + KrbConfigConfigMap *apiv1.ConfigMapKeySelector `json:"krbConfigConfigMap,omitempty"` + + // KrbServicePrincipalName is the principal name of Kerberos service + // It must be set if either ccache or keytab is used. + KrbServicePrincipalName string `json:"krbServicePrincipalName,omitempty"` +} + +func (a *HDFSArtifact) String() string { + var cred string + if a.HDFSUser != "" { + cred = fmt.Sprintf("HDFS user %s", a.HDFSUser) + } else if a.KrbCCacheSecret != nil { + cred = fmt.Sprintf("ccache %v", a.KrbCCacheSecret.Name) + } else if a.KrbKeytabSecret != nil { + cred = fmt.Sprintf("keytab %v (%s/%s)", a.KrbKeytabSecret.Name, a.KrbUsername, a.KrbRealm) + } + return fmt.Sprintf("hdfs://%s/%s with %s", strings.Join(a.Addresses, ", "), a.Path, cred) +} + // RawArtifact allows raw string content to be placed as an artifact in a container type RawArtifact struct { // Data is the string contents of the artifact Data string `json:"data"` } +func (r *RawArtifact) HasLocation() bool { + return r != nil +} + // HTTPArtifact allows an file served on HTTP to be placed as an input artifact in a container type HTTPArtifact struct { // URL of the artifact URL string `json:"url"` } +func (h *HTTPArtifact) HasLocation() bool { + return h != nil && h.URL != "" +} + // ScriptTemplate is a template subtype to enable scripting through code steps type ScriptTemplate struct { apiv1.Container `json:",inline"` @@ -681,6 +847,10 @@ type ResourceTemplate struct { // Must be one of: get, create, apply, delete, replace Action string `json:"action"` + // MergeStrategy is the strategy used to merge a patch. It defaults to "strategic" + // Must be one of: strategic, merge, json + MergeStrategy string `json:"mergeStrategy,omitempty"` + // Manifest contains the kubernetes manifest Manifest string `json:"manifest"` @@ -786,6 +956,10 @@ type DAGTask struct { // When is an expression in which the task should conditionally execute When string `json:"when,omitempty"` + + // ContinueOn makes argo to proceed with the following step even if this step fails. + // Errors and Failed states can be specified + ContinueOn *ContinueOn `json:"continueOn,omitempty"` } // SuspendTemplate is a template subtype to suspend a workflow at a predetermined point in time @@ -859,7 +1033,13 @@ func (args *Arguments) GetParameterByName(name string) *Parameter { // HasLocation whether or not an artifact has a location defined func (a *Artifact) HasLocation() bool { - return a.S3 != nil || a.Git != nil || a.HTTP != nil || a.Artifactory != nil || a.Raw != nil || a.GCS != nil + return a.S3.HasLocation() || + a.Git.HasLocation() || + a.HTTP.HasLocation() || + a.Artifactory.HasLocation() || + a.Raw.HasLocation() || + a.HDFS.HasLocation() || + a.GCS.HasLocation() } // GetTemplate retrieves a defined template by its name @@ -881,3 +1061,35 @@ func (wf *Workflow) NodeID(name string) string { _, _ = h.Write([]byte(name)) return fmt.Sprintf("%s-%v", wf.ObjectMeta.Name, h.Sum32()) } + +// ContinueOn defines if a workflow should continue even if a task or step fails/errors. +// It can be specified if the workflow should continue when the pod errors, fails or both. +type ContinueOn struct { + // +optional + Error bool `json:"error,omitempty"` + // +optional + Failed bool `json:"failed,omitempty"` +} + +func continues(c *ContinueOn, phase NodePhase) bool { + if c == nil { + return false + } + if c.Error == true && phase == NodeError { + return true + } + if c.Failed == true && phase == NodeFailed { + return true + } + return false +} + +// ContinuesOn returns whether the DAG should be proceeded if the task fails or errors. +func (t *DAGTask) ContinuesOn(phase NodePhase) bool { + return continues(t.ContinueOn, phase) +} + +// ContinuesOn returns whether the StepGroup should be proceeded if the task fails or errors. +func (s *WorkflowStep) ContinuesOn(phase NodePhase) bool { + return continues(s.ContinueOn, phase) +} diff --git a/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go index 0135feb7dd45..9f9c336ae482 100644 --- a/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go @@ -120,6 +120,11 @@ func (in *ArtifactLocation) DeepCopyInto(out *ArtifactLocation) { *out = new(ArtifactoryArtifact) (*in).DeepCopyInto(*out) } + if in.HDFS != nil { + in, out := &in.HDFS, &out.HDFS + *out = new(HDFSArtifact) + (*in).DeepCopyInto(*out) + } if in.Raw != nil { in, out := &in.Raw, &out.Raw *out = new(RawArtifact) @@ -128,7 +133,7 @@ func (in *ArtifactLocation) DeepCopyInto(out *ArtifactLocation) { if in.GCS != nil { in, out := &in.GCS, &out.GCS *out = new(GCSArtifact) - **out = **in + (*in).DeepCopyInto(*out) } return } @@ -186,6 +191,22 @@ func (in *ArtifactoryAuth) DeepCopy() *ArtifactoryAuth { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContinueOn) DeepCopyInto(out *ContinueOn) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContinueOn. +func (in *ContinueOn) DeepCopy() *ContinueOn { + if in == nil { + return nil + } + out := new(ContinueOn) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DAGTask) DeepCopyInto(out *DAGTask) { *out = *in @@ -207,6 +228,11 @@ func (in *DAGTask) DeepCopyInto(out *DAGTask) { *out = new(Sequence) **out = **in } + if in.ContinueOn != nil { + in, out := &in.ContinueOn, &out.ContinueOn + *out = new(ContinueOn) + **out = **in + } return } @@ -278,7 +304,7 @@ func (in *ExceptionResult) DeepCopy() *ExceptionResult { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCSArtifact) DeepCopyInto(out *GCSArtifact) { *out = *in - out.GCSBucket = in.GCSBucket + in.GCSBucket.DeepCopyInto(&out.GCSBucket) return } @@ -295,6 +321,7 @@ func (in *GCSArtifact) DeepCopy() *GCSArtifact { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCSBucket) DeepCopyInto(out *GCSBucket) { *out = *in + in.CredentialsSecret.DeepCopyInto(&out.CredentialsSecret) return } @@ -339,6 +366,76 @@ func (in *GitArtifact) DeepCopy() *GitArtifact { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HDFSArtifact) DeepCopyInto(out *HDFSArtifact) { + *out = *in + in.HDFSConfig.DeepCopyInto(&out.HDFSConfig) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HDFSArtifact. +func (in *HDFSArtifact) DeepCopy() *HDFSArtifact { + if in == nil { + return nil + } + out := new(HDFSArtifact) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HDFSConfig) DeepCopyInto(out *HDFSConfig) { + *out = *in + in.HDFSKrbConfig.DeepCopyInto(&out.HDFSKrbConfig) + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HDFSConfig. +func (in *HDFSConfig) DeepCopy() *HDFSConfig { + if in == nil { + return nil + } + out := new(HDFSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HDFSKrbConfig) DeepCopyInto(out *HDFSKrbConfig) { + *out = *in + if in.KrbCCacheSecret != nil { + in, out := &in.KrbCCacheSecret, &out.KrbCCacheSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.KrbKeytabSecret != nil { + in, out := &in.KrbKeytabSecret, &out.KrbKeytabSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.KrbConfigConfigMap != nil { + in, out := &in.KrbConfigConfigMap, &out.KrbConfigConfigMap + *out = new(v1.ConfigMapKeySelector) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HDFSKrbConfig. +func (in *HDFSKrbConfig) DeepCopy() *HDFSKrbConfig { + if in == nil { + return nil + } + out := new(HDFSKrbConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPArtifact) DeepCopyInto(out *HTTPArtifact) { *out = *in @@ -676,28 +773,6 @@ func (in *Sequence) DeepCopy() *Sequence { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Sidecar) DeepCopyInto(out *Sidecar) { - *out = *in - in.Container.DeepCopyInto(&out.Container) - if in.MirrorVolumeMounts != nil { - in, out := &in.MirrorVolumeMounts, &out.MirrorVolumeMounts - *out = new(bool) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Sidecar. -func (in *Sidecar) DeepCopy() *Sidecar { - if in == nil { - return nil - } - out := new(Sidecar) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SuspendTemplate) DeepCopyInto(out *SuspendTemplate) { *out = *in @@ -791,9 +866,23 @@ func (in *Template) DeepCopyInto(out *Template) { *out = new(SuspendTemplate) **out = **in } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.InitContainers != nil { + in, out := &in.InitContainers, &out.InitContainers + *out = make([]UserContainer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Sidecars != nil { in, out := &in.Sidecars, &out.Sidecars - *out = make([]Sidecar, len(*in)) + *out = make([]UserContainer, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -825,6 +914,11 @@ func (in *Template) DeepCopyInto(out *Template) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Priority != nil { + in, out := &in.Priority, &out.Priority + *out = new(int32) + **out = **in + } if in.Errors != nil { in, out := &in.Errors, &out.Errors *out = make([]ExceptionCondition, len(*in)) @@ -848,6 +942,28 @@ func (in *Template) DeepCopy() *Template { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserContainer) DeepCopyInto(out *UserContainer) { + *out = *in + in.Container.DeepCopyInto(&out.Container) + if in.MirrorVolumeMounts != nil { + in, out := &in.MirrorVolumeMounts, &out.MirrorVolumeMounts + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserContainer. +func (in *UserContainer) DeepCopy() *UserContainer { + if in == nil { + return nil + } + out := new(UserContainer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ValueFrom) DeepCopyInto(out *ValueFrom) { *out = *in @@ -984,6 +1100,21 @@ func (in *WorkflowSpec) DeepCopyInto(out *WorkflowSpec) { *out = make([]v1.LocalObjectReference, len(*in)) copy(*out, *in) } + if in.HostNetwork != nil { + in, out := &in.HostNetwork, &out.HostNetwork + *out = new(bool) + **out = **in + } + if in.DNSPolicy != nil { + in, out := &in.DNSPolicy, &out.DNSPolicy + *out = new(v1.DNSPolicy) + **out = **in + } + if in.DNSConfig != nil { + in, out := &in.DNSConfig, &out.DNSConfig + *out = new(v1.PodDNSConfig) + (*in).DeepCopyInto(*out) + } if in.TTLSecondsAfterFinished != nil { in, out := &in.TTLSecondsAfterFinished, &out.TTLSecondsAfterFinished *out = new(int32) @@ -994,6 +1125,16 @@ func (in *WorkflowSpec) DeepCopyInto(out *WorkflowSpec) { *out = new(int64) **out = **in } + if in.Priority != nil { + in, out := &in.Priority, &out.Priority + *out = new(int32) + **out = **in + } + if in.PodPriority != nil { + in, out := &in.PodPriority, &out.PodPriority + *out = new(int32) + **out = **in + } return } @@ -1070,6 +1211,11 @@ func (in *WorkflowStep) DeepCopyInto(out *WorkflowStep) { *out = new(Sequence) **out = **in } + if in.ContinueOn != nil { + in, out := &in.ContinueOn, &out.ContinueOn + *out = new(ContinueOn) + **out = **in + } return } diff --git a/test/e2e/expectedfailures/disallow-unknown.json b/test/e2e/expectedfailures/disallow-unknown.json deleted file mode 100644 index 659d97d24dec..000000000000 --- a/test/e2e/expectedfailures/disallow-unknown.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "apiVersion": "argoproj.io/v1alpha1", - "kind": "Workflow", - "metadata": { - "generateName": "hello-world-" - }, - "spec": { - "entrypoint": "whalesay", - "templates": [ - { - "name": "whalesay", - "container": { - "image": "docker/whalesay:latest", - "command": [ - "cowsay" - ], - "args": [ - "hello world" - ], - "someExtraField": "foo" - } - } - ] - } -} diff --git a/test/e2e/expectedfailures/failed-retries.yaml b/test/e2e/expectedfailures/failed-retries.yaml new file mode 100644 index 000000000000..2930d2677f88 --- /dev/null +++ b/test/e2e/expectedfailures/failed-retries.yaml @@ -0,0 +1,30 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: failed-retries- +spec: + entrypoint: failed-retries + + templates: + - name: failed-retries + steps: + - - name: fail + template: fail + - name: delayed-fail + template: delayed-fail + + - name: fail + retryStrategy: + limit: 1 + container: + image: alpine:latest + command: [sh, -c] + args: ["exit 1"] + + - name: delayed-fail + retryStrategy: + limit: 1 + container: + image: alpine:latest + command: [sh, -c] + args: ["sleep 1; exit 1"] diff --git a/test/e2e/expectedfailures/input-artifact-not-optional.yaml b/test/e2e/expectedfailures/input-artifact-not-optional.yaml new file mode 100644 index 000000000000..e1a3615c71ad --- /dev/null +++ b/test/e2e/expectedfailures/input-artifact-not-optional.yaml @@ -0,0 +1,22 @@ +# This example demonstrates the input artifacts not optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: input-artifact-not-optional- +spec: + entrypoint: http-artifact-example + templates: + - name: http-artifact-example + inputs: + artifacts: + - name: kubectl + path: /bin/kubectl + mode: 0755 + optional: false + http: + url: "" + container: + image: debian:9.4 + command: [sh, -c] + args: ["echo NoKubectl"] diff --git a/test/e2e/expectedfailures/output-artifact-not-optional.yaml b/test/e2e/expectedfailures/output-artifact-not-optional.yaml new file mode 100644 index 000000000000..d6fe97da86b6 --- /dev/null +++ b/test/e2e/expectedfailures/output-artifact-not-optional.yaml @@ -0,0 +1,24 @@ +# This example demonstrates the output artifacts not optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: output-artifact-not-optional- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay hello world | tee /tmp/hello_world12.txt"] + outputs: + artifacts: + - name: hello-art + optional: false + path: /tmp/hello_world.txt diff --git a/test/e2e/expectedfailures/pns/pns-output-artifacts.yaml b/test/e2e/expectedfailures/pns/pns-output-artifacts.yaml new file mode 100644 index 000000000000..9680ef096507 --- /dev/null +++ b/test/e2e/expectedfailures/pns/pns-output-artifacts.yaml @@ -0,0 +1,39 @@ +# Workflow specifically designed for testing process namespace sharing with output artifacts +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: pns-output-artifacts- +spec: + entrypoint: pns-output-artifacts + templates: + - name: pns-output-artifacts + archiveLocation: + archiveLogs: true + container: + image: debian:9.2 + command: [sh, -c] + args: [" + echo hello world > /mnt/workdir/foo && + echo stdout && + echo '' && + echo stderr >&2 && + sleep 1 + "] + volumeMounts: + - name: workdir + mountPath: /mnt/workdir + outputs: + artifacts: + - name: etc + path: /etc + - name: mnt + path: /mnt + - name: workdir + path: /mnt/workdir + sidecars: + - name: nginx + image: nginx:latest + + volumes: + - name: workdir + emptyDir: {} diff --git a/test/e2e/expectedfailures/pns/pns-quick-exit-output-art.yaml b/test/e2e/expectedfailures/pns/pns-quick-exit-output-art.yaml new file mode 100644 index 000000000000..286a82846e26 --- /dev/null +++ b/test/e2e/expectedfailures/pns/pns-quick-exit-output-art.yaml @@ -0,0 +1,30 @@ +# Workflow specifically designed for testing process namespace sharing with output artifacts +# This fails because the main container exits before the wait sidecar is able to establish the file +# handle of the main container's root filesystem. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: pns-quick-exit-output-art- +spec: + entrypoint: pns-quick-exit-output-art + templates: + - name: pns-quick-exit-output-art + archiveLocation: + archiveLogs: true + container: + image: debian:9.2 + command: [sh, -x, -c] + args: [" + touch /mnt/workdir/foo + "] + volumeMounts: + - name: workdir + mountPath: /mnt/workdir + outputs: + artifacts: + - name: mnt + path: /mnt + + volumes: + - name: workdir + emptyDir: {} diff --git a/test/e2e/functional/artifact-disable-archive.yaml b/test/e2e/functional/artifact-disable-archive.yaml deleted file mode 100644 index f1f41e1bad39..000000000000 --- a/test/e2e/functional/artifact-disable-archive.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# This tests the disabling of archive, and ability to recursively copy a directory -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: artifact-disable-archive- -spec: - entrypoint: artifact-example - templates: - - name: artifact-example - steps: - - - name: generate-artifact - template: whalesay - - - name: consume-artifact - template: print-message - arguments: - artifacts: - - name: etc - from: "{{steps.generate-artifact.outputs.artifacts.etc}}" - - name: hello-txt - from: "{{steps.generate-artifact.outputs.artifacts.hello-txt}}" - - - name: whalesay - container: - image: docker/whalesay:latest - command: [sh, -c] - args: ["cowsay hello world | tee /tmp/hello_world.txt"] - outputs: - artifacts: - - name: etc - path: /etc - archive: - none: {} - - name: hello-txt - path: /tmp/hello_world.txt - archive: - none: {} - - - name: print-message - inputs: - artifacts: - - name: etc - path: /tmp/etc - - name: hello-txt - path: /tmp/hello.txt - container: - image: alpine:latest - command: [sh, -c] - args: - - cat /tmp/hello.txt && cd /tmp/etc && find . diff --git a/test/e2e/functional/artifact-disable-archive.yaml b/test/e2e/functional/artifact-disable-archive.yaml new file mode 120000 index 000000000000..109a8c619867 --- /dev/null +++ b/test/e2e/functional/artifact-disable-archive.yaml @@ -0,0 +1 @@ +../../../examples/artifact-disable-archive.yaml \ No newline at end of file diff --git a/test/e2e/functional/continue-on-fail.yaml b/test/e2e/functional/continue-on-fail.yaml new file mode 120000 index 000000000000..3bb5bfc75322 --- /dev/null +++ b/test/e2e/functional/continue-on-fail.yaml @@ -0,0 +1 @@ +../../../examples/continue-on-fail.yaml \ No newline at end of file diff --git a/test/e2e/functional/custom_template_variable.yaml b/test/e2e/functional/custom_template_variable.yaml new file mode 100644 index 000000000000..f9ee8fca8df2 --- /dev/null +++ b/test/e2e/functional/custom_template_variable.yaml @@ -0,0 +1,32 @@ +# This template demonstrates the customer variable suppport. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: custom-template-variable- +spec: + entrypoint: hello-hello-hello + + templates: + - name: hello-hello-hello + steps: + - - name: hello1 + template: whalesay + arguments: + parameters: [{name: message, value: "hello1"}] + - - name: hello2a + template: whalesay + arguments: + parameters: [{name: message, value: "hello2a"}] + - name: hello2b + template: whalesay + arguments: + parameters: [{name: message, value: "hello2b"}] + + - name: whalesay + inputs: + parameters: + - name: message + container: + image: docker/whalesay + command: [cowsay] + args: ["{{custom.variable}}"] diff --git a/test/e2e/functional/dag-argument-passing.yaml b/test/e2e/functional/dag-argument-passing.yaml index c1e51a6bb61c..24f5c7aa8c8b 100644 --- a/test/e2e/functional/dag-argument-passing.yaml +++ b/test/e2e/functional/dag-argument-passing.yaml @@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - generateName: dag-arg-passing- + generateName: dag-argument-passing- spec: entrypoint: dag-arg-passing templates: @@ -16,7 +16,7 @@ spec: container: image: alpine:3.7 command: [sh, -c, -x] - args: ['echo "{{inputs.parameters.message}}"; cat /tmp/passthrough'] + args: ['sleep 1; echo "{{inputs.parameters.message}}"; cat /tmp/passthrough'] outputs: parameters: - name: hosts diff --git a/test/e2e/functional/global-outputs-dag.yaml b/test/e2e/functional/global-outputs-dag.yaml index fa7eeb449847..cea147513f7b 100644 --- a/test/e2e/functional/global-outputs-dag.yaml +++ b/test/e2e/functional/global-outputs-dag.yaml @@ -21,7 +21,7 @@ spec: container: image: alpine:3.7 command: [sh, -c] - args: ["echo -n hello world > /tmp/hello_world.txt"] + args: ["sleep 1; echo -n hello world > /tmp/hello_world.txt"] outputs: parameters: # export a global parameter. The parameter will be programatically available in the completed diff --git a/test/e2e/functional/global-outputs-variable.yaml b/test/e2e/functional/global-outputs-variable.yaml index eed27afd1cc0..ca2222e6f61a 100644 --- a/test/e2e/functional/global-outputs-variable.yaml +++ b/test/e2e/functional/global-outputs-variable.yaml @@ -23,7 +23,7 @@ spec: container: image: alpine:3.7 command: [sh, -c] - args: ["echo -n hello world > /tmp/hello_world.txt"] + args: ["sleep 1; echo -n hello world > /tmp/hello_world.txt"] outputs: parameters: - name: hello-param diff --git a/test/e2e/functional/init-container.yaml b/test/e2e/functional/init-container.yaml new file mode 120000 index 000000000000..fe78772b05ed --- /dev/null +++ b/test/e2e/functional/init-container.yaml @@ -0,0 +1 @@ +../../../examples/init-container.yaml \ No newline at end of file diff --git a/test/e2e/functional/input-artifact-optional.yaml b/test/e2e/functional/input-artifact-optional.yaml new file mode 100644 index 000000000000..9b7a8a051b19 --- /dev/null +++ b/test/e2e/functional/input-artifact-optional.yaml @@ -0,0 +1,22 @@ +# This example demonstrates the input artifacts optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: input-artifact-optional- +spec: + entrypoint: http-artifact-example + templates: + - name: http-artifact-example + inputs: + artifacts: + - name: kubectl + path: /bin/kubectl + mode: 0755 + optional: true + http: + url: "" + container: + image: debian:9.4 + command: [sh, -c] + args: ["echo NoKubectl"] diff --git a/test/e2e/functional/dag-outputs.yaml b/test/e2e/functional/nested-dag-outputs.yaml similarity index 99% rename from test/e2e/functional/dag-outputs.yaml rename to test/e2e/functional/nested-dag-outputs.yaml index 89ecc41130cc..8cc92c5003da 100644 --- a/test/e2e/functional/dag-outputs.yaml +++ b/test/e2e/functional/nested-dag-outputs.yaml @@ -38,6 +38,7 @@ spec: image: docker/whalesay:latest command: [sh, -c] args: [" + sleep 1; cowsay hello world | tee /tmp/my-output-artifact.txt && echo 'my-output-parameter' > /tmp/my-output-parameter.txt "] diff --git a/test/e2e/functional/output-artifact-optional.yaml b/test/e2e/functional/output-artifact-optional.yaml new file mode 100644 index 000000000000..803289d6ca85 --- /dev/null +++ b/test/e2e/functional/output-artifact-optional.yaml @@ -0,0 +1,24 @@ +# This example demonstrates the output artifacts optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: output-artifact-optional- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["sleep 1; cowsay hello world | tee /tmp/hello_world12.txt"] + outputs: + artifacts: + - name: hello-art + optional: true + path: /tmp/hello_world.txt diff --git a/test/e2e/functional/output-input-artifact-optional.yaml b/test/e2e/functional/output-input-artifact-optional.yaml new file mode 100644 index 000000000000..f1519df74d4e --- /dev/null +++ b/test/e2e/functional/output-input-artifact-optional.yaml @@ -0,0 +1,40 @@ +# This example demonstrates the output and input artifacts are optionals +# from one step to the next. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: output-input-artifact-optional- +spec: + entrypoint: artifact-example + templates: + - name: artifact-example + steps: + - - name: generate-artifact + template: whalesay + - - name: consume-artifact + template: print-message + arguments: + artifacts: + - name: message + from: "{{steps.generate-artifact.outputs.artifacts.hello-art}}" + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["sleep 1; cowsay hello world | tee /tmp/hello_world123.txt"] + outputs: + artifacts: + - name: hello-art + optional: true + path: /tmp/hello_world.txt + + - name: print-message + inputs: + artifacts: + - name: message + path: /tmp/message + optional: true + container: + image: alpine:latest + command: [sh, -c] + args: ["echo /tmp/message"] diff --git a/test/e2e/functional/output-param-different-uid.yaml b/test/e2e/functional/output-param-different-uid.yaml new file mode 100644 index 000000000000..dbb7942fc945 --- /dev/null +++ b/test/e2e/functional/output-param-different-uid.yaml @@ -0,0 +1,27 @@ +# Tests PNS ability to capture output artifact when user id is different +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: pns-output-parameter-different-user- +spec: + entrypoint: multi-whalesay + templates: + - name: multi-whalesay + steps: + - - name: whalesay + template: whalesay + withSequence: + count: "10" + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["sleep 1; cowsay hello world | tee /tmp/hello_world.txt"] + securityContext: + runAsUser: 1234 + outputs: + parameters: + - name: hello-art + valueFrom: + path: /tmp/hello_world.txt \ No newline at end of file diff --git a/test/e2e/functional/pns-output-params.yaml b/test/e2e/functional/pns-output-params.yaml new file mode 100644 index 000000000000..fe0001d38322 --- /dev/null +++ b/test/e2e/functional/pns-output-params.yaml @@ -0,0 +1,71 @@ +# Workflow specifically designed for testing process namespace sharing with output parameters +# This exercises the copy out regular files from volume mounted paths, or base image layer paths, +# including overlaps between the two. +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: pns-outputs-params- +spec: + entrypoint: output-parameter + templates: + - name: output-parameter + steps: + - - name: generate-parameter + template: whalesay + - - name: consume-parameter + template: print-message + arguments: + parameters: + - { name: A, value: "{{steps.generate-parameter.outputs.parameters.A}}" } + - { name: B, value: "{{steps.generate-parameter.outputs.parameters.B}}" } + - { name: C, value: "{{steps.generate-parameter.outputs.parameters.C}}" } + - { name: D, value: "{{steps.generate-parameter.outputs.parameters.D}}" } + + - name: whalesay + container: + image: docker/whalesay:latest + command: [sh, -x, -c] + args: [" + sleep 1; + echo -n A > /tmp/A && + echo -n B > /mnt/outer/inner/B && + echo -n C > /tmp/C && + echo -n D > /mnt/outer/D + "] + volumeMounts: + - name: outer + mountPath: /mnt/outer + - name: inner + mountPath: /mnt/outer/inner + outputs: + parameters: + - name: A + valueFrom: + path: /tmp/A + - name: B + valueFrom: + path: /mnt/outer/inner/B + - name: C + valueFrom: + path: /tmp/C + - name: D + valueFrom: + path: /mnt/outer/D + + - name: print-message + inputs: + parameters: + - name: A + - name: B + - name: C + - name: D + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["{{inputs.parameters.A}} {{inputs.parameters.B}} {{inputs.parameters.C}} {{inputs.parameters.D}}"] + + volumes: + - name: outer + emptyDir: {} + - name: inner + emptyDir: {} diff --git a/test/e2e/functional/retry-with-artifacts.yaml b/test/e2e/functional/retry-with-artifacts.yaml index 7aa5dcd37421..4a509d568504 100644 --- a/test/e2e/functional/retry-with-artifacts.yaml +++ b/test/e2e/functional/retry-with-artifacts.yaml @@ -23,7 +23,7 @@ spec: container: image: docker/whalesay:latest command: [sh, -c] - args: ["cowsay hello world | tee /tmp/hello_world.txt"] + args: ["sleep 1; cowsay hello world | tee /tmp/hello_world.txt"] outputs: artifacts: - name: hello-art diff --git a/test/e2e/lintfail/disallow-unknown.yaml b/test/e2e/lintfail/disallow-unknown.yaml new file mode 100644 index 000000000000..4d7c349cbf7c --- /dev/null +++ b/test/e2e/lintfail/disallow-unknown.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: disallow-unknown- +spec: + entrypoint: whalesay + templates: + - name: whalesay + container: + image: docker/whalesay:latest + command: + - cowsay + args: + - hello world + someExtraField: foo diff --git a/test/e2e/expectedfailures/invalid-spec.yaml b/test/e2e/lintfail/invalid-spec.yaml similarity index 100% rename from test/e2e/expectedfailures/invalid-spec.yaml rename to test/e2e/lintfail/invalid-spec.yaml diff --git a/test/e2e/expectedfailures/maformed-spec.yaml b/test/e2e/lintfail/malformed-spec.yaml similarity index 100% rename from test/e2e/expectedfailures/maformed-spec.yaml rename to test/e2e/lintfail/malformed-spec.yaml diff --git a/test/e2e/ui/ui-dag-with-params.yaml b/test/e2e/ui/ui-dag-with-params.yaml index a954c0a8bb94..9756cda593e3 100644 --- a/test/e2e/ui/ui-dag-with-params.yaml +++ b/test/e2e/ui/ui-dag-with-params.yaml @@ -3,24 +3,53 @@ kind: Workflow metadata: generateName: ui-dag-with-params- spec: - entrypoint: diamond + entrypoint: pipeline + templates: - - name: diamond - dag: - tasks: - - name: A - template: nested-diamond - arguments: - parameters: [{name: message, value: A}] - - name: nested-diamond + - name: echo inputs: parameters: - name: message + container: + image: alpine:latest + command: [echo, "{{inputs.parameters.message}}"] + + - name: subpipeline-a dag: tasks: - - name: A + - name: A1 template: echo - - name: echo - container: - image: alpine:3.7 - command: [echo, "hello"] + arguments: + parameters: [{name: message, value: "Hello World!"}] + - name: A2 + template: echo + arguments: + parameters: [{name: message, value: "Hello World!"}] + + - name: subpipeline-b + dag: + tasks: + - name: B1 + template: echo + arguments: + parameters: [{name: message, value: "Hello World!"}] + - name: B2 + template: echo + dependencies: [B1] + arguments: + parameters: [{name: message, value: "Hello World!"}] + withItems: + - 0 + - 1 + + - name: pipeline + dag: + tasks: + - name: A + template: subpipeline-a + withItems: + - 0 + - 1 + - name: B + dependencies: [A] + template: subpipeline-b diff --git a/test/e2e/ui/ui-nested-steps.yaml b/test/e2e/ui/ui-nested-steps.yaml index aeb03da41e1f..c091c6827a24 100644 --- a/test/e2e/ui/ui-nested-steps.yaml +++ b/test/e2e/ui/ui-nested-steps.yaml @@ -5,6 +5,9 @@ metadata: generateName: ui-nested-steps- spec: entrypoint: ui-nested-steps + volumes: + - name: workdir + emptyDir: {} templates: - name: ui-nested-steps steps: @@ -24,14 +27,17 @@ spec: - name: locate-faces container: image: alpine:latest - command: ["sh", "-c"] + command: [sh, -c] args: - - echo '[1, 2, 3]' > /result.json + - echo '[1, 2, 3]' > /workdir/result.json + volumeMounts: + - name: workdir + mountPath: /workdir outputs: parameters: - name: imagemagick-commands valueFrom: - path: /result.json + path: /workdir/result.json - name: handle-individual-faces steps: diff --git a/util/archive/archive.go b/util/archive/archive.go new file mode 100644 index 000000000000..e10820777fed --- /dev/null +++ b/util/archive/archive.go @@ -0,0 +1,131 @@ +package archive + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + + "github.com/cyrusbiotechnology/argo/errors" + "github.com/cyrusbiotechnology/argo/util" + log "github.com/sirupsen/logrus" +) + +type flusher interface { + Flush() error +} + +// TarGzToWriter tar.gz's the source path to the supplied writer +func TarGzToWriter(sourcePath string, w io.Writer) error { + sourcePath, err := filepath.Abs(sourcePath) + if err != nil { + return errors.InternalErrorf("getting absolute path: %v", err) + } + log.Infof("Taring %s", sourcePath) + sourceFi, err := os.Stat(sourcePath) + if err != nil { + if os.IsNotExist(err) { + return errors.New(errors.CodeNotFound, err.Error()) + } + return errors.InternalWrapError(err) + } + if !sourceFi.Mode().IsRegular() && !sourceFi.IsDir() { + return errors.InternalErrorf("%s is not a regular file or directory", sourcePath) + } + if flush, ok := w.(flusher); ok { + defer func() { _ = flush.Flush() }() + } + gzw := gzip.NewWriter(w) + defer util.Close(gzw) + tw := tar.NewWriter(gzw) + defer util.Close(tw) + + if sourceFi.IsDir() { + return tarDir(sourcePath, tw) + } + return tarFile(sourcePath, tw) +} + +func tarDir(sourcePath string, tw *tar.Writer) error { + baseName := filepath.Base(sourcePath) + return filepath.Walk(sourcePath, func(fpath string, info os.FileInfo, err error) error { + if err != nil { + return errors.InternalWrapError(err) + } + // build the name to be used in the archive + nameInArchive, err := filepath.Rel(sourcePath, fpath) + if err != nil { + return errors.InternalWrapError(err) + } + nameInArchive = filepath.Join(baseName, nameInArchive) + log.Infof("writing %s", nameInArchive) + + var header *tar.Header + if (info.Mode() & os.ModeSymlink) != 0 { + linkTarget, err := os.Readlink(fpath) + if err != nil { + return errors.InternalWrapError(err) + } + header, err = tar.FileInfoHeader(info, filepath.ToSlash(linkTarget)) + if err != nil { + return errors.InternalWrapError(err) + } + } else { + header, err = tar.FileInfoHeader(info, info.Name()) + if err != nil { + return errors.InternalWrapError(err) + } + } + header.Name = nameInArchive + + err = tw.WriteHeader(header) + if err != nil { + return errors.InternalWrapError(err) + } + if !info.Mode().IsRegular() { + return nil + } + f, err := os.Open(fpath) + if err != nil { + return errors.InternalWrapError(err) + } + + // copy file data into tar writer + _, err = io.Copy(tw, f) + closeErr := f.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + return nil + }) +} + +func tarFile(sourcePath string, tw *tar.Writer) error { + f, err := os.Open(sourcePath) + if err != nil { + return errors.InternalWrapError(err) + } + defer util.Close(f) + info, err := f.Stat() + if err != nil { + return errors.InternalWrapError(err) + } + header, err := tar.FileInfoHeader(info, f.Name()) + if err != nil { + return errors.InternalWrapError(err) + } + header.Name = filepath.Base(sourcePath) + err = tw.WriteHeader(header) + if err != nil { + return errors.InternalWrapError(err) + } + _, err = io.Copy(tw, f) + if err != nil { + return err + } + return nil +} diff --git a/util/archive/archive_test.go b/util/archive/archive_test.go new file mode 100644 index 000000000000..2b4766b01fe0 --- /dev/null +++ b/util/archive/archive_test.go @@ -0,0 +1,60 @@ +package archive + +import ( + "bufio" + "crypto/rand" + "encoding/hex" + "os" + "path/filepath" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func tempFile(dir, prefix, suffix string) (*os.File, error) { + if dir == "" { + dir = os.TempDir() + } else { + os.MkdirAll(dir, 0700) + } + randBytes := make([]byte, 16) + rand.Read(randBytes) + filePath := filepath.Join(dir, prefix+hex.EncodeToString(randBytes)+suffix) + return os.Create(filePath) +} + +func TestTarDirectory(t *testing.T) { + f, err := tempFile(os.TempDir()+"/argo-test", "dir-", ".tgz") + assert.Nil(t, err) + log.Infof("Taring to %s", f.Name()) + w := bufio.NewWriter(f) + + err = TarGzToWriter("../../test/e2e", w) + assert.Nil(t, err) + + err = f.Close() + assert.Nil(t, err) +} + +func TestTarFile(t *testing.T) { + data, err := tempFile(os.TempDir()+"/argo-test", "file-", "") + assert.Nil(t, err) + _, err = data.WriteString("hello world") + assert.Nil(t, err) + data.Close() + + dataTarPath := data.Name() + ".tgz" + f, err := os.Create(dataTarPath) + assert.Nil(t, err) + log.Infof("Taring to %s", f.Name()) + w := bufio.NewWriter(f) + + err = TarGzToWriter(data.Name(), w) + assert.Nil(t, err) + err = os.Remove(data.Name()) + assert.Nil(t, err) + + err = f.Close() + assert.Nil(t, err) +} diff --git a/util/file/fileutil.go b/util/file/fileutil.go new file mode 100644 index 000000000000..37f6a56179c2 --- /dev/null +++ b/util/file/fileutil.go @@ -0,0 +1,87 @@ +package file + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/base64" + "io" + "io/ioutil" + "strings" + + log "github.com/sirupsen/logrus" +) + +type TarReader interface { + Next() (*tar.Header, error) +} + +// ExistsInTar return true if file or directory exists in tar +func ExistsInTar(sourcePath string, tarReader TarReader) bool { + sourcePath = strings.Trim(sourcePath, "/") + for { + hdr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return false + } + if hdr.FileInfo().IsDir() && strings.Contains(sourcePath, strings.Trim(hdr.Name, "/")) { + return true + } + if strings.Contains(sourcePath, hdr.Name) && hdr.Size > 0 { + return true + } + } + return false +} + +//Close the file +func close(f io.Closer) { + err := f.Close() + if err != nil { + log.Warnf("Failed to close the file/writer/reader. %v", err) + } +} + +// CompressEncodeString will return the compressed string with base64 encoded +func CompressEncodeString(content string) string { + return base64.StdEncoding.EncodeToString(CompressContent([]byte(content))) +} + +// DecodeDecompressString will return decode and decompress the +func DecodeDecompressString(content string) (string, error) { + + buf, err := base64.StdEncoding.DecodeString(content) + if err != nil { + return "", err + } + dBuf, err := DecompressContent(buf) + if err != nil { + return "", err + } + return string(dBuf), nil +} + +// CompressContent will compress the byte array using zip writer +func CompressContent(content []byte) []byte { + var buf bytes.Buffer + zipWriter := gzip.NewWriter(&buf) + + _, err := zipWriter.Write(content) + if err != nil { + log.Warnf("Error in compressing: %v", err) + } + close(zipWriter) + return buf.Bytes() +} + +// DecompressContent will return the uncompressed content +func DecompressContent(content []byte) ([]byte, error) { + + buf := bytes.NewReader(content) + gZipReader, _ := gzip.NewReader(buf) + defer close(gZipReader) + return ioutil.ReadAll(gZipReader) +} diff --git a/util/file/fileutil_test.go b/util/file/fileutil_test.go new file mode 100644 index 000000000000..f5b1fd1bae82 --- /dev/null +++ b/util/file/fileutil_test.go @@ -0,0 +1,121 @@ +package file_test + +import ( + "archive/tar" + "bytes" + "github.com/cyrusbiotechnology/argo/util/file" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +// TestResubmitWorkflowWithOnExit ensures we do not carry over the onExit node even if successful +func TestCompressContentString(t *testing.T) { + content := "{\"pod-limits-rrdm8-591645159\":{\"id\":\"pod-limits-rrdm8-591645159\",\"name\":\"pod-limits-rrdm8[0]." + + "run-pod(0:0)\",\"displayName\":\"run-pod(0:0)\",\"type\":\"Pod\",\"templateName\":\"run-pod\",\"phase\":" + + "\"Succeeded\",\"boundaryID\":\"pod-limits-rrdm8\",\"startedAt\":\"2019-03-07T19:14:50Z\",\"finishedAt\":" + + "\"2019-03-07T19:14:55Z\"}}" + + compString := file.CompressEncodeString(content) + + resultString, _ := file.DecodeDecompressString(compString) + + assert.Equal(t, content, resultString) +} + +func TestExistsInTar(t *testing.T) { + type fakeFile struct { + name, body string + isDir bool + } + + newTarReader := func(t *testing.T, files []fakeFile) *tar.Reader { + var buf bytes.Buffer + writer := tar.NewWriter(&buf) + for _, f := range files { + mode := os.FileMode(0600) + if f.isDir { + mode |= os.ModeDir + } + hdr := tar.Header{Name: f.name, Mode: int64(mode), Size: int64(len(f.body))} + err := writer.WriteHeader(&hdr) + assert.Nil(t, err) + _, err = writer.Write([]byte(f.body)) + assert.Nil(t, err) + } + err := writer.Close() + assert.Nil(t, err) + return tar.NewReader(&buf) + } + + type TestCase struct { + sourcePath string + expected bool + files []fakeFile + } + + tests := []TestCase{ + { + sourcePath: "/root.txt", expected: true, + files: []fakeFile{{name: "root.txt", body: "file in the root"}}, + }, + { + sourcePath: "/tmp/file/in/subfolder.txt", expected: true, + files: []fakeFile{{name: "subfolder.txt", body: "a file in a subfolder"}}, + }, + { + sourcePath: "/root", expected: true, + files: []fakeFile{ + {name: "root/", isDir: true}, + {name: "root/a.txt", body: "a"}, + {name: "root/b.txt", body: "b"}, + }, + }, + { + sourcePath: "/tmp/subfolder", expected: true, + files: []fakeFile{ + {name: "subfolder/", isDir: true}, + {name: "subfolder/a.txt", body: "a"}, + {name: "subfolder/b.txt", body: "b"}, + }, + }, + { + // should an empty tar return true?? + sourcePath: "/tmp/empty", expected: true, + files: []fakeFile{ + {name: "empty/", isDir: true}, + }, + }, + { + sourcePath: "/tmp/folder/that", expected: false, + files: []fakeFile{ + {name: "this/", isDir: true}, + {name: "this/a.txt", body: "a"}, + {name: "this/b.txt", body: "b"}, + }, + }, + { + sourcePath: "/empty.txt", expected: false, + files: []fakeFile{ + // fails because empty.txt is empty + {name: "empty.txt", body: ""}, + }, + }, + { + sourcePath: "/tmp/empty.txt", expected: false, + files: []fakeFile{ + // fails because empty.txt is empty + {name: "empty.txt", body: ""}, + }, + }, + } + for _, tc := range tests { + tc := tc + t.Run("source path "+tc.sourcePath, func(t *testing.T) { + t.Parallel() + tarReader := newTarReader(t, tc.files) + actual := file.ExistsInTar(tc.sourcePath, tarReader) + assert.Equalf(t, tc.expected, actual, "sourcePath %s not found", tc.sourcePath) + }) + } +} diff --git a/util/unstructured/unstructured.go b/util/unstructured/unstructured.go index b3073c4e82f3..e2c798c984b2 100644 --- a/util/unstructured/unstructured.go +++ b/util/unstructured/unstructured.go @@ -6,6 +6,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/informers/internalinterfaces" @@ -15,27 +16,27 @@ import ( // NewUnstructuredInformer constructs a new informer for Unstructured type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. -func NewUnstructuredInformer(resource *metav1.APIResource, client dynamic.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { +func NewUnstructuredInformer(resource schema.GroupVersionResource, client dynamic.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { return NewFilteredUnstructuredInformer(resource, client, namespace, resyncPeriod, indexers, nil) } // NewFilteredUnstructuredInformer constructs a new informer for Unstructured type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. -func NewFilteredUnstructuredInformer(resource *metav1.APIResource, client dynamic.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { +func NewFilteredUnstructuredInformer(resource schema.GroupVersionResource, client dynamic.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.Resource(resource, namespace).List(options) + return client.Resource(resource).Namespace(namespace).List(options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.Resource(resource, namespace).Watch(options) + return client.Resource(resource).Namespace(namespace).Watch(options) }, }, &unstructured.Unstructured{}, diff --git a/workflow/artifacts/gcs/gcs.go b/workflow/artifacts/gcs/gcs.go index cba41e09b70c..d0404e70ddef 100644 --- a/workflow/artifacts/gcs/gcs.go +++ b/workflow/artifacts/gcs/gcs.go @@ -6,18 +6,22 @@ import ( "errors" argoErrors "github.com/cyrusbiotechnology/argo/errors" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/util" log "github.com/sirupsen/logrus" + "google.golang.org/api/option" "io" "os" ) type GCSArtifactDriver struct { - Context context.Context + Context context.Context + CredsJSONData []byte } func (gcsDriver *GCSArtifactDriver) newGcsClient() (client *storage.Client, err error) { gcsDriver.Context = context.Background() - client, err = storage.NewClient(gcsDriver.Context) + + client, err = storage.NewClient(gcsDriver.Context, option.WithCredentialsJSON(gcsDriver.CredsJSONData)) if err != nil { return nil, argoErrors.InternalWrapError(err) } @@ -56,7 +60,7 @@ func (gcsDriver *GCSArtifactDriver) saveToFile(inputArtifact *wfv1.Artifact, fil if err != nil { return err } - defer r.Close() + defer util.Close(r) _, err = io.Copy(outputFile, r) if err != nil { @@ -94,7 +98,7 @@ func (gcsDriver *GCSArtifactDriver) saveToGCS(outputArtifact *wfv1.Artifact, fil return errors.New("only single files can be saved to GCS, not entire directories") } - defer inputFile.Close() + defer util.Close(inputFile) bucket := gcsClient.Bucket(outputArtifact.GCS.Bucket) object := bucket.Object(outputArtifact.GCS.Key) diff --git a/workflow/artifacts/git/git.go b/workflow/artifacts/git/git.go index 0eb31549c558..e5403dea355f 100644 --- a/workflow/artifacts/git/git.go +++ b/workflow/artifacts/git/git.go @@ -1,7 +1,11 @@ package git import ( + "fmt" + "io/ioutil" + "os" "os/exec" + "os/user" "strings" log "github.com/sirupsen/logrus" @@ -17,9 +21,10 @@ import ( // GitArtifactDriver is the artifact driver for a git repo type GitArtifactDriver struct { - Username string - Password string - SSHPrivateKey string + Username string + Password string + SSHPrivateKey string + InsecureIgnoreHostKey bool } // Load download artifacts from an git URL @@ -30,14 +35,16 @@ func (g *GitArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) erro return errors.InternalWrapError(err) } auth := &ssh2.PublicKeys{User: "git", Signer: signer} - auth.HostKeyCallback = ssh.InsecureIgnoreHostKey() - return gitClone(path, inputArtifact, auth) + if g.InsecureIgnoreHostKey { + auth.HostKeyCallback = ssh.InsecureIgnoreHostKey() + } + return gitClone(path, inputArtifact, auth, g.SSHPrivateKey) } if g.Username != "" || g.Password != "" { auth := &http.BasicAuth{Username: g.Username, Password: g.Password} - return gitClone(path, inputArtifact, auth) + return gitClone(path, inputArtifact, auth, "") } - return gitClone(path, inputArtifact, nil) + return gitClone(path, inputArtifact, nil, "") } // Save is unsupported for git output artifacts @@ -45,7 +52,35 @@ func (g *GitArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) err return errors.Errorf(errors.CodeBadRequest, "Git output artifacts unsupported") } -func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMethod) error { +func writePrivateKey(key string, insecureIgnoreHostKey bool) error { + usr, err := user.Current() + if err != nil { + return errors.InternalWrapError(err) + } + sshDir := fmt.Sprintf("%s/.ssh", usr.HomeDir) + err = os.Mkdir(sshDir, 0700) + if err != nil { + return errors.InternalWrapError(err) + } + + if insecureIgnoreHostKey { + sshConfig := `Host * + StrictHostKeyChecking no + UserKnownHostsFile /dev/null` + err = ioutil.WriteFile(fmt.Sprintf("%s/config", sshDir), []byte(sshConfig), 0644) + if err != nil { + return errors.InternalWrapError(err) + } + } + err = ioutil.WriteFile(fmt.Sprintf("%s/id_rsa", sshDir), []byte(key), 0600) + if err != nil { + return errors.InternalWrapError(err) + } + + return nil +} + +func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMethod, privateKey string) error { cloneOptions := git.CloneOptions{ URL: inputArtifact.Git.Repo, RecurseSubmodules: git.DefaultSubmoduleRecursionDepth, @@ -70,6 +105,23 @@ func gitClone(path string, inputArtifact *wfv1.Artifact, auth transport.AuthMeth return errors.InternalWrapError(err) } log.Errorf("`%s` stdout:\n%s", cmd.Args, string(output)) + if privateKey != "" { + err := writePrivateKey(privateKey, inputArtifact.Git.InsecureIgnoreHostKey) + if err != nil { + return errors.InternalWrapError(err) + } + } + submodulesCmd := exec.Command("git", "submodule", "update", "--init", "--recursive", "--force") + submodulesCmd.Dir = path + submoduleOutput, err := submodulesCmd.Output() + if err != nil { + if exErr, ok := err.(*exec.ExitError); ok { + log.Errorf("`%s` stderr:\n%s", submodulesCmd.Args, string(exErr.Stderr)) + return errors.InternalError(strings.Split(string(exErr.Stderr), "\n")[0]) + } + return errors.InternalWrapError(err) + } + log.Errorf("`%s` stdout:\n%s", submodulesCmd.Args, string(submoduleOutput)) } return nil } diff --git a/workflow/artifacts/hdfs/hdfs.go b/workflow/artifacts/hdfs/hdfs.go new file mode 100644 index 000000000000..8d31c8971841 --- /dev/null +++ b/workflow/artifacts/hdfs/hdfs.go @@ -0,0 +1,217 @@ +package hdfs + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/argoproj/pkg/file" + "gopkg.in/jcmturner/gokrb5.v5/credentials" + "gopkg.in/jcmturner/gokrb5.v5/keytab" + + "github.com/cyrusbiotechnology/argo/errors" + wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/util" + "github.com/cyrusbiotechnology/argo/workflow/common" +) + +// ArtifactDriver is a driver for HDFS +type ArtifactDriver struct { + Addresses []string // comma-separated name nodes + Path string + Force bool + HDFSUser string + KrbOptions *KrbOptions +} + +// KrbOptions is options for Kerberos +type KrbOptions struct { + CCacheOptions *CCacheOptions + KeytabOptions *KeytabOptions + Config string + ServicePrincipalName string +} + +// CCacheOptions is options for ccache +type CCacheOptions struct { + CCache credentials.CCache +} + +// KeytabOptions is options for keytab +type KeytabOptions struct { + Keytab keytab.Keytab + Username string + Realm string +} + +// ValidateArtifact validates HDFS artifact +func ValidateArtifact(errPrefix string, art *wfv1.HDFSArtifact) error { + if len(art.Addresses) == 0 { + return errors.Errorf(errors.CodeBadRequest, "%s.addresses is required", errPrefix) + } + if art.Path == "" { + return errors.Errorf(errors.CodeBadRequest, "%s.path is required", errPrefix) + } + if !filepath.IsAbs(art.Path) { + return errors.Errorf(errors.CodeBadRequest, "%s.path must be a absolute file path", errPrefix) + } + + hasKrbCCache := art.KrbCCacheSecret != nil + hasKrbKeytab := art.KrbKeytabSecret != nil + + if art.HDFSUser == "" && !hasKrbCCache && !hasKrbKeytab { + return errors.Errorf(errors.CodeBadRequest, "either %s.hdfsUser, %s.krbCCacheSecret or %s.krbKeytabSecret is required", errPrefix, errPrefix, errPrefix) + } + if hasKrbKeytab && (art.KrbServicePrincipalName == "" || art.KrbConfigConfigMap == nil || art.KrbUsername == "" || art.KrbRealm == "") { + return errors.Errorf(errors.CodeBadRequest, "%s.krbServicePrincipalName, %s.krbConfigConfigMap, %s.krbUsername and %s.krbRealm are required with %s.krbKeytabSecret", errPrefix, errPrefix, errPrefix, errPrefix, errPrefix) + } + if hasKrbCCache && (art.KrbServicePrincipalName == "" || art.KrbConfigConfigMap == nil) { + return errors.Errorf(errors.CodeBadRequest, "%s.krbServicePrincipalName and %s.krbConfigConfigMap are required with %s.krbCCacheSecret", errPrefix, errPrefix, errPrefix) + } + return nil +} + +// CreateDriver constructs ArtifactDriver +func CreateDriver(ci common.ResourceInterface, art *wfv1.HDFSArtifact) (*ArtifactDriver, error) { + var krbConfig string + var krbOptions *KrbOptions + var err error + + namespace := ci.GetNamespace() + + if art.KrbConfigConfigMap != nil && art.KrbConfigConfigMap.Name != "" { + krbConfig, err = ci.GetConfigMapKey(namespace, art.KrbConfigConfigMap.Name, art.KrbConfigConfigMap.Key) + if err != nil { + return nil, err + } + } + if art.KrbCCacheSecret != nil && art.KrbCCacheSecret.Name != "" { + bytes, err := ci.GetSecretFromVolMount(art.KrbCCacheSecret.Name, art.KrbCCacheSecret.Key) + if err != nil { + return nil, err + } + ccache, err := credentials.ParseCCache(bytes) + if err != nil { + return nil, err + } + krbOptions = &KrbOptions{ + CCacheOptions: &CCacheOptions{ + CCache: ccache, + }, + Config: krbConfig, + ServicePrincipalName: art.KrbServicePrincipalName, + } + } + if art.KrbKeytabSecret != nil && art.KrbKeytabSecret.Name != "" { + bytes, err := ci.GetSecretFromVolMount(art.KrbKeytabSecret.Name, art.KrbKeytabSecret.Key) + if err != nil { + return nil, err + } + ktb, err := keytab.Parse(bytes) + if err != nil { + return nil, err + } + krbOptions = &KrbOptions{ + KeytabOptions: &KeytabOptions{ + Keytab: ktb, + Username: art.KrbUsername, + Realm: art.KrbRealm, + }, + Config: krbConfig, + ServicePrincipalName: art.KrbServicePrincipalName, + } + } + + driver := ArtifactDriver{ + Addresses: art.Addresses, + Path: art.Path, + Force: art.Force, + HDFSUser: art.HDFSUser, + KrbOptions: krbOptions, + } + return &driver, nil +} + +// Load downloads artifacts from HDFS compliant storage +func (driver *ArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) error { + hdfscli, err := createHDFSClient(driver.Addresses, driver.HDFSUser, driver.KrbOptions) + if err != nil { + return err + } + defer util.Close(hdfscli) + + srcStat, err := hdfscli.Stat(driver.Path) + if err != nil { + return err + } + if srcStat.IsDir() { + return fmt.Errorf("HDFS artifact does not suppot directory copy") + } + + _, err = os.Stat(path) + if err != nil && !os.IsNotExist(err) { + return err + } + + if os.IsNotExist(err) { + dirPath := filepath.Dir(driver.Path) + if dirPath != "." && dirPath != "/" { + // Follow umask for the permission + err = os.MkdirAll(dirPath, 0777) + if err != nil { + return err + } + } + } else { + if driver.Force { + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) { + return err + } + } + } + + return hdfscli.CopyToLocal(driver.Path, path) +} + +// Save saves an artifact to HDFS compliant storage +func (driver *ArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) error { + hdfscli, err := createHDFSClient(driver.Addresses, driver.HDFSUser, driver.KrbOptions) + if err != nil { + return err + } + defer util.Close(hdfscli) + + isDir, err := file.IsDirectory(path) + if err != nil { + return err + } + if isDir { + return fmt.Errorf("HDFS artifact does not suppot directory copy") + } + + _, err = hdfscli.Stat(driver.Path) + if err != nil && !os.IsNotExist(err) { + return err + } + + if os.IsNotExist(err) { + dirPath := filepath.Dir(driver.Path) + if dirPath != "." && dirPath != "/" { + // Follow umask for the permission + err = hdfscli.MkdirAll(dirPath, 0777) + if err != nil { + return err + } + } + } else { + if driver.Force { + err = hdfscli.Remove(driver.Path) + if err != nil && !os.IsNotExist(err) { + return err + } + } + } + + return hdfscli.CopyToRemote(path, driver.Path) +} diff --git a/workflow/artifacts/hdfs/util.go b/workflow/artifacts/hdfs/util.go new file mode 100644 index 000000000000..3af330ae012e --- /dev/null +++ b/workflow/artifacts/hdfs/util.go @@ -0,0 +1,53 @@ +package hdfs + +import ( + "fmt" + + "github.com/colinmarc/hdfs" + krb "gopkg.in/jcmturner/gokrb5.v5/client" + "gopkg.in/jcmturner/gokrb5.v5/config" +) + +func createHDFSClient(addresses []string, user string, krbOptions *KrbOptions) (*hdfs.Client, error) { + options := hdfs.ClientOptions{ + Addresses: addresses, + } + + if krbOptions != nil { + krbClient, err := createKrbClient(krbOptions) + if err != nil { + return nil, err + } + options.KerberosClient = krbClient + options.KerberosServicePrincipleName = krbOptions.ServicePrincipalName + } else { + options.User = user + } + + return hdfs.NewClient(options) +} + +func createKrbClient(krbOptions *KrbOptions) (*krb.Client, error) { + krbConfig, err := config.NewConfigFromString(krbOptions.Config) + if err != nil { + return nil, err + } + + if krbOptions.CCacheOptions != nil { + client, err := krb.NewClientFromCCache(krbOptions.CCacheOptions.CCache) + if err != nil { + return nil, err + } + return client.WithConfig(krbConfig), nil + } else if krbOptions.KeytabOptions != nil { + client := krb.NewClientWithKeytab(krbOptions.KeytabOptions.Username, krbOptions.KeytabOptions.Realm, krbOptions.KeytabOptions.Keytab) + client = *client.WithConfig(krbConfig) + err = client.Login() + if err != nil { + return nil, err + } + return &client, nil + } + + return nil, fmt.Errorf("Failed to get a Kerberos client") +} diff --git a/workflow/artifacts/s3/s3.go b/workflow/artifacts/s3/s3.go index 7245ebea0e9a..2a03f2bc18c1 100644 --- a/workflow/artifacts/s3/s3.go +++ b/workflow/artifacts/s3/s3.go @@ -1,10 +1,13 @@ package s3 import ( - "github.com/argoproj/pkg/file" - argos3 "github.com/argoproj/pkg/s3" + "time" + log "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" + "github.com/argoproj/pkg/file" + argos3 "github.com/argoproj/pkg/s3" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" ) @@ -31,42 +34,70 @@ func (s3Driver *S3ArtifactDriver) newS3Client() (argos3.S3Client, error) { // Load downloads artifacts from S3 compliant storage func (s3Driver *S3ArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) error { - s3cli, err := s3Driver.newS3Client() - if err != nil { - return err - } - origErr := s3cli.GetFile(inputArtifact.S3.Bucket, inputArtifact.S3.Key, path) - if origErr == nil { - return nil - } - if !argos3.IsS3ErrCode(origErr, "NoSuchKey") { - return origErr - } - // If we get here, the error was a NoSuchKey. The key might be a s3 "directory" - isDir, err := s3cli.IsDirectory(inputArtifact.S3.Bucket, inputArtifact.S3.Key) - if err != nil { - log.Warnf("Failed to test if %s is a directory: %v", inputArtifact.S3.Bucket, err) - return origErr - } - if !isDir { - // It's neither a file, nor a directory. Return the original NoSuchKey error - return origErr - } - return s3cli.GetDirectory(inputArtifact.S3.Bucket, inputArtifact.S3.Key, path) + err := wait.ExponentialBackoff(wait.Backoff{Duration: time.Second * 2, Factor: 2.0, Steps: 5, Jitter: 0.1}, + func() (bool, error) { + log.Infof("S3 Load path: %s, key: %s", path, inputArtifact.S3.Key) + s3cli, err := s3Driver.newS3Client() + if err != nil { + log.Warnf("Failed to create new S3 client: %v", err) + return false, nil + } + origErr := s3cli.GetFile(inputArtifact.S3.Bucket, inputArtifact.S3.Key, path) + if origErr == nil { + return true, nil + } + if !argos3.IsS3ErrCode(origErr, "NoSuchKey") { + log.Warnf("Failed get file: %v", origErr) + return false, nil + } + // If we get here, the error was a NoSuchKey. The key might be a s3 "directory" + isDir, err := s3cli.IsDirectory(inputArtifact.S3.Bucket, inputArtifact.S3.Key) + if err != nil { + log.Warnf("Failed to test if %s is a directory: %v", inputArtifact.S3.Bucket, err) + return false, nil + } + if !isDir { + // It's neither a file, nor a directory. Return the original NoSuchKey error + return false, origErr + } + + if err = s3cli.GetDirectory(inputArtifact.S3.Bucket, inputArtifact.S3.Key, path); err != nil { + log.Warnf("Failed get directory: %v", err) + return false, nil + } + return true, nil + }) + + return err } // Save saves an artifact to S3 compliant storage func (s3Driver *S3ArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) error { - s3cli, err := s3Driver.newS3Client() - if err != nil { - return err - } - isDir, err := file.IsDirectory(path) - if err != nil { - return err - } - if isDir { - return s3cli.PutDirectory(outputArtifact.S3.Bucket, outputArtifact.S3.Key, path) - } - return s3cli.PutFile(outputArtifact.S3.Bucket, outputArtifact.S3.Key, path) + err := wait.ExponentialBackoff(wait.Backoff{Duration: time.Second * 2, Factor: 2.0, Steps: 5, Jitter: 0.1}, + func() (bool, error) { + log.Infof("S3 Save path: %s, key: %s", path, outputArtifact.S3.Key) + s3cli, err := s3Driver.newS3Client() + if err != nil { + log.Warnf("Failed to create new S3 client: %v", err) + return false, nil + } + isDir, err := file.IsDirectory(path) + if err != nil { + log.Warnf("Failed to test if %s is a directory: %v", path, err) + return false, nil + } + if isDir { + if err = s3cli.PutDirectory(outputArtifact.S3.Bucket, outputArtifact.S3.Key, path); err != nil { + log.Warnf("Failed to put directory: %v", err) + return false, nil + } + } else { + if err = s3cli.PutFile(outputArtifact.S3.Bucket, outputArtifact.S3.Key, path); err != nil { + log.Warnf("Failed to put file: %v", err) + return false, nil + } + } + return true, nil + }) + return err } diff --git a/workflow/common/common.go b/workflow/common/common.go index d5f9d9604fb0..da6ac92b33d2 100644 --- a/workflow/common/common.go +++ b/workflow/common/common.go @@ -30,10 +30,6 @@ const ( // PodMetadataAnnotationsPath is the file path containing pod metadata annotations. Examined by executor PodMetadataAnnotationsPath = PodMetadataMountPath + "/" + PodMetadataAnnotationsVolumePath - // DockerLibVolumeName is the volume name for the /var/lib/docker host path volume - DockerLibVolumeName = "docker-lib" - // DockerLibHostPath is the host directory path containing docker runtime state - DockerLibHostPath = "/var/lib/docker" // DockerSockVolumeName is the volume name for the /var/run/docker.sock host path volume DockerSockVolumeName = "docker-sock" @@ -75,10 +71,11 @@ const ( // Each artifact will be named according to its input name (e.g: /argo/inputs/artifacts/CODE) ExecutorArtifactBaseDir = "/argo/inputs/artifacts" - // InitContainerMainFilesystemDir is a path made available to the init container such that the init container - // can access the same volume mounts used in the main container. This is used for the purposes of artifact loading - // (when there is overlapping paths between artifacts and volume mounts) - InitContainerMainFilesystemDir = "/mainctrfs" + // ExecutorMainFilesystemDir is a path made available to the init/wait containers such that they + // can access the same volume mounts used in the main container. This is used for the purposes + // of artifact loading (when there is overlapping paths between artifacts and volume mounts), + // as well as artifact collection by the wait container. + ExecutorMainFilesystemDir = "/mainctrfs" // ExecutorStagingEmptyDir is the path of the emptydir which is used as a staging area to transfer a file between init/main container for script/resource templates ExecutorStagingEmptyDir = "/argo/staging" @@ -91,7 +88,6 @@ const ( // EnvVarPodName contains the name of the pod (currently unused) EnvVarPodName = "ARGO_POD_NAME" - // EnvVarContainerRuntimeExecutor contains the name of the container runtime executor to use, empty is equal to "docker" EnvVarContainerRuntimeExecutor = "ARGO_CONTAINER_RUNTIME_EXECUTOR" // EnvVarDownwardAPINodeIP is the envvar used to get the `status.hostIP` @@ -107,6 +103,12 @@ const ( // ContainerRuntimeExecutorKubelet to use the kubelet as container runtime executor ContainerRuntimeExecutorKubelet = "kubelet" + // ContainerRuntimeExecutorK8sAPI to use the Kubernetes API server as container runtime executor + ContainerRuntimeExecutorK8sAPI = "k8sapi" + + // ContainerRuntimeExecutorPNS indicates to use process namespace sharing as the container runtime executor + ContainerRuntimeExecutorPNS = "pns" + // Variables that are added to the scope during template execution and can be referenced using {{}} syntax // GlobalVarWorkflowName is a global workflow variable referencing the workflow's metadata.name field @@ -121,8 +123,15 @@ const ( GlobalVarWorkflowCreationTimestamp = "workflow.creationTimestamp" // LocalVarPodName is a step level variable that references the name of the pod LocalVarPodName = "pod.name" + + KubeConfigDefaultMountPath = "/kube/config" + KubeConfigDefaultVolumeName = "kubeconfig" + SecretVolMountPath = "/argo/secret" ) +// GlobalVarWorkflowRootTags is a list of root tags in workflow which could be used for variable reference +var GlobalVarValidWorkflowVariablePrefix = []string{"item.", "steps.", "inputs.", "outputs.", "pod.", "workflow.", "tasks."} + var ( GoogleSecretName = os.Getenv(EnvVarGoogleSecret) ) @@ -134,3 +143,10 @@ type ExecutionControl struct { // used to support workflow or steps/dag level timeouts. Deadline *time.Time `json:"deadline,omitempty"` } + +type ResourceInterface interface { + GetNamespace() string + GetSecrets(namespace, name, key string) ([]byte, error) + GetSecretFromVolMount(name, key string) ([]byte, error) + GetConfigMapKey(namespace, name, key string) (string, error) +} diff --git a/workflow/common/util.go b/workflow/common/util.go index 8ca11933f23c..14b04d3a8289 100644 --- a/workflow/common/util.go +++ b/workflow/common/util.go @@ -5,16 +5,16 @@ import ( "encoding/json" "fmt" "io" + "net/http" "os/exec" "regexp" "strconv" "strings" "time" - "github.com/cyrusbiotechnology/argo/errors" "github.com/cyrusbiotechnology/argo/pkg/apis/workflow" - wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" "github.com/ghodss/yaml" + "github.com/gorilla/websocket" log "github.com/sirupsen/logrus" "github.com/valyala/fasttemplate" apiv1 "k8s.io/api/core/v1" @@ -23,11 +23,15 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" + + "github.com/cyrusbiotechnology/argo/errors" + wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/util" ) // FindOverlappingVolume looks an artifact path, checks if it overlaps with any // user specified volumeMounts in the template, and returns the deepest volumeMount -// (if any). +// (if any). A return value of nil indicates the path is not under any volumeMount. func FindOverlappingVolume(tmpl *wfv1.Template, path string) *apiv1.VolumeMount { if tmpl.Container == nil { return nil @@ -66,6 +70,126 @@ func KillPodContainer(restConfig *rest.Config, namespace string, pod string, con return nil } +// ContainerLogStream returns an io.ReadCloser for a container's log stream using the websocket +// interface. This was implemented in the hopes that we could selectively choose stdout from stderr, +// but due to https://github.com/kubernetes/kubernetes/issues/28167, it is not possible to discern +// stdout from stderr using the K8s API server, so this function is unused, instead preferring the +// pod logs interface from client-go. It's left as a reference for when issue #28167 is eventually +// resolved. +func ContainerLogStream(config *rest.Config, namespace string, pod string, container string) (io.ReadCloser, error) { + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, errors.InternalWrapError(err) + } + logRequest := clientset.CoreV1().RESTClient().Get(). + Resource("pods"). + Name(pod). + Namespace(namespace). + SubResource("log"). + Param("container", container) + u := logRequest.URL() + switch u.Scheme { + case "https": + u.Scheme = "wss" + case "http": + u.Scheme = "ws" + default: + return nil, errors.Errorf("Malformed URL %s", u.String()) + } + + log.Info(u.String()) + wsrc := websocketReadCloser{ + &bytes.Buffer{}, + } + + wrappedRoundTripper, err := roundTripperFromConfig(config, wsrc.WebsocketCallback) + if err != nil { + return nil, errors.InternalWrapError(err) + } + + // Send the request and let the callback do its work + req := &http.Request{ + Method: http.MethodGet, + URL: u, + } + _, err = wrappedRoundTripper.RoundTrip(req) + if err != nil && !websocket.IsCloseError(err, websocket.CloseNormalClosure) { + return nil, errors.InternalWrapError(err) + } + return &wsrc, nil +} + +type RoundTripCallback func(conn *websocket.Conn, resp *http.Response, err error) error + +type WebsocketRoundTripper struct { + Dialer *websocket.Dialer + Do RoundTripCallback +} + +func (d *WebsocketRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + conn, resp, err := d.Dialer.Dial(r.URL.String(), r.Header) + if err == nil { + defer util.Close(conn) + } + return resp, d.Do(conn, resp, err) +} + +func (w *websocketReadCloser) WebsocketCallback(ws *websocket.Conn, resp *http.Response, err error) error { + if err != nil { + if resp != nil && resp.StatusCode != http.StatusOK { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(resp.Body) + return errors.InternalErrorf("Can't connect to log endpoint (%d): %s", resp.StatusCode, buf.String()) + } + return errors.InternalErrorf("Can't connect to log endpoint: %s", err.Error()) + } + + for { + _, body, err := ws.ReadMessage() + if len(body) > 0 { + //log.Debugf("%d: %s", msgType, string(body)) + _, writeErr := w.Write(body) + if writeErr != nil { + return writeErr + } + } + if err != nil { + if err == io.EOF { + log.Infof("websocket closed: %v", err) + return nil + } + log.Warnf("websocket error: %v", err) + return err + } + } +} + +func roundTripperFromConfig(config *rest.Config, callback RoundTripCallback) (http.RoundTripper, error) { + tlsConfig, err := rest.TLSConfigFor(config) + if err != nil { + return nil, err + } + // Create a roundtripper which will pass in the final underlying websocket connection to a callback + wsrt := &WebsocketRoundTripper{ + Do: callback, + Dialer: &websocket.Dialer{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: tlsConfig, + }, + } + // Make sure we inherit all relevant security headers + return rest.HTTPWrappersForConfig(config, wsrt) +} + +type websocketReadCloser struct { + *bytes.Buffer +} + +func (w *websocketReadCloser) Close() error { + //return w.conn.Close() + return nil +} + // ExecPodContainer runs a command in a container in a pod and returns the remotecommand.Executor func ExecPodContainer(restConfig *rest.Config, namespace string, pod string, container string, stdout bool, stderr bool, command ...string) (remotecommand.Executor, error) { clientset, err := kubernetes.NewForConfig(restConfig) @@ -96,7 +220,7 @@ func ExecPodContainer(restConfig *rest.Config, namespace string, pod string, con } // GetExecutorOutput returns the output of an remotecommand.Executor -func GetExecutorOutput(exec remotecommand.Executor) (string, string, error) { +func GetExecutorOutput(exec remotecommand.Executor) (*bytes.Buffer, *bytes.Buffer, error) { var stdOut bytes.Buffer var stdErr bytes.Buffer err := exec.Stream(remotecommand.StreamOptions{ @@ -105,9 +229,9 @@ func GetExecutorOutput(exec remotecommand.Executor) (string, string, error) { Tty: false, }) if err != nil { - return "", "", errors.InternalWrapError(err) + return nil, nil, errors.InternalWrapError(err) } - return stdOut.String(), stdErr.String(), nil + return &stdOut, &stdErr, nil } // ProcessArgs sets in the inputs, the values either passed via arguments, or the hardwired values @@ -146,17 +270,23 @@ func ProcessArgs(tmpl *wfv1.Template, args wfv1.Arguments, globalParams, localPa newInputArtifacts[i] = inArt continue } - // artifact must be supplied argArt := args.GetArtifactByName(inArt.Name) - if argArt == nil { - return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s was not supplied", inArt.Name) + if !inArt.Optional { + // artifact must be supplied + if argArt == nil { + return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s was not supplied", inArt.Name) + } + if !argArt.HasLocation() && !validateOnly { + return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s missing location information", inArt.Name) + } } - if !argArt.HasLocation() && !validateOnly { - return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s missing location information", inArt.Name) + if argArt != nil { + argArt.Path = inArt.Path + argArt.Mode = inArt.Mode + newInputArtifacts[i] = *argArt + } else { + newInputArtifacts[i] = inArt } - argArt.Path = inArt.Path - argArt.Mode = inArt.Mode - newInputArtifacts[i] = *argArt } tmpl.Inputs.Artifacts = newInputArtifacts @@ -195,6 +325,22 @@ func substituteParams(tmpl *wfv1.Template, globalParams, localParams map[string] } replaceMap["inputs.parameters."+inParam.Name] = *inParam.Value } + for _, inArt := range globalReplacedTmpl.Inputs.Artifacts { + if inArt.Path != "" { + replaceMap["inputs.artifacts."+inArt.Name+".path"] = inArt.Path + } + } + for _, outArt := range globalReplacedTmpl.Outputs.Artifacts { + if outArt.Path != "" { + replaceMap["outputs.artifacts."+outArt.Name+".path"] = outArt.Path + } + } + for _, param := range globalReplacedTmpl.Outputs.Parameters { + if param.ValueFrom != nil && param.ValueFrom.Path != "" { + replaceMap["outputs.parameters."+param.Name+".path"] = param.ValueFrom.Path + } + } + fstTmpl = fasttemplate.New(globalReplacedTmplStr, "{{", "}}") s, err := Replace(fstTmpl, replaceMap, true) if err != nil { @@ -243,10 +389,12 @@ func RunCommand(name string, arg ...string) error { log.Info(cmdStr) _, err := cmd.Output() if err != nil { - exErr := err.(*exec.ExitError) - errOutput := string(exErr.Stderr) - log.Errorf("`%s` failed: %s", cmdStr, errOutput) - return errors.InternalError(strings.TrimSpace(errOutput)) + if exErr, ok := err.(*exec.ExitError); ok { + errOutput := string(exErr.Stderr) + log.Errorf("`%s` failed: %s", cmdStr, errOutput) + return errors.InternalError(strings.TrimSpace(errOutput)) + } + return errors.InternalWrapError(err) } return nil } diff --git a/workflow/controller/config.go b/workflow/controller/config.go index 80c7480bbb7d..123a51967abe 100644 --- a/workflow/controller/config.go +++ b/workflow/controller/config.go @@ -23,14 +23,23 @@ import ( // WorkflowControllerConfig contain the configuration settings for the workflow controller type WorkflowControllerConfig struct { // ExecutorImage is the image name of the executor to use when running pods + // DEPRECATED: use --executor-image flag to workflow-controller instead ExecutorImage string `json:"executorImage,omitempty"` // ExecutorImagePullPolicy is the imagePullPolicy of the executor to use when running pods + // DEPRECATED: use `executor.imagePullPolicy` in configmap instead ExecutorImagePullPolicy string `json:"executorImagePullPolicy,omitempty"` + // Executor holds container customizations for the executor to use when running pods + Executor *apiv1.Container `json:"executor,omitempty"` + // ExecutorResources specifies the resource requirements that will be used for the executor sidecar + // DEPRECATED: use `executor.resources` in configmap instead ExecutorResources *apiv1.ResourceRequirements `json:"executorResources,omitempty"` + // KubeConfig specifies a kube config file for the wait & init containers + KubeConfig *KubeConfig `json:"kubeConfig,omitempty"` + // ContainerRuntimeExecutor specifies the container runtime interface to use, default is docker ContainerRuntimeExecutor string `json:"containerRuntimeExecutor,omitempty"` @@ -58,6 +67,24 @@ type WorkflowControllerConfig struct { MetricsConfig metrics.PrometheusConfig `json:"metricsConfig,omitempty"` TelemetryConfig metrics.PrometheusConfig `json:"telemetryConfig,omitempty"` + + // Parallelism limits the max total parallel workflows that can execute at the same time + Parallelism int `json:"parallelism,omitempty"` +} + +// KubeConfig is used for wait & init sidecar containers to communicate with a k8s apiserver by a outofcluster method, +// it is used when the workflow controller is in a different cluster with the workflow workloads +type KubeConfig struct { + // SecretName of the kubeconfig secret + // may not be empty if kuebConfig specified + SecretName string `json:"secretName"` + // SecretKey of the kubeconfig in the secret + // may not be empty if kubeConfig specified + SecretKey string `json:"secretKey"` + // VolumeName of kubeconfig, default to 'kubeconfig' + VolumeName string `json:"volumeName,omitempty"` + // MountPath of the kubeconfig secret, default to '/kube/config' + MountPath string `json:"mountPath,omitempty"` } // ArtifactRepository represents a artifact repository in which a controller will store its artifacts @@ -68,7 +95,9 @@ type ArtifactRepository struct { S3 *S3ArtifactRepository `json:"s3,omitempty"` // Artifactory stores artifacts to JFrog Artifactory Artifactory *ArtifactoryArtifactRepository `json:"artifactory,omitempty"` - GCS *GCSArtifactRepository `json:"gcs,omitempty"` + // HDFS stores artifacts in HDFS + HDFS *HDFSArtifactRepository `json:"hdfs,omitempty"` + GCS *GCSArtifactRepository `json:"gcs,omitempty"` } // S3ArtifactRepository defines the controller configuration for an S3 artifact repository @@ -90,6 +119,17 @@ type ArtifactoryArtifactRepository struct { RepoURL string `json:"repoURL,omitempty"` } +// HDFSArtifactRepository defines the controller configuration for an HDFS artifact repository +type HDFSArtifactRepository struct { + wfv1.HDFSConfig `json:",inline"` + + // PathFormat is defines the format of path to store a file. Can reference workflow variables + PathFormat string `json:"pathFormat,omitempty"` + + // Force copies a file forcibly even if it exists (default: false) + Force bool `json:"force,omitempty"` +} + // GCSArtifactRepository defines the controller configuration for a GCS artifact repository type GCSArtifactRepository struct { wfv1.GCSBucket `json:",inline"` @@ -143,6 +183,7 @@ func (wfc *WorkflowController) updateConfig(configString string) error { return errors.Errorf(errors.CodeBadRequest, "ConfigMap '%s' does not have executorImage", wfc.configMap) } wfc.Config = config + wfc.throttler.SetParallelism(config.Parallelism) return nil } @@ -156,13 +197,13 @@ func (wfc *WorkflowController) executorImage() string { // executorImagePullPolicy returns the imagePullPolicy to use for the workflow executor func (wfc *WorkflowController) executorImagePullPolicy() apiv1.PullPolicy { - var policy string if wfc.cliExecutorImagePullPolicy != "" { - policy = wfc.cliExecutorImagePullPolicy + return apiv1.PullPolicy(wfc.cliExecutorImagePullPolicy) + } else if wfc.Config.Executor != nil && wfc.Config.Executor.ImagePullPolicy != "" { + return wfc.Config.Executor.ImagePullPolicy } else { - policy = wfc.Config.ExecutorImagePullPolicy + return apiv1.PullPolicy(wfc.Config.ExecutorImagePullPolicy) } - return apiv1.PullPolicy(policy) } func (wfc *WorkflowController) watchControllerConfigMap(ctx context.Context) (cache.Controller, error) { diff --git a/workflow/controller/controller.go b/workflow/controller/controller.go index 2febc7d556e4..1cffd9563085 100644 --- a/workflow/controller/controller.go +++ b/workflow/controller/controller.go @@ -58,6 +58,7 @@ type WorkflowController struct { wfQueue workqueue.RateLimitingInterface podQueue workqueue.RateLimitingInterface completedPods chan string + throttler Throttler } const ( @@ -91,6 +92,7 @@ func NewWorkflowController( podQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), completedPods: make(chan string, 512), } + wfc.throttler = NewThrottler(0, wfc.wfQueue) return &wfc } @@ -227,22 +229,45 @@ func (wfc *WorkflowController) processNextItem() bool { log.Warnf("Key '%s' in index is not an unstructured", key) return true } + + if key, ok = wfc.throttler.Next(key); !ok { + log.Warnf("Workflow %s processing has been postponed due to max parallelism limit", key) + return true + } + wf, err := util.FromUnstructured(un) if err != nil { log.Warnf("Failed to unmarshal key '%s' to workflow object: %v", key, err) woc := newWorkflowOperationCtx(wf, wfc) woc.markWorkflowFailed(fmt.Sprintf("invalid spec: %s", err.Error())) woc.persistUpdates() + wfc.throttler.Remove(key) return true } if wf.ObjectMeta.Labels[common.LabelKeyCompleted] == "true" { + wfc.throttler.Remove(key) // can get here if we already added the completed=true label, // but we are still draining the controller's workflow workqueue return true } + woc := newWorkflowOperationCtx(wf, wfc) + + // Decompress the node if it is compressed + err = util.DecompressWorkflow(woc.wf) + if err != nil { + woc.log.Warnf("workflow decompression failed: %v", err) + woc.markWorkflowFailed(fmt.Sprintf("workflow decompression failed: %s", err.Error())) + woc.persistUpdates() + wfc.throttler.Remove(key) + return true + } woc.operate() + if woc.wf.Status.Completed() { + wfc.throttler.Remove(key) + } + // TODO: operate should return error if it was unable to operate properly // so we can requeue the work for a later time // See: https://github.com/kubernetes/client-go/blob/master/examples/workqueue/main.go @@ -317,6 +342,22 @@ func (wfc *WorkflowController) tweakWorkflowMetricslist(options *metav1.ListOpti options.LabelSelector = labelSelector.String() } +func getWfPriority(obj interface{}) (int32, time.Time) { + un, ok := obj.(*unstructured.Unstructured) + if !ok { + return 0, time.Now() + } + priority, hasPriority, err := unstructured.NestedInt64(un.Object, "spec", "priority") + if err != nil { + return 0, un.GetCreationTimestamp().Time + } + if !hasPriority { + priority = 0 + } + + return int32(priority), un.GetCreationTimestamp().Time +} + func (wfc *WorkflowController) addWorkflowInformerHandler() { wfc.wfInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ @@ -324,12 +365,16 @@ func (wfc *WorkflowController) addWorkflowInformerHandler() { key, err := cache.MetaNamespaceKeyFunc(obj) if err == nil { wfc.wfQueue.Add(key) + priority, creation := getWfPriority(obj) + wfc.throttler.Add(key, priority, creation) } }, UpdateFunc: func(old, new interface{}) { key, err := cache.MetaNamespaceKeyFunc(new) if err == nil { wfc.wfQueue.Add(key) + priority, creation := getWfPriority(new) + wfc.throttler.Add(key, priority, creation) } }, DeleteFunc: func(obj interface{}) { @@ -338,6 +383,7 @@ func (wfc *WorkflowController) addWorkflowInformerHandler() { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) if err == nil { wfc.wfQueue.Add(key) + wfc.throttler.Remove(key) } }, }, diff --git a/workflow/controller/dag.go b/workflow/controller/dag.go index 109e58595b8f..99ef6e10d977 100644 --- a/workflow/controller/dag.go +++ b/workflow/controller/dag.go @@ -79,7 +79,9 @@ func (d *dagContext) assessDAGPhase(targetTasks []string, nodes map[string]wfv1. unsuccessfulPhase = node.Phase } if node.Type == wfv1.NodeTypeRetry { - if hasMoreRetries(&node, d.wf) { + if node.Successful() { + retriesExhausted = false + } else if hasMoreRetries(&node, d.wf) { retriesExhausted = false } } @@ -106,6 +108,10 @@ func (d *dagContext) assessDAGPhase(targetTasks []string, nodes map[string]wfv1. } func hasMoreRetries(node *wfv1.NodeStatus, wf *wfv1.Workflow) bool { + if node.Phase == wfv1.NodeSucceeded { + return false + } + if len(node.Children) == 0 { return true } @@ -126,7 +132,7 @@ func (woc *wfOperationCtx) executeDAG(nodeName string, tmpl *wfv1.Template, boun } defer func() { if node != nil && woc.wf.Status.Nodes[node.ID].Completed() { - _ = woc.killDeamonedChildren(node.ID) + _ = woc.killDaemonedChildren(node.ID) } }() @@ -227,7 +233,7 @@ func (woc *wfOperationCtx) executeDAGTask(dagCtx *dagContext, taskName string) { depNode := dagCtx.getTaskNode(depName) if depNode != nil { if depNode.Completed() { - if !depNode.Successful() { + if !depNode.Successful() && !dagCtx.getTask(depName).ContinuesOn(depNode.Phase) { dependenciesSuccessful = false } continue @@ -251,12 +257,21 @@ func (woc *wfOperationCtx) executeDAGTask(dagCtx *dagContext, taskName string) { // All our dependencies were satisfied and successful. It's our turn to run + taskGroupNode := woc.getNodeByName(nodeName) + if taskGroupNode != nil && taskGroupNode.Type != wfv1.NodeTypeTaskGroup { + taskGroupNode = nil + } // connectDependencies is a helper to connect our dependencies to current task as children connectDependencies := func(taskNodeName string) { - if len(task.Dependencies) == 0 { + if len(task.Dependencies) == 0 || taskGroupNode != nil { // if we had no dependencies, then we are a root task, and we should connect the // boundary node as our parent - woc.addChildNode(dagCtx.boundaryName, taskNodeName) + if taskGroupNode == nil { + woc.addChildNode(dagCtx.boundaryName, taskNodeName) + } else { + woc.addChildNode(taskGroupNode.Name, taskNodeName) + } + } else { // Otherwise, add all outbound nodes of our dependencies as parents to this node for _, depName := range task.Dependencies { @@ -287,6 +302,16 @@ func (woc *wfOperationCtx) executeDAGTask(dagCtx *dagContext, taskName string) { return } + // If DAG task has withParam of with withSequence then we need to create virtual node of type TaskGroup. + // For example, if we had task A with withItems of ['foo', 'bar'] which expanded to ['A(0:foo)', 'A(1:bar)'], we still + // need to create a node for A. + if len(task.WithItems) > 0 || task.WithParam != "" || task.WithSequence != nil { + if taskGroupNode == nil { + connectDependencies(nodeName) + taskGroupNode = woc.initializeNode(nodeName, wfv1.NodeTypeTaskGroup, task.Template, dagCtx.boundaryID, wfv1.NodeRunning, "") + } + } + for _, t := range expandedTasks { node = dagCtx.getTaskNode(t.Name) taskNodeName := dagCtx.taskNodeName(t.Name) @@ -311,12 +336,8 @@ func (woc *wfOperationCtx) executeDAGTask(dagCtx *dagContext, taskName string) { _, _ = woc.executeTemplate(t.Template, t.Arguments, taskNodeName, dagCtx.boundaryID) } - // If we expanded the task, we still need to create the task entry for the non-expanded node, - // since dependant tasks will look to it, when deciding when to execute. For example, if we had - // task A with withItems of ['foo', 'bar'] which expanded to ['A(0:foo)', 'A(1:bar)'], we still - // need to create a node for A, after the withItems have completed. - if len(task.WithItems) > 0 || task.WithParam != "" || task.WithSequence != nil { - nodeStatus := wfv1.NodeSucceeded + if taskGroupNode != nil { + groupPhase := wfv1.NodeSucceeded for _, t := range expandedTasks { // Add the child relationship from our dependency's outbound nodes to this node. node := dagCtx.getTaskNode(t.Name) @@ -324,17 +345,10 @@ func (woc *wfOperationCtx) executeDAGTask(dagCtx *dagContext, taskName string) { return } if !node.Successful() { - nodeStatus = node.Phase + groupPhase = node.Phase } } - woc.initializeNode(nodeName, wfv1.NodeTypeTaskGroup, task.Template, dagCtx.boundaryID, nodeStatus, "") - if len(expandedTasks) > 0 { - for _, t := range expandedTasks { - woc.addChildNode(dagCtx.taskNodeName(t.Name), nodeName) - } - } else { - connectDependencies(nodeName) - } + woc.markNodePhase(taskGroupNode.Name, groupPhase) } } @@ -366,6 +380,13 @@ func (woc *wfOperationCtx) resolveDependencyReferences(dagCtx *dagContext, task } // Perform replacement + // Replace woc.volumes + err := woc.substituteParamsInVolumes(scope.replaceMap()) + if err != nil { + return nil, err + } + + // Replace task's parameters taskBytes, err := json.Marshal(task) if err != nil { return nil, errors.InternalWrapError(err) diff --git a/workflow/controller/exec_control.go b/workflow/controller/exec_control.go index 90fee860ed5e..9991f22882d8 100644 --- a/workflow/controller/exec_control.go +++ b/workflow/controller/exec_control.go @@ -3,6 +3,7 @@ package controller import ( "encoding/json" "fmt" + "sync" "time" apiv1 "k8s.io/api/core/v1" @@ -15,7 +16,10 @@ import ( // applyExecutionControl will ensure a pod's execution control annotation is up-to-date // kills any pending pods when workflow has reached it's deadline -func (woc *wfOperationCtx) applyExecutionControl(pod *apiv1.Pod) error { +func (woc *wfOperationCtx) applyExecutionControl(pod *apiv1.Pod, wfNodesLock *sync.RWMutex) error { + if pod == nil { + return nil + } switch pod.Status.Phase { case apiv1.PodSucceeded, apiv1.PodFailed: // Skip any pod which are already completed @@ -27,6 +31,8 @@ func (woc *wfOperationCtx) applyExecutionControl(pod *apiv1.Pod) error { woc.log.Infof("Deleting Pending pod %s/%s which has exceeded workflow deadline %s", pod.Namespace, pod.Name, woc.workflowDeadline) err := woc.controller.kubeclientset.CoreV1().Pods(pod.Namespace).Delete(pod.Name, &metav1.DeleteOptions{}) if err == nil { + wfNodesLock.Lock() + defer wfNodesLock.Unlock() node := woc.wf.Status.Nodes[pod.Name] var message string if woc.workflowDeadline.IsZero() { @@ -60,13 +66,19 @@ func (woc *wfOperationCtx) applyExecutionControl(pod *apiv1.Pod) error { return nil } } + if podExecCtl.Deadline != nil && podExecCtl.Deadline.IsZero() { + // If the pod has already been explicitly signaled to terminate, then do nothing. + // This can happen when daemon steps are terminated. + woc.log.Infof("Skipping sync of execution control of pod %s. pod has been signaled to terminate", pod.Name) + return nil + } woc.log.Infof("Execution control for pod %s out-of-sync desired: %v, actual: %v", pod.Name, desiredExecCtl.Deadline, podExecCtl.Deadline) return woc.updateExecutionControl(pod.Name, desiredExecCtl) } -// killDeamonedChildren kill any daemoned pods of a steps or DAG template node. -func (woc *wfOperationCtx) killDeamonedChildren(nodeID string) error { - woc.log.Infof("Checking deamoned children of %s", nodeID) +// killDaemonedChildren kill any daemoned pods of a steps or DAG template node. +func (woc *wfOperationCtx) killDaemonedChildren(nodeID string) error { + woc.log.Infof("Checking daemoned children of %s", nodeID) var firstErr error execCtl := common.ExecutionControl{ Deadline: &time.Time{}, @@ -116,7 +128,7 @@ func (woc *wfOperationCtx) updateExecutionControl(podName string, execCtl common woc.log.Infof("Signalling %s of updates", podName) exec, err := common.ExecPodContainer( woc.controller.restConfig, woc.wf.ObjectMeta.Namespace, podName, - common.WaitContainerName, true, true, "sh", "-c", "kill -s USR2 1", + common.WaitContainerName, true, true, "sh", "-c", "kill -s USR2 $(pidof argoexec)", ) if err != nil { return err diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go index 257085a0730a..e5dd72663b75 100644 --- a/workflow/controller/operator.go +++ b/workflow/controller/operator.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" argokubeerr "github.com/argoproj/pkg/kube/errors" @@ -20,10 +21,12 @@ import ( apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" + "k8s.io/utils/pointer" "github.com/cyrusbiotechnology/argo/errors" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/util/file" "github.com/cyrusbiotechnology/argo/util/retry" "github.com/cyrusbiotechnology/argo/workflow/common" "github.com/cyrusbiotechnology/argo/workflow/util" @@ -46,6 +49,9 @@ type wfOperationCtx struct { // globalParams holds any parameters that are available to be referenced // in the global scope (e.g. workflow.parameters.XXX). globalParams map[string]string + // volumes holds a DeepCopy of wf.Spec.Volumes to perform substitutions. + // It is then used in addVolumeReferences() when creating a pod. + volumes []apiv1.Volume // map of pods which need to be labeled with completed=true completedPods map[string]bool // deadline is the dealine time in which this operation should relinquish @@ -72,6 +78,9 @@ var ( // for before requeuing the workflow onto the workqueue. const maxOperationTime time.Duration = 10 * time.Second +//maxWorkflowSize is the maximum size for workflow.yaml +const maxWorkflowSize int = 1024 * 1024 + // newWorkflowOperationCtx creates and initializes a new wfOperationCtx object. func newWorkflowOperationCtx(wf *wfv1.Workflow, wfc *WorkflowController) *wfOperationCtx { // NEVER modify objects from the store. It's a read-only, local cache. @@ -87,6 +96,7 @@ func newWorkflowOperationCtx(wf *wfv1.Workflow, wfc *WorkflowController) *wfOper }), controller: wfc, globalParams: make(map[string]string), + volumes: wf.Spec.DeepCopy().Volumes, completedPods: make(map[string]bool), deadline: time.Now().UTC().Add(maxOperationTime), } @@ -103,7 +113,12 @@ func newWorkflowOperationCtx(wf *wfv1.Workflow, wfc *WorkflowController) *wfOper // TODO: an error returned by this method should result in requeuing the workflow to be retried at a // later time func (woc *wfOperationCtx) operate() { - defer woc.persistUpdates() + defer func() { + if woc.wf.Status.Completed() { + _ = woc.killDaemonedChildren("") + } + woc.persistUpdates() + }() defer func() { if r := recover(); r != nil { if rerr, ok := r.(error); ok { @@ -114,11 +129,14 @@ func (woc *wfOperationCtx) operate() { woc.log.Errorf("Recovered from panic: %+v\n%s", r, debug.Stack()) } }() + woc.log.Infof("Processing workflow") + // Perform one-time workflow validation if woc.wf.Status.Phase == "" { woc.markWorkflowRunning() - err := validate.ValidateWorkflow(woc.wf) + validateOpts := validate.ValidateOpts{ContainerRuntimeExecutor: woc.controller.Config.ContainerRuntimeExecutor} + err := validate.ValidateWorkflow(woc.wf, validateOpts) if err != nil { woc.markWorkflowFailed(fmt.Sprintf("invalid spec: %s", err.Error())) return @@ -144,7 +162,14 @@ func (woc *wfOperationCtx) operate() { woc.setGlobalParameters() - err := woc.createPVCs() + err := woc.substituteParamsInVolumes(woc.globalParams) + if err != nil { + woc.log.Errorf("%s volumes global param substitution error: %+v", woc.wf.ObjectMeta.Name, err) + woc.markWorkflowError(err, true) + return + } + + err = woc.createPVCs() if err != nil { woc.log.Errorf("%s pvc create error: %+v", woc.wf.ObjectMeta.Name, err) woc.markWorkflowError(err, true) @@ -245,6 +270,12 @@ func (woc *wfOperationCtx) setGlobalParameters() { for _, param := range woc.wf.Spec.Arguments.Parameters { woc.globalParams["workflow.parameters."+param.Name] = *param.Value } + for k, v := range woc.wf.ObjectMeta.Annotations { + woc.globalParams["workflow.annotations."+k] = v + } + for k, v := range woc.wf.ObjectMeta.Labels { + woc.globalParams["workflow.labels."+k] = v + } if woc.wf.Status.Outputs != nil { for _, param := range woc.wf.Status.Outputs.Parameters { woc.globalParams["workflow.outputs.parameters."+param.Name] = *param.Value @@ -270,9 +301,18 @@ func (woc *wfOperationCtx) persistUpdates() { return } wfClient := woc.controller.wfclientset.ArgoprojV1alpha1().Workflows(woc.wf.ObjectMeta.Namespace) - _, err := wfClient.Update(woc.wf) + err := woc.checkAndCompress() if err != nil { - woc.log.Warnf("Error updating workflow: %v", err) + woc.log.Warnf("Error compressing workflow: %v", err) + woc.markWorkflowFailed(err.Error()) + } + if woc.wf.Status.CompressedNodes != "" { + woc.wf.Status.Nodes = nil + } + + _, err = wfClient.Update(woc.wf) + if err != nil { + woc.log.Warnf("Error updating workflow: %v %s", err, apierr.ReasonForError(err)) if argokubeerr.IsRequestEntityTooLargeErr(err) { woc.persistWorkflowSizeLimitErr(wfClient, err) return @@ -462,18 +502,34 @@ func (woc *wfOperationCtx) podReconciliation() error { return err } seenPods := make(map[string]bool) + seenPodLock := &sync.Mutex{} + wfNodesLock := &sync.RWMutex{} performAssessment := func(pod *apiv1.Pod) error { + if pod == nil { + return nil + } nodeNameForPod := pod.Annotations[common.AnnotationKeyNodeName] nodeID := woc.wf.NodeID(nodeNameForPod) + seenPodLock.Lock() seenPods[nodeID] = true + seenPodLock.Unlock() + + wfNodesLock.Lock() + defer wfNodesLock.Unlock() if node, ok := woc.wf.Status.Nodes[nodeID]; ok { if newState := assessNodeStatus(pod, &node); newState != nil { woc.wf.Status.Nodes[nodeID] = *newState woc.addOutputsToScope("workflow", node.Outputs, nil) woc.updated = true } - if woc.wf.Status.Nodes[pod.ObjectMeta.Name].Completed() { + node := woc.wf.Status.Nodes[pod.ObjectMeta.Name] + if node.Completed() && !node.IsDaemoned() { + if tmpVal, tmpOk := pod.Labels[common.LabelKeyCompleted]; tmpOk { + if tmpVal == "true" { + return nil + } + } woc.completedPods[pod.ObjectMeta.Name] = true err := woc.collectPodErrorsAndWarnings(pod) if err != nil { @@ -484,17 +540,28 @@ func (woc *wfOperationCtx) podReconciliation() error { return nil } + parallelPodNum := make(chan string, 500) + var wg sync.WaitGroup + for _, pod := range podList.Items { - err = performAssessment(&pod) - if err != nil { - woc.log.Errorf("Failed to collect extended errors and warnings from pod %s: %s", pod.Name, err.Error()) - } - err = woc.applyExecutionControl(&pod) - if err != nil { - woc.log.Warnf("Failed to apply execution control to pod %s", pod.Name) - } + parallelPodNum <- pod.Name + wg.Add(1) + go func(tmpPod apiv1.Pod) { + defer wg.Done() + err = performAssessment(&tmpPod) + if err != nil { + woc.log.Errorf("Failed to collect extended errors and warnings from pod %s: %s", pod.Name, err.Error()) + } + err = woc.applyExecutionControl(&tmpPod, wfNodesLock) + if err != nil { + woc.log.Warnf("Failed to apply execution control to pod %s", tmpPod.Name) + } + <-parallelPodNum + }(pod) } + wg.Wait() + // Now check for deleted pods. Iterate our nodes. If any one of our nodes does not show up in // the seen list it implies that the pod was deleted without the controller seeing the event. // It is now impossible to infer pod status. The only thing we can do at this point is to mark @@ -539,6 +606,31 @@ func (woc *wfOperationCtx) countActivePods(boundaryIDs ...string) int64 { return activePods } +// countActiveChildren counts the number of active (Pending/Running) children nodes of parent parentName +func (woc *wfOperationCtx) countActiveChildren(boundaryIDs ...string) int64 { + var boundaryID = "" + if len(boundaryIDs) > 0 { + boundaryID = boundaryIDs[0] + } + var activeChildren int64 + // if we care about parallelism, count the active pods at the template level + for _, node := range woc.wf.Status.Nodes { + if boundaryID != "" && node.BoundaryID != boundaryID { + continue + } + switch node.Type { + case wfv1.NodeTypePod, wfv1.NodeTypeSteps, wfv1.NodeTypeDAG: + default: + continue + } + switch node.Phase { + case wfv1.NodePending, wfv1.NodeRunning: + activeChildren++ + } + } + return activeChildren +} + // getAllWorkflowPods returns all pods related to the current workflow func (woc *wfOperationCtx) getAllWorkflowPods() (*apiv1.PodList, error) { options := metav1.ListOptions{ @@ -560,19 +652,24 @@ func assessNodeStatus(pod *apiv1.Pod, node *wfv1.NodeStatus) *wfv1.NodeStatus { var newDaemonStatus *bool var message string updated := false - f := false switch pod.Status.Phase { case apiv1.PodPending: newPhase = wfv1.NodePending - newDaemonStatus = &f + newDaemonStatus = pointer.BoolPtr(false) message = getPendingReason(pod) case apiv1.PodSucceeded: + newPhase = wfv1.NodeSucceeded // A pod can exit with a successful status and still fail an exception condition check newPhase, message = handlePodFailures(pod) - newDaemonStatus = &f + newDaemonStatus = pointer.BoolPtr(false) case apiv1.PodFailed: - newPhase, message = handlePodFailures(pod) - newDaemonStatus = &f + // ignore pod failure for daemoned steps + if node.IsDaemoned() { + newPhase = wfv1.NodeSucceeded + } else { + newPhase, message = handlePodFailures(pod) + } + newDaemonStatus = pointer.BoolPtr(false) case apiv1.PodRunning: newPhase = wfv1.NodeRunning tmplStr, ok := pod.Annotations[common.AnnotationKeyTemplate] @@ -593,10 +690,9 @@ func assessNodeStatus(pod *apiv1.Pod, node *wfv1.NodeStatus) *wfv1.NodeStatus { return nil } } - // proceed to mark node status as succeeded (and daemoned) - newPhase = wfv1.NodeSucceeded - t := true - newDaemonStatus = &t + // proceed to mark node status as running (and daemoned) + newPhase = wfv1.NodeRunning + newDaemonStatus = pointer.BoolPtr(true) log.Infof("Processing ready daemon pod: %v", pod.ObjectMeta.SelfLink) } default: @@ -779,9 +875,10 @@ func handlePodFailures(pod *apiv1.Pod) (wfv1.NodePhase, string) { } errMsg := fmt.Sprintf("failed with exit code %d", ctr.State.Terminated.ExitCode) if ctr.Name != common.MainContainerName { - if ctr.State.Terminated.ExitCode == 137 { + if ctr.State.Terminated.ExitCode == 137 || ctr.State.Terminated.ExitCode == 143 { // if the sidecar was SIGKILL'd (exit code 137) assume it was because argoexec // forcibly killed the container, which we ignore the error for. + // Java code 143 is a normal exit 128 + 15 https://github.com/elastic/elasticsearch/issues/31847 log.Infof("Ignoring %d exit code of sidecar '%s'", ctr.State.Terminated.ExitCode, ctr.Name) continue } @@ -933,7 +1030,8 @@ func (woc *wfOperationCtx) getLastChildNode(node *wfv1.NodeStatus) (*wfv1.NodeSt // nodeName is the name to be used as the name of the node, and boundaryID indicates which template // boundary this node belongs to. func (woc *wfOperationCtx) executeTemplate(templateName string, args wfv1.Arguments, nodeName string, boundaryID string) (*wfv1.NodeStatus, error) { - woc.log.Debugf("Evaluating node %s: template: %s", nodeName, templateName) + woc.log.Debugf("Evaluating node %s: template: %s, boundaryID: %s", nodeName, templateName, boundaryID) + node := woc.getNodeByName(nodeName) if node != nil && node.Completed() { woc.log.Debugf("Node %s already completed", nodeName) @@ -1059,7 +1157,8 @@ func (woc *wfOperationCtx) markWorkflowPhase(phase wfv1.NodePhase, markCompleted switch phase { case wfv1.NodeSucceeded, wfv1.NodeFailed, wfv1.NodeError: - if markCompleted { + // wait for all daemon nodes to get terminated before marking workflow completed + if markCompleted && !woc.hasDaemonNodes() { woc.log.Infof("Marking workflow completed") woc.wf.Status.FinishedAt = metav1.Time{Time: time.Now().UTC()} if woc.wf.ObjectMeta.Labels == nil { @@ -1071,6 +1170,15 @@ func (woc *wfOperationCtx) markWorkflowPhase(phase wfv1.NodePhase, markCompleted } } +func (woc *wfOperationCtx) hasDaemonNodes() bool { + for _, node := range woc.wf.Status.Nodes { + if node.IsDaemoned() { + return true + } + } + return false +} + func (woc *wfOperationCtx) markWorkflowRunning() { woc.markWorkflowPhase(wfv1.NodeRunning, false) } @@ -1156,6 +1264,14 @@ func (woc *wfOperationCtx) markNodePhase(nodeName string, phase wfv1.NodePhase, return node } +// markNodeErrorClearOuput is a convenience method to mark a node with an error and clear the output +func (woc *wfOperationCtx) markNodeErrorClearOuput(nodeName string, err error) *wfv1.NodeStatus { + nodeStatus := woc.markNodeError(nodeName, err) + nodeStatus.Outputs = nil + woc.wf.Status.Nodes[nodeStatus.ID] = *nodeStatus + return nodeStatus +} + // markNodeError is a convenience method to mark a node with an error and set the message from the error func (woc *wfOperationCtx) markNodeError(nodeName string, err error) *wfv1.NodeStatus { return woc.markNodePhase(nodeName, wfv1.NodeError, err.Error()) @@ -1178,16 +1294,17 @@ func (woc *wfOperationCtx) checkParallelism(tmpl *wfv1.Template, node *wfv1.Node return ErrParallelismReached } } + fallthrough default: // if we are about to execute a pod, make our parent hasn't reached it's limit - if boundaryID != "" { + if boundaryID != "" && (node == nil || (node.Phase != wfv1.NodePending && node.Phase != wfv1.NodeRunning)) { boundaryNode := woc.wf.Status.Nodes[boundaryID] boundaryTemplate := woc.wf.GetTemplate(boundaryNode.TemplateName) if boundaryTemplate.Parallelism != nil { - templateActivePods := woc.countActivePods(boundaryID) - woc.log.Debugf("counted %d/%d active pods in boundary %s", templateActivePods, *boundaryTemplate.Parallelism, boundaryID) - if templateActivePods >= *boundaryTemplate.Parallelism { - woc.log.Infof("template (node %s) active pod parallelism reached %d/%d", boundaryID, templateActivePods, *boundaryTemplate.Parallelism) + activeSiblings := woc.countActiveChildren(boundaryID) + woc.log.Debugf("counted %d/%d active children in boundary %s", activeSiblings, *boundaryTemplate.Parallelism, boundaryID) + if activeSiblings >= *boundaryTemplate.Parallelism { + woc.log.Infof("template (node %s) active children parallelism reached %d/%d", boundaryID, activeSiblings, *boundaryTemplate.Parallelism) return ErrParallelismReached } } @@ -1213,8 +1330,17 @@ func (woc *wfOperationCtx) executeContainer(nodeName string, tmpl *wfv1.Template func (woc *wfOperationCtx) getOutboundNodes(nodeID string) []string { node := woc.wf.Status.Nodes[nodeID] switch node.Type { - case wfv1.NodeTypePod, wfv1.NodeTypeSkipped, wfv1.NodeTypeSuspend, wfv1.NodeTypeTaskGroup: + case wfv1.NodeTypePod, wfv1.NodeTypeSkipped, wfv1.NodeTypeSuspend: return []string{node.ID} + case wfv1.NodeTypeTaskGroup: + if len(node.Children) == 0 { + return []string{node.ID} + } + outboundNodes := make([]string, 0) + for _, child := range node.Children { + outboundNodes = append(outboundNodes, woc.getOutboundNodes(child)...) + } + return outboundNodes case wfv1.NodeTypeRetry: numChildren := len(node.Children) if numChildren > 0 { @@ -1314,7 +1440,7 @@ func (woc *wfOperationCtx) addOutputsToScope(prefix string, outputs *wfv1.Output if scope != nil { scope.addArtifactToScope(key, art) } - woc.addArtifactToGlobalScope(art) + woc.addArtifactToGlobalScope(art, scope) } } @@ -1421,7 +1547,7 @@ func (woc *wfOperationCtx) addParamToGlobalScope(param wfv1.Parameter) { // addArtifactToGlobalScope exports any desired node outputs to the global scope // Optionally adds to a local scope if supplied -func (woc *wfOperationCtx) addArtifactToGlobalScope(art wfv1.Artifact) { +func (woc *wfOperationCtx) addArtifactToGlobalScope(art wfv1.Artifact, scope *wfScope) { if art.GlobalName == "" { return } @@ -1435,6 +1561,9 @@ func (woc *wfOperationCtx) addArtifactToGlobalScope(art wfv1.Artifact) { art.Path = "" if !reflect.DeepEqual(woc.wf.Status.Outputs.Artifacts[i], art) { woc.wf.Status.Outputs.Artifacts[i] = art + if scope != nil { + scope.addArtifactToScope(globalArtName, art) + } woc.log.Infof("overwriting %s: %v", globalArtName, art) woc.updated = true } @@ -1450,6 +1579,9 @@ func (woc *wfOperationCtx) addArtifactToGlobalScope(art wfv1.Artifact) { art.Path = "" woc.log.Infof("setting %s: %v", globalArtName, art) woc.wf.Status.Outputs.Artifacts = append(woc.wf.Status.Outputs.Artifacts, art) + if scope != nil { + scope.addArtifactToScope(globalArtName, art) + } woc.updated = true } @@ -1479,16 +1611,12 @@ func (woc *wfOperationCtx) executeResource(nodeName string, tmpl *wfv1.Template, if node != nil { return node } - mainCtr := apiv1.Container{ - Image: woc.controller.executorImage(), - Command: []string{"argoexec"}, - Args: []string{"resource", tmpl.Resource.Action}, - VolumeMounts: []apiv1.VolumeMount{ - volumeMountPodMetadata, - }, - Env: execEnvVars, + mainCtr := woc.newExecContainer(common.MainContainerName) + mainCtr.Command = []string{"argoexec", "resource", tmpl.Resource.Action} + mainCtr.VolumeMounts = []apiv1.VolumeMount{ + volumeMountPodMetadata, } - _, err := woc.createWorkflowPod(nodeName, mainCtr, tmpl) + _, err := woc.createWorkflowPod(nodeName, *mainCtr, tmpl) if err != nil { return woc.initializeNode(nodeName, wfv1.NodeTypePod, tmpl.Name, boundaryID, wfv1.NodeError, err.Error()) } @@ -1577,3 +1705,66 @@ func expandSequence(seq *wfv1.Sequence) ([]wfv1.Item, error) { } return items, nil } + +// getSize return the entire workflow json string size +func (woc *wfOperationCtx) getSize() int { + nodeContent, err := json.Marshal(woc.wf) + if err != nil { + return -1 + } + + compressNodeSize := len(woc.wf.Status.CompressedNodes) + + if compressNodeSize > 0 { + nodeStatus, err := json.Marshal(woc.wf.Status.Nodes) + if err != nil { + return -1 + } + return len(nodeContent) - len(nodeStatus) + } + return len(nodeContent) +} + +// checkAndCompress will check the workflow size and compress node status if total workflow size is more than maxWorkflowSize. +// The compressed content will be assign to compressedNodes element and clear the nodestatus map. +func (woc *wfOperationCtx) checkAndCompress() error { + + if woc.wf.Status.CompressedNodes != "" || (woc.wf.Status.CompressedNodes == "" && woc.getSize() >= maxWorkflowSize) { + nodeContent, err := json.Marshal(woc.wf.Status.Nodes) + if err != nil { + return errors.InternalWrapError(err) + } + buff := string(nodeContent) + woc.wf.Status.CompressedNodes = file.CompressEncodeString(buff) + } + + if woc.wf.Status.CompressedNodes != "" && woc.getSize() >= maxWorkflowSize { + return errors.InternalError(fmt.Sprintf("Workflow is longer than maximum allowed size. Size=%d", woc.getSize())) + } + + return nil +} + +func (woc *wfOperationCtx) substituteParamsInVolumes(params map[string]string) error { + if woc.volumes == nil { + return nil + } + + volumes := woc.volumes + volumesBytes, err := json.Marshal(volumes) + if err != nil { + return errors.InternalWrapError(err) + } + fstTmpl := fasttemplate.New(string(volumesBytes), "{{", "}}") + newVolumesStr, err := common.Replace(fstTmpl, params, true) + if err != nil { + return err + } + var newVolumes []apiv1.Volume + err = json.Unmarshal([]byte(newVolumesStr), &newVolumes) + if err != nil { + return errors.InternalWrapError(err) + } + woc.volumes = newVolumes + return nil +} diff --git a/workflow/controller/operator_test.go b/workflow/controller/operator_test.go index 48009a5b6d98..a5fc682a62fc 100644 --- a/workflow/controller/operator_test.go +++ b/workflow/controller/operator_test.go @@ -225,11 +225,13 @@ func TestWorkflowParallelismLimit(t *testing.T) { assert.Equal(t, 2, len(pods.Items)) // operate again and make sure we don't schedule any more pods makePodsRunning(t, controller.kubeclientset, wf.ObjectMeta.Namespace) + assert.Equal(t, int64(2), woc.countActivePods()) wf, err = wfcset.Get(wf.ObjectMeta.Name, metav1.GetOptions{}) assert.Nil(t, err) // wfBytes, _ := json.MarshalIndent(wf, "", " ") // log.Printf("%s", wfBytes) woc = newWorkflowOperationCtx(wf, controller) + assert.Equal(t, int64(2), woc.countActivePods()) woc.operate() pods, err = controller.kubeclientset.CoreV1().Pods("").List(metav1.ListOptions{}) assert.Nil(t, err) @@ -435,14 +437,16 @@ func TestNestedTemplateParallelismLimit(t *testing.T) { // TestSidecarResourceLimits verifies resource limits on the sidecar can be set in the controller config func TestSidecarResourceLimits(t *testing.T) { controller := newController() - controller.Config.ExecutorResources = &apiv1.ResourceRequirements{ - Limits: apiv1.ResourceList{ - apiv1.ResourceCPU: resource.MustParse("0.5"), - apiv1.ResourceMemory: resource.MustParse("512Mi"), - }, - Requests: apiv1.ResourceList{ - apiv1.ResourceCPU: resource.MustParse("0.1"), - apiv1.ResourceMemory: resource.MustParse("64Mi"), + controller.Config.Executor = &apiv1.Container{ + Resources: apiv1.ResourceRequirements{ + Limits: apiv1.ResourceList{ + apiv1.ResourceCPU: resource.MustParse("0.5"), + apiv1.ResourceMemory: resource.MustParse("512Mi"), + }, + Requests: apiv1.ResourceList{ + apiv1.ResourceCPU: resource.MustParse("0.1"), + apiv1.ResourceMemory: resource.MustParse("64Mi"), + }, }, } wf := unmarshalWF(helloWorldWf) @@ -765,19 +769,19 @@ func TestAddGlobalArtifactToScope(t *testing.T) { }, } // Make sure if the artifact is not global, don't add to scope - woc.addArtifactToGlobalScope(art) + woc.addArtifactToGlobalScope(art, nil) assert.Nil(t, woc.wf.Status.Outputs) // Now mark it as global. Verify it is added to workflow outputs art.GlobalName = "global-art" - woc.addArtifactToGlobalScope(art) + woc.addArtifactToGlobalScope(art, nil) assert.Equal(t, 1, len(woc.wf.Status.Outputs.Artifacts)) assert.Equal(t, art.GlobalName, woc.wf.Status.Outputs.Artifacts[0].Name) assert.Equal(t, "some/key", woc.wf.Status.Outputs.Artifacts[0].S3.Key) // Change the value and verify update is reflected art.S3.Key = "new/key" - woc.addArtifactToGlobalScope(art) + woc.addArtifactToGlobalScope(art, nil) assert.Equal(t, 1, len(woc.wf.Status.Outputs.Artifacts)) assert.Equal(t, art.GlobalName, woc.wf.Status.Outputs.Artifacts[0].Name) assert.Equal(t, "new/key", woc.wf.Status.Outputs.Artifacts[0].S3.Key) @@ -785,7 +789,7 @@ func TestAddGlobalArtifactToScope(t *testing.T) { // Add a new global artifact art.GlobalName = "global-art2" art.S3.Key = "new/new/key" - woc.addArtifactToGlobalScope(art) + woc.addArtifactToGlobalScope(art, nil) assert.Equal(t, 2, len(woc.wf.Status.Outputs.Artifacts)) assert.Equal(t, art.GlobalName, woc.wf.Status.Outputs.Artifacts[1].Name) assert.Equal(t, "new/new/key", woc.wf.Status.Outputs.Artifacts[1].S3.Key) @@ -886,3 +890,118 @@ func TestExpandWithSequence(t *testing.T) { assert.Equal(t, "testuser01", items[0].Value.(string)) assert.Equal(t, "testuser0A", items[9].Value.(string)) } + +var metadataTemplate = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: metadata-template + labels: + image: foo:bar + annotations: + k8s-webhook-handler.io/repo: "git@github.com:argoproj/argo.git" + k8s-webhook-handler.io/revision: 1e111caa1d2cc672b3b53c202b96a5f660a7e9b2 +spec: + entrypoint: foo + templates: + - name: foo + container: + image: "{{workflow.labels.image}}" + env: + - name: REPO + value: "{{workflow.annotations.k8s-webhook-handler.io/repo}}" + - name: REVISION + value: "{{workflow.annotations.k8s-webhook-handler.io/revision}}" + command: [sh, -c] + args: ["echo hello world"] +` + +func TestMetadataPassing(t *testing.T) { + controller := newController() + wfcset := controller.wfclientset.ArgoprojV1alpha1().Workflows("") + wf := unmarshalWF(metadataTemplate) + wf, err := wfcset.Create(wf) + assert.Nil(t, err) + wf, err = wfcset.Get(wf.ObjectMeta.Name, metav1.GetOptions{}) + assert.Nil(t, err) + woc := newWorkflowOperationCtx(wf, controller) + woc.operate() + assert.Equal(t, wfv1.NodeRunning, woc.wf.Status.Phase) + pods, err := controller.kubeclientset.CoreV1().Pods(wf.ObjectMeta.Namespace).List(metav1.ListOptions{}) + assert.Nil(t, err) + assert.True(t, len(pods.Items) > 0, "pod was not created successfully") + + var ( + pod = pods.Items[0] + container = pod.Spec.Containers[1] + foundRepo = false + foundRev = false + ) + for _, ev := range container.Env { + switch ev.Name { + case "REPO": + assert.Equal(t, "git@github.com:argoproj/argo.git", ev.Value) + foundRepo = true + case "REVISION": + assert.Equal(t, "1e111caa1d2cc672b3b53c202b96a5f660a7e9b2", ev.Value) + foundRev = true + } + } + assert.True(t, foundRepo) + assert.True(t, foundRev) + assert.Equal(t, "foo:bar", container.Image) +} + +var ioPathPlaceholders = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: artifact-path-placeholders- +spec: + entrypoint: head-lines + arguments: + parameters: + - name: lines-count + value: 3 + artifacts: + - name: text + raw: + data: | + 1 + 2 + 3 + 4 + 5 + templates: + - name: head-lines + inputs: + parameters: + - name: lines-count + artifacts: + - name: text + path: /inputs/text/data + outputs: + parameters: + - name: actual-lines-count + valueFrom: + path: /outputs/actual-lines-count/data + artifacts: + - name: text + path: /outputs/text/data + container: + image: busybox + command: [sh, -c, 'head -n {{inputs.parameters.lines-count}} <"{{inputs.artifacts.text.path}}" | tee "{{outputs.artifacts.text.path}}" | wc -l > "{{outputs.parameters.actual-lines-count.path}}"'] +` + +func TestResolveIOPathPlaceholders(t *testing.T) { + wf := unmarshalWF(ioPathPlaceholders) + woc := newWoc(*wf) + woc.controller.Config.ArtifactRepository.S3 = new(S3ArtifactRepository) + woc.operate() + assert.Equal(t, wfv1.NodeRunning, woc.wf.Status.Phase) + pods, err := woc.controller.kubeclientset.CoreV1().Pods(wf.ObjectMeta.Namespace).List(metav1.ListOptions{}) + assert.Nil(t, err) + assert.True(t, len(pods.Items) > 0, "pod was not created successfully") + + assert.Equal(t, []string{"sh", "-c", "head -n 3 <\"/inputs/text/data\" | tee \"/outputs/text/data\" | wc -l > \"/outputs/actual-lines-count/data\""}, pods.Items[0].Spec.Containers[1].Command) +} diff --git a/workflow/controller/steps.go b/workflow/controller/steps.go index 039031013a24..bb9e6a175081 100644 --- a/workflow/controller/steps.go +++ b/workflow/controller/steps.go @@ -28,7 +28,7 @@ func (woc *wfOperationCtx) executeSteps(nodeName string, tmpl *wfv1.Template, bo } defer func() { if woc.wf.Status.Nodes[node.ID].Completed() { - _ = woc.killDeamonedChildren(node.ID) + _ = woc.killDaemonedChildren(node.ID) } }() stepsCtx := stepsContext{ @@ -155,6 +155,7 @@ func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNod woc.log.Debugf("Step group node %v already marked completed", node) return node } + // First, resolve any references to outputs from previous steps, and perform substitution stepGroup, err := woc.resolveReferences(stepGroup, stepsCtx.scope) if err != nil { @@ -167,6 +168,9 @@ func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNod return woc.markNodeError(sgNodeName, err) } + // Maps nodes to their steps + nodeSteps := make(map[string]wfv1.WorkflowStep) + // Kick off all parallel steps in the group for _, step := range stepGroup { childNodeName := fmt.Sprintf("%s.%s", sgNodeName, step.Name) @@ -174,6 +178,7 @@ func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNod // Check the step's when clause to decide if it should execute proceed, err := shouldExecute(step.When) if err != nil { + woc.initializeNode(childNodeName, wfv1.NodeTypeSkipped, "", stepsCtx.boundaryID, wfv1.NodeError, err.Error()) woc.addChildNode(sgNodeName, childNodeName) woc.markNodeError(childNodeName, err) return woc.markNodeError(sgNodeName, err) @@ -201,10 +206,8 @@ func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNod } } if childNode != nil { + nodeSteps[childNodeName] = step woc.addChildNode(sgNodeName, childNodeName) - if childNode.Completed() && !childNode.Successful() { - break - } } } @@ -218,7 +221,8 @@ func (woc *wfOperationCtx) executeStepGroup(stepGroup []wfv1.WorkflowStep, sgNod // All children completed. Determine step group status as a whole for _, childNodeID := range node.Children { childNode := woc.wf.Status.Nodes[childNodeID] - if !childNode.Successful() { + step := nodeSteps[childNode.Name] + if !childNode.Successful() && !step.ContinuesOn(childNode.Phase) { failMessage := fmt.Sprintf("child '%s' failed", childNodeID) woc.log.Infof("Step group node %s deemed failed: %s", node, failMessage) return woc.markNodePhase(node.Name, wfv1.NodeFailed, failMessage) @@ -274,6 +278,12 @@ func shouldExecute(when string) (bool, error) { func (woc *wfOperationCtx) resolveReferences(stepGroup []wfv1.WorkflowStep, scope *wfScope) ([]wfv1.WorkflowStep, error) { newStepGroup := make([]wfv1.WorkflowStep, len(stepGroup)) + // Step 0: replace all parameter scope references for volumes + err := woc.substituteParamsInVolumes(scope.replaceMap()) + if err != nil { + return nil, err + } + for i, step := range stepGroup { // Step 1: replace all parameter scope references in the step // TODO: improve this @@ -281,15 +291,8 @@ func (woc *wfOperationCtx) resolveReferences(stepGroup []wfv1.WorkflowStep, scop if err != nil { return nil, errors.InternalWrapError(err) } - replaceMap := make(map[string]string) - for key, val := range scope.scope { - valStr, ok := val.(string) - if ok { - replaceMap[key] = valStr - } - } fstTmpl := fasttemplate.New(string(stepBytes), "{{", "}}") - newStepStr, err := common.Replace(fstTmpl, replaceMap, true) + newStepStr, err := common.Replace(fstTmpl, scope.replaceMap(), true) if err != nil { return nil, err } diff --git a/workflow/controller/steps_test.go b/workflow/controller/steps_test.go new file mode 100644 index 000000000000..5f55d556a8f8 --- /dev/null +++ b/workflow/controller/steps_test.go @@ -0,0 +1,18 @@ +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/test" +) + +// TestStepsFailedRetries ensures a steps template will recognize exhausted retries +func TestStepsFailedRetries(t *testing.T) { + wf := test.LoadTestWorkflow("testdata/steps-failed-retries.yaml") + woc := newWoc(*wf) + woc.operate() + assert.Equal(t, string(wfv1.NodeFailed), string(woc.wf.Status.Phase)) +} diff --git a/workflow/controller/testdata/steps-failed-retries.yaml b/workflow/controller/testdata/steps-failed-retries.yaml new file mode 100644 index 000000000000..bd249586e311 --- /dev/null +++ b/workflow/controller/testdata/steps-failed-retries.yaml @@ -0,0 +1,153 @@ +metadata: + creationTimestamp: "2018-12-28T19:21:20Z" + generateName: failed-retries- + generation: 1 + labels: + workflows.argoproj.io/phase: Running + name: failed-retries-tjjsc + namespace: default + resourceVersion: "85216" + selfLink: /apis/argoproj.io/v1alpha1/namespaces/default/workflows/failed-retries-tjjsc + uid: c18bba2a-0ad5-11e9-b44e-ea782c392741 +spec: + arguments: {} + entrypoint: failed-retries + templates: + - inputs: {} + metadata: {} + name: failed-retries + outputs: {} + steps: + - - arguments: {} + name: fail + template: fail + - arguments: {} + name: delayed-fail + template: delayed-fail + - container: + args: + - exit 1 + command: + - sh + - -c + image: alpine:latest + name: "" + resources: {} + inputs: {} + metadata: {} + name: fail + outputs: {} + retryStrategy: + limit: 1 + - container: + args: + - sleep 1; exit 1 + command: + - sh + - -c + image: alpine:latest + name: "" + resources: {} + inputs: {} + metadata: {} + name: delayed-fail + outputs: {} + retryStrategy: + limit: 1 +status: + finishedAt: null + nodes: + failed-retries-tjjsc: + children: + - failed-retries-tjjsc-2095973878 + displayName: failed-retries-tjjsc + finishedAt: null + id: failed-retries-tjjsc + name: failed-retries-tjjsc + phase: Running + startedAt: "2019-01-03T01:23:18Z" + templateName: failed-retries + type: Steps + failed-retries-tjjsc-20069324: + boundaryID: failed-retries-tjjsc + children: + - failed-retries-tjjsc-1229492679 + - failed-retries-tjjsc-759866442 + displayName: fail + finishedAt: "2019-01-03T01:23:32Z" + id: failed-retries-tjjsc-20069324 + message: No more retries left + name: failed-retries-tjjsc[0].fail + phase: Failed + startedAt: "2019-01-03T01:23:18Z" + type: Retry + failed-retries-tjjsc-759866442: + boundaryID: failed-retries-tjjsc + displayName: fail(1) + finishedAt: "2018-12-28T19:21:32Z" + id: failed-retries-tjjsc-759866442 + message: failed with exit code 1 + name: failed-retries-tjjsc[0].fail(1) + phase: Failed + startedAt: "2019-01-03T01:23:27Z" + templateName: fail + type: Pod + failed-retries-tjjsc-1229492679: + boundaryID: failed-retries-tjjsc + displayName: fail(0) + finishedAt: "2018-12-28T19:21:26Z" + id: failed-retries-tjjsc-1229492679 + message: failed with exit code 1 + name: failed-retries-tjjsc[0].fail(0) + phase: Failed + startedAt: "2019-01-03T01:23:18Z" + templateName: fail + type: Pod + failed-retries-tjjsc-1375221696: + boundaryID: failed-retries-tjjsc + displayName: delayed-fail(0) + finishedAt: "2018-12-28T19:21:27Z" + id: failed-retries-tjjsc-1375221696 + message: failed with exit code 1 + name: failed-retries-tjjsc[0].delayed-fail(0) + phase: Failed + startedAt: "2019-01-03T01:23:18Z" + templateName: delayed-fail + type: Pod + failed-retries-tjjsc-1574533273: + boundaryID: failed-retries-tjjsc + children: + - failed-retries-tjjsc-1375221696 + - failed-retries-tjjsc-2113289837 + displayName: delayed-fail + finishedAt: null + id: failed-retries-tjjsc-1574533273 + name: failed-retries-tjjsc[0].delayed-fail + phase: Running + startedAt: "2019-01-03T01:23:18Z" + type: Retry + failed-retries-tjjsc-2095973878: + boundaryID: failed-retries-tjjsc + children: + - failed-retries-tjjsc-20069324 + - failed-retries-tjjsc-1574533273 + displayName: '[0]' + finishedAt: null + id: failed-retries-tjjsc-2095973878 + name: failed-retries-tjjsc[0] + phase: Running + startedAt: "2019-01-03T01:23:18Z" + type: StepGroup + failed-retries-tjjsc-2113289837: + boundaryID: failed-retries-tjjsc + displayName: delayed-fail(1) + finishedAt: "2018-12-28T19:21:33Z" + id: failed-retries-tjjsc-2113289837 + message: failed with exit code 1 + name: failed-retries-tjjsc[0].delayed-fail(1) + phase: Failed + startedAt: "2019-01-03T01:23:28Z" + templateName: delayed-fail + type: Pod + phase: Running + startedAt: "2019-01-03T01:23:18Z" diff --git a/workflow/controller/throttler.go b/workflow/controller/throttler.go new file mode 100644 index 000000000000..8224f4af900e --- /dev/null +++ b/workflow/controller/throttler.go @@ -0,0 +1,153 @@ +package controller + +import ( + "container/heap" + "sync" + "time" + + "k8s.io/client-go/util/workqueue" +) + +// Throttler allows CRD controller to limit number of items it is processing in parallel. +type Throttler interface { + Add(key interface{}, priority int32, creationTime time.Time) + // Next returns true if item should be processed by controller now or return false. + Next(key interface{}) (interface{}, bool) + // Remove notifies throttler that item processing is done. In responses the throttler triggers processing of previously throttled items. + Remove(key interface{}) + // SetParallelism update throttler parallelism limit. + SetParallelism(parallelism int) +} + +type throttler struct { + queue workqueue.RateLimitingInterface + inProgress map[interface{}]bool + pending *priorityQueue + lock *sync.Mutex + parallelism int +} + +func NewThrottler(parallelism int, queue workqueue.RateLimitingInterface) Throttler { + return &throttler{ + queue: queue, + inProgress: make(map[interface{}]bool), + lock: &sync.Mutex{}, + parallelism: parallelism, + pending: &priorityQueue{itemByKey: make(map[interface{}]*item)}, + } +} + +func (t *throttler) SetParallelism(parallelism int) { + t.lock.Lock() + defer t.lock.Unlock() + if t.parallelism != parallelism { + t.parallelism = parallelism + t.queueThrottled() + } +} + +func (t *throttler) Add(key interface{}, priority int32, creationTime time.Time) { + t.lock.Lock() + defer t.lock.Unlock() + t.pending.add(key, priority, creationTime) +} + +func (t *throttler) Next(key interface{}) (interface{}, bool) { + t.lock.Lock() + defer t.lock.Unlock() + + if _, isInProgress := t.inProgress[key]; isInProgress || t.pending.Len() == 0 { + return key, true + } + if t.parallelism < 1 || t.parallelism > len(t.inProgress) { + next := t.pending.pop() + t.inProgress[next.key] = true + return next.key, true + } + return key, false + +} + +func (t *throttler) Remove(key interface{}) { + t.lock.Lock() + defer t.lock.Unlock() + delete(t.inProgress, key) + t.pending.remove(key) + + t.queueThrottled() +} + +func (t *throttler) queueThrottled() { + for t.pending.Len() > 0 && (t.parallelism < 1 || t.parallelism > len(t.inProgress)) { + next := t.pending.pop() + t.inProgress[next.key] = true + t.queue.Add(next.key) + } +} + +type item struct { + key interface{} + creationTime time.Time + priority int32 + index int +} + +type priorityQueue struct { + items []*item + itemByKey map[interface{}]*item +} + +func (pq *priorityQueue) pop() *item { + return heap.Pop(pq).(*item) +} + +func (pq *priorityQueue) add(key interface{}, priority int32, creationTime time.Time) { + if res, ok := pq.itemByKey[key]; ok { + if res.priority != priority { + res.priority = priority + heap.Fix(pq, res.index) + } + } else { + heap.Push(pq, &item{key: key, priority: priority, creationTime: creationTime}) + } +} + +func (pq *priorityQueue) remove(key interface{}) { + if item, ok := pq.itemByKey[key]; ok { + heap.Remove(pq, item.index) + delete(pq.itemByKey, key) + } +} + +func (pq priorityQueue) Len() int { return len(pq.items) } + +func (pq priorityQueue) Less(i, j int) bool { + if pq.items[i].priority == pq.items[j].priority { + return pq.items[i].creationTime.Before(pq.items[j].creationTime) + } + return pq.items[i].priority > pq.items[j].priority +} + +func (pq priorityQueue) Swap(i, j int) { + pq.items[i], pq.items[j] = pq.items[j], pq.items[i] + pq.items[i].index = i + pq.items[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(pq.items) + item := x.(*item) + item.index = n + pq.items = append(pq.items, item) + pq.itemByKey[item.key] = item +} + +func (pq *priorityQueue) Pop() interface{} { + old := pq.items + n := len(old) + item := old[n-1] + item.index = -1 + pq.items = old[0 : n-1] + delete(pq.itemByKey, item.key) + return item +} diff --git a/workflow/controller/throttler_test.go b/workflow/controller/throttler_test.go new file mode 100644 index 000000000000..454a6bca1392 --- /dev/null +++ b/workflow/controller/throttler_test.go @@ -0,0 +1,86 @@ +package controller + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "k8s.io/client-go/util/workqueue" +) + +func TestNoParallelismSamePriority(t *testing.T) { + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + throttler := NewThrottler(0, queue) + + throttler.Add("c", 0, time.Now().Add(2*time.Hour)) + throttler.Add("b", 0, time.Now().Add(1*time.Hour)) + throttler.Add("a", 0, time.Now()) + + next, ok := throttler.Next("b") + assert.True(t, ok) + assert.Equal(t, "a", next) + + next, ok = throttler.Next("c") + assert.True(t, ok) + assert.Equal(t, "b", next) +} + +func TestWithParallelismLimitAndPriority(t *testing.T) { + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + throttler := NewThrottler(2, queue) + + throttler.Add("a", 1, time.Now()) + throttler.Add("b", 2, time.Now()) + throttler.Add("c", 3, time.Now()) + throttler.Add("d", 4, time.Now()) + + next, ok := throttler.Next("a") + assert.True(t, ok) + assert.Equal(t, "d", next) + + next, ok = throttler.Next("a") + assert.True(t, ok) + assert.Equal(t, "c", next) + + _, ok = throttler.Next("a") + assert.False(t, ok) + + next, ok = throttler.Next("c") + assert.True(t, ok) + assert.Equal(t, "c", next) + + throttler.Remove("c") + + assert.Equal(t, 1, queue.Len()) + queued, _ := queue.Get() + assert.Equal(t, "b", queued) +} + +func TestChangeParallelism(t *testing.T) { + queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + throttler := NewThrottler(1, queue) + + throttler.Add("a", 1, time.Now()) + throttler.Add("b", 2, time.Now()) + throttler.Add("c", 3, time.Now()) + throttler.Add("d", 4, time.Now()) + + next, ok := throttler.Next("a") + assert.True(t, ok) + assert.Equal(t, "d", next) + + _, ok = throttler.Next("b") + assert.False(t, ok) + + _, ok = throttler.Next("c") + assert.False(t, ok) + + throttler.SetParallelism(3) + + assert.Equal(t, 2, queue.Len()) + queued, _ := queue.Get() + assert.Equal(t, "c", queued) + queued, _ = queue.Get() + assert.Equal(t, "b", queued) +} diff --git a/workflow/controller/workflowpod.go b/workflow/controller/workflowpod.go index 47c579620068..28dfc3e51418 100644 --- a/workflow/controller/workflowpod.go +++ b/workflow/controller/workflowpod.go @@ -5,16 +5,19 @@ import ( "fmt" "io" "path" + "path/filepath" "strconv" - "github.com/cyrusbiotechnology/argo/errors" - wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" - "github.com/cyrusbiotechnology/argo/workflow/common" log "github.com/sirupsen/logrus" "github.com/valyala/fasttemplate" apiv1 "k8s.io/api/core/v1" apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + "github.com/cyrusbiotechnology/argo/errors" + wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/workflow/common" ) // Reusable k8s pod spec portions used in workflow pods @@ -43,27 +46,8 @@ var ( MountPath: common.PodMetadataMountPath, } - hostPathDir = apiv1.HostPathDirectory hostPathSocket = apiv1.HostPathSocket - // volumeDockerLib provides the wait container access to the minion's host docker containers - // runtime files (e.g. /var/lib/docker/container). This is used by the executor to access - // the main container's logs (and potentially storage to upload output artifacts) - volumeDockerLib = apiv1.Volume{ - Name: common.DockerLibVolumeName, - VolumeSource: apiv1.VolumeSource{ - HostPath: &apiv1.HostPathVolumeSource{ - Path: common.DockerLibHostPath, - Type: &hostPathDir, - }, - }, - } - volumeMountDockerLib = apiv1.VolumeMount{ - Name: volumeDockerLib.Name, - MountPath: volumeDockerLib.VolumeSource.HostPath.Path, - ReadOnly: true, - } - // volumeDockerSock provides the wait container direct access to the minion's host docker daemon. // The primary purpose of this is to make available `docker cp` to collect an output artifact // from a container. Alternatively, we could use `kubectl cp`, but `docker cp` avoids the extra @@ -82,45 +66,8 @@ var ( MountPath: "/var/run/docker.sock", ReadOnly: true, } - - // execEnvVars exposes various pod information as environment variables to the exec container - execEnvVars = []apiv1.EnvVar{ - envFromField(common.EnvVarPodName, "metadata.name"), - } - - volumeMountGoogleSecret = apiv1.VolumeMount{ - Name: common.GoogleSecretVolumeName, - MountPath: "/var/secrets/google", - } - - googleCredentialSecretEnvVar = apiv1.EnvVar{ - Name: "GOOGLE_APPLICATION_CREDENTIALS", - Value: "/var/secrets/google/key.json", - } - - volumeGoogleSecret = apiv1.Volume{ - Name: common.GoogleSecretVolumeName, - VolumeSource: apiv1.VolumeSource{ - Secret: &apiv1.SecretVolumeSource{ - SecretName: common.GoogleSecretName, - }, - }, - } ) -// envFromField is a helper to return a EnvVar with the name and field -func envFromField(envVarName, fieldPath string) apiv1.EnvVar { - return apiv1.EnvVar{ - Name: envVarName, - ValueFrom: &apiv1.EnvVarSource{ - FieldRef: &apiv1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: fieldPath, - }, - }, - } -} - func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Container, tmpl *wfv1.Template) (*apiv1.Pod, error) { nodeID := woc.wf.NodeID(nodeName) woc.log.Debugf("Creating Pod: %s (%s)", nodeName, nodeID) @@ -129,7 +76,8 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont mainCtr.Name = common.MainContainerName pod := &apiv1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: nodeID, + Name: nodeID, + Namespace: woc.wf.ObjectMeta.Namespace, Labels: map[string]string{ common.LabelKeyWorkflow: woc.wf.ObjectMeta.Name, // Allows filtering by pods related to specific workflow common.LabelKeyCompleted: "false", // Allows filtering by incomplete workflow pods @@ -142,22 +90,36 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont }, }, Spec: apiv1.PodSpec{ - RestartPolicy: apiv1.RestartPolicyNever, - Containers: []apiv1.Container{ - mainCtr, - }, + RestartPolicy: apiv1.RestartPolicyNever, Volumes: woc.createVolumes(), ActiveDeadlineSeconds: tmpl.ActiveDeadlineSeconds, ServiceAccountName: woc.wf.Spec.ServiceAccountName, ImagePullSecrets: woc.wf.Spec.ImagePullSecrets, }, } + + if woc.wf.Spec.HostNetwork != nil { + pod.Spec.HostNetwork = *woc.wf.Spec.HostNetwork + } + + if woc.wf.Spec.DNSPolicy != nil { + pod.Spec.DNSPolicy = *woc.wf.Spec.DNSPolicy + } + + if woc.wf.Spec.DNSConfig != nil { + pod.Spec.DNSConfig = woc.wf.Spec.DNSConfig + } + if woc.controller.Config.InstanceID != "" { pod.ObjectMeta.Labels[common.LabelKeyControllerInstanceID] = woc.controller.Config.InstanceID } + if woc.controller.Config.ContainerRuntimeExecutor == common.ContainerRuntimeExecutorPNS { + pod.Spec.ShareProcessNamespace = pointer.BoolPtr(true) + } - if common.GoogleSecretName != "" { - pod.Spec.Volumes = append(pod.Spec.Volumes, volumeGoogleSecret) + err := woc.addArchiveLocation(pod, tmpl) + if err != nil { + return nil, err } if tmpl.GetType() != wfv1.TemplateTypeResource { @@ -170,6 +132,11 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont } pod.Spec.Containers = append(pod.Spec.Containers, *waitCtr) } + // NOTE: the order of the container list is significant. kubelet will pull, create, and start + // each container sequentially in the order that they appear in this list. For PNS we want the + // wait container to start before the main, so that it always has the chance to see the main + // container's PID and root filesystem. + pod.Spec.Containers = append(pod.Spec.Containers, mainCtr) // Add init container only if it needs input artifacts. This is also true for // script templates (which needs to populate the script) @@ -181,7 +148,7 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont addSchedulingConstraints(pod, wfSpec, tmpl) woc.addMetadata(pod, tmpl) - err := addVolumeReferences(pod, wfSpec, tmpl, woc.wf.Status.PersistentVolumeClaims) + err = addVolumeReferences(pod, woc.volumes, tmpl, woc.wf.Status.PersistentVolumeClaims) if err != nil { return nil, err } @@ -191,21 +158,21 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont return nil, err } - err = woc.addArchiveLocation(pod, tmpl) - if err != nil { - return nil, err - } - if tmpl.GetType() == wfv1.TemplateTypeScript { - addExecutorStagingVolume(pod) + addScriptStagingVolume(pod) } - // addSidecars should be called after all volumes have been manipulated - // in the main container (in case sidecar requires volume mount mirroring) + // addInitContainers, addSidecars and addOutputArtifactsVolumes should be called after all + // volumes have been manipulated in the main container since volumeMounts are mirrored + err = addInitContainers(pod, tmpl) + if err != nil { + return nil, err + } err = addSidecars(pod, tmpl) if err != nil { return nil, err } + addOutputArtifactsVolumes(pod, tmpl) // Set the container template JSON in pod annotations, which executor examines for things like // artifact location/path. @@ -216,8 +183,16 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont pod.ObjectMeta.Annotations[common.AnnotationKeyTemplate] = string(tmplBytes) // Perform one last variable substitution here. Some variables come from the from workflow - // configmap (e.g. archive location), and were not substituted in executeTemplate. - pod, err = substituteGlobals(pod, woc.globalParams) + // configmap (e.g. archive location) or volumes attribute, and were not substituted + // in executeTemplate. + podParams := make(map[string]string) + for gkey, gval := range woc.globalParams { + podParams[gkey] = gval + } + for _, inParam := range tmpl.Inputs.Parameters { + podParams["inputs.parameters."+inParam.Name] = *inParam.Value + } + pod, err = substitutePodParams(pod, podParams) if err != nil { return nil, err } @@ -253,20 +228,20 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont return created, nil } -// substituteGlobals returns a pod spec with global parameter references substituted as well as pod.name -func substituteGlobals(pod *apiv1.Pod, globalParams map[string]string) (*apiv1.Pod, error) { - newGlobalParams := make(map[string]string) - for k, v := range globalParams { - newGlobalParams[k] = v +// substitutePodParams returns a pod spec with parameter references substituted as well as pod.name +func substitutePodParams(pod *apiv1.Pod, podParams map[string]string) (*apiv1.Pod, error) { + newPodParams := make(map[string]string) + for k, v := range podParams { + newPodParams[k] = v } - newGlobalParams[common.LocalVarPodName] = pod.Name - globalParams = newGlobalParams + newPodParams[common.LocalVarPodName] = pod.Name + podParams = newPodParams specBytes, err := json.Marshal(pod) if err != nil { return nil, err } fstTmpl := fasttemplate.New(string(specBytes), "{{", "}}") - newSpecBytes, err := common.Replace(fstTmpl, globalParams, true) + newSpecBytes, err := common.Replace(fstTmpl, podParams, true) if err != nil { return nil, err } @@ -279,27 +254,79 @@ func substituteGlobals(pod *apiv1.Pod, globalParams map[string]string) (*apiv1.P } func (woc *wfOperationCtx) newInitContainer(tmpl *wfv1.Template) apiv1.Container { - ctr := woc.newExecContainer(common.InitContainerName, false) - ctr.Command = []string{"argoexec"} - ctr.Args = []string{"init"} - ctr.VolumeMounts = append(ctr.VolumeMounts, volumeMountPodMetadata) - + ctr := woc.newExecContainer(common.InitContainerName) + ctr.Command = []string{"argoexec", "init"} return *ctr } func (woc *wfOperationCtx) newWaitContainer(tmpl *wfv1.Template) (*apiv1.Container, error) { - ctr := woc.newExecContainer(common.WaitContainerName, false) - ctr.Command = []string{"argoexec"} - ctr.Args = []string{"wait"} - ctr.VolumeMounts = append(ctr.VolumeMounts, woc.createVolumeMounts()...) - + ctr := woc.newExecContainer(common.WaitContainerName) + ctr.Command = []string{"argoexec", "wait"} + switch woc.controller.Config.ContainerRuntimeExecutor { + case common.ContainerRuntimeExecutorPNS: + ctr.SecurityContext = &apiv1.SecurityContext{ + Capabilities: &apiv1.Capabilities{ + Add: []apiv1.Capability{ + // necessary to access main's root filesystem when run with a different user id + apiv1.Capability("SYS_PTRACE"), + }, + }, + } + if hasPrivilegedContainers(tmpl) { + // if the main or sidecar is privileged, the wait sidecar must also run privileged, + // in order to SIGTERM/SIGKILL the pid + ctr.SecurityContext.Privileged = pointer.BoolPtr(true) + } + case "", common.ContainerRuntimeExecutorDocker: + ctr.VolumeMounts = append(ctr.VolumeMounts, volumeMountDockerSock) + } return ctr, nil } +// hasPrivilegedContainers tests if the main container or sidecars is privileged +func hasPrivilegedContainers(tmpl *wfv1.Template) bool { + if containerIsPrivileged(tmpl.Container) { + return true + } + for _, side := range tmpl.Sidecars { + if containerIsPrivileged(&side.Container) { + return true + } + } + return false +} + +func containerIsPrivileged(ctr *apiv1.Container) bool { + if ctr != nil && ctr.SecurityContext != nil && ctr.SecurityContext.Privileged != nil && *ctr.SecurityContext.Privileged { + return true + } + return false +} + func (woc *wfOperationCtx) createEnvVars() []apiv1.EnvVar { + var execEnvVars []apiv1.EnvVar + execEnvVars = append(execEnvVars, apiv1.EnvVar{ + Name: common.EnvVarPodName, + ValueFrom: &apiv1.EnvVarSource{ + FieldRef: &apiv1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.name", + }, + }, + }) + if woc.controller.Config.Executor != nil { + execEnvVars = append(execEnvVars, woc.controller.Config.Executor.Env...) + } switch woc.controller.Config.ContainerRuntimeExecutor { + case common.ContainerRuntimeExecutorK8sAPI: + execEnvVars = append(execEnvVars, + apiv1.EnvVar{ + Name: common.EnvVarContainerRuntimeExecutor, + Value: woc.controller.Config.ContainerRuntimeExecutor, + }, + ) case common.ContainerRuntimeExecutorKubelet: - return append(execEnvVars, + execEnvVars = append(execEnvVars, apiv1.EnvVar{ Name: common.EnvVarContainerRuntimeExecutor, Value: woc.controller.Config.ContainerRuntimeExecutor, @@ -321,57 +348,87 @@ func (woc *wfOperationCtx) createEnvVars() []apiv1.EnvVar { Value: strconv.FormatBool(woc.controller.Config.KubeletInsecure), }, ) - default: - return execEnvVars - } -} - -func (woc *wfOperationCtx) createVolumeMounts() []apiv1.VolumeMount { - volumeMounts := []apiv1.VolumeMount{ - volumeMountPodMetadata, - } - switch woc.controller.Config.ContainerRuntimeExecutor { - case common.ContainerRuntimeExecutorKubelet: - return volumeMounts - default: - return append(volumeMounts, volumeMountDockerLib, volumeMountDockerSock) + case common.ContainerRuntimeExecutorPNS: + execEnvVars = append(execEnvVars, + apiv1.EnvVar{ + Name: common.EnvVarContainerRuntimeExecutor, + Value: woc.controller.Config.ContainerRuntimeExecutor, + }, + ) } + return execEnvVars } func (woc *wfOperationCtx) createVolumes() []apiv1.Volume { volumes := []apiv1.Volume{ volumePodMetadata, } + if woc.controller.Config.KubeConfig != nil { + name := woc.controller.Config.KubeConfig.VolumeName + if name == "" { + name = common.KubeConfigDefaultVolumeName + } + volumes = append(volumes, apiv1.Volume{ + Name: name, + VolumeSource: apiv1.VolumeSource{ + Secret: &apiv1.SecretVolumeSource{ + SecretName: woc.controller.Config.KubeConfig.SecretName, + }, + }, + }) + } switch woc.controller.Config.ContainerRuntimeExecutor { - case common.ContainerRuntimeExecutorKubelet: + case common.ContainerRuntimeExecutorKubelet, common.ContainerRuntimeExecutorK8sAPI, common.ContainerRuntimeExecutorPNS: return volumes default: - return append(volumes, volumeDockerLib, volumeDockerSock) + return append(volumes, volumeDockerSock) } } -func (woc *wfOperationCtx) newExecContainer(name string, privileged bool) *apiv1.Container { +func (woc *wfOperationCtx) newExecContainer(name string) *apiv1.Container { exec := apiv1.Container{ Name: name, Image: woc.controller.executorImage(), ImagePullPolicy: woc.controller.executorImagePullPolicy(), - VolumeMounts: []apiv1.VolumeMount{}, Env: woc.createEnvVars(), - SecurityContext: &apiv1.SecurityContext{ - Privileged: &privileged, + VolumeMounts: []apiv1.VolumeMount{ + volumeMountPodMetadata, }, } - if woc.controller.Config.ExecutorResources != nil { + if woc.controller.Config.Executor != nil { + exec.Args = woc.controller.Config.Executor.Args + } + if isResourcesSpecified(woc.controller.Config.Executor) { + exec.Resources = woc.controller.Config.Executor.Resources + } else if woc.controller.Config.ExecutorResources != nil { exec.Resources = *woc.controller.Config.ExecutorResources } - - if common.GoogleSecretName != "" { - exec.VolumeMounts = append(exec.VolumeMounts, volumeMountGoogleSecret) - exec.Env = append(exec.Env, googleCredentialSecretEnvVar) + if woc.controller.Config.KubeConfig != nil { + path := woc.controller.Config.KubeConfig.MountPath + if path == "" { + path = common.KubeConfigDefaultMountPath + } + name := woc.controller.Config.KubeConfig.VolumeName + if name == "" { + name = common.KubeConfigDefaultVolumeName + } + exec.VolumeMounts = []apiv1.VolumeMount{{ + Name: name, + MountPath: path, + ReadOnly: true, + SubPath: woc.controller.Config.KubeConfig.SecretKey, + }, + } + exec.Args = append(exec.Args, "--kubeconfig="+path) } + return &exec } +func isResourcesSpecified(ctr *apiv1.Container) bool { + return ctr != nil && (ctr.Resources.Limits.Cpu() != nil || ctr.Resources.Limits.Memory() != nil) +} + // addMetadata applies metadata specified in the template func (woc *wfOperationCtx) addMetadata(pod *apiv1.Pod, tmpl *wfv1.Template) { for k, v := range tmpl.Metadata.Annotations { @@ -412,11 +469,36 @@ func addSchedulingConstraints(pod *apiv1.Pod, wfSpec *wfv1.WorkflowSpec, tmpl *w } else if len(wfSpec.Tolerations) > 0 { pod.Spec.Tolerations = wfSpec.Tolerations } + + // Set scheduler name (if specified) + if tmpl.SchedulerName != "" { + pod.Spec.SchedulerName = tmpl.SchedulerName + } else if wfSpec.SchedulerName != "" { + pod.Spec.SchedulerName = wfSpec.SchedulerName + } + // Set priorityClass (if specified) + if tmpl.PriorityClassName != "" { + pod.Spec.PriorityClassName = tmpl.PriorityClassName + } else if wfSpec.PodPriorityClassName != "" { + pod.Spec.PriorityClassName = wfSpec.PodPriorityClassName + } + // Set priority (if specified) + if tmpl.Priority != nil { + pod.Spec.Priority = tmpl.Priority + } else if wfSpec.PodPriority != nil { + pod.Spec.Priority = wfSpec.PodPriority + } + // Set schedulerName (if specified) + if tmpl.SchedulerName != "" { + pod.Spec.SchedulerName = tmpl.SchedulerName + } else if wfSpec.SchedulerName != "" { + pod.Spec.SchedulerName = wfSpec.SchedulerName + } } // addVolumeReferences adds any volumeMounts that a container/sidecar is referencing, to the pod.spec.volumes // These are either specified in the workflow.spec.volumes or the workflow.spec.volumeClaimTemplate section -func addVolumeReferences(pod *apiv1.Pod, wfSpec *wfv1.WorkflowSpec, tmpl *wfv1.Template, pvcs []apiv1.Volume) error { +func addVolumeReferences(pod *apiv1.Pod, vols []apiv1.Volume, tmpl *wfv1.Template, pvcs []apiv1.Volume) error { switch tmpl.GetType() { case wfv1.TemplateTypeContainer, wfv1.TemplateTypeScript: default: @@ -425,7 +507,7 @@ func addVolumeReferences(pod *apiv1.Pod, wfSpec *wfv1.WorkflowSpec, tmpl *wfv1.T // getVolByName is a helper to retrieve a volume by its name, either from the volumes or claims section getVolByName := func(name string) *apiv1.Volume { - for _, vol := range wfSpec.Volumes { + for _, vol := range vols { if vol.Name == name { return &vol } @@ -460,6 +542,7 @@ func addVolumeReferences(pod *apiv1.Pod, wfSpec *wfv1.WorkflowSpec, tmpl *wfv1.T } return nil } + if tmpl.Container != nil { err := addVolumeRef(tmpl.Container.VolumeMounts) if err != nil { @@ -472,12 +555,30 @@ func addVolumeReferences(pod *apiv1.Pod, wfSpec *wfv1.WorkflowSpec, tmpl *wfv1.T return err } } + for _, sidecar := range tmpl.Sidecars { err := addVolumeRef(sidecar.VolumeMounts) if err != nil { return err } } + + volumes, volumeMounts := createSecretVolumes(tmpl) + pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) + + for idx, container := range pod.Spec.Containers { + if container.Name == common.WaitContainerName { + pod.Spec.Containers[idx].VolumeMounts = append(pod.Spec.Containers[idx].VolumeMounts, volumeMounts...) + break + } + } + for idx, container := range pod.Spec.InitContainers { + if container.Name == common.InitContainerName { + pod.Spec.InitContainers[idx].VolumeMounts = append(pod.Spec.InitContainers[idx].VolumeMounts, volumeMounts...) + break + } + } + return nil } @@ -521,7 +622,7 @@ func (woc *wfOperationCtx) addInputArtifactsVolumes(pod *apiv1.Pod, tmpl *wfv1.T // instead of the artifacts volume if tmpl.Container != nil { for _, mnt := range tmpl.Container.VolumeMounts { - mnt.MountPath = path.Join(common.InitContainerMainFilesystemDir, mnt.MountPath) + mnt.MountPath = filepath.Join(common.ExecutorMainFilesystemDir, mnt.MountPath) initCtr.VolumeMounts = append(initCtr.VolumeMounts, mnt) } } @@ -530,19 +631,19 @@ func (woc *wfOperationCtx) addInputArtifactsVolumes(pod *apiv1.Pod, tmpl *wfv1.T } } - mainCtrIndex := 0 - var mainCtr *apiv1.Container + mainCtrIndex := -1 for i, ctr := range pod.Spec.Containers { - if ctr.Name == common.MainContainerName { + switch ctr.Name { + case common.MainContainerName: mainCtrIndex = i - mainCtr = &pod.Spec.Containers[i] + break } } - if mainCtr == nil { - panic("Could not find main container in pod spec") + if mainCtrIndex == -1 { + panic("Could not find main or wait container in pod spec") } - // TODO: the order in which we construct the volume mounts may matter, - // especially if they are overlapping. + mainCtr := &pod.Spec.Containers[mainCtrIndex] + for _, art := range tmpl.Inputs.Artifacts { if art.Path == "" { return errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s did not specify a path", art.Name) @@ -570,31 +671,77 @@ func (woc *wfOperationCtx) addInputArtifactsVolumes(pod *apiv1.Pod, tmpl *wfv1.T return nil } -// addArchiveLocation updates the template with the default artifact repository information -// configured in the controller. This is skipped for templates which have explicitly set an archive -// location in the template. -func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Template) error { - if tmpl.ArchiveLocation == nil { - tmpl.ArchiveLocation = &wfv1.ArtifactLocation{ - ArchiveLogs: woc.controller.Config.ArtifactRepository.ArchiveLogs, +// addOutputArtifactsVolumes mirrors any volume mounts in the main container to the wait sidecar. +// For any output artifacts that were produced in mounted volumes (e.g. PVCs, emptyDirs), the +// wait container will collect the artifacts directly from volumeMount instead of `docker cp`-ing +// them to the wait sidecar. In order for this to work, we mirror all volume mounts in the main +// container under a well-known path. +func addOutputArtifactsVolumes(pod *apiv1.Pod, tmpl *wfv1.Template) { + if tmpl.GetType() == wfv1.TemplateTypeResource { + return + } + mainCtrIndex := -1 + waitCtrIndex := -1 + var mainCtr *apiv1.Container + for i, ctr := range pod.Spec.Containers { + switch ctr.Name { + case common.MainContainerName: + mainCtrIndex = i + case common.WaitContainerName: + waitCtrIndex = i } } - if tmpl.ArchiveLocation.S3 != nil || tmpl.ArchiveLocation.Artifactory != nil { - // User explicitly set the location. nothing else to do. - return nil + if mainCtrIndex == -1 || waitCtrIndex == -1 { + panic("Could not find main or wait container in pod spec") + } + mainCtr = &pod.Spec.Containers[mainCtrIndex] + waitCtr := &pod.Spec.Containers[waitCtrIndex] + + for _, mnt := range mainCtr.VolumeMounts { + mnt.MountPath = filepath.Join(common.ExecutorMainFilesystemDir, mnt.MountPath) + // ReadOnly is needed to be false for overlapping volume mounts + mnt.ReadOnly = false + waitCtr.VolumeMounts = append(waitCtr.VolumeMounts, mnt) } + pod.Spec.Containers[waitCtrIndex] = *waitCtr +} + +// addArchiveLocation conditionally updates the template with the default artifact repository +// information configured in the controller, for the purposes of archiving outputs. This is skipped +// for templates which do not need to archive anything, or have explicitly set an archive location +// in the template. +func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Template) error { // needLocation keeps track if the workflow needs to have an archive location set. // If so, and one was not supplied (or defaulted), we will return error var needLocation bool - if tmpl.ArchiveLocation.ArchiveLogs != nil && *tmpl.ArchiveLocation.ArchiveLogs { - needLocation = true - } + if tmpl.ArchiveLocation != nil { + if tmpl.ArchiveLocation.S3 != nil || tmpl.ArchiveLocation.Artifactory != nil || tmpl.ArchiveLocation.HDFS != nil { + // User explicitly set the location. nothing else to do. + return nil + } + if tmpl.ArchiveLocation.ArchiveLogs != nil && *tmpl.ArchiveLocation.ArchiveLogs { + needLocation = true + } + } + for _, art := range tmpl.Outputs.Artifacts { + if !art.HasLocation() { + needLocation = true + break + } + } + if !needLocation { + woc.log.Debugf("archive location unnecessary") + return nil + } + tmpl.ArchiveLocation = &wfv1.ArtifactLocation{ + ArchiveLogs: woc.controller.Config.ArtifactRepository.ArchiveLogs, + } // artifact location is defaulted using the following formula: // //.tgz // (e.g. myworkflowartifacts/argo-wf-fhljp/argo-wf-fhljp-123291312382/src.tgz) if s3Location := woc.controller.Config.ArtifactRepository.S3; s3Location != nil { - log.Debugf("Setting s3 artifact repository information") + woc.log.Debugf("Setting s3 artifact repository information") artLocationKey := s3Location.KeyFormat // NOTE: we use unresolved variables, will get substituted later if artLocationKey == "" { @@ -605,7 +752,7 @@ func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Templat Key: artLocationKey, } } else if woc.controller.Config.ArtifactRepository.Artifactory != nil { - log.Debugf("Setting artifactory artifact repository information") + woc.log.Debugf("Setting artifactory artifact repository information") repoURL := "" if woc.controller.Config.ArtifactRepository.Artifactory.RepoURL != "" { repoURL = woc.controller.Config.ArtifactRepository.Artifactory.RepoURL + "/" @@ -615,6 +762,13 @@ func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Templat ArtifactoryAuth: woc.controller.Config.ArtifactRepository.Artifactory.ArtifactoryAuth, URL: artURL, } + } else if hdfsLocation := woc.controller.Config.ArtifactRepository.HDFS; hdfsLocation != nil { + woc.log.Debugf("Setting HDFS artifact repository information") + tmpl.ArchiveLocation.HDFS = &wfv1.HDFSArtifact{ + HDFSConfig: hdfsLocation.HDFSConfig, + Path: hdfsLocation.PathFormat, + Force: hdfsLocation.Force, + } } else if woc.controller.Config.ArtifactRepository.GCS != nil { log.Debugf("Setting GCS artifact repository information") artLocationKey := fmt.Sprintf("%s/%s", woc.wf.ObjectMeta.Name, pod.ObjectMeta.Name) @@ -623,22 +777,14 @@ func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Templat Key: artLocationKey, } } else { - for _, art := range tmpl.Outputs.Artifacts { - if !art.HasLocation() { - needLocation = true - break - } - } - if needLocation { - return errors.Errorf(errors.CodeBadRequest, "controller is not configured with a default archive location") - } + return errors.Errorf(errors.CodeBadRequest, "controller is not configured with a default archive location") } return nil } -// addExecutorStagingVolume sets up a shared staging volume between the init container +// addScriptStagingVolume sets up a shared staging volume between the init container // and main container for the purpose of holding the script source code for script templates -func addExecutorStagingVolume(pod *apiv1.Pod) { +func addScriptStagingVolume(pod *apiv1.Pod) { volName := "argo-staging" stagingVol := apiv1.Volume{ Name: volName, @@ -666,11 +812,7 @@ func addExecutorStagingVolume(pod *apiv1.Pod) { Name: volName, MountPath: common.ExecutorStagingEmptyDir, } - if ctr.VolumeMounts == nil { - ctr.VolumeMounts = []apiv1.VolumeMount{volMount} - } else { - ctr.VolumeMounts = append(ctr.VolumeMounts, volMount) - } + ctr.VolumeMounts = append(ctr.VolumeMounts, volMount) pod.Spec.Containers[i] = ctr found = true break @@ -681,31 +823,40 @@ func addExecutorStagingVolume(pod *apiv1.Pod) { } } +// addInitContainers adds all init containers to the pod spec of the step +// Optionally volume mounts from the main container to the init containers +func addInitContainers(pod *apiv1.Pod, tmpl *wfv1.Template) error { + if len(tmpl.InitContainers) == 0 { + return nil + } + mainCtr := findMainContainer(pod) + if mainCtr == nil { + panic("Unable to locate main container") + } + for _, ctr := range tmpl.InitContainers { + log.Debugf("Adding init container %s", ctr.Name) + if ctr.MirrorVolumeMounts != nil && *ctr.MirrorVolumeMounts { + mirrorVolumeMounts(mainCtr, &ctr.Container) + } + pod.Spec.InitContainers = append(pod.Spec.InitContainers, ctr.Container) + } + return nil +} + // addSidecars adds all sidecars to the pod spec of the step. // Optionally volume mounts from the main container to the sidecar func addSidecars(pod *apiv1.Pod, tmpl *wfv1.Template) error { if len(tmpl.Sidecars) == 0 { return nil } - var mainCtr *apiv1.Container - for _, ctr := range pod.Spec.Containers { - if ctr.Name != common.MainContainerName { - continue - } - mainCtr = &ctr - break - } + mainCtr := findMainContainer(pod) if mainCtr == nil { panic("Unable to locate main container") } for _, sidecar := range tmpl.Sidecars { + log.Debugf("Adding sidecar container %s", sidecar.Name) if sidecar.MirrorVolumeMounts != nil && *sidecar.MirrorVolumeMounts { - for _, volMnt := range mainCtr.VolumeMounts { - if sidecar.VolumeMounts == nil { - sidecar.VolumeMounts = make([]apiv1.VolumeMount, 0) - } - sidecar.VolumeMounts = append(sidecar.VolumeMounts, volMnt) - } + mirrorVolumeMounts(mainCtr, &sidecar.Container) } pod.Spec.Containers = append(pod.Spec.Containers, sidecar.Container) } @@ -726,3 +877,130 @@ func verifyResolvedVariables(obj interface{}) error { }) return unresolvedErr } + +// createSecretVolumes will retrieve and create Volumes and Volumemount object for Pod +func createSecretVolumes(tmpl *wfv1.Template) ([]apiv1.Volume, []apiv1.VolumeMount) { + var allVolumesMap = make(map[string]apiv1.Volume) + var uniqueKeyMap = make(map[string]bool) + var secretVolumes []apiv1.Volume + var secretVolMounts []apiv1.VolumeMount + + createArchiveLocationSecret(tmpl, allVolumesMap, uniqueKeyMap) + + for _, art := range tmpl.Outputs.Artifacts { + createSecretVolume(allVolumesMap, art, uniqueKeyMap) + } + for _, art := range tmpl.Inputs.Artifacts { + createSecretVolume(allVolumesMap, art, uniqueKeyMap) + } + + for volMountName, val := range allVolumesMap { + secretVolumes = append(secretVolumes, val) + secretVolMounts = append(secretVolMounts, apiv1.VolumeMount{ + Name: volMountName, + MountPath: common.SecretVolMountPath + "/" + val.Name, + ReadOnly: true, + }) + } + + return secretVolumes, secretVolMounts +} + +func createArchiveLocationSecret(tmpl *wfv1.Template, volMap map[string]apiv1.Volume, uniqueKeyMap map[string]bool) { + if tmpl.ArchiveLocation == nil { + return + } + if s3ArtRepo := tmpl.ArchiveLocation.S3; s3ArtRepo != nil { + createSecretVal(volMap, &s3ArtRepo.AccessKeySecret, uniqueKeyMap) + createSecretVal(volMap, &s3ArtRepo.SecretKeySecret, uniqueKeyMap) + } else if hdfsArtRepo := tmpl.ArchiveLocation.HDFS; hdfsArtRepo != nil { + createSecretVal(volMap, hdfsArtRepo.KrbKeytabSecret, uniqueKeyMap) + createSecretVal(volMap, hdfsArtRepo.KrbCCacheSecret, uniqueKeyMap) + } else if artRepo := tmpl.ArchiveLocation.Artifactory; artRepo != nil { + createSecretVal(volMap, artRepo.UsernameSecret, uniqueKeyMap) + createSecretVal(volMap, artRepo.PasswordSecret, uniqueKeyMap) + } else if gitRepo := tmpl.ArchiveLocation.Git; gitRepo != nil { + createSecretVal(volMap, gitRepo.UsernameSecret, uniqueKeyMap) + createSecretVal(volMap, gitRepo.PasswordSecret, uniqueKeyMap) + createSecretVal(volMap, gitRepo.SSHPrivateKeySecret, uniqueKeyMap) + } else if gcsRepo := tmpl.ArchiveLocation.GCS; gcsRepo != nil { + createSecretVal(volMap, &gcsRepo.CredentialsSecret, uniqueKeyMap) + } +} + +func createSecretVolume(volMap map[string]apiv1.Volume, art wfv1.Artifact, keyMap map[string]bool) { + if art.S3 != nil { + createSecretVal(volMap, &art.S3.AccessKeySecret, keyMap) + createSecretVal(volMap, &art.S3.SecretKeySecret, keyMap) + } else if art.Git != nil { + createSecretVal(volMap, art.Git.UsernameSecret, keyMap) + createSecretVal(volMap, art.Git.PasswordSecret, keyMap) + createSecretVal(volMap, art.Git.SSHPrivateKeySecret, keyMap) + } else if art.Artifactory != nil { + createSecretVal(volMap, art.Artifactory.UsernameSecret, keyMap) + createSecretVal(volMap, art.Artifactory.PasswordSecret, keyMap) + } else if art.HDFS != nil { + createSecretVal(volMap, art.HDFS.KrbCCacheSecret, keyMap) + createSecretVal(volMap, art.HDFS.KrbKeytabSecret, keyMap) + } else if art.GCS != nil { + createSecretVal(volMap, &art.GCS.CredentialsSecret, keyMap) + } +} + +func createSecretVal(volMap map[string]apiv1.Volume, secret *apiv1.SecretKeySelector, keyMap map[string]bool) { + if secret == nil { + return + } + if vol, ok := volMap[secret.Name]; ok { + key := apiv1.KeyToPath{ + Key: secret.Key, + Path: secret.Key, + } + if val, _ := keyMap[secret.Name+"-"+secret.Key]; !val { + keyMap[secret.Name+"-"+secret.Key] = true + vol.Secret.Items = append(vol.Secret.Items, key) + } + } else { + volume := apiv1.Volume{ + Name: secret.Name, + VolumeSource: apiv1.VolumeSource{ + Secret: &apiv1.SecretVolumeSource{ + SecretName: secret.Name, + Items: []apiv1.KeyToPath{ + { + Key: secret.Key, + Path: secret.Key, + }, + }, + }, + }, + } + keyMap[secret.Name+"-"+secret.Key] = true + volMap[secret.Name] = volume + } +} + +// findMainContainer finds main container +func findMainContainer(pod *apiv1.Pod) *apiv1.Container { + var mainCtr *apiv1.Container + for _, ctr := range pod.Spec.Containers { + if ctr.Name != common.MainContainerName { + continue + } + mainCtr = &ctr + break + } + return mainCtr +} + +// mirrorVolumeMounts mirrors volumeMounts of source container to target container +func mirrorVolumeMounts(sourceContainer, targetContainer *apiv1.Container) { + for _, volMnt := range sourceContainer.VolumeMounts { + if targetContainer.VolumeMounts == nil { + targetContainer.VolumeMounts = make([]apiv1.VolumeMount, 0) + } + log.Debugf("Adding volume mount %v to container %v", volMnt.Name, targetContainer.Name) + targetContainer.VolumeMounts = append(targetContainer.VolumeMounts, volMnt) + + } +} diff --git a/workflow/controller/workflowpod_test.go b/workflow/controller/workflowpod_test.go index 9efc3e92b0c6..32cabe9a9c71 100644 --- a/workflow/controller/workflowpod_test.go +++ b/workflow/controller/workflowpod_test.go @@ -1,9 +1,12 @@ package controller import ( + "encoding/json" + "fmt" "testing" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/workflow/common" "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" @@ -180,7 +183,16 @@ func TestWorkflowControllerArchiveConfig(t *testing.T) { // TestWorkflowControllerArchiveConfigUnresolvable verifies workflow fails when archive location has // unresolvable variables func TestWorkflowControllerArchiveConfigUnresolvable(t *testing.T) { - woc := newWoc() + wf := unmarshalWF(helloWorldWf) + wf.Spec.Templates[0].Outputs = wfv1.Outputs{ + Artifacts: []wfv1.Artifact{ + { + Name: "foo", + Path: "/tmp/file", + }, + }, + } + woc := newWoc(*wf) woc.controller.Config.ArtifactRepository.S3 = &S3ArtifactRepository{ S3Bucket: wfv1.S3Bucket{ Bucket: "foo", @@ -192,3 +204,327 @@ func TestWorkflowControllerArchiveConfigUnresolvable(t *testing.T) { _, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) assert.Error(t, err) } + +// TestConditionalNoAddArchiveLocation verifies we do not add archive location if it is not needed +func TestConditionalNoAddArchiveLocation(t *testing.T) { + woc := newWoc() + woc.controller.Config.ArtifactRepository.S3 = &S3ArtifactRepository{ + S3Bucket: wfv1.S3Bucket{ + Bucket: "foo", + }, + KeyFormat: "path/in/bucket", + } + woc.operate() + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.NoError(t, err) + var tmpl wfv1.Template + err = json.Unmarshal([]byte(pod.Annotations[common.AnnotationKeyTemplate]), &tmpl) + assert.NoError(t, err) + assert.Nil(t, tmpl.ArchiveLocation) +} + +// TestConditionalNoAddArchiveLocation verifies we add archive location when it is needed +func TestConditionalArchiveLocation(t *testing.T) { + wf := unmarshalWF(helloWorldWf) + wf.Spec.Templates[0].Outputs = wfv1.Outputs{ + Artifacts: []wfv1.Artifact{ + { + Name: "foo", + Path: "/tmp/file", + }, + }, + } + woc := newWoc() + woc.controller.Config.ArtifactRepository.S3 = &S3ArtifactRepository{ + S3Bucket: wfv1.S3Bucket{ + Bucket: "foo", + }, + KeyFormat: "path/in/bucket", + } + woc.operate() + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.NoError(t, err) + var tmpl wfv1.Template + err = json.Unmarshal([]byte(pod.Annotations[common.AnnotationKeyTemplate]), &tmpl) + assert.NoError(t, err) + assert.Nil(t, tmpl.ArchiveLocation) +} + +// TestVolumeAndVolumeMounts verifies the ability to carry forward volumes and volumeMounts from workflow.spec +func TestVolumeAndVolumeMounts(t *testing.T) { + volumes := []apiv1.Volume{ + { + Name: "volume-name", + VolumeSource: apiv1.VolumeSource{ + EmptyDir: &apiv1.EmptyDirVolumeSource{}, + }, + }, + } + volumeMounts := []apiv1.VolumeMount{ + { + Name: "volume-name", + MountPath: "/test", + }, + } + + // For Docker executor + { + woc := newWoc() + woc.volumes = volumes + woc.wf.Spec.Templates[0].Container.VolumeMounts = volumeMounts + woc.controller.Config.ContainerRuntimeExecutor = common.ContainerRuntimeExecutorDocker + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, 3, len(pod.Spec.Volumes)) + assert.Equal(t, "podmetadata", pod.Spec.Volumes[0].Name) + assert.Equal(t, "docker-sock", pod.Spec.Volumes[1].Name) + assert.Equal(t, "volume-name", pod.Spec.Volumes[2].Name) + assert.Equal(t, 1, len(pod.Spec.Containers[1].VolumeMounts)) + assert.Equal(t, "volume-name", pod.Spec.Containers[1].VolumeMounts[0].Name) + } + + // For Kubelet executor + { + woc := newWoc() + woc.volumes = volumes + woc.wf.Spec.Templates[0].Container.VolumeMounts = volumeMounts + woc.controller.Config.ContainerRuntimeExecutor = common.ContainerRuntimeExecutorKubelet + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, 2, len(pod.Spec.Volumes)) + assert.Equal(t, "podmetadata", pod.Spec.Volumes[0].Name) + assert.Equal(t, "volume-name", pod.Spec.Volumes[1].Name) + assert.Equal(t, 1, len(pod.Spec.Containers[1].VolumeMounts)) + assert.Equal(t, "volume-name", pod.Spec.Containers[1].VolumeMounts[0].Name) + } + + // For K8sAPI executor + { + woc := newWoc() + woc.volumes = volumes + woc.wf.Spec.Templates[0].Container.VolumeMounts = volumeMounts + woc.controller.Config.ContainerRuntimeExecutor = common.ContainerRuntimeExecutorK8sAPI + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, 2, len(pod.Spec.Volumes)) + assert.Equal(t, "podmetadata", pod.Spec.Volumes[0].Name) + assert.Equal(t, "volume-name", pod.Spec.Volumes[1].Name) + assert.Equal(t, 1, len(pod.Spec.Containers[1].VolumeMounts)) + assert.Equal(t, "volume-name", pod.Spec.Containers[1].VolumeMounts[0].Name) + } +} + +func TestVolumesPodSubstitution(t *testing.T) { + volumes := []apiv1.Volume{ + { + Name: "volume-name", + VolumeSource: apiv1.VolumeSource{ + PersistentVolumeClaim: &apiv1.PersistentVolumeClaimVolumeSource{ + ClaimName: "{{inputs.parameters.volume-name}}", + }, + }, + }, + } + volumeMounts := []apiv1.VolumeMount{ + { + Name: "volume-name", + MountPath: "/test", + }, + } + tmpStr := "test-name" + inputParameters := []wfv1.Parameter{ + { + Name: "volume-name", + Value: &tmpStr, + }, + } + + woc := newWoc() + woc.volumes = volumes + woc.wf.Spec.Templates[0].Container.VolumeMounts = volumeMounts + woc.wf.Spec.Templates[0].Inputs.Parameters = inputParameters + woc.controller.Config.ContainerRuntimeExecutor = common.ContainerRuntimeExecutorDocker + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, 3, len(pod.Spec.Volumes)) + assert.Equal(t, "volume-name", pod.Spec.Volumes[2].Name) + assert.Equal(t, "test-name", pod.Spec.Volumes[2].PersistentVolumeClaim.ClaimName) + assert.Equal(t, 1, len(pod.Spec.Containers[1].VolumeMounts)) + assert.Equal(t, "volume-name", pod.Spec.Containers[1].VolumeMounts[0].Name) +} + +func TestOutOfCluster(t *testing.T) { + + verifyKubeConfigVolume := func(ctr apiv1.Container, volName, mountPath string) { + for _, vol := range ctr.VolumeMounts { + if vol.Name == volName && vol.MountPath == mountPath { + for _, arg := range ctr.Args { + if arg == fmt.Sprintf("--kubeconfig=%s", mountPath) { + return + } + } + } + } + t.Fatalf("%v does not have kubeconfig mounted properly (name: %s, mountPath: %s)", ctr, volName, mountPath) + } + + // default mount path & volume name + { + woc := newWoc() + woc.controller.Config.KubeConfig = &KubeConfig{ + SecretName: "foo", + SecretKey: "bar", + } + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + + assert.Nil(t, err) + assert.Equal(t, "kubeconfig", pod.Spec.Volumes[1].Name) + assert.Equal(t, "foo", pod.Spec.Volumes[1].VolumeSource.Secret.SecretName) + + waitCtr := pod.Spec.Containers[0] + verifyKubeConfigVolume(waitCtr, "kubeconfig", "/kube/config") + } + + // custom mount path & volume name, in case name collision + { + woc := newWoc() + woc.controller.Config.KubeConfig = &KubeConfig{ + SecretName: "foo", + SecretKey: "bar", + MountPath: "/some/path/config", + VolumeName: "kube-config-secret", + } + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + + assert.Nil(t, err) + assert.Equal(t, "kube-config-secret", pod.Spec.Volumes[1].Name) + assert.Equal(t, "foo", pod.Spec.Volumes[1].VolumeSource.Secret.SecretName) + + // kubeconfig volume is the last one + waitCtr := pod.Spec.Containers[0] + verifyKubeConfigVolume(waitCtr, "kube-config-secret", "/some/path/config") + } +} + +// TestPriority verifies the ability to carry forward priorityClassName and priority. +func TestPriority(t *testing.T) { + priority := int32(15) + woc := newWoc() + woc.wf.Spec.Templates[0].PriorityClassName = "foo" + woc.wf.Spec.Templates[0].Priority = &priority + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, pod.Spec.PriorityClassName, "foo") + assert.Equal(t, pod.Spec.Priority, &priority) +} + +// TestSchedulerName verifies the ability to carry forward schedulerName. +func TestSchedulerName(t *testing.T) { + woc := newWoc() + woc.wf.Spec.Templates[0].SchedulerName = "foo" + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, pod.Spec.SchedulerName, "foo") +} + +// TestInitContainers verifies the ability to set up initContainers +func TestInitContainers(t *testing.T) { + volumes := []apiv1.Volume{ + { + Name: "volume-name", + VolumeSource: apiv1.VolumeSource{ + EmptyDir: &apiv1.EmptyDirVolumeSource{}, + }, + }, + } + volumeMounts := []apiv1.VolumeMount{ + { + Name: "volume-name", + MountPath: "/test", + }, + } + mirrorVolumeMounts := true + + woc := newWoc() + woc.volumes = volumes + woc.wf.Spec.Templates[0].Container.VolumeMounts = volumeMounts + woc.wf.Spec.Templates[0].InitContainers = []wfv1.UserContainer{ + { + MirrorVolumeMounts: &mirrorVolumeMounts, + Container: apiv1.Container{ + Name: "init-foo", + }, + }, + } + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, 1, len(pod.Spec.InitContainers)) + assert.Equal(t, "init-foo", pod.Spec.InitContainers[0].Name) +} + +// TestSidecars verifies the ability to set up sidecars +func TestSidecars(t *testing.T) { + volumes := []apiv1.Volume{ + { + Name: "volume-name", + VolumeSource: apiv1.VolumeSource{ + EmptyDir: &apiv1.EmptyDirVolumeSource{}, + }, + }, + } + volumeMounts := []apiv1.VolumeMount{ + { + Name: "volume-name", + MountPath: "/test", + }, + } + mirrorVolumeMounts := true + + woc := newWoc() + woc.volumes = volumes + woc.wf.Spec.Templates[0].Container.VolumeMounts = volumeMounts + woc.wf.Spec.Templates[0].Sidecars = []wfv1.UserContainer{ + { + MirrorVolumeMounts: &mirrorVolumeMounts, + Container: apiv1.Container{ + Name: "side-foo", + }, + }, + } + + woc.executeContainer(woc.wf.Spec.Entrypoint, &woc.wf.Spec.Templates[0], "") + podName := getPodName(woc.wf) + pod, err := woc.controller.kubeclientset.CoreV1().Pods("").Get(podName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, 3, len(pod.Spec.Containers)) + assert.Equal(t, "wait", pod.Spec.Containers[0].Name) + assert.Equal(t, "main", pod.Spec.Containers[1].Name) + assert.Equal(t, "side-foo", pod.Spec.Containers[2].Name) +} diff --git a/workflow/executor/common/common.go b/workflow/executor/common/common.go new file mode 100644 index 000000000000..dd218c447305 --- /dev/null +++ b/workflow/executor/common/common.go @@ -0,0 +1,140 @@ +package common + +import ( + "bytes" + "compress/gzip" + "fmt" + "os" + "strings" + "syscall" + "time" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" +) + +const ( + containerShimPrefix = "://" +) + +// killGracePeriod is the time in seconds after sending SIGTERM before +// forcefully killing the sidecar with SIGKILL (value matches k8s) +const KillGracePeriod = 10 + +// GetContainerID returns container ID of a ContainerStatus resource +func GetContainerID(container *v1.ContainerStatus) string { + i := strings.Index(container.ContainerID, containerShimPrefix) + if i == -1 { + return "" + } + return container.ContainerID[i+len(containerShimPrefix):] +} + +// KubernetesClientInterface is the interface to implement getContainerStatus method +type KubernetesClientInterface interface { + GetContainerStatus(containerID string) (*v1.Pod, *v1.ContainerStatus, error) + KillContainer(pod *v1.Pod, container *v1.ContainerStatus, sig syscall.Signal) error + CreateArchive(containerID, sourcePath string) (*bytes.Buffer, error) +} + +// WaitForTermination of the given containerID, set the timeout to 0 to discard it +func WaitForTermination(c KubernetesClientInterface, containerID string, timeout time.Duration) error { + ticker := time.NewTicker(time.Second * 1) + defer ticker.Stop() + timer := time.NewTimer(timeout) + if timeout == 0 { + timer.Stop() + } else { + defer timer.Stop() + } + + log.Infof("Starting to wait completion of containerID %s ...", containerID) + for { + select { + case <-ticker.C: + _, containerStatus, err := c.GetContainerStatus(containerID) + if err != nil { + return err + } + if containerStatus.State.Terminated == nil { + continue + } + log.Infof("ContainerID %q is terminated: %v", containerID, containerStatus.String()) + return nil + case <-timer.C: + return fmt.Errorf("timeout after %s", timeout.String()) + } + } +} + +// TerminatePodWithContainerID invoke the given SIG against the PID1 of the container. +// No-op if the container is on the hostPID +func TerminatePodWithContainerID(c KubernetesClientInterface, containerID string, sig syscall.Signal) error { + pod, container, err := c.GetContainerStatus(containerID) + if err != nil { + return err + } + if container.State.Terminated != nil { + log.Infof("Container %s is already terminated: %v", container.ContainerID, container.State.Terminated.String()) + return nil + } + if pod.Spec.HostPID { + return fmt.Errorf("cannot terminate a hostPID Pod %s", pod.Name) + } + if pod.Spec.RestartPolicy != "Never" { + return fmt.Errorf("cannot terminate pod with a %q restart policy", pod.Spec.RestartPolicy) + } + return c.KillContainer(pod, container, sig) +} + +// KillGracefully kills a container gracefully. +func KillGracefully(c KubernetesClientInterface, containerID string) error { + log.Infof("SIGTERM containerID %q: %s", containerID, syscall.SIGTERM.String()) + err := TerminatePodWithContainerID(c, containerID, syscall.SIGTERM) + if err != nil { + return err + } + err = WaitForTermination(c, containerID, time.Second*KillGracePeriod) + if err == nil { + log.Infof("ContainerID %q successfully killed", containerID) + return nil + } + log.Infof("SIGKILL containerID %q: %s", containerID, syscall.SIGKILL.String()) + err = TerminatePodWithContainerID(c, containerID, syscall.SIGKILL) + if err != nil { + return err + } + err = WaitForTermination(c, containerID, time.Second*KillGracePeriod) + if err != nil { + return err + } + log.Infof("ContainerID %q successfully killed", containerID) + return nil +} + +// CopyArchive downloads files and directories as a tarball and saves it to a specified path. +func CopyArchive(c KubernetesClientInterface, containerID, sourcePath, destPath string) error { + log.Infof("Archiving %s:%s to %s", containerID, sourcePath, destPath) + b, err := c.CreateArchive(containerID, sourcePath) + if err != nil { + return err + } + f, err := os.OpenFile(destPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + return err + } + w := gzip.NewWriter(f) + _, err = w.Write(b.Bytes()) + if err != nil { + return err + } + err = w.Flush() + if err != nil { + return err + } + err = w.Close() + if err != nil { + return err + } + return nil +} diff --git a/workflow/executor/docker/docker.go b/workflow/executor/docker/docker.go index 3e06cba11a11..bee8550bdf24 100644 --- a/workflow/executor/docker/docker.go +++ b/workflow/executor/docker/docker.go @@ -1,23 +1,23 @@ package docker import ( + "archive/tar" + "compress/gzip" "fmt" + "io" "os" "os/exec" - "strings" "time" - "github.com/cyrusbiotechnology/argo/util" + log "github.com/sirupsen/logrus" "github.com/cyrusbiotechnology/argo/errors" + "github.com/cyrusbiotechnology/argo/util" + "github.com/cyrusbiotechnology/argo/util/file" "github.com/cyrusbiotechnology/argo/workflow/common" - log "github.com/sirupsen/logrus" + execcommon "github.com/cyrusbiotechnology/argo/workflow/executor/common" ) -// killGracePeriod is the time in seconds after sending SIGTERM before -// forcefully killing the sidecar with SIGKILL (value matches k8s) -const killGracePeriod = 10 - type DockerExecutor struct{} func NewDockerExecutor() (*DockerExecutor, error) { @@ -51,38 +51,48 @@ func (d *DockerExecutor) CopyFile(containerID string, sourcePath string, destPat if err != nil { return err } + copiedFile, err := os.Open(destPath) + if err != nil { + return err + } + defer util.Close(copiedFile) + gzipReader, err := gzip.NewReader(copiedFile) + if err != nil { + return err + } + if !file.ExistsInTar(sourcePath, tar.NewReader(gzipReader)) { + errMsg := fmt.Sprintf("path %s does not exist (or %s is empty) in archive %s", sourcePath, sourcePath, destPath) + log.Warn(errMsg) + return errors.Errorf(errors.CodeNotFound, errMsg) + } log.Infof("Archiving completed") return nil } -// GetOutput returns the entirety of the container output as a string -// Used to capturing script results as an output parameter -func (d *DockerExecutor) GetOutput(containerID string) (string, error) { +func (d *DockerExecutor) GetOutputStream(containerID string, combinedOutput bool) (io.ReadCloser, error) { cmd := exec.Command("docker", "logs", containerID) log.Info(cmd.Args) - outBytes, _ := cmd.Output() - return strings.TrimSpace(string(outBytes)), nil -} - -// Wait for the container to complete -func (d *DockerExecutor) Wait(containerID string) error { - return common.RunCommand("docker", "wait", containerID) -} - -// Logs captures the logs of a container to a file -func (d *DockerExecutor) Logs(containerID string, path string) error { - cmd := exec.Command("docker", "logs", containerID) - outfile, err := os.Create(path) + if combinedOutput { + cmd.Stderr = cmd.Stdout + } + reader, err := cmd.StdoutPipe() if err != nil { - return errors.InternalWrapError(err) + return nil, errors.InternalWrapError(err) } - defer util.Close(outfile) - cmd.Stdout = outfile err = cmd.Start() if err != nil { - return errors.InternalWrapError(err) + return nil, errors.InternalWrapError(err) } - return cmd.Wait() + return reader, nil +} + +func (d *DockerExecutor) WaitInit() error { + return nil +} + +// Wait for the container to complete +func (d *DockerExecutor) Wait(containerID string) error { + return common.RunCommand("docker", "wait", containerID) } // killContainers kills a list of containerIDs first with a SIGTERM then with a SIGKILL after a grace period @@ -101,8 +111,8 @@ func (d *DockerExecutor) Kill(containerIDs []string) error { // waitCmd.Wait() might return error "signal: killed" when we SIGKILL the process // We ignore errors in this case //ignoreWaitError := false - timer := time.AfterFunc(killGracePeriod*time.Second, func() { - log.Infof("Timed out (%ds) for containers to terminate gracefully. Killing forcefully", killGracePeriod) + timer := time.AfterFunc(execcommon.KillGracePeriod*time.Second, func() { + log.Infof("Timed out (%ds) for containers to terminate gracefully. Killing forcefully", execcommon.KillGracePeriod) forceKillArgs := append([]string{"kill", "--signal", "KILL"}, containerIDs...) forceKillCmd := exec.Command("docker", forceKillArgs...) log.Info(forceKillCmd.Args) diff --git a/workflow/executor/executor.go b/workflow/executor/executor.go index ae494940ad7c..9c13caf139ec 100644 --- a/workflow/executor/executor.go +++ b/workflow/executor/executor.go @@ -22,27 +22,38 @@ import ( "time" argofile "github.com/argoproj/pkg/file" + log "github.com/sirupsen/logrus" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "github.com/cyrusbiotechnology/argo/errors" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/util/archive" "github.com/cyrusbiotechnology/argo/util/retry" artifact "github.com/cyrusbiotechnology/argo/workflow/artifacts" "github.com/cyrusbiotechnology/argo/workflow/artifacts/artifactory" "github.com/cyrusbiotechnology/argo/workflow/artifacts/gcs" "github.com/cyrusbiotechnology/argo/workflow/artifacts/git" + "github.com/cyrusbiotechnology/argo/workflow/artifacts/hdfs" "github.com/cyrusbiotechnology/argo/workflow/artifacts/http" "github.com/cyrusbiotechnology/argo/workflow/artifacts/raw" "github.com/cyrusbiotechnology/argo/workflow/artifacts/s3" "github.com/cyrusbiotechnology/argo/workflow/common" - "github.com/fsnotify/fsnotify" - log "github.com/sirupsen/logrus" - apiv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" +) + +const ( + // This directory temporarily stores the tarballs of the artifacts before uploading + tempOutArtDir = "/argo/outputs/artifacts" ) // WorkflowExecutor is program which runs as the init/wait container type WorkflowExecutor struct { + common.ResourceInterface + PodName string Template wfv1.Template ClientSet kubernetes.Interface @@ -53,8 +64,10 @@ type WorkflowExecutor struct { // memoized container ID to prevent multiple lookups mainContainerID string + // memoized configmaps + memoizedConfigMaps map[string]string // memoized secrets - memoizedSecrets map[string]string + memoizedSecrets map[string][]byte // list of errors that occurred during execution. // the first of these is used as the overall message of the node errors []error @@ -68,29 +81,32 @@ type ContainerRuntimeExecutor interface { // CopyFile copies a source file in a container to a local path CopyFile(containerID string, sourcePath string, destPath string) error - // GetOutput returns the entirety of the container output as a string - // Used to capturing script results as an output parameter - GetOutput(containerID string) (string, error) + // GetOutputStream returns the entirety of the container output as a io.Reader + // Used to capture script results as an output parameter, and to archive container logs + GetOutputStream(containerID string, combinedOutput bool) (io.ReadCloser, error) - // Wait for the container to complete - Wait(containerID string) error + // WaitInit is called before Wait() to signal the executor about an impending Wait call. + // For most executors this is a noop, and is only used by the the PNS executor + WaitInit() error - // Copy logs to a given path - Logs(containerID string, path string) error + // Wait waits for the container to complete + Wait(containerID string) error // Kill a list of containerIDs first with a SIGTERM then with a SIGKILL after a grace period Kill(containerIDs []string) error } // NewExecutor instantiates a new workflow executor -func NewExecutor(clientset kubernetes.Interface, podName, namespace, podAnnotationsPath string, cre ContainerRuntimeExecutor) WorkflowExecutor { +func NewExecutor(clientset kubernetes.Interface, podName, namespace, podAnnotationsPath string, cre ContainerRuntimeExecutor, template wfv1.Template) WorkflowExecutor { return WorkflowExecutor{ PodName: podName, ClientSet: clientset, Namespace: namespace, PodAnnotationsPath: podAnnotationsPath, RuntimeExecutor: cre, - memoizedSecrets: map[string]string{}, + Template: template, + memoizedConfigMaps: map[string]string{}, + memoizedSecrets: map[string][]byte{}, errors: []error{}, } } @@ -107,12 +123,22 @@ func (we *WorkflowExecutor) HandleError() { } } -// LoadArtifacts loads aftifacts from location to a container path +// LoadArtifacts loads artifacts from location to a container path func (we *WorkflowExecutor) LoadArtifacts() error { log.Infof("Start loading input artifacts...") for _, art := range we.Template.Inputs.Artifacts { + log.Infof("Downloading artifact: %s", art.Name) + + if !art.HasLocation() { + if art.Optional { + log.Warnf("Ignoring optional artifact '%s' which was not supplied", art.Name) + continue + } else { + return errors.New("required artifact %s not supplied", art.Name) + } + } artDriver, err := we.InitDriver(art) if err != nil { return err @@ -132,7 +158,7 @@ func (we *WorkflowExecutor) LoadArtifacts() error { // as opposed to the `input-artifacts` volume that is an implementation detail // unbeknownst to the user. log.Infof("Specified artifact path %s overlaps with volume mount at %s. Extracting to volume mount", art.Path, mnt.MountPath) - artPath = path.Join(common.InitContainerMainFilesystemDir, art.Path) + artPath = path.Join(common.ExecutorMainFilesystemDir, art.Path) } // The artifact is downloaded to a temporary location, after which we determine if @@ -199,15 +225,13 @@ func (we *WorkflowExecutor) SaveArtifacts() error { return err } - // This directory temporarily stores the tarballs of the artifacts before uploading - tempOutArtDir := "/argo/outputs/artifacts" err = os.MkdirAll(tempOutArtDir, os.ModePerm) if err != nil { return errors.InternalWrapError(err) } for i, art := range we.Template.Outputs.Artifacts { - err := we.saveArtifact(tempOutArtDir, mainCtrID, &art) + err := we.saveArtifact(mainCtrID, &art) if err != nil { return err } @@ -216,27 +240,19 @@ func (we *WorkflowExecutor) SaveArtifacts() error { return nil } -func (we *WorkflowExecutor) saveArtifact(tempOutArtDir string, mainCtrID string, art *wfv1.Artifact) error { - log.Infof("Saving artifact: %s", art.Name) +func (we *WorkflowExecutor) saveArtifact(mainCtrID string, art *wfv1.Artifact) error { // Determine the file path of where to find the artifact if art.Path == "" { return errors.InternalErrorf("Artifact %s did not specify a path", art.Name) } - - // fileName is incorporated into the final path when uploading it to the artifact repo - fileName := fmt.Sprintf("%s.tgz", art.Name) - // localArtPath is the final staging location of the file (or directory) which we will pass - // to the SaveArtifacts call - localArtPath := path.Join(tempOutArtDir, fileName) - err := we.RuntimeExecutor.CopyFile(mainCtrID, art.Path, localArtPath) - if err != nil { - return err - } - fileName, localArtPath, err = stageArchiveFile(fileName, localArtPath, art) + fileName, localArtPath, err := we.stageArchiveFile(mainCtrID, art) if err != nil { + if art.Optional && errors.IsCode(errors.CodeNotFound, err) { + log.Warnf("Ignoring optional artifact '%s' which does not exist in path '%s': %v", art.Name, art.Path, err) + return nil + } return err } - if !art.HasLocation() { // If user did not explicitly set an artifact destination location in the template, // use the default archive location (appended with the filename). @@ -256,6 +272,10 @@ func (we *WorkflowExecutor) saveArtifact(tempOutArtDir string, mainCtrID string, } artifactoryURL.Path = path.Join(artifactoryURL.Path, fileName) art.Artifactory.URL = artifactoryURL.String() + } else if we.Template.ArchiveLocation.HDFS != nil { + shallowCopy := *we.Template.ArchiveLocation.HDFS + art.HDFS = &shallowCopy + art.HDFS.Path = path.Join(art.HDFS.Path, fileName) } else if we.Template.ArchiveLocation.GCS != nil { shallowCopy := *we.Template.ArchiveLocation.GCS art.GCS = &shallowCopy @@ -283,7 +303,13 @@ func (we *WorkflowExecutor) saveArtifact(tempOutArtDir string, mainCtrID string, return nil } -func stageArchiveFile(fileName, localArtPath string, art *wfv1.Artifact) (string, string, error) { +// stageArchiveFile stages a path in a container for archiving from the wait sidecar. +// Returns a filename and a local path for the upload. +// The filename is incorporated into the final path when uploading it to the artifact repo. +// The local path is the final staging location of the file (or directory) which we will pass +// to the SaveArtifacts call and may be a directory or file. +func (we *WorkflowExecutor) stageArchiveFile(mainCtrID string, art *wfv1.Artifact) (string, string, error) { + log.Infof("Staging artifact: %s", art.Name) strategy := art.Archive if strategy == nil { // If no strategy is specified, default to the tar strategy @@ -291,44 +317,83 @@ func stageArchiveFile(fileName, localArtPath string, art *wfv1.Artifact) (string Tar: &wfv1.TarStrategy{}, } } - tempOutArtDir := filepath.Dir(localArtPath) - if strategy.None != nil { - log.Info("Disabling archive before upload") - unarchivedArtPath := path.Join(tempOutArtDir, art.Name) - err := untar(localArtPath, unarchivedArtPath) - if err != nil { - return "", "", err + + if !we.isBaseImagePath(art.Path) { + // If we get here, we are uploading an artifact from a mirrored volume mount which the wait + // sidecar has direct access to. We can upload directly from the shared volume mount, + // instead of copying it from the container. + mountedArtPath := filepath.Join(common.ExecutorMainFilesystemDir, art.Path) + log.Infof("Staging %s from mirrored volume mount %s", art.Path, mountedArtPath) + if strategy.None != nil { + fileName := filepath.Base(art.Path) + log.Infof("No compression strategy needed. Staging skipped") + return fileName, mountedArtPath, nil } - // Delete the tarball - err = os.Remove(localArtPath) + fileName := fmt.Sprintf("%s.tgz", art.Name) + localArtPath := filepath.Join(tempOutArtDir, fileName) + f, err := os.Create(localArtPath) if err != nil { return "", "", errors.InternalWrapError(err) } - isDir, err := argofile.IsDirectory(unarchivedArtPath) + w := bufio.NewWriter(f) + err = archive.TarGzToWriter(mountedArtPath, w) if err != nil { - return "", "", errors.InternalWrapError(err) + return "", "", err } - fileName = filepath.Base(art.Path) - if isDir { - localArtPath = unarchivedArtPath - } else { - // If we are uploading a single file, we need to preserve original filename so that - // 1. minio client can infer its mime-type, based on file extension - // 2. the original filename is incorporated into the final path - localArtPath = path.Join(tempOutArtDir, fileName) - err = os.Rename(unarchivedArtPath, localArtPath) - if err != nil { - return "", "", errors.InternalWrapError(err) - } + log.Infof("Successfully staged %s from mirrored volume mount %s", art.Path, mountedArtPath) + return fileName, localArtPath, nil + } + + fileName := fmt.Sprintf("%s.tgz", art.Name) + localArtPath := filepath.Join(tempOutArtDir, fileName) + log.Infof("Copying %s from container base image layer to %s", art.Path, localArtPath) + + err := we.RuntimeExecutor.CopyFile(mainCtrID, art.Path, localArtPath) + if err != nil { + return "", "", err + } + if strategy.Tar != nil { + // NOTE we already tar gzip the file in the executor. So this is a noop. + return fileName, localArtPath, nil + } + // localArtPath now points to a .tgz file, and the archive strategy is *not* tar. We need to untar it + log.Infof("Untaring %s archive before upload", localArtPath) + unarchivedArtPath := path.Join(filepath.Dir(localArtPath), art.Name) + err = untar(localArtPath, unarchivedArtPath) + if err != nil { + return "", "", err + } + // Delete the tarball + err = os.Remove(localArtPath) + if err != nil { + return "", "", errors.InternalWrapError(err) + } + isDir, err := argofile.IsDirectory(unarchivedArtPath) + if err != nil { + return "", "", errors.InternalWrapError(err) + } + fileName = filepath.Base(art.Path) + if isDir { + localArtPath = unarchivedArtPath + } else { + // If we are uploading a single file, we need to preserve original filename so that + // 1. minio client can infer its mime-type, based on file extension + // 2. the original filename is incorporated into the final path + localArtPath = path.Join(tempOutArtDir, fileName) + err = os.Rename(unarchivedArtPath, localArtPath) + if err != nil { + return "", "", errors.InternalWrapError(err) } - } else if strategy.Tar != nil { - // NOTE we already tar gzip the file in the executor. So this is a noop. In the future, if - // we were to support other compression formats (e.g. bzip2) or options, the logic would go - // here, and compression would be moved out of the executors. } + // In the future, if we were to support other compression formats (e.g. bzip2) or options + // the logic would go here, and compression would be moved out of the executors return fileName, localArtPath, nil } +func (we *WorkflowExecutor) isBaseImagePath(path string) bool { + return common.FindOverlappingVolume(&we.Template, path) == nil +} + // SaveParameters will save the content in the specified file path as output parameter value func (we *WorkflowExecutor) SaveParameters() error { if len(we.Template.Outputs.Parameters) == 0 { @@ -347,10 +412,24 @@ func (we *WorkflowExecutor) SaveParameters() error { if param.ValueFrom == nil || param.ValueFrom.Path == "" { continue } - output, err := we.RuntimeExecutor.GetFileContents(mainCtrID, param.ValueFrom.Path) - if err != nil { - return err + + var output string + if we.isBaseImagePath(param.ValueFrom.Path) { + log.Infof("Copying %s from base image layer", param.ValueFrom.Path) + output, err = we.RuntimeExecutor.GetFileContents(mainCtrID, param.ValueFrom.Path) + if err != nil { + return err + } + } else { + log.Infof("Copying %s from from volume mount", param.ValueFrom.Path) + mountedPath := filepath.Join(common.ExecutorMainFilesystemDir, param.ValueFrom.Path) + out, err := ioutil.ReadFile(mountedPath) + if err != nil { + return err + } + output = string(out) } + outputLen := len(output) // Trims off a single newline for user convenience if outputLen > 0 && output[outputLen-1] == '\n' { @@ -373,7 +452,7 @@ func (we *WorkflowExecutor) saveLogsToPath(logDir string, fileName string) (outp return } outputPath = path.Join(logDir, fileName) - err = we.RuntimeExecutor.Logs(mainCtrID, outputPath) + err = we.saveLogToFile(mainCtrID, outputPath) if err != nil { return } @@ -411,6 +490,10 @@ func (we *WorkflowExecutor) SaveLogs() (*wfv1.Artifact, error) { } artifactoryURL.Path = path.Join(artifactoryURL.Path, fileName) art.Artifactory.URL = artifactoryURL.String() + } else if we.Template.ArchiveLocation.HDFS != nil { + shallowCopy := *we.Template.ArchiveLocation.HDFS + art.HDFS = &shallowCopy + art.HDFS.Path = path.Join(art.HDFS.Path, fileName) } else { return nil, errors.Errorf(errors.CodeBadRequest, "Unable to determine path to store %s. Archive location provided no information", art.Name) } @@ -426,6 +509,30 @@ func (we *WorkflowExecutor) SaveLogs() (*wfv1.Artifact, error) { return &art, nil } +// GetSecretFromVolMount will retrive the Secrets from VolumeMount +func (we *WorkflowExecutor) GetSecretFromVolMount(accessKeyName string, accessKey string) ([]byte, error) { + return ioutil.ReadFile(filepath.Join(common.SecretVolMountPath, accessKeyName, accessKey)) +} + +// saveLogToFile saves the entire log output of a container to a local file +func (we *WorkflowExecutor) saveLogToFile(mainCtrID, path string) error { + outFile, err := os.Create(path) + if err != nil { + return errors.InternalWrapError(err) + } + defer func() { _ = outFile.Close() }() + reader, err := we.RuntimeExecutor.GetOutputStream(mainCtrID, true) + if err != nil { + return err + } + defer func() { _ = reader.Close() }() + _, err = io.Copy(outFile, reader) + if err != nil { + return errors.InternalWrapError(err) + } + return nil +} + // InitDriver initializes an instance of an artifact driver func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriver, error) { if art.S3 != nil { @@ -433,15 +540,16 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv var secretKey string if art.S3.AccessKeySecret.Name != "" { - var err error - accessKey, err = we.getSecrets(we.Namespace, art.S3.AccessKeySecret.Name, art.S3.AccessKeySecret.Key) + accessKeyBytes, err := we.GetSecretFromVolMount(art.S3.AccessKeySecret.Name, art.S3.AccessKeySecret.Key) if err != nil { return nil, err } - secretKey, err = we.getSecrets(we.Namespace, art.S3.SecretKeySecret.Name, art.S3.SecretKeySecret.Key) + accessKey = string(accessKeyBytes) + secretKeyBytes, err := we.GetSecretFromVolMount(art.S3.SecretKeySecret.Name, art.S3.SecretKeySecret.Key) if err != nil { return nil, err } + secretKey = string(secretKeyBytes) } driver := s3.S3ArtifactDriver{ @@ -453,6 +561,16 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv } return &driver, nil } + if art.GCS != nil { + credsJSONData, err := we.GetSecretFromVolMount(art.GCS.CredentialsSecret.Name, art.GCS.CredentialsSecret.Key) + if err != nil { + return nil, err + } + driver := gcs.GCSArtifactDriver{ + CredsJSONData: credsJSONData, + } + return &driver, nil + } if art.GCS != nil { driver := gcs.GCSArtifactDriver{} return &driver, nil @@ -461,50 +579,56 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv return &http.HTTPArtifactDriver{}, nil } if art.Git != nil { - gitDriver := git.GitArtifactDriver{} + gitDriver := git.GitArtifactDriver{ + InsecureIgnoreHostKey: art.Git.InsecureIgnoreHostKey, + } if art.Git.UsernameSecret != nil { - username, err := we.getSecrets(we.Namespace, art.Git.UsernameSecret.Name, art.Git.UsernameSecret.Key) + usernameBytes, err := we.GetSecretFromVolMount(art.Git.UsernameSecret.Name, art.Git.UsernameSecret.Key) if err != nil { return nil, err } - gitDriver.Username = username + gitDriver.Username = string(usernameBytes) } if art.Git.PasswordSecret != nil { - password, err := we.getSecrets(we.Namespace, art.Git.PasswordSecret.Name, art.Git.PasswordSecret.Key) + passwordBytes, err := we.GetSecretFromVolMount(art.Git.PasswordSecret.Name, art.Git.PasswordSecret.Key) if err != nil { return nil, err } - gitDriver.Password = password + gitDriver.Password = string(passwordBytes) } if art.Git.SSHPrivateKeySecret != nil { - sshPrivateKey, err := we.getSecrets(we.Namespace, art.Git.SSHPrivateKeySecret.Name, art.Git.SSHPrivateKeySecret.Key) + sshPrivateKeyBytes, err := we.GetSecretFromVolMount(art.Git.SSHPrivateKeySecret.Name, art.Git.SSHPrivateKeySecret.Key) if err != nil { return nil, err } - gitDriver.SSHPrivateKey = sshPrivateKey + gitDriver.SSHPrivateKey = string(sshPrivateKeyBytes) } return &gitDriver, nil } if art.Artifactory != nil { - username, err := we.getSecrets(we.Namespace, art.Artifactory.UsernameSecret.Name, art.Artifactory.UsernameSecret.Key) + usernameBytes, err := we.GetSecretFromVolMount(art.Artifactory.UsernameSecret.Name, art.Artifactory.UsernameSecret.Key) if err != nil { return nil, err } - password, err := we.getSecrets(we.Namespace, art.Artifactory.PasswordSecret.Name, art.Artifactory.PasswordSecret.Key) + passwordBytes, err := we.GetSecretFromVolMount(art.Artifactory.PasswordSecret.Name, art.Artifactory.PasswordSecret.Key) if err != nil { return nil, err } driver := artifactory.ArtifactoryArtifactDriver{ - Username: username, - Password: password, + Username: string(usernameBytes), + Password: string(passwordBytes), } return &driver, nil } + if art.HDFS != nil { + return hdfs.CreateDriver(we, art.HDFS) + } if art.Raw != nil { return &raw.RawArtifactDriver{}, nil } + return nil, errors.Errorf(errors.CodeBadRequest, "Unsupported artifact driver for %s", art.Name) } @@ -530,8 +654,48 @@ func (we *WorkflowExecutor) getPod() (*apiv1.Pod, error) { return pod, nil } -// getSecrets retrieves a secret value and memoizes the result -func (we *WorkflowExecutor) getSecrets(namespace, name, key string) (string, error) { +// GetNamespace returns the namespace +func (we *WorkflowExecutor) GetNamespace() string { + return we.Namespace +} + +// GetConfigMapKey retrieves a configmap value and memoizes the result +func (we *WorkflowExecutor) GetConfigMapKey(namespace, name, key string) (string, error) { + cachedKey := fmt.Sprintf("%s/%s/%s", namespace, name, key) + if val, ok := we.memoizedConfigMaps[cachedKey]; ok { + return val, nil + } + configmapsIf := we.ClientSet.CoreV1().ConfigMaps(namespace) + var configmap *apiv1.ConfigMap + var err error + _ = wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) { + configmap, err = configmapsIf.Get(name, metav1.GetOptions{}) + if err != nil { + log.Warnf("Failed to get configmap '%s': %v", name, err) + if !retry.IsRetryableKubeAPIError(err) { + return false, err + } + return false, nil + } + return true, nil + }) + if err != nil { + return "", errors.InternalWrapError(err) + } + // memoize all keys in the configmap since it's highly likely we will need to get a + // subsequent key in the configmap (e.g. username + password) and we can save an API call + for k, v := range configmap.Data { + we.memoizedConfigMaps[fmt.Sprintf("%s/%s/%s", namespace, name, k)] = v + } + val, ok := we.memoizedConfigMaps[cachedKey] + if !ok { + return "", errors.Errorf(errors.CodeBadRequest, "configmap '%s' does not have the key '%s'", name, key) + } + return val, nil +} + +// GetSecrets retrieves a secret value and memoizes the result +func (we *WorkflowExecutor) GetSecrets(namespace, name, key string) ([]byte, error) { cachedKey := fmt.Sprintf("%s/%s/%s", namespace, name, key) if val, ok := we.memoizedSecrets[cachedKey]; ok { return val, nil @@ -551,16 +715,16 @@ func (we *WorkflowExecutor) getSecrets(namespace, name, key string) (string, err return true, nil }) if err != nil { - return "", errors.InternalWrapError(err) + return []byte{}, errors.InternalWrapError(err) } // memoize all keys in the secret since it's highly likely we will need to get a // subsequent key in the secret (e.g. username + password) and we can save an API call for k, v := range secret.Data { - we.memoizedSecrets[fmt.Sprintf("%s/%s/%s", namespace, name, k)] = string(v) + we.memoizedSecrets[fmt.Sprintf("%s/%s/%s", namespace, name, k)] = v } val, ok := we.memoizedSecrets[cachedKey] if !ok { - return "", errors.Errorf(errors.CodeBadRequest, "secret '%s' does not have the key '%s'", name, key) + return []byte{}, errors.Errorf(errors.CodeBadRequest, "secret '%s' does not have the key '%s'", name, key) } return val, nil } @@ -605,10 +769,21 @@ func (we *WorkflowExecutor) CaptureScriptResult() error { if err != nil { return err } - out, err := we.RuntimeExecutor.GetOutput(mainContainerID) + reader, err := we.RuntimeExecutor.GetOutputStream(mainContainerID, false) if err != nil { return err } + defer func() { _ = reader.Close() }() + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return errors.InternalWrapError(err) + } + out := string(bytes) + // Trims off a single newline for user convenience + outputLen := len(out) + if outputLen > 0 && out[outputLen-1] == '\n' { + out = out[0 : outputLen-1] + } we.Template.Outputs.Result = &out return nil } @@ -633,6 +808,7 @@ func (we *WorkflowExecutor) AnnotateOutputs(logArt *wfv1.Artifact) error { // AddError adds an error to the list of encountered errors durign execution func (we *WorkflowExecutor) AddError(err error) { + log.Errorf("executor error: %+v", err) we.errors = append(we.errors, err) } @@ -850,20 +1026,13 @@ func containerID(ctrID string) string { // Wait is the sidecar container logic which waits for the main container to complete. // Also monitors for updates in the pod annotations which may change (e.g. terminate) // Upon completion, kills any sidecars after it finishes. -func (we *WorkflowExecutor) Wait() (err error) { - defer func() { - killSidecarsErr := we.killSidecars() - if killSidecarsErr != nil { - log.Errorf("Failed to kill sidecars: %v", killSidecarsErr) - if err == nil { - // set error only if not already set - err = killSidecarsErr - } - } - }() +func (we *WorkflowExecutor) Wait() error { + err := we.RuntimeExecutor.WaitInit() + if err != nil { + return err + } log.Infof("Waiting on main container") - var mainContainerID string - mainContainerID, err = we.waitMainContainerStart() + mainContainerID, err := we.waitMainContainerStart() if err != nil { return err } @@ -875,49 +1044,87 @@ func (we *WorkflowExecutor) Wait() (err error) { go we.monitorDeadline(ctx, annotationUpdatesCh) err = we.RuntimeExecutor.Wait(mainContainerID) + if err != nil { + return err + } log.Infof("Main container completed") - return + return nil } // waitMainContainerStart waits for the main container to start and returns its container ID. func (we *WorkflowExecutor) waitMainContainerStart() (string, error) { for { - ctrStatus, err := we.GetMainContainerStatus() + podsIf := we.ClientSet.CoreV1().Pods(we.Namespace) + fieldSelector := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", we.PodName)) + opts := metav1.ListOptions{ + FieldSelector: fieldSelector.String(), + } + watchIf, err := podsIf.Watch(opts) if err != nil { - return "", err + return "", errors.InternalWrapErrorf(err, "Failed to establish pod watch: %v", err) } - if ctrStatus != nil { - log.Debug(ctrStatus) - if ctrStatus.ContainerID != "" { - we.mainContainerID = containerID(ctrStatus.ContainerID) - return containerID(ctrStatus.ContainerID), nil - } else if ctrStatus.State.Waiting == nil && ctrStatus.State.Running == nil && ctrStatus.State.Terminated == nil { - // status still not ready, wait - time.Sleep(1 * time.Second) - } else if ctrStatus.State.Waiting != nil { - // main container is still in waiting status - time.Sleep(1 * time.Second) - } else { - // main container in running or terminated state but missing container ID - return "", errors.InternalError("Main container ID cannot be found") + for watchEv := range watchIf.ResultChan() { + if watchEv.Type == watch.Error { + return "", errors.InternalErrorf("Pod watch error waiting for main to start: %v", watchEv.Object) + } + pod, ok := watchEv.Object.(*apiv1.Pod) + if !ok { + log.Warnf("Pod watch returned non pod object: %v", watchEv.Object) + continue + } + for _, ctrStatus := range pod.Status.ContainerStatuses { + if ctrStatus.Name == common.MainContainerName { + log.Debug(ctrStatus) + if ctrStatus.ContainerID != "" { + we.mainContainerID = containerID(ctrStatus.ContainerID) + return containerID(ctrStatus.ContainerID), nil + } else if ctrStatus.State.Waiting == nil && ctrStatus.State.Running == nil && ctrStatus.State.Terminated == nil { + // status still not ready, wait + } else if ctrStatus.State.Waiting != nil { + // main container is still in waiting status + } else { + // main container in running or terminated state but missing container ID + return "", errors.InternalError("Main container ID cannot be found") + } + } } } + log.Warnf("Pod watch closed unexpectedly") } } +func watchFileChanges(ctx context.Context, pollInterval time.Duration, filePath string) <-chan struct{} { + res := make(chan struct{}) + go func() { + defer close(res) + + var modTime *time.Time + for { + select { + case <-ctx.Done(): + return + default: + } + + file, err := os.Stat(filePath) + if err != nil { + log.Fatal(err) + } + newModTime := file.ModTime() + if modTime != nil && !modTime.Equal(file.ModTime()) { + res <- struct{}{} + } + modTime = &newModTime + time.Sleep(pollInterval) + } + }() + return res +} + // monitorAnnotations starts a goroutine which monitors for any changes to the pod annotations. // Emits an event on the returned channel upon any updates func (we *WorkflowExecutor) monitorAnnotations(ctx context.Context) <-chan struct{} { log.Infof("Starting annotations monitor") - // Create a fsnotify watcher on the local annotations file to listen for updates from the Downward API - watcher, err := fsnotify.NewWatcher() - if err != nil { - log.Fatal(err) - } - err = watcher.Add(we.PodAnnotationsPath) - if err != nil { - log.Fatal(err) - } // Create a channel to listen for a SIGUSR2. Upon receiving of the signal, we force reload our annotations // directly from kubernetes API. The controller uses this to fast-track notification of annotations @@ -930,12 +1137,12 @@ func (we *WorkflowExecutor) monitorAnnotations(ctx context.Context) <-chan struc // Create a channel which will notify a listener on new updates to the annotations annotationUpdateCh := make(chan struct{}) + annotationChanges := watchFileChanges(ctx, 10*time.Second, we.PodAnnotationsPath) go func() { for { select { case <-ctx.Done(): log.Infof("Annotations monitor stopped") - _ = watcher.Close() signal.Stop(sigs) close(sigs) close(annotationUpdateCh) @@ -944,7 +1151,7 @@ func (we *WorkflowExecutor) monitorAnnotations(ctx context.Context) <-chan struc log.Infof("Received update signal. Reloading annotations from API") annotationUpdateCh <- struct{}{} we.setExecutionControl() - case <-watcher.Events: + case <-annotationChanges: log.Infof("%s updated", we.PodAnnotationsPath) err := we.LoadExecutionControl() if err != nil { @@ -1023,8 +1230,8 @@ func (we *WorkflowExecutor) monitorDeadline(ctx context.Context, annotationsUpda } } -// killSidecars kills any sidecars to the main container -func (we *WorkflowExecutor) killSidecars() error { +// KillSidecars kills any sidecars to the main container +func (we *WorkflowExecutor) KillSidecars() error { if len(we.Template.Sidecars) == 0 { log.Infof("No sidecars") return nil @@ -1052,15 +1259,6 @@ func (we *WorkflowExecutor) killSidecars() error { return we.RuntimeExecutor.Kill(sidecarIDs) } -// LoadTemplate reads the template definition from the the Kubernetes downward api annotations volume file -func (we *WorkflowExecutor) LoadTemplate() error { - err := unmarshalAnnotationField(we.PodAnnotationsPath, common.AnnotationKeyTemplate, &we.Template) - if err != nil { - return err - } - return nil -} - // LoadExecutionControl reads the execution control definition from the the Kubernetes downward api annotations volume file func (we *WorkflowExecutor) LoadExecutionControl() error { err := unmarshalAnnotationField(we.PodAnnotationsPath, common.AnnotationKeyExecutionControl, &we.ExecutionControl) @@ -1073,6 +1271,16 @@ func (we *WorkflowExecutor) LoadExecutionControl() error { return nil } +// LoadTemplate reads the template definition from the the Kubernetes downward api annotations volume file +func LoadTemplate(path string) (*wfv1.Template, error) { + var tmpl wfv1.Template + err := unmarshalAnnotationField(path, common.AnnotationKeyTemplate, &tmpl) + if err != nil { + return nil, err + } + return &tmpl, nil +} + // unmarshalAnnotationField unmarshals the value of an annotation key into the supplied interface // from the downward api annotation volume file func unmarshalAnnotationField(filePath string, key string, into interface{}) error { diff --git a/workflow/executor/k8sapi/client.go b/workflow/executor/k8sapi/client.go new file mode 100644 index 000000000000..fcc8ab9ae14a --- /dev/null +++ b/workflow/executor/k8sapi/client.go @@ -0,0 +1,151 @@ +package k8sapi + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "syscall" + "time" + + "github.com/cyrusbiotechnology/argo/util" + + "github.com/cyrusbiotechnology/argo/errors" + "github.com/cyrusbiotechnology/argo/workflow/common" + execcommon "github.com/cyrusbiotechnology/argo/workflow/executor/common" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" +) + +type k8sAPIClient struct { + clientset *kubernetes.Clientset + config *restclient.Config + podName string + namespace string +} + +var _ execcommon.KubernetesClientInterface = &k8sAPIClient{} + +func newK8sAPIClient(clientset *kubernetes.Clientset, config *restclient.Config, podName, namespace string) (*k8sAPIClient, error) { + return &k8sAPIClient{ + clientset: clientset, + config: config, + podName: podName, + namespace: namespace, + }, nil +} + +func (c *k8sAPIClient) getFileContents(containerID, sourcePath string) (string, error) { + _, containerStatus, err := c.GetContainerStatus(containerID) + if err != nil { + return "", err + } + command := []string{"cat", sourcePath} + exec, err := common.ExecPodContainer(c.config, c.namespace, c.podName, containerStatus.Name, true, false, command...) + if err != nil { + return "", err + } + stdOut, _, err := common.GetExecutorOutput(exec) + if err != nil { + return "", err + } + return stdOut.String(), nil +} + +func (c *k8sAPIClient) CreateArchive(containerID, sourcePath string) (*bytes.Buffer, error) { + _, containerStatus, err := c.GetContainerStatus(containerID) + if err != nil { + return nil, err + } + command := []string{"tar", "cf", "-", sourcePath} + exec, err := common.ExecPodContainer(c.config, c.namespace, c.podName, containerStatus.Name, true, false, command...) + if err != nil { + return nil, err + } + stdOut, _, err := common.GetExecutorOutput(exec) + if err != nil { + return nil, err + } + return stdOut, nil +} + +func (c *k8sAPIClient) getLogsAsStream(containerID string) (io.ReadCloser, error) { + _, containerStatus, err := c.GetContainerStatus(containerID) + if err != nil { + return nil, err + } + return c.clientset.CoreV1().Pods(c.namespace). + GetLogs(c.podName, &v1.PodLogOptions{Container: containerStatus.Name, SinceTime: &metav1.Time{}}).Stream() +} + +func (c *k8sAPIClient) getLogs(containerID string) (string, error) { + reader, err := c.getLogsAsStream(containerID) + if err != nil { + return "", err + } + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return "", errors.InternalWrapError(err) + } + return string(bytes), nil +} + +func (c *k8sAPIClient) saveLogs(containerID, path string) error { + reader, err := c.getLogsAsStream(containerID) + if err != nil { + return err + } + outFile, err := os.Create(path) + if err != nil { + return errors.InternalWrapError(err) + } + defer util.Close(outFile) + _, err = io.Copy(outFile, reader) + if err != nil { + return errors.InternalWrapError(err) + } + return nil +} + +func (c *k8sAPIClient) getPod() (*v1.Pod, error) { + return c.clientset.CoreV1().Pods(c.namespace).Get(c.podName, metav1.GetOptions{}) +} + +func (c *k8sAPIClient) GetContainerStatus(containerID string) (*v1.Pod, *v1.ContainerStatus, error) { + pod, err := c.getPod() + if err != nil { + return nil, nil, err + } + for _, containerStatus := range pod.Status.ContainerStatuses { + if execcommon.GetContainerID(&containerStatus) != containerID { + continue + } + return pod, &containerStatus, nil + } + return nil, nil, errors.New(errors.CodeNotFound, fmt.Sprintf("containerID %q is not found in the pod %s", containerID, c.podName)) +} + +func (c *k8sAPIClient) waitForTermination(containerID string, timeout time.Duration) error { + return execcommon.WaitForTermination(c, containerID, timeout) +} + +func (c *k8sAPIClient) KillContainer(pod *v1.Pod, container *v1.ContainerStatus, sig syscall.Signal) error { + command := []string{"/bin/sh", "-c", fmt.Sprintf("kill -%d 1", sig)} + exec, err := common.ExecPodContainer(c.config, c.namespace, c.podName, container.Name, false, false, command...) + if err != nil { + return err + } + _, _, err = common.GetExecutorOutput(exec) + return err +} + +func (c *k8sAPIClient) killGracefully(containerID string) error { + return execcommon.KillGracefully(c, containerID) +} + +func (c *k8sAPIClient) copyArchive(containerID, sourcePath, destPath string) error { + return execcommon.CopyArchive(c, containerID, sourcePath, destPath) +} diff --git a/workflow/executor/k8sapi/k8sapi.go b/workflow/executor/k8sapi/k8sapi.go new file mode 100644 index 000000000000..51832f815529 --- /dev/null +++ b/workflow/executor/k8sapi/k8sapi.go @@ -0,0 +1,64 @@ +package k8sapi + +import ( + "io" + + log "github.com/sirupsen/logrus" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + + "github.com/cyrusbiotechnology/argo/errors" +) + +type K8sAPIExecutor struct { + client *k8sAPIClient +} + +func NewK8sAPIExecutor(clientset *kubernetes.Clientset, config *restclient.Config, podName, namespace string) (*K8sAPIExecutor, error) { + log.Infof("Creating a K8sAPI executor") + client, err := newK8sAPIClient(clientset, config, podName, namespace) + if err != nil { + return nil, errors.InternalWrapError(err) + } + return &K8sAPIExecutor{ + client: client, + }, nil +} + +func (k *K8sAPIExecutor) GetFileContents(containerID string, sourcePath string) (string, error) { + return "", errors.Errorf(errors.CodeNotImplemented, "GetFileContents() is not implemented in the k8sapi executor.") +} + +func (k *K8sAPIExecutor) CopyFile(containerID string, sourcePath string, destPath string) error { + return errors.Errorf(errors.CodeNotImplemented, "CopyFile() is not implemented in the k8sapi executor.") +} + +func (k *K8sAPIExecutor) GetOutputStream(containerID string, combinedOutput bool) (io.ReadCloser, error) { + log.Infof("Getting output of %s", containerID) + if !combinedOutput { + log.Warn("non combined output unsupported") + } + return k.client.getLogsAsStream(containerID) +} + +func (k *K8sAPIExecutor) WaitInit() error { + return nil +} + +// Wait for the container to complete +func (k *K8sAPIExecutor) Wait(containerID string) error { + log.Infof("Waiting for container %s to complete", containerID) + return k.client.waitForTermination(containerID, 0) +} + +// Kill kills a list of containerIDs first with a SIGTERM then with a SIGKILL after a grace period +func (k *K8sAPIExecutor) Kill(containerIDs []string) error { + log.Infof("Killing containers %s", containerIDs) + for _, containerID := range containerIDs { + err := k.client.killGracefully(containerID) + if err != nil { + return err + } + } + return nil +} diff --git a/workflow/executor/kubelet/client.go b/workflow/executor/kubelet/client.go index 13e5aebe8c43..e708acdff71f 100644 --- a/workflow/executor/kubelet/client.go +++ b/workflow/executor/kubelet/client.go @@ -12,14 +12,12 @@ import ( "net/url" "os" "strconv" - "strings" "syscall" "time" - "github.com/cyrusbiotechnology/argo/util" - "github.com/cyrusbiotechnology/argo/errors" "github.com/cyrusbiotechnology/argo/workflow/common" + execcommon "github.com/cyrusbiotechnology/argo/workflow/executor/common" "github.com/gorilla/websocket" log "github.com/sirupsen/logrus" "k8s.io/api/core/v1" @@ -27,7 +25,6 @@ import ( const ( readWSResponseTimeout = time.Minute * 1 - containerShimPrefix = "://" ) type kubeletClient struct { @@ -41,6 +38,8 @@ type kubeletClient struct { kubeletEndpoint string } +var _ execcommon.KubernetesClientInterface = &kubeletClient{} + func newKubeletClient() (*kubeletClient, error) { kubeletHost := os.Getenv(common.EnvVarDownwardAPINodeIP) if kubeletHost == "" { @@ -126,6 +125,26 @@ func (k *kubeletClient) getPodList() (*v1.PodList, error) { return podList, resp.Body.Close() } +func (k *kubeletClient) GetLogStream(containerID string) (io.ReadCloser, error) { + podList, err := k.getPodList() + if err != nil { + return nil, err + } + for _, pod := range podList.Items { + for _, container := range pod.Status.ContainerStatuses { + if execcommon.GetContainerID(&container) != containerID { + continue + } + resp, err := k.doRequestLogs(pod.Namespace, pod.Name, container.Name) + if err != nil { + return nil, err + } + return resp.Body, nil + } + } + return nil, errors.New(errors.CodeNotFound, fmt.Sprintf("containerID %q is not found in the pod list", containerID)) +} + func (k *kubeletClient) doRequestLogs(namespace, podName, containerName string) (*http.Response, error) { u, err := url.ParseRequestURI(fmt.Sprintf("https://%s/containerLogs/%s/%s/%s", k.kubeletEndpoint, namespace, podName, containerName)) if err != nil { @@ -146,92 +165,20 @@ func (k *kubeletClient) doRequestLogs(namespace, podName, containerName string) return resp, nil } -func (k *kubeletClient) getLogs(namespace, podName, containerName string) (string, error) { - resp, err := k.doRequestLogs(namespace, podName, containerName) - if resp != nil { - defer func() { _ = resp.Body.Close() }() - } - if err != nil { - return "", err - } - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", errors.InternalWrapError(err) - } - return string(b), resp.Body.Close() -} - -func (k *kubeletClient) saveLogsToFile(namespace, podName, containerName, path string) error { - resp, err := k.doRequestLogs(namespace, podName, containerName) - if resp != nil { - defer func() { _ = resp.Body.Close() }() - } - if err != nil { - return err - } - outFile, err := os.Create(path) - if err != nil { - return errors.InternalWrapError(err) - } - defer util.Close(outFile) - _, err = io.Copy(outFile, resp.Body) - return err -} - -func getContainerID(container *v1.ContainerStatus) string { - i := strings.Index(container.ContainerID, containerShimPrefix) - if i == -1 { - return "" - } - return container.ContainerID[i+len(containerShimPrefix):] -} - -func (k *kubeletClient) getContainerStatus(containerID string) (*v1.ContainerStatus, error) { +func (k *kubeletClient) GetContainerStatus(containerID string) (*v1.Pod, *v1.ContainerStatus, error) { podList, err := k.getPodList() if err != nil { - return nil, errors.InternalWrapError(err) + return nil, nil, errors.InternalWrapError(err) } for _, pod := range podList.Items { for _, container := range pod.Status.ContainerStatuses { - if getContainerID(&container) != containerID { + if execcommon.GetContainerID(&container) != containerID { continue } - return &container, nil + return &pod, &container, nil } } - return nil, errors.New(errors.CodeNotFound, fmt.Sprintf("containerID %q is not found in the pod list", containerID)) -} - -func (k *kubeletClient) GetContainerLogs(containerID string) (string, error) { - podList, err := k.getPodList() - if err != nil { - return "", errors.InternalWrapError(err) - } - for _, pod := range podList.Items { - for _, container := range pod.Status.ContainerStatuses { - if getContainerID(&container) != containerID { - continue - } - return k.getLogs(pod.Namespace, pod.Name, container.Name) - } - } - return "", errors.New(errors.CodeNotFound, fmt.Sprintf("containerID %q is not found in the pod list", containerID)) -} - -func (k *kubeletClient) SaveLogsToFile(containerID, path string) error { - podList, err := k.getPodList() - if err != nil { - return errors.InternalWrapError(err) - } - for _, pod := range podList.Items { - for _, container := range pod.Status.ContainerStatuses { - if getContainerID(&container) != containerID { - continue - } - return k.saveLogsToFile(pod.Namespace, pod.Name, container.Name, path) - } - } - return errors.New(errors.CodeNotFound, fmt.Sprintf("containerID %q is not found in the pod list", containerID)) + return nil, nil, errors.New(errors.CodeNotFound, fmt.Sprintf("containerID %q is not found in the pod list", containerID)) } func (k *kubeletClient) exec(u *url.URL) (*url.URL, error) { @@ -294,40 +241,7 @@ func (k *kubeletClient) readFileContents(u *url.URL) (*bytes.Buffer, error) { } } -// TerminatePodWithContainerID invoke the given SIG against the PID1 of the container. -// No-op if the container is on the hostPID -func (k *kubeletClient) TerminatePodWithContainerID(containerID string, sig syscall.Signal) error { - podList, err := k.getPodList() - if err != nil { - return errors.InternalWrapError(err) - } - for _, pod := range podList.Items { - for _, container := range pod.Status.ContainerStatuses { - if getContainerID(&container) != containerID { - continue - } - if container.State.Terminated != nil { - log.Infof("Container %s is already terminated: %v", container.ContainerID, container.State.Terminated.String()) - return nil - } - if pod.Spec.HostPID { - return fmt.Errorf("cannot terminate a hostPID Pod %s", pod.Name) - } - if pod.Spec.RestartPolicy != "Never" { - return fmt.Errorf("cannot terminate pod with a %q restart policy", pod.Spec.RestartPolicy) - } - u, err := url.ParseRequestURI(fmt.Sprintf("wss://%s/exec/%s/%s/%s?command=/bin/sh&&command=-c&command=kill+-%d+1&output=1&error=1", k.kubeletEndpoint, pod.Namespace, pod.Name, container.Name, sig)) - if err != nil { - return errors.InternalWrapError(err) - } - _, err = k.exec(u) - return err - } - } - return errors.New(errors.CodeNotFound, fmt.Sprintf("containerID %q is not found in the pod list", containerID)) -} - -// CreateArchive exec in the given containerID and create a tarball of the given sourcePath. Works with directory +// createArchive exec in the given containerID and create a tarball of the given sourcePath. Works with directory func (k *kubeletClient) CreateArchive(containerID, sourcePath string) (*bytes.Buffer, error) { return k.getCommandOutput(containerID, fmt.Sprintf("command=tar&command=-cf&command=-&command=%s&output=1", sourcePath)) } @@ -344,7 +258,7 @@ func (k *kubeletClient) getCommandOutput(containerID, command string) (*bytes.Bu } for _, pod := range podList.Items { for _, container := range pod.Status.ContainerStatuses { - if getContainerID(&container) != containerID { + if execcommon.GetContainerID(&container) != containerID { continue } if container.State.Terminated != nil { @@ -367,30 +281,22 @@ func (k *kubeletClient) getCommandOutput(containerID, command string) (*bytes.Bu // WaitForTermination of the given containerID, set the timeout to 0 to discard it func (k *kubeletClient) WaitForTermination(containerID string, timeout time.Duration) error { - ticker := time.NewTicker(time.Second * 1) - defer ticker.Stop() - timer := time.NewTimer(timeout) - if timeout == 0 { - timer.Stop() - } else { - defer timer.Stop() - } + return execcommon.WaitForTermination(k, containerID, timeout) +} - log.Infof("Starting to wait completion of containerID %s ...", containerID) - for { - select { - case <-ticker.C: - containerStatus, err := k.getContainerStatus(containerID) - if err != nil { - return err - } - if containerStatus.State.Terminated == nil { - continue - } - log.Infof("ContainerID %q is terminated: %v", containerID, containerStatus.String()) - return nil - case <-timer.C: - return fmt.Errorf("timeout after %s", timeout.String()) - } +func (k *kubeletClient) KillContainer(pod *v1.Pod, container *v1.ContainerStatus, sig syscall.Signal) error { + u, err := url.ParseRequestURI(fmt.Sprintf("wss://%s/exec/%s/%s/%s?command=/bin/sh&&command=-c&command=kill+-%d+1&output=1&error=1", k.kubeletEndpoint, pod.Namespace, pod.Name, container.Name, sig)) + if err != nil { + return errors.InternalWrapError(err) } + _, err = k.exec(u) + return err +} + +func (k *kubeletClient) KillGracefully(containerID string) error { + return execcommon.KillGracefully(k, containerID) +} + +func (k *kubeletClient) CopyArchive(containerID, sourcePath, destPath string) error { + return execcommon.CopyArchive(k, containerID, sourcePath, destPath) } diff --git a/workflow/executor/kubelet/kubelet.go b/workflow/executor/kubelet/kubelet.go index fc96bf56c8b3..41b9d6689fd0 100644 --- a/workflow/executor/kubelet/kubelet.go +++ b/workflow/executor/kubelet/kubelet.go @@ -1,19 +1,12 @@ package kubelet import ( - "compress/gzip" - "os" - "syscall" - "time" + "io" "github.com/cyrusbiotechnology/argo/errors" log "github.com/sirupsen/logrus" ) -// killGracePeriod is the time in seconds after sending SIGTERM before -// forcefully killing the sidecar with SIGKILL (value matches k8s) -const killGracePeriod = 10 - type KubeletExecutor struct { cli *kubeletClient } @@ -30,48 +23,22 @@ func NewKubeletExecutor() (*KubeletExecutor, error) { } func (k *KubeletExecutor) GetFileContents(containerID string, sourcePath string) (string, error) { - b, err := k.cli.GetFileContents(containerID, sourcePath) - if err != nil { - return "", err - } - return b.String(), nil + return "", errors.Errorf(errors.CodeNotImplemented, "GetFileContents() is not implemented in the kubelet executor.") } func (k *KubeletExecutor) CopyFile(containerID string, sourcePath string, destPath string) error { - log.Infof("Archiving %s:%s to %s", containerID, sourcePath, destPath) - b, err := k.cli.CreateArchive(containerID, sourcePath) - if err != nil { - return err - } - f, err := os.OpenFile(destPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) - if err != nil { - return err - } - w := gzip.NewWriter(f) - _, err = w.Write(b.Bytes()) - if err != nil { - return err - } - err = w.Flush() - if err != nil { - return err - } - err = w.Close() - if err != nil { - return err - } - return f.Close() + return errors.Errorf(errors.CodeNotImplemented, "CopyFile() is not implemented in the kubelet executor.") } -// GetOutput returns the entirety of the container output as a string -// Used to capturing script results as an output parameter -func (k *KubeletExecutor) GetOutput(containerID string) (string, error) { - return k.cli.GetContainerLogs(containerID) +func (k *KubeletExecutor) GetOutputStream(containerID string, combinedOutput bool) (io.ReadCloser, error) { + if !combinedOutput { + log.Warn("non combined output unsupported") + } + return k.cli.GetLogStream(containerID) } -// Logs copies logs to a given path -func (k *KubeletExecutor) Logs(containerID, path string) error { - return k.cli.SaveLogsToFile(containerID, path) +func (k *KubeletExecutor) WaitInit() error { + return nil } // Wait for the container to complete @@ -82,26 +49,10 @@ func (k *KubeletExecutor) Wait(containerID string) error { // Kill kills a list of containerIDs first with a SIGTERM then with a SIGKILL after a grace period func (k *KubeletExecutor) Kill(containerIDs []string) error { for _, containerID := range containerIDs { - log.Infof("SIGTERM containerID %q: %s", containerID, syscall.SIGTERM.String()) - err := k.cli.TerminatePodWithContainerID(containerID, syscall.SIGTERM) - if err != nil { - return err - } - err = k.cli.WaitForTermination(containerID, time.Second*killGracePeriod) - if err == nil { - log.Infof("ContainerID %q successfully killed", containerID) - continue - } - log.Infof("SIGKILL containerID %q: %s", containerID, syscall.SIGKILL.String()) - err = k.cli.TerminatePodWithContainerID(containerID, syscall.SIGKILL) - if err != nil { - return err - } - err = k.cli.WaitForTermination(containerID, time.Second*killGracePeriod) + err := k.cli.KillGracefully(containerID) if err != nil { return err } - log.Infof("ContainerID %q successfully killed", containerID) } return nil } diff --git a/workflow/executor/mocks/ContainerRuntimeExecutor.go b/workflow/executor/mocks/ContainerRuntimeExecutor.go index df574d2da817..55046f8fe877 100644 --- a/workflow/executor/mocks/ContainerRuntimeExecutor.go +++ b/workflow/executor/mocks/ContainerRuntimeExecutor.go @@ -1,6 +1,8 @@ -// Code generated by mockery v1.0.0 +// Code generated by mockery v1.0.0. DO NOT EDIT. + package mocks +import io "io" import mock "github.com/stretchr/testify/mock" // ContainerRuntimeExecutor is an autogenerated mock type for the ContainerRuntimeExecutor type @@ -43,20 +45,22 @@ func (_m *ContainerRuntimeExecutor) GetFileContents(containerID string, sourcePa return r0, r1 } -// GetOutput provides a mock function with given fields: containerID -func (_m *ContainerRuntimeExecutor) GetOutput(containerID string) (string, error) { - ret := _m.Called(containerID) +// GetOutputStream provides a mock function with given fields: containerID, combinedOutput +func (_m *ContainerRuntimeExecutor) GetOutputStream(containerID string, combinedOutput bool) (io.ReadCloser, error) { + ret := _m.Called(containerID, combinedOutput) - var r0 string - if rf, ok := ret.Get(0).(func(string) string); ok { - r0 = rf(containerID) + var r0 io.ReadCloser + if rf, ok := ret.Get(0).(func(string, bool) io.ReadCloser); ok { + r0 = rf(containerID, combinedOutput) } else { - r0 = ret.Get(0).(string) + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.ReadCloser) + } } var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(containerID) + if rf, ok := ret.Get(1).(func(string, bool) error); ok { + r1 = rf(containerID, combinedOutput) } else { r1 = ret.Error(1) } @@ -78,13 +82,13 @@ func (_m *ContainerRuntimeExecutor) Kill(containerIDs []string) error { return r0 } -// Logs provides a mock function with given fields: containerID, path -func (_m *ContainerRuntimeExecutor) Logs(containerID string, path string) error { - ret := _m.Called(containerID, path) +// Wait provides a mock function with given fields: containerID +func (_m *ContainerRuntimeExecutor) Wait(containerID string) error { + ret := _m.Called(containerID) var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(containerID, path) + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(containerID) } else { r0 = ret.Error(0) } @@ -92,13 +96,13 @@ func (_m *ContainerRuntimeExecutor) Logs(containerID string, path string) error return r0 } -// Wait provides a mock function with given fields: containerID -func (_m *ContainerRuntimeExecutor) Wait(containerID string) error { - ret := _m.Called(containerID) +// WaitInit provides a mock function with given fields: +func (_m *ContainerRuntimeExecutor) WaitInit() error { + ret := _m.Called() var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(containerID) + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() } else { r0 = ret.Error(0) } diff --git a/workflow/executor/pns/pns.go b/workflow/executor/pns/pns.go new file mode 100644 index 000000000000..a85b59e9a388 --- /dev/null +++ b/workflow/executor/pns/pns.go @@ -0,0 +1,385 @@ +package pns + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "sync" + "syscall" + "time" + + executil "github.com/argoproj/pkg/exec" + gops "github.com/mitchellh/go-ps" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + + "github.com/cyrusbiotechnology/argo/errors" + "github.com/cyrusbiotechnology/argo/util/archive" + "github.com/cyrusbiotechnology/argo/workflow/common" + execcommon "github.com/cyrusbiotechnology/argo/workflow/executor/common" +) + +type PNSExecutor struct { + clientset *kubernetes.Clientset + podName string + namespace string + + // ctrIDToPid maps a containerID to a process ID + ctrIDToPid map[string]int + // pidToCtrID maps a process ID to a container ID + pidToCtrID map[int]string + + // pidFileHandles holds file handles to all root containers + pidFileHandles map[int]*fileInfo + + // thisPID is the pid of this process + thisPID int + // mainPID holds the main container's pid + mainPID int + // mainFS holds a file descriptor to the main filesystem, allowing the executor to access the + // filesystem after the main process exited + mainFS *os.File + // rootFS holds a file descriptor to the root filesystem, allowing the executor to exit out of a chroot + rootFS *os.File + // debug enables additional debugging + debug bool + // hasOutputs indicates if the template has outputs. determines if we need to + hasOutputs bool +} + +type fileInfo struct { + file os.File + info os.FileInfo +} + +func NewPNSExecutor(clientset *kubernetes.Clientset, podName, namespace string, hasOutputs bool) (*PNSExecutor, error) { + thisPID := os.Getpid() + log.Infof("Creating PNS executor (namespace: %s, pod: %s, pid: %d, hasOutputs: %v)", namespace, podName, thisPID, hasOutputs) + if thisPID == 1 { + return nil, errors.New(errors.CodeBadRequest, "process namespace sharing is not enabled on pod") + } + return &PNSExecutor{ + clientset: clientset, + podName: podName, + namespace: namespace, + ctrIDToPid: make(map[string]int), + pidToCtrID: make(map[int]string), + pidFileHandles: make(map[int]*fileInfo), + thisPID: thisPID, + debug: log.GetLevel() == log.DebugLevel, + hasOutputs: hasOutputs, + }, nil +} + +func (p *PNSExecutor) GetFileContents(containerID string, sourcePath string) (string, error) { + err := p.enterChroot() + if err != nil { + return "", err + } + defer func() { _ = p.exitChroot() }() + out, err := ioutil.ReadFile(sourcePath) + if err != nil { + return "", err + } + return string(out), nil +} + +// enterChroot enters chroot of the main container +func (p *PNSExecutor) enterChroot() error { + if p.mainFS == nil { + return errors.InternalErrorf("could not chroot into main for artifact collection: container may have exited too quickly") + } + if err := p.mainFS.Chdir(); err != nil { + return errors.InternalWrapErrorf(err, "failed to chdir to main filesystem: %v", err) + } + err := syscall.Chroot(".") + if err != nil { + return errors.InternalWrapErrorf(err, "failed to chroot to main filesystem: %v", err) + } + return nil +} + +// exitChroot exits chroot +func (p *PNSExecutor) exitChroot() error { + if err := p.rootFS.Chdir(); err != nil { + return errors.InternalWrapError(err) + } + err := syscall.Chroot(".") + if err != nil { + return errors.InternalWrapError(err) + } + return nil +} + +// CopyFile copies a source file in a container to a local path +func (p *PNSExecutor) CopyFile(containerID string, sourcePath string, destPath string) (err error) { + destFile, err := os.Create(destPath) + if err != nil { + return err + } + defer func() { + // exit chroot and close the file. preserve the original error + deferErr := p.exitChroot() + if err == nil && deferErr != nil { + err = errors.InternalWrapError(deferErr) + } + deferErr = destFile.Close() + if err == nil && deferErr != nil { + err = errors.InternalWrapError(deferErr) + } + }() + w := bufio.NewWriter(destFile) + err = p.enterChroot() + if err != nil { + return err + } + + err = archive.TarGzToWriter(sourcePath, w) + if err != nil { + return err + } + + return nil +} + +func (p *PNSExecutor) WaitInit() error { + if !p.hasOutputs { + return nil + } + go p.pollRootProcesses(time.Minute) + // Secure a filehandle on our own root. This is because we will chroot back and forth from + // the main container's filesystem, to our own. + rootFS, err := os.Open("/") + if err != nil { + return errors.InternalWrapError(err) + } + p.rootFS = rootFS + return nil +} + +// Wait for the container to complete +func (p *PNSExecutor) Wait(containerID string) error { + mainPID, err := p.getContainerPID(containerID) + if err != nil { + if !p.hasOutputs { + log.Warnf("Ignoring wait failure: %v. Process assumed to have completed", err) + return nil + } + return err + } + log.Infof("Main pid identified as %d", mainPID) + p.mainPID = mainPID + for pid, f := range p.pidFileHandles { + if pid == p.mainPID { + log.Info("Successfully secured file handle on main container root filesystem") + p.mainFS = &f.file + } else { + log.Infof("Closing root filehandle for non-main pid %d", pid) + _ = f.file.Close() + } + } + if p.mainFS == nil { + log.Warn("Failed to secure file handle on main container's root filesystem. Output artifacts from base image layer will fail") + } + + // wait for pid to complete + log.Infof("Waiting for main pid %d to complete", mainPID) + err = executil.WaitPID(mainPID) + if err != nil { + return err + } + log.Infof("Main pid %d completed", mainPID) + return nil +} + +// pollRootProcesses will poll /proc for root pids (pids without parents) in a tight loop, for the +// purpose of securing an open file handle against /proc//root as soon as possible. +// It opens file handles on all root pids because at this point, we do not yet know which pid is the +// "main" container. +// Polling is necessary because it is not possible to use something like fsnotify against procfs. +func (p *PNSExecutor) pollRootProcesses(timeout time.Duration) { + log.Warnf("Polling root processes (%v)", timeout) + deadline := time.Now().Add(timeout) + for { + p.updateCtrIDMap() + if p.mainFS != nil { + log.Info("Stopped root processes polling due to successful securing of main root fs") + break + } + if time.Now().After(deadline) { + log.Warnf("Polling root processes timed out (%v)", timeout) + break + } + time.Sleep(50 * time.Millisecond) + } +} + +func (p *PNSExecutor) GetOutputStream(containerID string, combinedOutput bool) (io.ReadCloser, error) { + if !combinedOutput { + log.Warn("non combined output unsupported") + } + opts := v1.PodLogOptions{ + Container: common.MainContainerName, + } + return p.clientset.CoreV1().Pods(p.namespace).GetLogs(p.podName, &opts).Stream() +} + +// Kill a list of containerIDs first with a SIGTERM then with a SIGKILL after a grace period +func (p *PNSExecutor) Kill(containerIDs []string) error { + var asyncErr error + wg := sync.WaitGroup{} + for _, cid := range containerIDs { + wg.Add(1) + go func(containerID string) { + err := p.killContainer(containerID) + if err != nil && asyncErr != nil { + asyncErr = err + } + wg.Done() + }(cid) + } + wg.Wait() + return asyncErr +} + +func (p *PNSExecutor) killContainer(containerID string) error { + pid, err := p.getContainerPID(containerID) + if err != nil { + log.Warnf("Ignoring kill container failure of %s: %v. Process assumed to have completed", containerID, err) + return nil + } + // On Unix systems, FindProcess always succeeds and returns a Process + // for the given pid, regardless of whether the process exists. + proc, _ := os.FindProcess(pid) + log.Infof("Sending SIGTERM to pid %d", pid) + err = proc.Signal(syscall.SIGTERM) + if err != nil { + log.Warnf("Failed to SIGTERM pid %d: %v", pid, err) + } + + waitPIDOpts := executil.WaitPIDOpts{Timeout: execcommon.KillGracePeriod * time.Second} + err = executil.WaitPID(pid, waitPIDOpts) + if err == nil { + log.Infof("PID %d completed", pid) + return nil + } + if err != executil.ErrWaitPIDTimeout { + return err + } + log.Warnf("Timed out (%v) waiting for pid %d to complete after SIGTERM. Issing SIGKILL", waitPIDOpts.Timeout, pid) + time.Sleep(30 * time.Minute) + err = proc.Signal(syscall.SIGKILL) + if err != nil { + log.Warnf("Failed to SIGKILL pid %d: %v", pid, err) + } + return err +} + +// getContainerPID returns the pid associated with the container id. Returns error if it was unable +// to be determined because no running root processes exist with that container ID +func (p *PNSExecutor) getContainerPID(containerID string) (int, error) { + pid, ok := p.ctrIDToPid[containerID] + if ok { + return pid, nil + } + p.updateCtrIDMap() + pid, ok = p.ctrIDToPid[containerID] + if !ok { + return -1, errors.InternalErrorf("Failed to determine pid for containerID %s: container may have exited too quickly", containerID) + } + return pid, nil +} + +// updateCtrIDMap updates the mapping between container IDs to PIDs +func (p *PNSExecutor) updateCtrIDMap() { + allProcs, err := gops.Processes() + if err != nil { + log.Warnf("Failed to list processes: %v", err) + return + } + for _, proc := range allProcs { + pid := proc.Pid() + if pid == 1 || pid == p.thisPID || proc.PPid() != 0 { + // ignore the pause container, our own pid, and non-root processes + continue + } + + // Useful code for debugging: + if p.debug { + if data, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/root", pid) + "/etc/os-release"); err == nil { + log.Infof("pid %d: %s", pid, string(data)) + _, _ = parseContainerID(pid) + } + } + + if p.hasOutputs && p.mainFS == nil { + rootPath := fmt.Sprintf("/proc/%d/root", pid) + currInfo, err := os.Stat(rootPath) + if err != nil { + log.Warnf("Failed to stat %s: %v", rootPath, err) + continue + } + log.Infof("pid %d: %v", pid, currInfo) + prevInfo := p.pidFileHandles[pid] + + // Secure the root filehandle of the process. NOTE if the file changed, it means that + // the main container may have switched (e.g. gone from busybox to the user's container) + if prevInfo == nil || !os.SameFile(prevInfo.info, currInfo) { + fs, err := os.Open(rootPath) + if err != nil { + log.Warnf("Failed to open %s: %v", rootPath, err) + continue + } + log.Infof("Secured filehandle on %s", rootPath) + p.pidFileHandles[pid] = &fileInfo{ + info: currInfo, + file: *fs, + } + if prevInfo != nil { + _ = prevInfo.file.Close() + } + } + } + + // Update maps of pids to container ids + if _, ok := p.pidToCtrID[pid]; !ok { + containerID, err := parseContainerID(pid) + if err != nil { + log.Warnf("Failed to identify containerID for process %d", pid) + continue + } + log.Infof("containerID %s mapped to pid %d", containerID, pid) + p.ctrIDToPid[containerID] = pid + p.pidToCtrID[pid] = containerID + } + } +} + +// parseContainerID parses the containerID of a pid +func parseContainerID(pid int) (string, error) { + cgroupPath := fmt.Sprintf("/proc/%d/cgroup", pid) + cgroupFile, err := os.OpenFile(cgroupPath, os.O_RDONLY, os.ModePerm) + if err != nil { + return "", errors.InternalWrapError(err) + } + defer func() { _ = cgroupFile.Close() }() + sc := bufio.NewScanner(cgroupFile) + for sc.Scan() { + // See https://www.systutorials.com/docs/linux/man/5-proc/ for /proc/XX/cgroup format. e.g.: + // 5:cpuacct,cpu,cpuset:/daemons + line := sc.Text() + log.Debugf("pid %d: %s", pid, line) + parts := strings.Split(line, "/") + if len(parts) > 1 { + if containerID := parts[len(parts)-1]; containerID != "" { + // need to check for empty string because the line may look like: 5:rdma:/ + return containerID, nil + } + } + } + return "", errors.InternalErrorf("Failed to parse container ID from %s", cgroupPath) +} diff --git a/workflow/executor/resource.go b/workflow/executor/resource.go index 848c62c9e4ab..7345f99d6cca 100644 --- a/workflow/executor/resource.go +++ b/workflow/executor/resource.go @@ -3,11 +3,15 @@ package executor import ( "bufio" "bytes" + "encoding/json" "fmt" + "io/ioutil" "os/exec" "strings" "time" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/cyrusbiotechnology/argo/errors" log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" @@ -16,28 +20,58 @@ import ( ) // ExecResource will run kubectl action against a manifest -func (we *WorkflowExecutor) ExecResource(action string, manifestPath string) (string, error) { +func (we *WorkflowExecutor) ExecResource(action string, manifestPath string, isDelete bool) (string, string, error) { args := []string{ action, } - if action == "delete" { + output := "json" + if isDelete { args = append(args, "--ignore-not-found") + output = "name" + } + + if action == "patch" { + mergeStrategy := "strategic" + if we.Template.Resource.MergeStrategy != "" { + mergeStrategy = we.Template.Resource.MergeStrategy + } + + args = append(args, "--type") + args = append(args, mergeStrategy) + + args = append(args, "-p") + buff, err := ioutil.ReadFile(manifestPath) + + if err != nil { + return "", "", errors.New(errors.CodeBadRequest, err.Error()) + } + + args = append(args, string(buff)) } + args = append(args, "-f") args = append(args, manifestPath) args = append(args, "-o") - args = append(args, "name") + args = append(args, output) cmd := exec.Command("kubectl", args...) log.Info(strings.Join(cmd.Args, " ")) out, err := cmd.Output() if err != nil { exErr := err.(*exec.ExitError) errMsg := strings.TrimSpace(string(exErr.Stderr)) - return "", errors.New(errors.CodeBadRequest, errMsg) + return "", "", errors.New(errors.CodeBadRequest, errMsg) + } + if action == "delete" { + return "", "", nil + } + obj := unstructured.Unstructured{} + err = json.Unmarshal(out, &obj) + if err != nil { + return "", "", err } - resourceName := strings.TrimSpace(string(out)) - log.Infof(resourceName) - return resourceName, nil + resourceName := fmt.Sprintf("%s.%s/%s", obj.GroupVersionKind().Kind, obj.GroupVersionKind().Group, obj.GetName()) + log.Infof("%s/%s", obj.GetNamespace(), resourceName) + return obj.GetNamespace(), resourceName, nil } // gjsonLabels is an implementation of labels.Labels interface @@ -58,7 +92,7 @@ func (g gjsonLabels) Get(label string) string { } // WaitResource waits for a specific resource to satisfy either the success or failure condition -func (we *WorkflowExecutor) WaitResource(resourceName string) error { +func (we *WorkflowExecutor) WaitResource(resourceNamespace string, resourceName string) error { if we.Template.Resource.SuccessCondition == "" && we.Template.Resource.FailureCondition == "" { return nil } @@ -82,12 +116,11 @@ func (we *WorkflowExecutor) WaitResource(resourceName string) error { failReqs, _ = failSelector.Requirements() } - // Start the condition result reader using ExponentialBackoff - // Exponential backoff is for steps of 0, 5, 20, 80, 320 seconds since the first step is without - // delay in the ExponentialBackoff - err := wait.ExponentialBackoff(wait.Backoff{Duration: (time.Second * 5), Factor: 4.0, Steps: 5}, + // Start the condition result reader using PollImmediateInfinite + // Poll intervall of 5 seconds serves as a backoff intervall in case of immediate result reader failure + err := wait.PollImmediateInfinite(time.Second*5, func() (bool, error) { - isErrRetry, err := checkResourceState(resourceName, successReqs, failReqs) + isErrRetry, err := checkResourceState(resourceNamespace, resourceName, successReqs, failReqs) if err == nil { log.Infof("Returning from successful wait for resource %s", resourceName) @@ -115,9 +148,9 @@ func (we *WorkflowExecutor) WaitResource(resourceName string) error { } // Function to do the kubectl get -w command and then waiting on json reading. -func checkResourceState(resourceName string, successReqs labels.Requirements, failReqs labels.Requirements) (bool, error) { +func checkResourceState(resourceNamespace string, resourceName string, successReqs labels.Requirements, failReqs labels.Requirements) (bool, error) { - cmd, reader, err := startKubectlWaitCmd(resourceName) + cmd, reader, err := startKubectlWaitCmd(resourceNamespace, resourceName) if err != nil { return false, err } @@ -180,8 +213,12 @@ func checkResourceState(resourceName string, successReqs labels.Requirements, fa } // Start Kubectl command Get with -w return error if unable to start command -func startKubectlWaitCmd(resourceName string) (*exec.Cmd, *bufio.Reader, error) { - cmd := exec.Command("kubectl", "get", resourceName, "-w", "-o", "json") +func startKubectlWaitCmd(resourceNamespace string, resourceName string) (*exec.Cmd, *bufio.Reader, error) { + args := []string{"get", resourceName, "-w", "-o", "json"} + if resourceNamespace != "" { + args = append(args, "-n", resourceNamespace) + } + cmd := exec.Command("kubectl", args...) stdout, err := cmd.StdoutPipe() if err != nil { return nil, nil, errors.InternalWrapError(err) @@ -217,7 +254,7 @@ func readJSON(reader *bufio.Reader) ([]byte, error) { } // SaveResourceParameters will save any resource output parameters -func (we *WorkflowExecutor) SaveResourceParameters(resourceName string) error { +func (we *WorkflowExecutor) SaveResourceParameters(resourceNamespace string, resourceName string) error { if len(we.Template.Outputs.Parameters) == 0 { log.Infof("No output parameters") return nil @@ -229,9 +266,17 @@ func (we *WorkflowExecutor) SaveResourceParameters(resourceName string) error { } var cmd *exec.Cmd if param.ValueFrom.JSONPath != "" { - cmd = exec.Command("kubectl", "get", resourceName, "-o", fmt.Sprintf("jsonpath='%s'", param.ValueFrom.JSONPath)) + args := []string{"get", resourceName, "-o", fmt.Sprintf("jsonpath=%s", param.ValueFrom.JSONPath)} + if resourceNamespace != "" { + args = append(args, "-n", resourceNamespace) + } + cmd = exec.Command("kubectl", args...) } else if param.ValueFrom.JQFilter != "" { - cmdStr := fmt.Sprintf("kubectl get %s -o json | jq -c '%s'", resourceName, param.ValueFrom.JQFilter) + resArgs := []string{resourceName} + if resourceNamespace != "" { + resArgs = append(resArgs, "-n", resourceNamespace) + } + cmdStr := fmt.Sprintf("kubectl get %s -o json | jq -c '%s'", strings.Join(resArgs, " "), param.ValueFrom.JQFilter) cmd = exec.Command("sh", "-c", cmdStr) } else { continue diff --git a/workflow/metrics/collector.go b/workflow/metrics/collector.go index a3b838490b5e..52590b3051e4 100644 --- a/workflow/metrics/collector.go +++ b/workflow/metrics/collector.go @@ -112,16 +112,12 @@ func (wc *workflowCollector) collectWorkflow(ch chan<- prometheus.Metric, wf wfv addGauge(descWorkflowInfo, 1, wf.Spec.Entrypoint, wf.Spec.ServiceAccountName, joinTemplates(wf.Spec.Templates)) - if phase := wf.Status.Phase; phase != "" { - // TODO: we do not have queuing feature yet so are not adding to a 'Pending' guague. - // Uncomment when we support queueing. - //addGauge(descWorkflowStatusPhase, boolFloat64(phase == wfv1.NodePending), string(wfv1.NodePending)) - addGauge(descWorkflowStatusPhase, boolFloat64(phase == wfv1.NodeRunning), string(wfv1.NodeRunning)) - addGauge(descWorkflowStatusPhase, boolFloat64(phase == wfv1.NodeSucceeded), string(wfv1.NodeSucceeded)) - addGauge(descWorkflowStatusPhase, boolFloat64(phase == wfv1.NodeSkipped), string(wfv1.NodeSkipped)) - addGauge(descWorkflowStatusPhase, boolFloat64(phase == wfv1.NodeFailed), string(wfv1.NodeFailed)) - addGauge(descWorkflowStatusPhase, boolFloat64(phase == wfv1.NodeError), string(wfv1.NodeError)) - } + addGauge(descWorkflowStatusPhase, boolFloat64(wf.Status.Phase == wfv1.NodePending || wf.Status.Phase == ""), string(wfv1.NodePending)) + addGauge(descWorkflowStatusPhase, boolFloat64(wf.Status.Phase == wfv1.NodeRunning), string(wfv1.NodeRunning)) + addGauge(descWorkflowStatusPhase, boolFloat64(wf.Status.Phase == wfv1.NodeSucceeded), string(wfv1.NodeSucceeded)) + addGauge(descWorkflowStatusPhase, boolFloat64(wf.Status.Phase == wfv1.NodeSkipped), string(wfv1.NodeSkipped)) + addGauge(descWorkflowStatusPhase, boolFloat64(wf.Status.Phase == wfv1.NodeFailed), string(wfv1.NodeFailed)) + addGauge(descWorkflowStatusPhase, boolFloat64(wf.Status.Phase == wfv1.NodeError), string(wfv1.NodeError)) if !wf.CreationTimestamp.IsZero() { addGauge(descWorkflowCreated, float64(wf.CreationTimestamp.Unix())) diff --git a/workflow/metrics/server.go b/workflow/metrics/server.go index 65153e23a044..f2ed0bf63d66 100644 --- a/workflow/metrics/server.go +++ b/workflow/metrics/server.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "fmt" "net/http" "github.com/prometheus/client_golang/prometheus" @@ -20,7 +21,7 @@ type PrometheusConfig struct { func RunServer(ctx context.Context, config PrometheusConfig, registry *prometheus.Registry) { mux := http.NewServeMux() mux.Handle(config.Path, promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) - srv := &http.Server{Addr: config.Port, Handler: mux} + srv := &http.Server{Addr: fmt.Sprintf(":%s", config.Port), Handler: mux} defer func() { if cerr := srv.Close(); cerr != nil { @@ -28,7 +29,7 @@ func RunServer(ctx context.Context, config PrometheusConfig, registry *prometheu } }() - log.Infof("Starting prometheus metrics server at 0.0.0.0%s%s", config.Port, config.Path) + log.Infof("Starting prometheus metrics server at 0.0.0.0:%s%s", config.Port, config.Path) if err := srv.ListenAndServe(); err != nil { panic(err) } diff --git a/workflow/ttlcontroller/ttlcontroller.go b/workflow/ttlcontroller/ttlcontroller.go index 40f569cf3302..320da041872b 100644 --- a/workflow/ttlcontroller/ttlcontroller.go +++ b/workflow/ttlcontroller/ttlcontroller.go @@ -130,7 +130,12 @@ func (c *Controller) processNextWorkItem() bool { // enqueueWF conditionally queues a workflow to the ttl queue if it is within the deletion period func (c *Controller) enqueueWF(obj interface{}) { - wf, err := util.FromUnstructured(obj.(*unstructured.Unstructured)) + un, ok := obj.(*unstructured.Unstructured) + if !ok { + log.Warnf("'%v' is not an unstructured", obj) + return + } + wf, err := util.FromUnstructured(un) if err != nil { log.Warnf("Failed to unmarshal workflow %v object: %v", obj, err) return diff --git a/workflow/util/util.go b/workflow/util/util.go index 543743aae91c..aaa2911472be 100644 --- a/workflow/util/util.go +++ b/workflow/util/util.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -25,40 +26,34 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + "k8s.io/utils/pointer" "github.com/cyrusbiotechnology/argo/errors" "github.com/cyrusbiotechnology/argo/pkg/apis/workflow" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" "github.com/cyrusbiotechnology/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1" cmdutil "github.com/cyrusbiotechnology/argo/util/cmd" + "github.com/cyrusbiotechnology/argo/util/file" "github.com/cyrusbiotechnology/argo/util/retry" unstructutil "github.com/cyrusbiotechnology/argo/util/unstructured" "github.com/cyrusbiotechnology/argo/workflow/common" "github.com/cyrusbiotechnology/argo/workflow/validate" ) -func NewDynamicWorkflowClient(config *rest.Config) (dynamic.Interface, error) { - dynClientPool := dynamic.NewDynamicClientPool(config) - return dynClientPool.ClientForGroupVersionKind(wfv1.SchemaGroupVersionKind) -} - // NewWorkflowInformer returns the workflow informer used by the controller. This is actually // a custom built UnstructuredInformer which is in actuality returning unstructured.Unstructured // objects. We no longer return WorkflowInformer due to: // https://github.com/kubernetes/kubernetes/issues/57705 // https://github.com/cyrusbiotechnology/argo/issues/632 func NewWorkflowInformer(cfg *rest.Config, ns string, resyncPeriod time.Duration, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - dclient, err := NewDynamicWorkflowClient(cfg) + dclient, err := dynamic.NewForConfig(cfg) if err != nil { panic(err) } - resource := &metav1.APIResource{ - Name: workflow.Plural, - SingularName: workflow.Singular, - Namespaced: true, - Group: workflow.Group, - Version: "v1alpha1", - ShortNames: []string{"wf"}, + resource := schema.GroupVersionResource{ + Group: workflow.Group, + Version: "v1alpha1", + Resource: "workflows", } informer := unstructutil.NewFilteredUnstructuredInformer( resource, @@ -139,13 +134,14 @@ func IsWorkflowCompleted(wf *wfv1.Workflow) bool { // SubmitOpts are workflow submission options type SubmitOpts struct { - Name string // --name - GenerateName string // --generate-name - InstanceID string // --instanceid - Entrypoint string // --entrypoint - Parameters []string // --parameter - ParameterFile string // --parameter-file - ServiceAccount string // --serviceaccount + Name string // --name + GenerateName string // --generate-name + InstanceID string // --instanceid + Entrypoint string // --entrypoint + Parameters []string // --parameter + ParameterFile string // --parameter-file + ServiceAccount string // --serviceaccount + OwnerReference *metav1.OwnerReference // useful if your custom controller creates argo workflow resources } // SubmitWorkflow validates and submit a single workflow and override some of the fields of the workflow @@ -240,7 +236,11 @@ func SubmitWorkflow(wfIf v1alpha1.WorkflowInterface, wf *wfv1.Workflow, opts *Su if opts.Name != "" { wf.ObjectMeta.Name = opts.Name } - err := validate.ValidateWorkflow(wf) + if opts.OwnerReference != nil { + wf.SetOwnerReferences(append(wf.GetOwnerReferences(), *opts.OwnerReference)) + } + + err := validate.ValidateWorkflow(wf, validate.ValidateOpts{}) if err != nil { return nil, err } @@ -258,8 +258,7 @@ func SuspendWorkflow(wfIf v1alpha1.WorkflowInterface, workflowName string) error return false, errSuspendedCompletedWorkflow } if wf.Spec.Suspend == nil || *wf.Spec.Suspend != true { - t := true - wf.Spec.Suspend = &t + wf.Spec.Suspend = pointer.BoolPtr(true) wf, err = wfIf.Update(wf) if err != nil { if apierr.IsConflict(err) { @@ -527,3 +526,19 @@ func TerminateWorkflow(wfClient v1alpha1.WorkflowInterface, name string) error { } return err } + +// DecompressWorkflow decompresses the compressed status of a workflow (if compressed) +func DecompressWorkflow(wf *wfv1.Workflow) error { + if wf.Status.CompressedNodes != "" { + nodeContent, err := file.DecodeDecompressString(wf.Status.CompressedNodes) + if err != nil { + return errors.InternalWrapError(err) + } + err = json.Unmarshal([]byte(nodeContent), &wf.Status.Nodes) + if err != nil { + return err + } + wf.Status.CompressedNodes = "" + } + return nil +} diff --git a/workflow/validate/lint.go b/workflow/validate/lint.go index 428f93252dfb..4545c99be70a 100644 --- a/workflow/validate/lint.go +++ b/workflow/validate/lint.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/argoproj/pkg/json" + "github.com/cyrusbiotechnology/argo/errors" "github.com/cyrusbiotechnology/argo/pkg/apis/workflow" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" @@ -60,7 +61,7 @@ func LintWorkflowFile(filePath string, strict bool) error { return errors.Errorf(errors.CodeBadRequest, "%s failed to parse: %v", filePath, err) } for _, wf := range workflows { - err = ValidateWorkflow(&wf, true) + err = ValidateWorkflow(&wf, ValidateOpts{Lint: true}) if err != nil { return errors.Errorf(errors.CodeBadRequest, "%s: %s", filePath, err.Error()) } diff --git a/workflow/validate/validate.go b/workflow/validate/validate.go index 324e512736fb..8b73e1b98dfb 100644 --- a/workflow/validate/validate.go +++ b/workflow/validate/validate.go @@ -8,15 +8,31 @@ import ( "regexp" "strings" + "github.com/valyala/fasttemplate" + apivalidation "k8s.io/apimachinery/pkg/util/validation" + "github.com/cyrusbiotechnology/argo/errors" wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/workflow/artifacts/hdfs" "github.com/cyrusbiotechnology/argo/workflow/common" - "github.com/valyala/fasttemplate" - apivalidation "k8s.io/apimachinery/pkg/util/validation" ) +// ValidateOpts provides options when linting +type ValidateOpts struct { + // Lint indicates if this is performing validation in the context of linting. If true, will + // skip some validations which is permissible during linting but not submission (e.g. missing + // input parameters to the workflow) + Lint bool + // ContainerRuntimeExecutor will trigger additional validation checks specific to different + // types of executors. For example, the inability of kubelet/k8s executors to copy artifacts + // out of the base image layer. If unspecified, will use docker executor validation + ContainerRuntimeExecutor string +} + // wfValidationCtx is the context for validating a workflow spec type wfValidationCtx struct { + ValidateOpts + wf *wfv1.Workflow // globalParams keeps track of variables which are available the global // scope and can be referenced from anywhere. @@ -35,21 +51,19 @@ const ( anyItemMagicValue = "item.*" ) -// ValidateWorkflow accepts a workflow and performs validation against it. If lint is specified as -// true, will skip some validations which is permissible during linting but not submission -func ValidateWorkflow(wf *wfv1.Workflow, lint ...bool) error { +// ValidateWorkflow accepts a workflow and performs validation against it. +func ValidateWorkflow(wf *wfv1.Workflow, opts ValidateOpts) error { ctx := wfValidationCtx{ + ValidateOpts: opts, wf: wf, globalParams: make(map[string]string), results: make(map[string]bool), } - linting := len(lint) > 0 && lint[0] - err := validateWorkflowFieldNames(wf.Spec.Templates) if err != nil { return errors.Errorf(errors.CodeBadRequest, "spec.templates%s", err.Error()) } - if linting { + if ctx.Lint { // if we are just linting we don't care if spec.arguments.parameters.XXX doesn't have an // explicit value. workflows without a default value is a desired use case err = validateArgumentsFieldNames("spec.arguments.", wf.Spec.Arguments) @@ -65,6 +79,14 @@ func ValidateWorkflow(wf *wfv1.Workflow, lint ...bool) error { for _, param := range ctx.wf.Spec.Arguments.Parameters { ctx.globalParams["workflow.parameters."+param.Name] = placeholderValue } + + for k := range ctx.wf.ObjectMeta.Annotations { + ctx.globalParams["workflow.annotations."+k] = placeholderValue + } + for k := range ctx.wf.ObjectMeta.Labels { + ctx.globalParams["workflow.labels."+k] = placeholderValue + } + if ctx.wf.Spec.Entrypoint == "" { return errors.New(errors.CodeBadRequest, "spec.entrypoint is required") } @@ -110,6 +132,19 @@ func (ctx *wfValidationCtx) validateTemplate(tmpl *wfv1.Template, args wfv1.Argu localParams[common.LocalVarPodName] = placeholderValue scope[common.LocalVarPodName] = placeholderValue } + if tmpl.IsLeaf() { + for _, art := range tmpl.Outputs.Artifacts { + if art.Path != "" { + scope[fmt.Sprintf("outputs.artifacts.%s.path", art.Name)] = true + } + } + for _, param := range tmpl.Outputs.Parameters { + if param.ValueFrom != nil && param.ValueFrom.Path != "" { + scope[fmt.Sprintf("outputs.parameters.%s.path", param.Name)] = true + } + } + } + _, err = common.ProcessArgs(tmpl, args, ctx.globalParams, localParams, true) if err != nil { return errors.Errorf(errors.CodeBadRequest, "templates.%s %s", tmpl.Name, err) @@ -132,6 +167,16 @@ func (ctx *wfValidationCtx) validateTemplate(tmpl *wfv1.Template, args wfv1.Argu if err != nil { return err } + err = ctx.validateBaseImageOutputs(tmpl) + if err != nil { + return err + } + if tmpl.ArchiveLocation != nil { + err = validateArtifactLocation("templates.archiveLocation", *tmpl.ArchiveLocation) + if err != nil { + return err + } + } return nil } @@ -174,6 +219,7 @@ func validateInputs(tmpl *wfv1.Template) (map[string]interface{}, error) { if art.Path == "" { return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path not specified", tmpl.Name, artRef) } + scope[fmt.Sprintf("inputs.artifacts.%s.path", art.Name)] = true } else { if art.Path != "" { return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path only valid in container/script templates", tmpl.Name, artRef) @@ -183,7 +229,7 @@ func validateInputs(tmpl *wfv1.Template) (map[string]interface{}, error) { return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.from not valid in inputs", tmpl.Name, artRef) } errPrefix := fmt.Sprintf("templates.%s.%s", tmpl.Name, artRef) - err = validateArtifactLocation(errPrefix, art) + err = validateArtifactLocation(errPrefix, art.ArtifactLocation) if err != nil { return nil, err } @@ -191,12 +237,18 @@ func validateInputs(tmpl *wfv1.Template) (map[string]interface{}, error) { return scope, nil } -func validateArtifactLocation(errPrefix string, art wfv1.Artifact) error { +func validateArtifactLocation(errPrefix string, art wfv1.ArtifactLocation) error { if art.Git != nil { if art.Git.Repo == "" { return errors.Errorf(errors.CodeBadRequest, "%s.git.repo is required", errPrefix) } } + if art.HDFS != nil { + err := hdfs.ValidateArtifact(fmt.Sprintf("%s.hdfs", errPrefix), art.HDFS) + if err != nil { + return err + } + } // TODO: validate other artifact locations return nil } @@ -208,6 +260,11 @@ func resolveAllVariables(scope map[string]interface{}, tmplStr string) error { fstTmpl := fasttemplate.New(tmplStr, "{{", "}}") fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + + // Skip the custom variable references + if !checkValidWorkflowVariablePrefix(tag) { + return 0, nil + } _, ok := scope[tag] if !ok && unresolvedErr == nil { if (tag == "item" || strings.HasPrefix(tag, "item.")) && allowAllItemRefs { @@ -223,6 +280,16 @@ func resolveAllVariables(scope map[string]interface{}, tmplStr string) error { return unresolvedErr } +// checkValidWorkflowVariablePrefix is a helper methood check variable starts workflow root elements +func checkValidWorkflowVariablePrefix(tag string) bool { + for _, rootTag := range common.GlobalVarValidWorkflowVariablePrefix { + if strings.HasPrefix(tag, rootTag) { + return true + } + } + return false +} + func validateNonLeaf(tmpl *wfv1.Template) error { if tmpl.ActiveDeadlineSeconds != nil { return errors.Errorf(errors.CodeBadRequest, "templates.%s.activeDeadlineSeconds is only valid for leaf templates", tmpl.Name) @@ -464,7 +531,7 @@ func validateOutputs(scope map[string]interface{}, tmpl *wfv1.Template) error { } } if art.GlobalName != "" && !isParameter(art.GlobalName) { - errs := isValidWorkflowFieldName(art.GlobalName) + errs := isValidParamOrArtifactName(art.GlobalName) if len(errs) > 0 { return errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.globalName: %s", tmpl.Name, artRef, errs[0]) } @@ -492,7 +559,7 @@ func validateOutputs(scope map[string]interface{}, tmpl *wfv1.Template) error { } } if param.GlobalName != "" && !isParameter(param.GlobalName) { - errs := isValidWorkflowFieldName(param.GlobalName) + errs := isValidParamOrArtifactName(param.GlobalName) if len(errs) > 0 { return errors.Errorf(errors.CodeBadRequest, "%s.globalName: %s", paramRef, errs[0]) } @@ -501,6 +568,51 @@ func validateOutputs(scope map[string]interface{}, tmpl *wfv1.Template) error { return nil } +// validateBaseImageOutputs detects if the template contains an output from +func (ctx *wfValidationCtx) validateBaseImageOutputs(tmpl *wfv1.Template) error { + switch ctx.ContainerRuntimeExecutor { + case "", common.ContainerRuntimeExecutorDocker: + // docker executor supports all modes of artifact outputs + case common.ContainerRuntimeExecutorPNS: + // pns supports copying from the base image, but only if there is no volume mount underneath it + errMsg := "pns executor does not support outputs from base image layer with volume mounts. must use emptyDir" + for _, out := range tmpl.Outputs.Artifacts { + if common.FindOverlappingVolume(tmpl, out.Path) == nil { + // output is in the base image layer. need to verify there are no volume mounts under it + if tmpl.Container != nil { + for _, volMnt := range tmpl.Container.VolumeMounts { + if strings.HasPrefix(volMnt.MountPath, out.Path+"/") { + return errors.Errorf(errors.CodeBadRequest, "templates.%s.outputs.artifacts.%s: %s", tmpl.Name, out.Name, errMsg) + } + } + + } + if tmpl.Script != nil { + for _, volMnt := range tmpl.Container.VolumeMounts { + if strings.HasPrefix(volMnt.MountPath, out.Path+"/") { + return errors.Errorf(errors.CodeBadRequest, "templates.%s.outputs.artifacts.%s: %s", tmpl.Name, out.Name, errMsg) + } + } + } + } + } + case common.ContainerRuntimeExecutorK8sAPI, common.ContainerRuntimeExecutorKubelet: + // for kubelet/k8s fail validation if we detect artifact is copied from base image layer + errMsg := fmt.Sprintf("%s executor does not support outputs from base image layer. must use emptyDir", ctx.ContainerRuntimeExecutor) + for _, out := range tmpl.Outputs.Artifacts { + if common.FindOverlappingVolume(tmpl, out.Path) == nil { + return errors.Errorf(errors.CodeBadRequest, "templates.%s.outputs.artifacts.%s: %s", tmpl.Name, out.Name, errMsg) + } + } + for _, out := range tmpl.Outputs.Parameters { + if out.ValueFrom != nil && common.FindOverlappingVolume(tmpl, out.ValueFrom.Path) == nil { + return errors.Errorf(errors.CodeBadRequest, "templates.%s.outputs.parameters.%s: %s", tmpl.Name, out.Name, errMsg) + } + } + } + return nil +} + // validateOutputParameter verifies that only one of valueFrom is defined in an output func validateOutputParameter(paramRef string, param *wfv1.Parameter) error { if param.ValueFrom == nil { @@ -556,7 +668,14 @@ func validateWorkflowFieldNames(slice interface{}) error { if name == "" { return errors.Errorf(errors.CodeBadRequest, "[%d].name is required", i) } - if errs := isValidWorkflowFieldName(name); len(errs) != 0 { + var errs []string + t := reflect.TypeOf(item) + if t == reflect.TypeOf(wfv1.Parameter{}) || t == reflect.TypeOf(wfv1.Artifact{}) { + errs = isValidParamOrArtifactName(name) + } else { + errs = isValidWorkflowFieldName(name) + } + if len(errs) != 0 { return errors.Errorf(errors.CodeBadRequest, "[%d].name: '%s' is invalid: %s", i, name, strings.Join(errs, ";")) } _, ok := names[name] @@ -715,13 +834,22 @@ func verifyNoCycles(tmpl *wfv1.Template, nameToTask map[string]wfv1.DAGTask) err var ( // paramRegex matches a parameter. e.g. {{inputs.parameters.blah}} - paramRegex = regexp.MustCompile(`{{[-a-zA-Z0-9]+(\.[-a-zA-Z0-9]+)*}}`) + paramRegex = regexp.MustCompile(`{{[-a-zA-Z0-9]+(\.[-a-zA-Z0-9_]+)*}}`) + paramOrArtifactNameRegex = regexp.MustCompile(`^[-a-zA-Z0-9_]+[-a-zA-Z0-9_]*$`) ) func isParameter(p string) bool { return paramRegex.MatchString(p) } +func isValidParamOrArtifactName(p string) []string { + var errs []string + if !paramOrArtifactNameRegex.MatchString(p) { + return append(errs, "Parameter/Artifact name must consist of alpha-numeric characters, '_' or '-' e.g. my_param_1, MY-PARAM-1") + } + return errs +} + const ( workflowFieldNameFmt string = "[a-zA-Z0-9][-a-zA-Z0-9]*" workflowFieldNameErrMsg string = "name must consist of alpha-numeric characters or '-', and must start with an alpha-numeric character" diff --git a/workflow/validate/validate_test.go b/workflow/validate/validate_test.go index 558a24717487..e3787c4ac3ba 100644 --- a/workflow/validate/validate_test.go +++ b/workflow/validate/validate_test.go @@ -3,17 +3,19 @@ package validate import ( "testing" - wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" - "github.com/cyrusbiotechnology/argo/test" "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" + + wfv1 "github.com/cyrusbiotechnology/argo/pkg/apis/workflow/v1alpha1" + "github.com/cyrusbiotechnology/argo/test" + "github.com/cyrusbiotechnology/argo/workflow/common" ) // validate is a test helper to accept YAML as a string and return // its validation result. func validate(yamlStr string) error { wf := unmarshalWf(yamlStr) - return ValidateWorkflow(wf) + return ValidateWorkflow(wf, ValidateOpts{}) } func unmarshalWf(yamlStr string) *wfv1.Workflow { @@ -163,6 +165,76 @@ func TestUnresolved(t *testing.T) { } } +var ioArtifactPaths = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: artifact-path-placeholders- +spec: + entrypoint: head-lines + arguments: + parameters: + - name: lines-count + value: 3 + artifacts: + - name: text + raw: + data: | + 1 + 2 + 3 + 4 + 5 + templates: + - name: head-lines + inputs: + parameters: + - name: lines-count + artifacts: + - name: text + path: /inputs/text/data + outputs: + parameters: + - name: actual-lines-count + valueFrom: + path: /outputs/actual-lines-count/data + artifacts: + - name: text + path: /outputs/text/data + container: + image: busybox + command: [sh, -c, 'head -n {{inputs.parameters.lines-count}} <"{{inputs.artifacts.text.path}}" | tee "{{outputs.artifacts.text.path}}" | wc -l > "{{outputs.parameters.actual-lines-count.path}}"'] +` + +func TestResolveIOArtifactPathPlaceholders(t *testing.T) { + err := validate(ioArtifactPaths) + assert.Nil(t, err) +} + +var outputParameterPath = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: get-current-date- +spec: + entrypoint: get-current-date + templates: + - name: get-current-date + outputs: + parameters: + - name: current-date + valueFrom: + path: /tmp/current-date + container: + image: busybox + command: [sh, -c, 'date > {{outputs.parameters.current-date.path}}'] +` + +func TestResolveOutputParameterPathPlaceholder(t *testing.T) { + err := validate(outputParameterPath) + assert.Nil(t, err) +} + var stepOutputReferences = ` apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -322,9 +394,7 @@ spec: func TestInvalidArgParamName(t *testing.T) { err := validate(invalidArgParamNames) - if assert.NotNil(t, err) { - assert.Contains(t, err.Error(), invalidErr) - } + assert.NotNil(t, err) } var invalidArgArtNames = ` @@ -336,7 +406,7 @@ spec: entrypoint: kubectl-input-artifact arguments: artifacts: - - name: -kubectl + - name: "&-kubectl" http: url: https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubectl @@ -344,7 +414,7 @@ spec: - name: kubectl-input-artifact inputs: artifacts: - - name: -kubectl + - name: "&-kubectl" path: /usr/local/bin/kubectl mode: 0755 container: @@ -423,7 +493,7 @@ spec: container: image: docker/whalesay command: [cowsay] - args: ["{{inputs.parameters.message}}"] + args: ["{{inputs.parameters.message+123}}"] ` func TestInvalidInputParamName(t *testing.T) { @@ -500,7 +570,7 @@ spec: args: ["cowsay hello world | tee /tmp/hello_world.txt"] outputs: artifacts: - - name: __1 + - name: "!1" path: /tmp/hello_world.txt ` @@ -751,13 +821,13 @@ spec: func TestVolumeMountArtifactPathCollision(t *testing.T) { // ensure we detect and reject path collisions wf := unmarshalWf(volumeMountArtifactPathCollision) - err := ValidateWorkflow(wf) + err := ValidateWorkflow(wf, ValidateOpts{}) if assert.NotNil(t, err) { assert.Contains(t, err.Error(), "already mounted") } // tweak the mount path and validation should now be successful wf.Spec.Templates[0].Container.VolumeMounts[0].MountPath = "/differentpath" - err = ValidateWorkflow(wf) + err = ValidateWorkflow(wf, ValidateOpts{}) assert.Nil(t, err) } @@ -1043,7 +1113,7 @@ func TestPodNameVariable(t *testing.T) { } func TestGlobalParamWithVariable(t *testing.T) { - err := ValidateWorkflow(test.LoadE2EWorkflow("functional/global-outputs-variable.yaml")) + err := ValidateWorkflow(test.LoadE2EWorkflow("functional/global-outputs-variable.yaml"), ValidateOpts{}) assert.Nil(t, err) } @@ -1068,12 +1138,47 @@ spec: // TestSpecArgumentNoValue we allow parameters to have no value at the spec level during linting func TestSpecArgumentNoValue(t *testing.T) { wf := unmarshalWf(specArgumentNoValue) - err := ValidateWorkflow(wf, true) + err := ValidateWorkflow(wf, ValidateOpts{Lint: true}) assert.Nil(t, err) - err = ValidateWorkflow(wf) + err = ValidateWorkflow(wf, ValidateOpts{}) assert.NotNil(t, err) } +var specArgumentSnakeCase = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: spec-arg-snake-case- +spec: + entrypoint: whalesay + arguments: + artifacts: + - name: __kubectl + http: + url: https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubectl + parameters: + - name: my_snake_case_param + value: "hello world" + templates: + - name: whalesay + inputs: + artifacts: + - name: __kubectl + path: /usr/local/bin/kubectl + mode: 0755 + container: + image: docker/whalesay:latest + command: [sh, -c] + args: ["cowsay {{workflow.parameters.my_snake_case_param}} | tee /tmp/hello_world.txt && ls /usr/local/bin/kubectl"] +` + +// TestSpecArgumentSnakeCase we allow parameter and artifact names to be snake case +func TestSpecArgumentSnakeCase(t *testing.T) { + wf := unmarshalWf(specArgumentSnakeCase) + err := ValidateWorkflow(wf, ValidateOpts{Lint: true}) + assert.Nil(t, err) +} + var specBadSequenceCountAndEnd = ` apiVersion: argoproj.io/v1alpha1 kind: Workflow @@ -1100,12 +1205,154 @@ spec: container: image: alpine:latest command: [echo, "{{inputs.parameters.num}}"] - ` // TestSpecBadSequenceCountAndEnd verifies both count and end cannot be defined func TestSpecBadSequenceCountAndEnd(t *testing.T) { wf := unmarshalWf(specBadSequenceCountAndEnd) - err := ValidateWorkflow(wf, true) + err := ValidateWorkflow(wf, ValidateOpts{Lint: true}) assert.Error(t, err) } + +var customVariableInput = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hello-world- +spec: + entrypoint: whalesay + templates: + - name: whalesay + container: + image: docker/whalesay:{{user.username}} +` + +// TestCustomTemplatVariable verifies custom template variable +func TestCustomTemplatVariable(t *testing.T) { + wf := unmarshalWf(customVariableInput) + err := ValidateWorkflow(wf, ValidateOpts{Lint: true}) + assert.Equal(t, err, nil) +} + +var baseImageOutputArtifact = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: base-image-out-art- +spec: + entrypoint: base-image-out-art + templates: + - name: base-image-out-art + container: + image: alpine:latest + command: [echo, hello] + outputs: + artifacts: + - name: tmp + path: /tmp +` + +var baseImageOutputParameter = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: base-image-out-art- +spec: + entrypoint: base-image-out-art + templates: + - name: base-image-out-art + container: + image: alpine:latest + command: [echo, hello] + outputs: + parameters: + - name: tmp + valueFrom: + path: /tmp/file +` + +var volumeMountOutputArtifact = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: base-image-out-art- +spec: + entrypoint: base-image-out-art + volumes: + - name: workdir + emptyDir: {} + templates: + - name: base-image-out-art + container: + image: alpine:latest + command: [echo, hello] + volumeMounts: + - name: workdir + mountPath: /mnt/vol + outputs: + artifacts: + - name: workdir + path: /mnt/vol +` + +var baseImageDirWithEmptyDirOutputArtifact = ` +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: base-image-out-art- +spec: + entrypoint: base-image-out-art + volumes: + - name: workdir + emptyDir: {} + templates: + - name: base-image-out-art + container: + image: alpine:latest + command: [echo, hello] + volumeMounts: + - name: workdir + mountPath: /mnt/vol + outputs: + artifacts: + - name: workdir + path: /mnt +` + +// TestBaseImageOutputVerify verifies we error when we detect the condition when the container +// runtime executor doesn't support output artifacts from a base image layer, and fails validation +func TestBaseImageOutputVerify(t *testing.T) { + wfBaseOutArt := unmarshalWf(baseImageOutputArtifact) + wfBaseOutParam := unmarshalWf(baseImageOutputParameter) + wfEmptyDirOutArt := unmarshalWf(volumeMountOutputArtifact) + wfBaseWithEmptyDirOutArt := unmarshalWf(baseImageDirWithEmptyDirOutputArtifact) + var err error + + for _, executor := range []string{common.ContainerRuntimeExecutorK8sAPI, common.ContainerRuntimeExecutorKubelet, common.ContainerRuntimeExecutorPNS, common.ContainerRuntimeExecutorDocker, ""} { + switch executor { + case common.ContainerRuntimeExecutorK8sAPI, common.ContainerRuntimeExecutorKubelet: + err = ValidateWorkflow(wfBaseOutArt, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.Error(t, err) + err = ValidateWorkflow(wfBaseOutParam, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.Error(t, err) + err = ValidateWorkflow(wfBaseWithEmptyDirOutArt, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.Error(t, err) + case common.ContainerRuntimeExecutorPNS: + err = ValidateWorkflow(wfBaseOutArt, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.NoError(t, err) + err = ValidateWorkflow(wfBaseOutParam, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.NoError(t, err) + err = ValidateWorkflow(wfBaseWithEmptyDirOutArt, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.Error(t, err) + case common.ContainerRuntimeExecutorDocker, "": + err = ValidateWorkflow(wfBaseOutArt, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.NoError(t, err) + err = ValidateWorkflow(wfBaseOutParam, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.NoError(t, err) + err = ValidateWorkflow(wfBaseWithEmptyDirOutArt, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.NoError(t, err) + } + err = ValidateWorkflow(wfEmptyDirOutArt, ValidateOpts{ContainerRuntimeExecutor: executor}) + assert.NoError(t, err) + } +}