diff --git a/README.md b/README.md index 1a37a77c19..8ac92045a2 100644 --- a/README.md +++ b/README.md @@ -61,88 +61,50 @@ You can watch industry experts talking about Volcano in different International ## Quick Start Guide -The easiest way to deploy Volcano is to use the Helm chart. Volcano can be deployed by cloning code and also by adding helm repo. +### Prerequisites -## Using Volcano Helm Repo +- Kubernetes 1.12+ with CRD support -Add helm repo using following command, +### Install with YAML files -``` -helm repo add volcano https://volcano-sh.github.io/charts -``` - -Install Volcano using following command, - -``` -helm install volcano/volcano --namespace --name - -e.g : -helm install volcano/volcano --namespace volcano-trial --name volcano-trial -``` - -## Cloning Code -### Pre-requisites - -First of all, clone the repo to your local path: +Install volcano k8s resources ``` -# mkdir -p $GOPATH/src/volcano.sh/ -# cd $GOPATH/src/volcano.sh/ -# git clone --recursive https://github.com/volcano-sh/volcano.git +kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/master/installer/volcano-development.yaml ``` -### 1. Volcano Image - -Official images are available on [DockerHub](https://hub.docker.com/u/volcanosh), however you can -build them locally with the command: +Install `default-queue` for volcano scheduler, note that the crd resources should be ready before this. ``` -cd $GOPATH/src/volcano.sh/volcano -make images - -## Verify your images -# docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -volcanosh/vc-admission latest a83338506638 8 seconds ago 41.4MB -volcanosh/vc-scheduler latest faa3c2a25ac3 9 seconds ago 49.6MB -volcanosh/vc-controllers latest 7b11606ebfb8 10 seconds ago 44.2MB +kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/master/installer/helm/chart/volcano/templates/default-queue.yaml ``` -**NOTE**: -1. You need ensure the images are correctly loaded in your kubernetes cluster, for -example, if you are using [kind cluster](https://github.com/kubernetes-sigs/kind), -try command ```kind load docker-image : ``` for each of the images. -2. When reinstall the volcano charts, since tiller server will not manage CRD resource, -you need to delete them manually eg: `kubectl delete crds xxxx` before reinstalling or try command with `--no-crd-hook` option. - -### 2. Helm charts - -Secondly, install helm chart. - -``` -helm install installer/helm/chart/volcano --namespace --name - -e.g : -helm install installer/helm/chart/volcano --namespace volcano-trial --name volcano-trial - -``` +Enjoy! Volcano will create the following resources in `volcano-system` namespace. -To verify your installation run the following commands: ``` -#1. Verify the Running Pods -# kubectl get pods --namespace -NAME READY STATUS RESTARTS AGE --admission-84fd9b9dd8-9trxn 1/1 Running 0 43s --controllers-75dcc8ff89-42v6r 1/1 Running 0 43s --scheduler-b94cdb867-89pm2 1/1 Running 0 43s ---admission-init-qbtmb 0/1 Completed 0 43s - -#2. Verify the Services -# kubectl get services --namespace -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE --admission-service ClusterIP 10.105.78.53 443/TCP 91s +NAME READY STATUS RESTARTS AGE +pod/volcano-admission-5bd5756f79-dnr4l 1/1 Running 0 96s +pod/volcano-admission-init-4hjpx 0/1 Completed 0 96s +pod/volcano-controllers-687948d9c8-nw4b4 1/1 Running 0 96s +pod/volcano-scheduler-94998fc64-4z8kh 1/1 Running 0 96s + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/volcano-admission-service ClusterIP 10.98.152.108 443/TCP 96s + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/volcano-admission 1/1 1 1 96s +deployment.apps/volcano-controllers 1/1 1 1 96s +deployment.apps/volcano-scheduler 1/1 1 1 96s + +NAME DESIRED CURRENT READY AGE +replicaset.apps/volcano-admission-5bd5756f79 1 1 1 96s +replicaset.apps/volcano-controllers-687948d9c8 1 1 1 96s +replicaset.apps/volcano-scheduler-94998fc64 1 1 1 96s + +NAME COMPLETIONS DURATION AGE +job.batch/volcano-admission-init 1/1 48s 96s ``` diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml new file mode 100644 index 0000000000..2bb2911ee9 --- /dev/null +++ b/installer/volcano-development.yaml @@ -0,0 +1,802 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: volcano-system +--- +# Source: volcano/templates/scheduler.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: volcano-scheduler-configmap + namespace: volcano-system +data: + kube-batch-ci.conf: | + actions: "enqueue, reclaim, allocate, backfill, preempt" + tiers: + - plugins: + - name: priority + - name: gang + - name: conformance + - plugins: + - name: drf + - name: predicates + - name: proportion + - name: nodeorder + kube-batch.conf: | + actions: "enqueue, allocate, backfill" + tiers: + - plugins: + - name: priority + - name: gang + - name: conformance + - plugins: + - name: drf + - name: predicates + - name: proportion + - name: nodeorder + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: volcano-scheduler + namespace: volcano-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volcano-scheduler + namespace: volcano-system +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "update", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["pods", "pods/status"] + verbs: ["create", "get", "list", "watch", "update", "bind", "updateStatus", "delete"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["list", "watch"] + - apiGroups: ["scheduling.incubator.k8s.io"] + resources: ["podgroups"] + verbs: ["list", "watch", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["list", "watch"] + - apiGroups: ["scheduling.incubator.k8s.io"] + resources: ["queues"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["scheduling.sigs.dev"] + resources: ["queues"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["scheduling.sigs.dev"] + resources: ["podgroups"] + verbs: ["list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volcano-scheduler-role + namespace: volcano-system +subjects: + - kind: ServiceAccount + name: volcano-scheduler + namespace: volcano-system +roleRef: + kind: ClusterRole + name: volcano-scheduler + apiGroup: rbac.authorization.k8s.io + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: volcano-scheduler + namespace: volcano-system + labels: + app: volcano-scheduler +spec: + replicas: 1 + selector: + matchLabels: + app: volcano-scheduler + template: + metadata: + labels: + app: volcano-scheduler + spec: + serviceAccount: volcano-scheduler + + containers: + - name: volcano-scheduler + image: volcanosh/vc-scheduler:latest + args: + - --alsologtostderr + - --scheduler-conf=/volcano.scheduler/kube-batch-ci.conf + - -v=3 + - 2>&1 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: scheduler-config + mountPath: /volcano.scheduler + volumes: + - name: scheduler-config + configMap: + name: volcano-scheduler-configmap + +--- +# Source: volcano/templates/admission.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: volcano-admission + namespace: volcano-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volcano-admission + namespace: volcano-system +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "create", "update"] + # Rules below is used generate admission service secret + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/approval"] + verbs: ["create", "update"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "patch"] + - apiGroups: ["scheduling.incubator.k8s.io"] + resources: ["queues"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volcano-admission-role + namespace: volcano-system +subjects: + - kind: ServiceAccount + name: volcano-admission + namespace: volcano-system +roleRef: + kind: ClusterRole + name: volcano-admission + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: volcano-admission + name: volcano-admission + namespace: volcano-system +spec: + replicas: 1 + selector: + matchLabels: + app: volcano-admission + template: + metadata: + labels: + app: volcano-admission + spec: + serviceAccount: volcano-admission + + containers: + - args: + - --tls-cert-file=/admission.local.config/certificates/tls.crt + - --tls-private-key-file=/admission.local.config/certificates/tls.key + - --ca-cert-file=/admission.local.config/certificates/ca.crt + - --webhook-namespace=volcano-system + - --webhook-service-name=volcano-admission-service + - --alsologtostderr + - --port=443 + - -v=4 + - 2>&1 + image: volcanosh/vc-admission:latest + imagePullPolicy: IfNotPresent + name: admission + volumeMounts: + - mountPath: /admission.local.config/certificates + name: admission-certs + readOnly: true + volumes: + - name: admission-certs + secret: + defaultMode: 420 + secretName: volcano-admission-secret + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: volcano-admission + name: volcano-admission-service + namespace: volcano-system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + app: volcano-admission + sessionAffinity: None + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: volcano-admission-init + namespace: volcano-system + labels: + app: volcano-admission-init +spec: + backoffLimit: 3 + template: + spec: + serviceAccountName: volcano-admission + restartPolicy: Never + containers: + - name: main + image: volcanosh/vc-admission:latest + imagePullPolicy: IfNotPresent + command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace", + "volcano-system", "--secret", "volcano-admission-secret"] + +--- +# Source: volcano/templates/controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: volcano-controllers + namespace: volcano-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volcano-controllers + namespace: volcano-system +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "delete", "update"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "update", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs/status"] + verbs: ["update", "patch"] + - apiGroups: ["bus.volcano.sh"] + resources: ["commands"] + verbs: ["get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "update", "bind", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["scheduling.incubator.k8s.io"] + resources: ["podgroups", "queues", "queues/status"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["get", "list", "watch", "create", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volcano-controllers-role + namespace: volcano-system +subjects: + - kind: ServiceAccount + name: volcano-controllers + namespace: volcano-system +roleRef: + kind: ClusterRole + name: volcano-controllers + apiGroup: rbac.authorization.k8s.io + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: volcano-controllers + namespace: volcano-system + labels: + app: volcano-controller +spec: + replicas: 1 + selector: + matchLabels: + app: volcano-controller + template: + metadata: + labels: + app: volcano-controller + spec: + serviceAccount: volcano-controllers + + containers: + - name: volcano-controllers + image: volcanosh/vc-controllers:latest + args: + - --alsologtostderr + - -v=4 + - 2>&1 + imagePullPolicy: "IfNotPresent" + +--- +# Source: volcano/templates/batch_v1alpha1_job.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: jobs.batch.volcano.sh + annotations: + "helm.sh/hook": crd-install +spec: + group: batch.volcano.sh + names: + kind: Job + plural: jobs + shortNames: + - vcjob + - vj + scope: Namespaced + validation: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of a cron job, including + the minAvailable + properties: + volumes: + description: The volumes for Job + items: + properties: + volumeClaim: + description: VolumeClaim defines the PVC used by the VolumeMount. + type: object + mountPath: + description: Path within the container at which the volume should be mounted. + Must not contain ':'. + type: string + volumeClaimName: + description: The name of the volume claim. + type: object + required: + - mountPath + type: array + minAvailable: + description: The minimal available pods to run for this Job + format: int32 + type: integer + policies: + description: Specifies the default lifecycle of tasks + items: + properties: + action: + description: The action that will be taken to the PodGroup according + to Event. One of "Restart", "None". Default to None. + type: string + event: + description: The Event recorded by scheduler; the controller takes + actions according to this Event. + type: string + events: + description: The Events recorded by scheduler; the controller takes + actions according to this Events. + type: array + timeout: + description: Timeout is the grace period for controller to take + actions. Default to nil (take action immediately). + type: object + type: object + type: array + schedulerName: + description: SchedulerName is the default value of `tasks.template.spec.schedulerName`. + type: string + plugins: + description: Enabled task plugins when creating job. + type: object + additionalProperties: + type: array + tasks: + description: Tasks specifies the task specification of Job + items: + properties: + name: + description: Name specifies the name of tasks + type: string + policies: + description: Specifies the lifecycle of task + items: + properties: + action: + description: The action that will be taken to the PodGroup + according to Event. One of "Restart", "None". Default + to None. + type: string + event: + description: The Event recorded by scheduler; the controller + takes actions according to this Event. + type: string + events: + description: The Events recorded by scheduler; the controller takes + actions according to this Events. + type: array + timeout: + description: Timeout is the grace period for controller + to take actions. Default to nil (take action immediately). + type: object + type: object + type: array + replicas: + description: Replicas specifies the replicas of this TaskSpec + in Job + format: int32 + type: integer + template: + description: Specifies the pod that will be created for this TaskSpec + when executing a Job + type: object + type: object + type: array + queue: + description: The name of the queue on which job should been created + type: string + maxRetry: + description: The limit for retrying submiting job, default is 3 + format: int32 + type: integer + type: object + status: + description: Current status of Job + properties: + Succeeded: + description: The number of pods which reached phase Succeeded. + format: int32 + type: integer + failed: + description: The number of pods which reached phase Failed. + format: int32 + type: integer + minAvailable: + description: The minimal available pods to run for this Job + format: int32 + type: integer + pending: + description: The number of pending pods. + format: int32 + type: integer + running: + description: The number of running pods. + format: int32 + type: integer + version: + description: Job's current version + format: int32 + type: integer + retryCount: + description: The number that volcano retried to submit the job. + format: int32 + type: integer + ControlledResources: + description: All of the resources that are controlled by this job. + type: object + additionalProperties: + type: string + state: + description: Current state of Job. + properties: + message: + description: Human-readable message indicating details about last + transition. + type: string + phase: + description: The phase of Job + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + type: object + type: object + version: v1alpha1 + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +# Source: volcano/templates/bus_v1alpha1_command.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: commands.bus.volcano.sh + annotations: + "helm.sh/hook": crd-install +spec: + group: bus.volcano.sh + names: + kind: Command + plural: commands + scope: Namespaced + validation: + openAPIV3Schema: + properties: + action: + description: Action defines the action that will be took to the target object. + type: string + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + message: + description: Human-readable message indicating details of this command. + type: string + metadata: + type: object + reason: + description: Unique, one-word, CamelCase reason for this command. + type: string + target: + description: TargetObject defines the target object of this command. + type: object + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +# Source: volcano/templates/scheduling_v1alpha1_podgroup.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: podgroups.scheduling.incubator.k8s.io + annotations: + "helm.sh/hook": crd-install +spec: + group: scheduling.incubator.k8s.io + names: + kind: PodGroup + plural: podgroups + scope: Namespaced + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + minMember: + format: int32 + type: integer + type: object + status: + properties: + succeeded: + format: int32 + type: integer + failed: + format: int32 + type: integer + running: + format: int32 + type: integer + type: object + type: object + version: v1alpha1 + +--- +# Source: volcano/templates/scheduling_v1alpha1_queue.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: queues.scheduling.incubator.k8s.io + annotations: + "helm.sh/hook": crd-install +spec: + group: scheduling.incubator.k8s.io + names: + kind: Queue + plural: queues + scope: Cluster + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + weight: + format: int32 + type: integer + type: object + type: object + version: v1alpha1 + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +# Source: volcano/templates/scheduling_v1alpha2_podgroup.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: podgroups.scheduling.sigs.dev +spec: + group: scheduling.sigs.dev + names: + kind: PodGroup + plural: podgroups + scope: Namespaced + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + minMember: + format: int32 + type: integer + queue: + type: string + priorityClassName: + type: string + type: object + status: + properties: + succeeded: + format: int32 + type: integer + failed: + format: int32 + type: integer + running: + format: int32 + type: integer + type: object + type: object + version: v1alpha2 + +--- +# Source: volcano/templates/scheduling_v1alpha2_queue.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: queues.scheduling.sigs.dev +spec: + group: scheduling.sigs.dev + names: + kind: Queue + plural: queues + scope: Cluster + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + weight: + format: int32 + type: integer + type: object + status: + properties: + unknown: + format: int32 + type: integer + pending: + format: int32 + type: integer + running: + format: int32 + type: integer + type: object + type: object + version: v1alpha2