diff --git a/DRAFT_RELEASE_NOTES.md b/DRAFT_RELEASE_NOTES.md index 3d60110b6..5f1ba2611 100644 --- a/DRAFT_RELEASE_NOTES.md +++ b/DRAFT_RELEASE_NOTES.md @@ -71,8 +71,25 @@ To start your aiSSEMBLE upgrade, update your project's pom.xml to use the 1.10.0 ``` +### Split Data Records for the Spark Pipeline +If your spark pipeline is using `aissemble-data-records-separate-module` profile for your data records, you must add the `` tag for +the `jackson-mapper-asl` dependency artifact in the root pom.xml file to enable the build. +```xml + + org.codehaus.jackson + jackson-mapper-asl + + ${version.jackson.mapper.asl} + +``` + + ## Conditional Steps +### Spark Operator Chart +The service account for pipeline invocation service is separated from spark operator. It's now configured solely for the pipeline invocation chart. +If you have custom configuration for the `spark-operator`, you will need to migrate the related changes to the `pipeline-invocation-service`. +Refer to Pipeline Invocation Helm Chart [README](https://github.com/boozallen/aissemble/blob/dev/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md) for detail. + ## Final Steps - Required for All Projects ### Finalizing the Upgrade 1. Run `./mvnw org.technologybrewery.baton:baton-maven-plugin:baton-migrate` to apply the automatic migrations diff --git a/build-parent/pom.xml b/build-parent/pom.xml index 180281ff3..4a73a8df7 100644 --- a/build-parent/pom.xml +++ b/build-parent/pom.xml @@ -67,6 +67,7 @@ 4.0.3 3.5.1 1.9.3 + 3.4.1 4.0.1 diff --git a/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile index 09eac4ecf..5d99ae2bd 100644 --- a/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-pipeline-invocation/src/main/resources/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi9/openjdk-11-runtime:1.20 AS builder +FROM registry.access.redhat.com/ubi9/openjdk-17-runtime:1.20 AS builder USER root RUN microdnf install -y openssl gzip && \ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash diff --git a/extensions/extensions-helm/aissemble-spark-application-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml b/extensions/extensions-helm/aissemble-spark-application-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml deleted file mode 100644 index d34353275..000000000 --- a/extensions/extensions-helm/aissemble-spark-application-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml +++ /dev/null @@ -1,4427 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - api-approved.kubernetes.io: https://github.com/kubeflow/spark-operator/pull/1298 - "helm.sh/hook": pre-install - name: sparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: SparkApplication - listKind: SparkApplicationList - plural: sparkapplications - shortNames: - - sparkapp - singular: sparkapplication - scope: Namespaced - versions: - - name: v1beta2 - served: true - storage: true - subresources: - status: { } - additionalPrinterColumns: - - jsonPath: .status.applicationState.state - name: Status - type: string - - jsonPath: .status.executionAttempts - name: Attempts - type: string - - jsonPath: .status.lastSubmissionAttemptTime - name: Start - type: string - - jsonPath: .status.terminationTime - name: Finish - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - deps: - properties: - excludePackages: - items: - type: string - type: array - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - packages: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - repositories: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostAliases: - items: - properties: - hostnames: - items: - type: string - type: array - ip: - type: string - type: object - type: array - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - kubernetesMaster: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - podSecurityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - serviceAccount: - type: string - serviceAnnotations: - additionalProperties: - type: string - type: object - shareProcessNamespace: - type: boolean - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - dynamicAllocation: - properties: - enabled: - type: boolean - initialExecutors: - format: int32 - type: integer - maxExecutors: - format: int32 - type: integer - minExecutors: - format: int32 - type: integer - shuffleTrackingTimeout: - format: int64 - type: integer - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostAliases: - items: - properties: - hostnames: - items: - type: string - type: array - ip: - type: string - type: object - type: array - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podSecurityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - serviceAccount: - type: string - shareProcessNamespace: - type: boolean - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - windowsOptions: - properties: - gmsaCredentialSpec: - type: string - gmsaCredentialSpecName: - type: string - runAsUserName: - type: string - type: object - type: object - startupProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - subPathExpr: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - portName: - type: string - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - proxyUser: - type: string - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkUIOptions: - properties: - serviceAnnotations: - additionalProperties: - type: string - type: object - ingressAnnotations: - additionalProperties: - type: string - type: object - ingressTLS: - items: - properties: - hosts: - items: - type: string - type: array - secretName: - type: string - type: object - type: array - servicePort: - format: int32 - type: integer - servicePortName: - type: string - serviceType: - type: string - type: object - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - csi: - properties: - driver: - type: string - fsType: - type: string - nodePublishSecretRef: - properties: - name: - type: string - type: object - readOnly: - type: boolean - volumeAttributes: - additionalProperties: - type: string - type: object - required: - - driver - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - properties: - volumeClaimTemplate: - properties: - spec: - properties: - accessModes: - items: - type: string - type: array - resources: - properties: - requests: - properties: - storage: - type: string - type: object - type: object - storageClassName: - type: string - type: object - type: object - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - tenant: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - status: - properties: - applicationState: - properties: - errorMessage: - type: string - state: - type: string - required: - - state - type: object - driverInfo: - properties: - podName: - type: string - webUIAddress: - type: string - webUIIngressAddress: - type: string - webUIIngressName: - type: string - webUIPort: - format: int32 - type: integer - webUIServiceName: - type: string - type: object - executionAttempts: - format: int32 - type: integer - executorState: - additionalProperties: - type: string - type: object - lastSubmissionAttemptTime: - format: date-time - nullable: true - type: string - sparkApplicationId: - type: string - submissionAttempts: - format: int32 - type: integer - submissionID: - type: string - terminationTime: - format: date-time - nullable: true - type: string - required: - - driverInfo - type: object - required: - - metadata - - spec - type: object -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/README.md b/extensions/extensions-helm/aissemble-spark-operator-chart/README.md index 96dd29a76..696a5a44d 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/README.md +++ b/extensions/extensions-helm/aissemble-spark-operator-chart/README.md @@ -50,20 +50,6 @@ aissemble-spark-operator-chart: | serviceAccounts.sparkoperator.name | Name for the spark service account | No | sparkoperator | | podSecurityContext | Pod security context | No | runAsUser: 185
runAsGroup: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch" | -## Migrated Properties -The following properties have been migrated from the `spark-operator` subchart to the `aissemble-spark-operator-chart` chart. -Any required overrides should be cognisant of the alternate path. For example: - -```yaml -aissemble-spark-operator-chart: - rbac: - createClusterRole: false -``` - -| Property | Description | Default | -|------------------------|-------------------------------------------------------------------------------|---------| -| rbac.createClusterRole | Create and use RBAC `ClusterRole` resources. Migrated to use modified rules. | true | - # Shared Ivy Cache Spark uses [Ivy](https://ant.apache.org/ivy/) to resolve and download dependencies for Spark applications. By default, diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml b/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml index b56db396d..17a671f76 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml +++ b/extensions/extensions-helm/aissemble-spark-operator-chart/values.template.yaml @@ -29,12 +29,6 @@ spark-operator: runAsGroup: 1000 fsGroup: 1000 fsGroupChangePolicy: "OnRootMismatch" - - rbac: - # -- Create and use RBAC `ClusterRole` resources - # -- Set to false in order to enable overriding with our own RBAC template - createClusterRole: false - # volumes - Operator volumes volumes: - name: spark-logging @@ -61,8 +55,4 @@ spark-operator: sparkoperator: # -- Optional name for the operator service account - name: "sparkoperator" - -rbac: - # -- Set to True in order to enable overriding with our own RBAC template - createClusterRole: True \ No newline at end of file + name: "sparkoperator" \ No newline at end of file diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md index 443a7850c..29ee313dd 100644 --- a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/README.md @@ -10,16 +10,18 @@ helm install pipeline-invocation-service oci://ghcr.io/boozallen/aissemble-pipel **Note**: *the version should match the aiSSEMBLE project version.* # Properties -| Property | Description | Required Override | Default | -|------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------| -| ingress.apiVersion | k8s API version to use | No | networking.k8s.io/v1 | -| ingress.enabled | k8s Whether to enable ingress | No | false | -| ingress.kind | Type of kubernetes entity | No | Ingress | -| ingress.metadata.name | Name of the ingress | No | pipeline-invocation-service-web | -| ingress.metadata.annotations.kubernetes.io/ingress.class | Ingress class name | No | nginx | -| ingress.metadata.annotations.ingress.metadata.annotations.nginx.ingress.kubernetes.io/server-snippet | Custom configurations for the nginx ingress class | No | gunzip on; gzip on; gzip_proxied any; gzip_types *; | -| ingress.spec.rules.hosts | A list of hosts for ingress to support, each with their own path definition | No | | -| ingress.status | Load balancer IP if required | No | None | +| Property | Description | Required Override | Default | +|------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|-------------------|---------------------------------------------------| +| ingress.apiVersion | k8s API version to use | No | networking.k8s.io/v1 | +| ingress.enabled | k8s Whether to enable ingress | No | false | +| ingress.kind | Type of kubernetes entity | No | Ingress | +| ingress.metadata.name | Name of the ingress | No | pipeline-invocation-service-web | +| ingress.metadata.annotations.kubernetes.io/ingress.class | Ingress class name | No | nginx | +| ingress.metadata.annotations.ingress.metadata.annotations.nginx.ingress.kubernetes.io/server-snippet | Custom configurations for the nginx ingress class | No | gunzip on; gzip on; gzip_proxied any; gzip_types *; | +| ingress.spec.rules.hosts | A list of hosts for ingress to support, each with their own path definition | No | | +| ingress.status | Load balancer IP if required | No | None | +| rbac.createClusterRole | Create and use RBAC `ClusterRole` resources. | No | true | +| rbac.serviceAccountName | The service account name used for the RBAC. This value must be the same as the `deployment.serviceAccountName` | No | pipeline-ivocation-service | # Quarkus Configuration @@ -30,3 +32,15 @@ The following configuration of the service is provided. Additional configuratio |---------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------| | kafka.bootstrap.servers | Specifies the kafka bootstrap server when using kafka for messaging | Any valid URI | | mp.messaging.incoming.pipeline-invocation.* | Specifies and configures the smallrye connector to use. Supported connectors are `smallrye-amqp`, `smallrye-kafka`, `smallrye-mqtt`, and `smallrye-rabbitmq` | See xref:messaging-details.adoc[the Messaging documentation] for more details | + + +## Migrated Properties +The service account for pipeline invocation service is separated from spark operator. It's now configured solely for the pipeline invocation chart. + +The following properties have been migrated from the `aissemble-spark-operator-chart` chart to the `aissemble-pipeline-invocation-app-chart` chart. +| Property | Description | Default | +|-------------------------|-------------------------------------------------------------------------------|----------------------------| +| rbac.createClusterRole | Create and use RBAC `ClusterRole` resources. | true | + + + diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/.gitkeep b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/rbac.yaml similarity index 74% rename from extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml rename to extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/rbac.yaml index 14d8d2746..a67141387 100644 --- a/extensions/extensions-helm/aissemble-spark-operator-chart/templates/rbac.yaml +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/templates/rbac.yaml @@ -1,24 +1,28 @@ {{- /* aiSSEMBLE Custom rbac.yaml -Required custom rbac.yaml file that grants the sparkoperator service account +Required custom rbac.yaml file that grants the pipeline-invocation-service service account create, delete, and update access to the apigroup apiextensions.k8s.io. This is necessary for the pipeline-invocation-service to create instances of the SparkApplication CRD to submit pipelines to the Spark Operator for execution. */}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.rbac.serviceAccountName | default "pipeline-invocation-service" }} + +--- -{{- if or .Values.rbac.create .Values.rbac.createClusterRole }} +{{ if .Values.rbac.createClusterRole }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "spark-operator.fullname" (index .Subcharts "spark-operator") }} + name: {{ .Values.rbac.serviceAccountName | default "pipeline-invocation-service" }}-clusterrole annotations: "helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-delete-policy": hook-failed, before-hook-creation "helm.sh/hook-weight": "-10" - labels: - {{- include "spark-operator.labels" (index .Subcharts "spark-operator") | nindent 4 }} rules: - apiGroups: - "" @@ -34,6 +38,7 @@ rules: - configmaps - secrets verbs: + - list - create - get - delete @@ -75,6 +80,7 @@ rules: resources: - customresourcedefinitions verbs: + - create - get - apiGroups: - admissionregistration.k8s.io @@ -97,7 +103,6 @@ rules: - scheduledsparkapplications/finalizers verbs: - "*" - {{- if .Values.batchScheduler.enable }} # required for the `volcano` batch scheduler - apiGroups: - scheduling.incubator.k8s.io @@ -107,34 +112,29 @@ rules: - podgroups verbs: - "*" - {{- end }} - {{ if .Values.webhook.enable }} - apiGroups: - batch resources: - jobs verbs: - delete - {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "spark-operator.fullname" (index .Subcharts "spark-operator") }} + name: {{ .Values.rbac.serviceAccountName | default "pipeline-invocation-service" }}-clusterrole-binding annotations: "helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-delete-policy": hook-failed, before-hook-creation "helm.sh/hook-weight": "-10" - labels: - {{- include "spark-operator.labels" (index .Subcharts "spark-operator") | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ include "spark-operator.serviceAccountName" (index .Subcharts "spark-operator") }} + name: {{ .Values.rbac.serviceAccountName | default "pipeline-invocation-service" }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: {{ include "spark-operator.fullname" (index .Subcharts "spark-operator") }} + name: {{ .Values.rbac.serviceAccountName | default "pipeline-invocation-service" }}-clusterrole apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/rbac_test.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/rbac_test.yaml new file mode 100644 index 000000000..7033008a2 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/tests/rbac_test.yaml @@ -0,0 +1,97 @@ +suite: Pipeline Invocation Service RBAC +templates: + - rbac.yaml +tests: + - it: Should contain ServiceAccount document + documentIndex: 0 + asserts: + - containsDocument: + kind: ServiceAccount + apiVersion: v1 + - it: Should contain ClusterRole document + documentIndex: 1 + asserts: + - containsDocument: + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + - it: Should contain ClusterRoleBinding document + documentIndex: 2 + asserts: + - containsDocument: + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + - it: Should be 3 documents in total + asserts: + - hasDocuments: + count: 3 + - it: Still contains service account if options are disabled + set: + rbac: + createClusterRole: false + asserts: + - hasDocuments: + count: 1 + - containsDocument: + kind: ServiceAccount + apiVersion: v1 + - it: ServiceAccount should include appropriate default values + documentIndex: 0 + asserts: + - equal: + path: metadata.name + value: pipeline-invocation-service + - it: ClusterRole should include appropriate default values + documentIndex: 1 + asserts: + - equal: + path: metadata.name + value: pipeline-invocation-service-clusterrole + - it: ClusterRoleBinding should include appropriate default values + documentIndex: 2 + release: + namespace: default + asserts: + - equal: + path: metadata.name + value: pipeline-invocation-service-clusterrole-binding + - contains: + path: subjects + content: + kind: ServiceAccount + name: pipeline-invocation-service + namespace: default + - equal: + path: roleRef.kind + value: ClusterRole + - equal: + path: roleRef.name + value: pipeline-invocation-service-clusterrole + - equal: + path: roleRef.apiGroup + value: rbac.authorization.k8s.io + - it: Should set values appropriately for the service account + set: + rbac: + serviceAccountName: test + documentIndex: 0 + asserts: + - equal: + path: metadata.name + value: test + - it: Should set values appropriately for the cluster role binding + set: + rbac: + serviceAccountName: test + release: + namespace: default + documentIndex: 2 + asserts: + - contains: + path: subjects + content: + kind: ServiceAccount + name: test + namespace: default + - equal: + path: metadata.name + value: test-clusterrole-binding \ No newline at end of file diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml index 370b2eafc..697cc451d 100644 --- a/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/aissemble-pipeline-invocation-app-chart/values.yaml @@ -12,7 +12,7 @@ aissemble-quarkus-chart: containerPort: 9000 protocol: TCP restartPolicy: Always - serviceAccountName: sparkoperator + serviceAccountName: pipeline-invocation-service automountServiceAccountToken: true supplementalVolumeMounts: @@ -60,3 +60,8 @@ aissemble-quarkus-chart: name: pipeline-invocation-service port: number: 8080 +rbac: + createClusterRole: true + # Note: the if the rbac.serviceAccountName is changed, the aissemble-quarkus-chart.deployment.serviceAccountName + # needs to be changed to match the value + serviceAccountName: pipeline-invocation-service diff --git a/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml b/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml index 060812070..6f2f706b5 100644 --- a/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml +++ b/extensions/extensions-helm/extensions-helm-pipeline-invocation/extensions-helm-pipeline-invocation-lib/pom.xml @@ -23,7 +23,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} run tests diff --git a/extensions/extensions-helm/pom.xml b/extensions/extensions-helm/pom.xml index d9a437361..286d51b8e 100644 --- a/extensions/extensions-helm/pom.xml +++ b/extensions/extensions-helm/pom.xml @@ -199,7 +199,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} run tests @@ -250,7 +250,6 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 run tests diff --git a/extensions/extensions-pipeline-invocation-service/pom.xml b/extensions/extensions-pipeline-invocation-service/pom.xml index b4d84ddc8..5e5e0eeaa 100644 --- a/extensions/extensions-pipeline-invocation-service/pom.xml +++ b/extensions/extensions-pipeline-invocation-service/pom.xml @@ -41,18 +41,6 @@ io.quarkus quarkus-kubernetes-client - - io.quarkus - quarkus-smallrye-fault-tolerance - - - io.quarkus - quarkus-smallrye-metrics - - - io.quarkus - quarkus-smallrye-health - io.quarkus quarkus-smallrye-reactive-messaging-kafka @@ -83,7 +71,7 @@ io.quarkus - quarkus-rest-client-reactive-jackson + quarkus-resteasy-reactive-jackson org.apache.commons diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java index dd9a2b406..60799cbad 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/PipelineInvocationAgent.java @@ -101,7 +101,7 @@ protected List buildFinalValuesOverrides(PipelineInvocationRequest reque List args = new ArrayList<>(); args.add("--set"); - args.add("spec.serviceEnabled=false"); + args.add("service.enabled=false"); for(Map.Entry entry : request.getOverrideValues().entrySet()) { args.add("--set"); diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java index ab7a933a5..01e7a0cea 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/HttpEndpoint.java @@ -30,7 +30,7 @@ @Path("/invoke-pipeline") @ApplicationScoped public class HttpEndpoint { - private static final Logger logger = Logger.getLogger(MessageEndpoint.class); + private static final Logger logger = Logger.getLogger(HttpEndpoint.class); @Inject private PipelineInvocationAgent pipelineInvocationAgent; diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java index 95dfce20d..ea7b495ab 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/endpoint/MessageEndpoint.java @@ -18,6 +18,9 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * Entrypoint for message-based requests for SparkApplication submission. @@ -38,13 +41,26 @@ public class MessageEndpoint { @Incoming(CHANNEL_NAME) @Acknowledgment(Acknowledgment.Strategy.PRE_PROCESSING) public void receivePipelineRequest(String rawRequest) { - try { - PipelineInvocationRequest pipelineInvocationRequest = PipelineInvocationRequest.fromString(rawRequest); - logger.info("Received message request to submit " + pipelineInvocationRequest.getApplicationName() + "."); - pipelineInvocationAgent.submitSparkApplication(pipelineInvocationRequest); - logger.info("Submitted " + pipelineInvocationRequest.getApplicationName() + " for processing."); - } catch (Exception failsafe) { - throw new RuntimeException(failsafe); - } + // Running the helm command asynchronously in a separated thread to avoid vertx event loop thread block issue + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(() -> { + try { + PipelineInvocationRequest pipelineInvocationRequest = PipelineInvocationRequest.fromString(rawRequest); + logger.info("Received message request to submit " + pipelineInvocationRequest.getApplicationName() + "."); + pipelineInvocationAgent.submitSparkApplication(pipelineInvocationRequest); + logger.info("Submitted " + pipelineInvocationRequest.getApplicationName() + " for processing."); + } catch (Exception failsafe) { + throw new RuntimeException(failsafe); + } finally { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(1500, TimeUnit.MILLISECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + } + } + }); } } diff --git a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java index 959b2582f..4e06c6d60 100644 --- a/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java +++ b/extensions/extensions-pipeline-invocation-service/src/main/java/com/boozallen/aissemble/pipeline/invocation/service/util/exec/HelmCommandExecutor.java @@ -68,7 +68,7 @@ private HelmCommandExecutor( * @throws ShellExecutionException */ public int executeAndLogOutput(List arguments) throws ShellExecutionException { - logger.error("Executing Helm command: {} {}", HELM_COMMAND, StringUtils.join(arguments, " ")); + logger.info("Executing Helm command: {} {}", HELM_COMMAND, StringUtils.join(arguments, " ")); ProcessExecutor executor = createHelmExecutor(arguments); return executor.executeAndRedirectOutput(logger); diff --git a/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml b/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml index 46ef25e3c..06890e14f 100644 --- a/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml +++ b/extensions/extensions-pipeline-invocation-service/src/test/resources/sampleValues/java-pipeline-debug-values.yaml @@ -1,5 +1,4 @@ spec: - serviceEnabled: "true" driver: javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:4747" executor: @@ -7,6 +6,7 @@ spec: service: + enabled: true spec: ports: - name: "debug" diff --git a/foundation/foundation-mda/pom.xml b/foundation/foundation-mda/pom.xml index 3bf9e0e72..5d06bc2b1 100644 --- a/foundation/foundation-mda/pom.xml +++ b/foundation/foundation-mda/pom.xml @@ -47,7 +47,7 @@ org.codehaus.mojo exec-maven-plugin - 3.4.1 + ${version.exec.maven.plugin} create-generator-usage-comments diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm index 32f4fc8be..b4f67a00a 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/argocd/spark-operator.yaml.vm @@ -23,5 +23,21 @@ spec: {{ end }} {{- with .Values.spec.syncPolicy }} syncPolicy: - {{- toYaml . | nindent 4 }} - {{- end }} \ No newline at end of file + {{- with .Values.spec.syncPolicy }} + {{- if .automated }} + automated: + {{- toYaml .automated | nindent 6}} + {{- else if hasKey . "automated" }} + automated: {} + {{- end }} + {{- end }} + {{/* This template comment explains how the logic below works*/}} + {{- /* This one trims the newlines so it doesn't affect output */ -}} + #these options are important + syncOptions: + - ServerSideApply=true # https://github.com/argoproj/argo-cd/issues/820#issuecomment-135463693 + {{- if .Values.spec.syncPolicy }} + {{- with .Values.spec.syncPolicy.syncOptions }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml index 662353df3..d4e4ed2a6 100644 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml +++ b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml @@ -96,7 +96,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart @@ -196,7 +196,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} ensure-docker-compose-installed diff --git a/test/test-mda-models/pom.xml b/test/test-mda-models/pom.xml index de7a6cbc9..ce56bf6b7 100644 --- a/test/test-mda-models/pom.xml +++ b/test/test-mda-models/pom.xml @@ -97,7 +97,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart diff --git a/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml b/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml index 061bb011e..e27ffb2c9 100644 --- a/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml +++ b/test/test-mda-models/test-data-delivery-spark-model-basic/pom.xml @@ -68,7 +68,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart diff --git a/test/test-mda-models/test-data-delivery-spark-model/pom.xml b/test/test-mda-models/test-data-delivery-spark-model/pom.xml index c66f09c98..fe205866e 100644 --- a/test/test-mda-models/test-data-delivery-spark-model/pom.xml +++ b/test/test-mda-models/test-data-delivery-spark-model/pom.xml @@ -77,7 +77,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + ${version.exec.maven.plugin} test-chart