spark-cluster-group.yaml.template

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: spark-master
spec:
  replicas: 1
  selector:
    matchLabels:
      component: spark-master
  template:
    metadata:
      labels:
        component: spark-master
    spec:
      priorityClassName: normal
      hostname: spark-master
      containers:
        - name: spark-master
          image: ucsdets/spark-notebook@sha256:fcc133c9860db1ee14e183305f41f053e1be80537a8d8ecb7debd9de58e247f3
#          image: scnakandala/dsc102-pa2-spark:latest
          imagePullPolicy: IfNotPresent
          env:
            - name: JUPYTER_TOKEN
              value: $JUPYTER_TOKEN
            - name: HOME
              value: /home
# https://stackoverflow.com/questions/42072638/spark-submit-how-to-set-the-user-name
            - name: SPARK_USER
              value: $K8S_USERNAME
          command: ["/spark-master"]
          ports:
            - containerPort: 7077
            - containerPort: 8080
            - containerPort: 4040
            - containerPort: 20002
            - containerPort: 50002
            - containerPort: 60002
            - containerPort: 60003
            - containerPort: 60004
          resources:
            limits:
              memory: "4Gi"
              cpu: 2
            requests:
              memory: "4Gi"
              cpu: 2
          volumeMounts:
            - name: dsc102-pa3-datasets
              mountPath: /dsc102-pa3-public
              subPath: "dsc102-fa20-public"
              readOnly: true
            - name: home
              mountPath: /home
            - name: teams
              mountPath: /teams
            - name: teams
              mountPath: /home/teams
      volumes:
        - name: dsc102-pa3-datasets
          persistentVolumeClaim:
            claimName: dsmlp-datasets-nfs
        - name: home
          persistentVolumeClaim:
            claimName: home-nfs
        - name: teams
          persistentVolumeClaim:
            claimName: teams
      securityContext:
        runAsUser: $K8S_UID
        fsGroup: $K8S_GID
---
kind: Service
apiVersion: v1
metadata:
  name: spark-master
spec:
  ports:
    - name: webui
      port: 8080
      targetPort: 8080
    - name: spark
      port: 7077
      targetPort: 7077
    - name: sparkui
      port: 4040
      targetPort: 4040
    - name: driverport
      port: 20002
      targetPort: 20002
    - name: blockmgrport
      port: 50002
      targetPort: 50002
    - name: fileserverport
      port: 60002
      targetPort: 60002
    - name: broadcastport
      port: 60003
      targetPort: 60003
    - name: replicaport
      port: 60004
      targetPort: 60004
  selector:
    component: spark-master
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: spark-worker
spec:
  replicas: $NUM_WORKERS
  selector:
    matchLabels:
      component: spark-worker
  template:
    metadata:
      labels:
        component: spark-worker
    spec:
      containers:
        - name: spark-worker
          image: ucsdets/spark-notebook@sha256:fcc133c9860db1ee14e183305f41f053e1be80537a8d8ecb7debd9de58e247f3
#          image: scnakandala/dsc102-pa2-spark:latest
          imagePullPolicy: IfNotPresent
          command: ["/spark-worker"]
          env:
            # https://stackoverflow.com/questions/42072638/spark-submit-how-to-set-the-user-name
            - name: SPARK_USER
              value: $K8S_USERNAME
          ports:
            - containerPort: 8081
            - containerPort: 30002
            - containerPort: 40002
            - containerPort: 50002
            - containerPort: 60002
            - containerPort: 60003
            - containerPort: 60004
          resources:
            limits:
              memory: "20Gi"
              cpu: 2
            requests:
              memory: "20Gi"
              cpu: 2
          volumeMounts:
            - name: dsc102-pa3-datasets
              mountPath: /dsc102-pa3-public
              subPath: "dsc102-fa20-public"
              readOnly: true
            - name: home
              mountPath: /home
            - name: teams
              mountPath: /teams
      volumes:
        - name: dsc102-pa3-datasets
          persistentVolumeClaim:
            claimName: dsmlp-datasets-nfs
        - name: home
          persistentVolumeClaim:
            claimName: home-nfs
        - name: teams
          persistentVolumeClaim:
            claimName: teams
      securityContext:
        runAsUser: $K8S_UID
        fsGroup: $K8S_GID
---
kind: Service
apiVersion: v1
metadata:
  name: spark-worker
spec:
  ports:
    - name: webui
      port: 8081
      targetPort: 8081
    - name: workerport
      port: 30002
      targetPort: 30002
    - name: executorport
      port: 40002
      targetPort: 40002
    - name: blockmgr
      port: 50002
      targetPort: 50002
    - name: fileserverport
      port: 60002
      targetPort: 60002
    - name: broadcastport
      port: 60003
      targetPort: 60003
    - name: replicaport
      port: 60004
      targetPort: 60004
  selector:
    component: spark-worker