From 5fb052e7fb95242f302fc3161bb9171cedf93f67 Mon Sep 17 00:00:00 2001 From: Peter McClonski Date: Wed, 12 Jun 2024 16:35:14 -0400 Subject: [PATCH] #127 V2 Metastore Service Chart Signed-off-by: Peter McClonski --- .../aissemble-hive-service/pom.xml | 53 +++++++++++++++++++ .../src/main/resources/docker/Dockerfile | 33 +++++------- .../README.md | 30 ----------- .../extensions-helm-hive/pom.xml | 1 - .../.helmignore | 0 .../Chart.yaml | 0 .../LICENSE.txt | 0 .../README.md | 30 +++++++++++ .../pom.xml | 2 +- .../templates/configmap.yaml | 8 +-- .../templates/deployment.yaml | 21 ++++++-- .../templates/service.yaml | 0 .../tests/configmap_test.yaml | 32 +---------- .../tests/deployment_test.yaml | 18 ++----- .../tests/service_test.yaml | 4 +- .../values.yaml | 33 +++++------- .../pom.xml | 2 + .../src/main/resources/profiles.json | 8 ++- .../hive-metastore-service-values.yaml.vm | 2 +- .../v2/spark.infrastructure.chart.yaml.vm | 3 ++ .../v2/spark.infrastructure.values.yaml.vm | 38 +++++++++++-- .../src/main/resources/migrations.json | 1 - 22 files changed, 185 insertions(+), 134 deletions(-) delete mode 100644 extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/.helmignore (100%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/Chart.yaml (100%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/LICENSE.txt (100%) create mode 100644 extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/pom.xml (93%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/templates/configmap.yaml (81%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/templates/deployment.yaml (77%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/templates/service.yaml (100%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml (75%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/tests/deployment_test.yaml (85%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/tests/service_test.yaml (87%) rename extensions/extensions-helm/{extensions-helm-hive => extensions-helm-spark-infrastructure}/aissemble-hive-metastore-service-chart/values.yaml (68%) diff --git a/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/pom.xml b/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/pom.xml index 84c902eb8..3dfaf5388 100644 --- a/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/pom.xml +++ b/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/pom.xml @@ -14,4 +14,57 @@ orphedomos + + target/dockerbuild/jars/ + 4.0.0 + 3.4.0 + + + + + io.delta + delta-hive_2.12 + 3.2.0 + + + mysql + mysql-connector-java + ${version.mysql-connector} + + + + + + + org.technologybrewery.orphedomos + orphedomos-maven-plugin + + + ${version.hadoop} + ${version.hive.metastore} + ${dockerbuild.jars.directory} + + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${dockerbuild.jars.directory} + jar + true + + + + unpack + prepare-package + + copy-dependencies + + + + + + + diff --git a/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/src/main/resources/docker/Dockerfile index 256634db7..1da67c6a5 100644 --- a/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-hive/aissemble-hive-service/src/main/resources/docker/Dockerfile @@ -1,32 +1,27 @@ -FROM openjdk:11-jre-slim +ARG METASTORE_VERSION +FROM apache/hive:${METASTORE_VERSION} as appsource + +FROM eclipse-temurin:17-jre as final LABEL org.opencontainers.image.source = "https://github.com/boozallen/aissemble" WORKDIR /opt -ENV HADOOP_VERSION=3.3.6 -ENV METASTORE_VERSION=3.0.0 - -ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION} -ENV HIVE_HOME=/opt/apache-hive-metastore-${METASTORE_VERSION}-bin +ARG JARS_DIR -RUN apt-get update && apt-get install curl -y +ENV HADOOP_HOME=/opt/hadoop +ENV HIVE_HOME=/opt/hive -RUN curl -L https://dlcdn.apache.org/hive/hive-standalone-metastore-${METASTORE_VERSION}/hive-standalone-metastore-${METASTORE_VERSION}-bin.tar.gz | tar zxf - && \ - curl -L https://dlcdn.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | tar zxf - - -RUN curl -L https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar \ - -o ${HIVE_HOME}/lib/mysql-connector-java-8.0.30.jar && \ - curl -L https://repo1.maven.org/maven2/com/sun/activation/javax.activation/1.2.0/javax.activation-1.2.0.jar \ - -o ${HIVE_HOME}/lib/javax.activation-1.2.0.jar && \ - curl https://github.com/delta-io/connectors/releases/download/v0.6.0/delta-hive-assembly_2.12-0.6.0.jar \ - -o ${HIVE_HOME}/lib/delta-hive-assembly_2.12-0.6.0.jar +COPY --from=appsource /opt/hadoop ${HADOOP_HOME} +COPY --from=appsource /opt/hive ${HIVE_HOME} RUN groupadd -r hive --gid=1000 && \ useradd --home ${HIVE_HOME} -g hive --shell /usr/sbin/nologin --uid 1000 hive && \ - chown hive:hive -R ${HIVE_HOME} + chown hive:hive -R ${HIVE_HOME} && \ + ln -s ${JAVA_HOME} /opt/jre + +ADD ${JARS_DIR}/* ${HIVE_HOME}/lib/ USER hive -EXPOSE 9083 -ENTRYPOINT ["/opt/apache-hive-metastore-3.0.0-bin/bin/start-metastore"] +ENTRYPOINT ["/opt/hive/bin/hive", "--skiphadoopversion", "--skiphbasecp", "--verbose", "--service", "metastore"] diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md b/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md deleted file mode 100644 index 15f19de51..000000000 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# aiSSEMBLE™ Hive Metastore Service Helm Chart -Baseline Helm chart for packaging and deploying Hive Metastore Services. Chart is built and managed during the normal Maven build lifecycle and placed in the **target/helm/repo** directory. See https://github.com/kokuwaio/helm-maven-plugin for more details. - -# Basic usage with Helm CLI -To use the module, perform [extension-helm setup](../README.md#leveraging-extensions-helm) and override the chart version with the desired aiSSEMBLE version. For example: -```bash -helm install hive-metastore-service ghcr.io/boozallen/aissemble-hive-metastore-service-chart --version -``` -**Note**: *the version should match the aiSSEMBLE project version.* - -# Properties -| Property | Description | Required Override | Default | -|-------------------------------------------------------|----------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| app.name | Sets label for app.kubernetes.io/name | No | Chart.Name (aissemble-hive-metastore-service-chart) | -| app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) | -| replicaCount | Sets desired number of replicas (instances) | No | 1 | -| hostname | The hostname for the application | No | hive-metastore-service | -| image.name | The image name | Yes | boozallen/aissemble-hive-service | -| image.imagePullPolicy | The image pull policy | No | IfNotPresent | -| image.dockerRepo | The image docker repository | No | NB: OSS: update with aissemble docker repository | -| image.tag | The image tag | No | Chart.AppVersion | -| deployment.ports | The deployment ports | No |   - name: "thrift"
  containerPort: 9083 | -| deployment.env | The environment variables for the pod | No |   - name: HADOOP_HOME
  value: /opt/hadoop-3.3.1
  - name: HADOOP_CLASSPATH
  value: $HADOOP_CLASSPATH:/opt/hadoop-3.3.1/share/hadoop/tools/lib/*.jar
  - name: JAVA_HOME
  value: /usr/local/openjdk-11
  - name: HIVE_HOME
  value: /opt/apache-hive-metastore-3.0.0-bin | -| deployment.restartPolicy | The deployment restart policy | No | Always | -| deployment.volumeMounts | The deployment volume mounts | No |   - name: metastore-service-config
  mountPath: /opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml
  subPath: metastore-site.xml | -| deployment.volumes | The deployment volumes | No |   - name: metastore-service-config
  configMap:
   name: metastore-service-config
   items:
     - key: metastore-site.xml
     path: metastore-site.xml | -| service.spec.ports | The service spec ports | No |   - name: "thrift"
  port: 9083
  targetPort: 9083 | -| configMap.metastoreServiceConfig.properties | Default configuration for the metastore service | No |   - name: metastore.thrift.uris
  value: thrift://0.0.0.0:9083
  description: Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.
  - name: metastore.task.threads.always
  value: org.apache.hadoop.hive.metastore.events.EventCleanerTask,org.apache.hadoop.hive.metastore.MaterializationsCacheCleanerTask
  - name: metastore.expression.proxy
  value: org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy
  - name: javax.jdo.option.ConnectionDriverName
  value: com.mysql.cj.jdbc.Driver
  - name: javax.jdo.option.ConnectionURL
  value: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true\&allowPublicKeyRetrieval=true\&useSSL=false
  description: JDBC connect string for a JDBC metastore
  - name: javax.jdo.option.ConnectionUserName
  value: hive
  description: Username to use against metastore database
  - name: javax.jdo.option.ConnectionPassword
  value: hive
  description: Password to use against metastore database
  - name: metastore.warehouse.dir
  value: s3a://spark-infrastructure/warehouse | -| configMap.metastoreServiceConfig.additionalProperties | Optional configuration for the metastore service. Properties added here will be included in the configuration without overriding the default properties | No | | - diff --git a/extensions/extensions-helm/extensions-helm-hive/pom.xml b/extensions/extensions-helm/extensions-helm-hive/pom.xml index 88e0477f3..f4b5629c2 100644 --- a/extensions/extensions-helm/extensions-helm-hive/pom.xml +++ b/extensions/extensions-helm/extensions-helm-hive/pom.xml @@ -15,7 +15,6 @@ aissemble-hive-metastore-db-chart - aissemble-hive-metastore-service-chart diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/.helmignore b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/.helmignore similarity index 100% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/.helmignore rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/.helmignore diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/Chart.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/Chart.yaml similarity index 100% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/Chart.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/Chart.yaml diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/LICENSE.txt b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/LICENSE.txt similarity index 100% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/LICENSE.txt rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/LICENSE.txt diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md new file mode 100644 index 000000000..c73bed951 --- /dev/null +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md @@ -0,0 +1,30 @@ +# aiSSEMBLE™ Hive Metastore Service Helm Chart +Baseline Helm chart for packaging and deploying Hive Metastore Services. Chart is built and managed during the normal Maven build lifecycle and placed in the **target/helm/repo** directory. See https://github.com/kokuwaio/helm-maven-plugin for more details. + +# Basic usage with Helm CLI +To use the module, perform [extension-helm setup](../README.md#leveraging-extensions-helm) and override the chart version with the desired aiSSEMBLE version. For example: +```bash +helm install hive-metastore-service ghcr.io/boozallen/aissemble-hive-metastore-service-chart --version +``` +**Note**: *the version should match the aiSSEMBLE project version.* + +# Properties +| Property | Description | Required Override | Default | +|-------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| app.name | Sets label for app.kubernetes.io/name | No | hive-metastore-service | +| app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) | +| replicaCount | Sets desired number of replicas (instances) | No | 1 | +| hostname | The hostname for the application | No | hive-metastore-service | +| image.name | The image name | Yes | boozallen/aissemble-hive-service | +| image.imagePullPolicy | The image pull policy | No | IfNotPresent | +| image.dockerRepo | The image docker repository | No | ghcr.io/ | +| image.tag | The image tag | No | Chart.AppVersion | +| deployment.ports | The deployment ports | No |   - name: "thrift"
  containerPort: 9083 | +| deployment.env | The environment variables for the pod | No | See [values.yaml](./values.yaml) | +| deployment.restartPolicy | The deployment restart policy | No | Always | +| deployment.volumeMounts | The deployment volume mounts | No |   - name: metastore-service-config
  mountPath: /opt/hive/conf/metastore-site.xml
  subPath: metastore-site.xml | +| deployment.volumes | The deployment volumes | No |   - name: metastore-service-config
  configMap:
   name: metastore-service-config
   items:
     - key: metastore-site.xml
     path: metastore-site.xml | +| service.spec.ports | The service spec ports | No |   - name: "thrift"
  port: 9083
  targetPort: 9083 | +| configMap.metastoreServiceConfig.baseProperties | Default configuration for the metastore service | No | See [values.yaml](./values.yaml) | +| configMap.metastoreServiceConfig.properties | Optional configuration for the metastore service. Properties added here will be included in the configuration without overriding the default properties | No | | + diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/pom.xml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/pom.xml similarity index 93% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/pom.xml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/pom.xml index fcca93767..8cb4c9045 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/pom.xml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/pom.xml @@ -4,7 +4,7 @@ com.boozallen.aissemble - extensions-helm-hive + extensions-helm-spark-infrastructure 1.8.0-SNAPSHOT diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/configmap.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml similarity index 81% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/configmap.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml index 4d9b700e2..8c9cfae60 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/configmap.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/configmap.yaml @@ -4,10 +4,10 @@ metadata: name: metastore-service-config data: # Add all the default properties from the local values.yaml to the ConfigMap - # Then check if theres any downstream properties and add them as well + # Then check if there are any downstream properties and add them as well metastore-site.xml: | - {{- range $property := .Values.configMap.metastoreServiceConfig.properties }} + {{- range $property := .Values.configMap.metastoreServiceConfig.baseProperties }} {{ $property.name }} {{ $property.value }} @@ -18,8 +18,8 @@ data: {{- end }} - {{- if .Values.configMap.metastoreServiceConfig.additionalProperties }} - {{- range $property := .Values.configMap.metastoreServiceConfig.additionalProperties }} + {{- if .Values.configMap.metastoreServiceConfig.properties }} + {{- range $property := .Values.configMap.metastoreServiceConfig.properties }} {{ $property.name }} {{ $property.value }} diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/deployment.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/deployment.yaml similarity index 77% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/deployment.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/deployment.yaml index 6491dc12d..b3098a537 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/deployment.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/deployment.yaml @@ -33,17 +33,28 @@ spec: nodeSelector: {{- toYaml . | nindent 12}} {{- end }} - {{- with .Values.deployment.initContainers }} initContainers: - {{- toYaml . | nindent 12 }} - {{- end }} + - name: "init-or-upgrade-schema" + image: {{ .Values.image.dockerRepo }}{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }} + imagePullPolicy: IfNotPresent + command: + - "/opt/hive/bin/schematool" + - "-dbType" + - {{ .Values.hive.dbType }} + - "-initOrUpgradeSchema" + - "--verbose" + volumeMounts: + - name: metastore-service-config + mountPath: /opt/hive/conf/metastore-site.xml + subPath: metastore-site.xml containers: - name: {{ .Values.app.name | default .Chart.Name }} image: {{ .Values.image.dockerRepo }}{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }} imagePullPolicy: {{ .Values.image.imagePullPolicy }} - {{- with .Values.deployment.env }} env: - {{- toYaml . | nindent 12 }} + {{- toYaml .Values.deployment.baseEnv | nindent 12}} + {{- if .Values.deployment.env}} + {{- toYaml .Values.deployment.env | nindent 12 }} {{- end }} {{- with .Values.deployment.args }} args: diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/service.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/service.yaml similarity index 100% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/templates/service.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/templates/service.yaml diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml similarity index 75% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml index 6f19aa4f0..1688d5d3f 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/configmap_test.yaml @@ -33,26 +33,12 @@ tests: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false JDBC connect string for a JDBC metastore - - javax.jdo.option.ConnectionUserName - hive - Username to use against metastore database - - - javax.jdo.option.ConnectionPassword - hive - Password to use against metastore database - - - metastore.warehouse.dir - s3a://spark-infrastructure/warehouse - - it: Should override default properties appropriately set: configMap: metastoreServiceConfig: - properties: + baseProperties: - name: propertyName1 value: value1 asserts: @@ -70,7 +56,7 @@ tests: set: configMap: metastoreServiceConfig: - additionalProperties: + properties: - name: propertyName1 value: value1 description: Optional description @@ -104,20 +90,6 @@ tests: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false JDBC connect string for a JDBC metastore - - javax.jdo.option.ConnectionUserName - hive - Username to use against metastore database - - - javax.jdo.option.ConnectionPassword - hive - Password to use against metastore database - - - metastore.warehouse.dir - s3a://spark-infrastructure/warehouse - propertyName1 value1 diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/deployment_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/deployment_test.yaml similarity index 85% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/deployment_test.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/deployment_test.yaml index 9c50dac06..550958647 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/deployment_test.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/deployment_test.yaml @@ -8,7 +8,7 @@ tests: of: Deployment - equal: path: metadata.name - value: aissemble-hive-metastore-service-chart + value: hive-metastore-service - equal: path: spec.template.spec.automountServiceAccountToken value: false @@ -21,31 +21,21 @@ tests: - equal: path: spec.template.spec.containers[0].imagePullPolicy value: IfNotPresent - - contains: - path: spec.template.spec.containers[0].env - content: - name: HADOOP_HOME - value: /opt/hadoop-3.3.6 - contains: path: spec.template.spec.containers[0].env content: name: HADOOP_CLASSPATH - value: $HADOOP_CLASSPATH:/opt/hadoop-3.3.6/share/hadoop/tools/lib/*.jar + value: $HADOOP_CLASSPATH:/opt/hadoop/share/hadoop/tools/lib/*.jar - contains: path: spec.template.spec.containers[0].env content: name: JAVA_HOME - value: /usr/local/openjdk-11 - - contains: - path: spec.template.spec.containers[0].env - content: - name: HIVE_HOME - value: /opt/apache-hive-metastore-3.0.0-bin + value: /opt/jre - contains: path: spec.template.spec.containers[0].volumeMounts content: name: metastore-service-config - mountPath: /opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml + mountPath: /opt/hive/conf/metastore-site.xml subPath: metastore-site.xml - contains: path: spec.template.spec.volumes diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/service_test.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/service_test.yaml similarity index 87% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/service_test.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/service_test.yaml index 834ee4a40..71412acf7 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/tests/service_test.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/tests/service_test.yaml @@ -8,10 +8,10 @@ tests: of: Service - equal: path: metadata.name - value: aissemble-hive-metastore-service-chart + value: hive-metastore-service - equal: path: spec.selector["app.kubernetes.io/name"] - value: aissemble-hive-metastore-service-chart + value: hive-metastore-service - contains: path: spec.ports content: diff --git a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/values.yaml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml similarity index 68% rename from extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/values.yaml rename to extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml index f94301f1d..d68ae5ddd 100644 --- a/extensions/extensions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/values.yaml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/values.yaml @@ -2,8 +2,8 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. app: - # Sets labels for app.kubernetes.io/name Default is Chart.name (aissemble-hive-metastore-service-chart) - name: '' + # Sets labels for app.kubernetes.io/name. Default is Chart.name (aissemble-hive-metastore-service-chart) + name: 'hive-metastore-service' # Sets the labels for app.kubernetes.io/version. Default is Chart.appVersion version: '' @@ -12,26 +12,26 @@ hostname: hive-metastore-service image: name: boozallen/aissemble-hive-service imagePullPolicy: IfNotPresent - dockerRepo: ghcr.io/ + dockerRepo: "ghcr.io/" + +hive: + dbType: "mysql" # hive-metastore-service Deployment deployment: ports: - name: "thrift" containerPort: 9083 - env: - - name: HADOOP_HOME - value: /opt/hadoop-3.3.6 + baseEnv: - name: HADOOP_CLASSPATH - value: $HADOOP_CLASSPATH:/opt/hadoop-3.3.6/share/hadoop/tools/lib/*.jar + value: $HADOOP_CLASSPATH:/opt/hadoop/share/hadoop/tools/lib/*.jar - name: JAVA_HOME - value: /usr/local/openjdk-11 - - name: HIVE_HOME - value: /opt/apache-hive-metastore-3.0.0-bin + value: /opt/jre + env: [] restartPolicy: Always volumeMounts: - name: metastore-service-config - mountPath: /opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml + mountPath: /opt/hive/conf/metastore-site.xml subPath: metastore-site.xml volumes: - name: metastore-service-config @@ -52,7 +52,7 @@ service: # hive-metastore-service Config Map configMap: metastoreServiceConfig: - properties: + baseProperties: - name: metastore.thrift.uris value: thrift://0.0.0.0:9083 description: Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore. @@ -65,11 +65,4 @@ configMap: - name: javax.jdo.option.ConnectionURL value: jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useSSL=false description: JDBC connect string for a JDBC metastore - - name: javax.jdo.option.ConnectionUserName - value: hive - description: Username to use against metastore database - - name: javax.jdo.option.ConnectionPassword - value: hive - description: Password to use against metastore database - - name: metastore.warehouse.dir - value: s3a://spark-infrastructure/warehouse + properties: {} \ No newline at end of file diff --git a/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml b/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml index f38cfb380..7d4748b97 100644 --- a/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml +++ b/extensions/extensions-helm/extensions-helm-spark-infrastructure/pom.xml @@ -16,7 +16,9 @@ aissemble-spark-history-chart aissemble-thrift-server-chart + aissemble-hive-metastore-service-chart + diff --git a/foundation/foundation-mda/src/main/resources/profiles.json b/foundation/foundation-mda/src/main/resources/profiles.json index fff444e69..8fdd038cb 100644 --- a/foundation/foundation-mda/src/main/resources/profiles.json +++ b/foundation/foundation-mda/src/main/resources/profiles.json @@ -290,7 +290,9 @@ { "name": "spark-infrastructure-kubernetes" } - ] + ], + "deprecated": "true", + "warningMessage": "Due to ongoing effort to create a Spark Infrastructure v2 chart and subsequent incompatibilities, v1 cannot be upgraded beyond version 1.7.x." }, { "name": "aissemble-spark-infrastructure-deploy-v2", @@ -1077,7 +1079,9 @@ { "name": "hive-metastore-service-kubernetes" } - ] + ], + "deprecated": "true", + "warningMessage": "Due to ongoing effort to create a Hive v2 chart and subsequent incompatibilities, v1 cannot be upgraded beyond version 1.7.x" }, { "name": "hive-metastore-service-deploy-v2", diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive-metastore-service-values.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive-metastore-service-values.yaml.vm index 9e3c4df02..cb76a2e32 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive-metastore-service-values.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/hive-metastore-service/hive-metastore-service-values.yaml.vm @@ -10,7 +10,7 @@ image: imagePullPolicy: IfNotPresent dockerRepo: ghcr.io/ # Overrides the image tag whose default is the chart appVersion. - tag: "${versionTag}" + tag: "1.7.0" # hive-metastore-service Deployment deployment: diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm index 98bf7b602..2e11aed6f 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.chart.yaml.vm @@ -35,4 +35,7 @@ dependencies: - name: aissemble-thrift-server-chart version: ${versionTag} repository: oci://ghcr.io/boozallen + - name: aissemble-hive-metastore-service-chart + version: ${versionTag} + repository: oci://ghcr.io/boozallen diff --git a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm index 553245d01..74f8b0ef6 100644 --- a/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm +++ b/foundation/foundation-mda/src/main/resources/templates/deployment/spark-infrastructure/v2/spark.infrastructure.values.yaml.vm @@ -12,11 +12,11 @@ aissemble-thrift-server-chart: deployment: envFromSecret: AWS_ACCESS_KEY_ID: - secretName: aws-s3-credentials - key: aws_access_key_id + secretName: remote-auth-config + key: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: - secretName: aws-s3-credentials - key: aws_secret_access_key + secretName: remote-auth-config + key: AWS_SECRET_ACCESS_KEY sparkConf: | spark.hadoop.fs.s3a.endpoint=http://s3-local:4566 @@ -40,3 +40,33 @@ aissemble-thrift-server-chart: Creates necessary schema on a startup if one does not exist + +aissemble-hive-metastore-service-chart: + deployment: + env: + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_SECRET_ACCESS_KEY + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: remote-auth-config + key: AWS_ACCESS_KEY_ID + + configMap: + metastoreServiceConfig: + properties: + - name: javax.jdo.option.ConnectionUserName + value: hive + description: Username to use against metastore database + - name: javax.jdo.option.ConnectionPassword + value: hive + description: Password to use against metastore database + - name: metastore.warehouse.dir + value: s3a://spark-infrastructure/warehouse + - name: fs.s3a.endpoint + value: http://s3-local:4566 + - name: fs.s3a.path.style.access + value: true \ No newline at end of file diff --git a/foundation/foundation-upgrade/src/main/resources/migrations.json b/foundation/foundation-upgrade/src/main/resources/migrations.json index 6b6f694f7..129af17f5 100644 --- a/foundation/foundation-upgrade/src/main/resources/migrations.json +++ b/foundation/foundation-upgrade/src/main/resources/migrations.json @@ -38,7 +38,6 @@ "**/jenkins/values.yaml", "**/kafka-cluster/values.yaml", "**/hive-metastore-db/values.yaml", - "**/hive-metastore-service/values.yaml", "**/metadata/values.yaml", "**/model-training-api/values.yaml", "**/pipeline-invocation-service/values.yaml"