Skip to content

Commit

Permalink
boozallen#127 boozallen#116 Hive Metastore Service v2 chart and Hive …
Browse files Browse the repository at this point in the history 
…upgrade

Signed-off-by: Peter McClonski <mcclonski_peter@bah.com>
  • Loading branch information
@peter-mcclonski
peter-mcclonski committed Jun 10, 2024
1 parent c7e773a commit b529c03
Show file tree
Hide file tree
Showing 19 changed files with 175 additions and 126 deletions.
53 changes: 53 additions & 0 deletions extensions/extensions-docker/aissemble-hive/aissemble-hive-service/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,57 @@

<packaging>orphedomos</packaging>

<properties>
<dockerbuild.jars.directory>target/dockerbuild/jars/</dockerbuild.jars.directory>
<version.hive.metastore>4.0.0</version.hive.metastore>
<version.hadoop>3.4.0</version.hadoop>
</properties>

<dependencies>
<dependency>
<groupId>io.delta</groupId>
<artifactId>delta-hive_2.12</artifactId>
<version>3.2.0</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${version.mysql-connector}</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.technologybrewery.orphedomos</groupId>
<artifactId>orphedomos-maven-plugin</artifactId>
<configuration>
<buildArgs>
<HADOOP_VERSION>${version.hadoop}</HADOOP_VERSION>
<METASTORE_VERSION>${version.hive.metastore}</METASTORE_VERSION>
<JARS_DIR>${dockerbuild.jars.directory}</JARS_DIR>
</buildArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<outputDirectory>${dockerbuild.jars.directory}</outputDirectory>
<includeTypes>jar</includeTypes>
<excludeTransitive>true</excludeTransitive>
</configuration>
<executions>
<execution>
<id>unpack</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
29 changes: 12 additions & 17 deletions ...ensions-docker/aissemble-hive/aissemble-hive-service/src/main/resources/docker/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,24 @@ LABEL org.opencontainers.image.source = "https://github.com/boozallen/aissemble"

WORKDIR /opt

ENV HADOOP_VERSION=3.3.6
ENV METASTORE_VERSION=3.0.0
ARG HADOOP_VERSION
ARG METASTORE_VERSION
ARG JARS_DIR

ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION}
ENV HIVE_HOME=/opt/apache-hive-metastore-${METASTORE_VERSION}-bin
ENV HIVE_HOME=/opt/apache-hive-${METASTORE_VERSION}-bin

RUN apt-get update && apt-get install curl -y
RUN apt-get update && apt-get install curl -y && \
groupadd -r hive --gid=1000 && \
mkdir ${HIVE_HOME} && \
useradd --home ${HIVE_HOME} -g hive --shell /usr/sbin/nologin --uid 1000 hive && \
chown hive:hive -R ${HIVE_HOME}

RUN curl -L https://dlcdn.apache.org/hive/hive-standalone-metastore-${METASTORE_VERSION}/hive-standalone-metastore-${METASTORE_VERSION}-bin.tar.gz | tar zxf - && \
RUN curl -L https://dlcdn.apache.org/hive/hive-${METASTORE_VERSION}/apache-hive-${METASTORE_VERSION}-bin.tar.gz | tar zxf - && \
curl -L https://dlcdn.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | tar zxf -

RUN curl -L https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar \
-o ${HIVE_HOME}/lib/mysql-connector-java-8.0.30.jar && \
curl -L https://repo1.maven.org/maven2/com/sun/activation/javax.activation/1.2.0/javax.activation-1.2.0.jar \
-o ${HIVE_HOME}/lib/javax.activation-1.2.0.jar && \
curl https://github.com/delta-io/connectors/releases/download/v0.6.0/delta-hive-assembly_2.12-0.6.0.jar \
-o ${HIVE_HOME}/lib/delta-hive-assembly_2.12-0.6.0.jar

RUN groupadd -r hive --gid=1000 && \
useradd --home ${HIVE_HOME} -g hive --shell /usr/sbin/nologin --uid 1000 hive && \
chown hive:hive -R ${HIVE_HOME}
ADD ${JARS_DIR}/* ${HIVE_HOME}/lib/

USER hive
EXPOSE 9083

ENTRYPOINT ["/opt/apache-hive-metastore-3.0.0-bin/bin/start-metastore"]
ENTRYPOINT ["/opt/apache-hive-4.0.0-bin/bin/hive", "--skiphadoopversion", "--skiphbasecp", "--verbose", "--service", "metastore"]
30 changes: 0 additions & 30 deletions ...ions-helm/extensions-helm-hive/aissemble-hive-metastore-service-chart/README.md
View file

This file was deleted.

1 change: 0 additions & 1 deletion extensions/extensions-helm/extensions-helm-hive/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

<modules>
<module>aissemble-hive-metastore-db-chart</module>
<module>aissemble-hive-metastore-service-chart</module>
</modules>
<build>
<plugins>
Expand Down
0 ...-hive-metastore-service-chart/.helmignore → ...-hive-metastore-service-chart/.helmignore
View file
File renamed without changes.
0 ...e-hive-metastore-service-chart/Chart.yaml → ...e-hive-metastore-service-chart/Chart.yaml
View file
File renamed without changes.
0 ...-hive-metastore-service-chart/LICENSE.txt → ...-hive-metastore-service-chart/LICENSE.txt
View file
File renamed without changes.
30 changes: 30 additions & 0 deletions ...ions-helm-spark-infrastructure/aissemble-hive-metastore-service-chart/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# aiSSEMBLE&trade; Hive Metastore Service Helm Chart
Baseline Helm chart for packaging and deploying Hive Metastore Services. Chart is built and managed during the normal Maven build lifecycle and placed in the **target/helm/repo** directory. See https://github.com/kokuwaio/helm-maven-plugin for more details.

# Basic usage with Helm CLI
To use the module, perform [extension-helm setup](../README.md#leveraging-extensions-helm) and override the chart version with the desired aiSSEMBLE version. For example:
```bash
helm install hive-metastore-service ghcr.io/boozallen/aissemble-hive-metastore-service-chart --version <AISSEMBLE-VERSION>
```
**Note**: *the version should match the aiSSEMBLE project version.*

# Properties
| Property | Description | Required Override | Default |
|-------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| app.name | Sets label for app.kubernetes.io/name | No | hive-metastore-service |
| app.version | Sets label for app.kubernetes.io/version | No | Chart.AppVersion (aiSSEMBLE project version) |
| replicaCount | Sets desired number of replicas (instances) | No | 1 |
| hostname | The hostname for the application | No | hive-metastore-service |
| image.name | The image name | Yes | boozallen/aissemble-hive-service |
| image.imagePullPolicy | The image pull policy | No | IfNotPresent |
| image.dockerRepo | The image docker repository | No | ghcr.io/ |
| image.tag | The image tag | No | Chart.AppVersion |
| deployment.ports | The deployment ports | No | &emsp; - name: "thrift" <br/>&emsp;&emsp;containerPort: 9083 |
| deployment.env | The environment variables for the pod | No | See values.yaml |
| deployment.restartPolicy | The deployment restart policy | No | Always |
| deployment.volumeMounts | The deployment volume mounts | No | &emsp; - name: metastore-service-config <br/>&emsp;&emsp;mountPath: /opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml <br/>&emsp;&emsp;subPath: metastore-site.xml |
| deployment.volumes | The deployment volumes | No | &emsp; - name: metastore-service-config <br/>&emsp;&emsp;configMap: <br/>&emsp;&emsp;&emsp;name: metastore-service-config <br/>&emsp;&emsp;&emsp;items: <br/>&emsp;&emsp;&emsp;&emsp; - key: metastore-site.xml <br/>&emsp;&emsp;&emsp;&emsp;&emsp;path: metastore-site.xml |
| service.spec.ports | The service spec ports | No | &emsp; - name: "thrift" <br/>&emsp;&emsp;port: 9083 <br/>&emsp;&emsp;targetPort: 9083 |
| configMap.metastoreServiceConfig.baseProperties | Default configuration for the metastore service | No | See values.yaml |
| configMap.metastoreServiceConfig.properties | Optional configuration for the metastore service. Properties added here will be included in the configuration without overriding the default properties | No | |

2 changes: 1 addition & 1 deletion ...mble-hive-metastore-service-chart/pom.xml → ...mble-hive-metastore-service-chart/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

<parent>
<groupId>com.boozallen.aissemble</groupId>
<artifactId>extensions-helm-hive</artifactId>
<artifactId>extensions-helm-spark-infrastructure</artifactId>
<version>1.7.0-SNAPSHOT</version>
</parent>

Expand Down
8 changes: 4 additions & 4 deletions ...re-service-chart/templates/configmap.yaml → ...re-service-chart/templates/configmap.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ metadata:
name: metastore-service-config
data:
# Add all the default properties from the local values.yaml to the ConfigMap
# Then check if theres any downstream properties and add them as well
# Then check if there are any downstream properties and add them as well
metastore-site.xml: |
<configuration>
{{- range $property := .Values.configMap.metastoreServiceConfig.properties }}
{{- range $property := .Values.configMap.metastoreServiceConfig.baseProperties }}
<property>
<name>{{ $property.name }}</name>
<value>{{ $property.value }}</value>
Expand All @@ -18,8 +18,8 @@ data:
</property>
{{- end }}
{{- if .Values.configMap.metastoreServiceConfig.additionalProperties }}
{{- range $property := .Values.configMap.metastoreServiceConfig.additionalProperties }}
{{- if .Values.configMap.metastoreServiceConfig.properties }}
{{- range $property := .Values.configMap.metastoreServiceConfig.properties }}
<property>
<name>{{ $property.name }}</name>
<value>{{ $property.value }}</value>
Expand Down
21 changes: 16 additions & 5 deletions ...e-service-chart/templates/deployment.yaml → ...e-service-chart/templates/deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,28 @@ spec:
nodeSelector:
{{- toYaml . | nindent 12}}
{{- end }}
{{- with .Values.deployment.initContainers }}
initContainers:
{{- toYaml . | nindent 12 }}
{{- end }}
- name: "init-or-upgrade-schema"
image: {{ .Values.image.dockerRepo }}{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: IfNotPresent
command:
- "/opt/apache-hive-4.0.0-bin/bin/schematool"
- "-dbType"
- {{ .Values.hive.dbType }}
- "-initOrUpgradeSchema"
- "--verbose"
volumeMounts:
- name: metastore-service-config
mountPath: /opt/apache-hive-4.0.0-bin/conf/metastore-site.xml
subPath: metastore-site.xml
containers:
- name: {{ .Values.app.name | default .Chart.Name }}
image: {{ .Values.image.dockerRepo }}{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}
imagePullPolicy: {{ .Values.image.imagePullPolicy }}
{{- with .Values.deployment.env }}
env:
{{- toYaml . | nindent 12 }}
{{- toYaml .Values.deployment.baseEnv | nindent 12}}
{{- if .Values.deployment.env}}
{{- toYaml .Values.deployment.env | nindent 12 }}
{{- end }}
{{- with .Values.deployment.args }}
args:
Expand Down
0 ...tore-service-chart/templates/service.yaml → ...tore-service-chart/templates/service.yaml
View file
File renamed without changes.
32 changes: 2 additions & 30 deletions ...e-service-chart/tests/configmap_test.yaml → ...e-service-chart/tests/configmap_test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,26 +33,12 @@ tests:
<value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
<description>JDBC connect string for a JDBC metastore</description>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>hive</value>
<description>Username to use against metastore database</description>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>hive</value>
<description>Password to use against metastore database</description>
</property>
<property>
<name>metastore.warehouse.dir</name>
<value>s3a://spark-infrastructure/warehouse</value>
</property>
</configuration>
- it: Should override default properties appropriately
set:
configMap:
metastoreServiceConfig:
properties:
baseProperties:
- name: propertyName1
value: value1
asserts:
Expand All @@ -70,7 +56,7 @@ tests:
set:
configMap:
metastoreServiceConfig:
additionalProperties:
properties:
- name: propertyName1
value: value1
description: Optional description
Expand Down Expand Up @@ -104,20 +90,6 @@ tests:
<value>jdbc:mysql://hive-metastore-db:3306/metastore?createDatabaseIfNotExist=true&amp;allowPublicKeyRetrieval=true&amp;useSSL=false</value>
<description>JDBC connect string for a JDBC metastore</description>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>hive</value>
<description>Username to use against metastore database</description>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>hive</value>
<description>Password to use against metastore database</description>
</property>
<property>
<name>metastore.warehouse.dir</name>
<value>s3a://spark-infrastructure/warehouse</value>
</property>
<property>
<name>propertyName1</name>
<value>value1</value>
Expand Down
Loading

0 comments on commit b529c03

Please sign in to comment.