New changes

.gitignore

@@ -6,4 +6,5 @@ radar-is.yml
 production.yaml
 .idea/
 RADAR-Kubernetes.iml
+postgreskeystore.jks
 environments.yaml

base.yaml

@@ -88,6 +88,15 @@ velero:
           aws_secret_access_key=secretKey
 
 # --------------------------------------------------------- 10-base.yaml ---------------------------------------------------------
+confluent_cloud:
+  enabled: false
+  cc:
+    bootstrapServerurl: confluentBootstrapServers
+    apiKey: ccApikey
+    apiSecret: ccApiSecret
+    schemaRegistryApiKey: srApiKey
+    schemaRegistryApiSecret: srApiSecret
+    schemaRegistryUrl: confluentSchemaRegistryUrl
 
 cp_zookeeper:
   _install: true
@@ -105,6 +114,8 @@ cp_schema_registry:
 catalog_server:
   _install: true
   replicaCount: 1
+  schema_registry: http://cp-schema-registry:8081
+
 
 postgresql:
   _install: true
@@ -114,6 +125,10 @@ postgresql:
 management_portal:
   _install: true
   replicaCount: 1  # should be 1
+  postgres:
+    ssl:
+      enabled: false
+      keystorepassword: keystorepassword
   managementportal:
     catalogue_server_enable_auto_import: true
     common_admin_password: password
@@ -132,12 +147,20 @@ management_portal:
     radar_rest_sources_backend: secret
     radarFitbitConnector: secret
     radarAppconfig: secret
+  extra_oauth_clients: []
+  smtp:
+    enabled: false
+    host: smtp
+    port: 25
+    username: username
+    password: secret
+    from: noreply@example.com
 
 app_config:
   _install: true
   replicaCount: 1  # should be 1
   clientSecret: secret
-  
+
 smtp:
   _install: true
   replicaCount: 1
@@ -231,14 +254,20 @@ ceph:
 radar_output:
   _install: true
   replicaCount: 1
-  num_threads: 2
-  target_type: s3
-  source_type: s3
-  hdfs_namenodes: "[hdfs-namenode-0, hdfs-namenode-1]"
-  sourceBucketName: radar-intermediate-storage
-  targetBucketName: radar-output-storage
-  redisUrl: redis://redis-master:6379
-  minimumFileAge: 900
+  source:
+    s3:
+      endpoint: http://minio:9000/
+      accessToken: AKIAIOSFODNN7EXAMPLE
+      secretKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+      bucket: radar-intermediate-storage
+  target:
+    s3:
+      endpoint: http://minio:9000/
+      accessToken: AKIAIOSFODNN7EXAMPLE
+      secretKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+      bucket: radar-output-storage
+  redis:
+    url: redis://redis-master:6379
 
 # --------------------------------------------------------- 20-ingestion.yaml ---------------------------------------------------------
 
@@ -323,7 +352,24 @@ radar_s3_connector:
   replicaCount: 1
   # The bucket name where intermediate data for cold storage should be written to.
   bucketName: radar_intermediate_storage
+  bucketAccessKey: AKIAIOSFODNN7EXAMPLE
+  bucketSecretKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  s3Endpoint: http://minio:9000/
+
+s3_proxy:
+  _install: false
+  replicaCount: 1
+  s3:
+    identity: AKIAIOSFODNN7EXAMPLE
+    credential: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  target:
+    provider: azureblob
 
+# --------------------------------------------------------- 30-confluent-cloud.yaml ---------------------------------------------------------
+
+ccSchemaRegistryProxy:
+  _install: false
+  externalName: schema-registry-domain
 
 # --------------------------------------------------------- 20-grafana.yaml ---------------------------------------------------------
 

charts/app-config/templates/configmap.yaml

@@ -16,13 +16,15 @@ data:
       enhancerFactory: org.radarbase.appconfig.inject.ManagementPortalEnhancerFactory
 
     authentication:
-      url: http://managementportal:8080/managementportal/
+      url: http://management-portal:8080/managementportal
       clientId: {{ .Values.clientId }}
       clientSecret: {{ .Values.clientSecret }}
       resourceName: res_appconfig
 
     jdbc:
       driver: "org.postgresql.Driver"
-      url: "jdbc:postgresql://{{ .Values.postgres.host }}:5432/appconfig"
-      user: {{ .Values.postgres.user }}
-      password: {{ .Values.postgres.password }}
+      url: "{{ .Values.jdbc.url }}"
+      user: {{ .Values.jdbc.user }}
+      password: {{ .Values.jdbc.password }}
+      properties:
+        hibernate.dialect: {{ .Values.jdbc.dialect }}

charts/app-config/templates/deployment.yaml

@@ -29,29 +29,38 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+          - name: JAVA_OPTS
+            value: "{{ .Values.javaOpts }}"
+          volumeMounts:
+          - name: config
+            mountPath: /etc/radar-app-config/
           ports:
             - name: http
               containerPort: 8090
               protocol: TCP
           livenessProbe:
             failureThreshold: 3
-            initialDelaySeconds: 5
+            initialDelaySeconds: 20
             periodSeconds: 60
             successThreshold: 1
             tcpSocket:
               port: http
             timeoutSeconds: 5
-            name: radar-integration
           readinessProbe:
             failureThreshold: 3
-            initialDelaySeconds: 5
+            initialDelaySeconds: 20
             periodSeconds: 60
             successThreshold: 1
             tcpSocket:
               port: http
             timeoutSeconds: 5
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "app-config.fullname" . }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/app-config/templates/ingress.yaml

@@ -35,6 +35,5 @@ spec:
             backend:
               serviceName: {{ $fullName }}
               servicePort: {{ $svcPort }}
-        {{- end }}
   {{- end }}
 {{- end }}

charts/app-config/values.yaml

@@ -52,22 +52,24 @@ resources:
   # resources, such as Minikube. If you do want to specify resources, uncomment the following
   # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
   limits:
-    cpu: 100m
-    memory: 512Mi
+    cpu: 2
   requests:
     cpu: 100m
-    memory: 128Mi
+    memory: 768Mi
 
 nodeSelector: {}
 
 tolerations: []
 
 affinity: {}
 
+javaOpts: "-Xmx550m"
 clientId: radar_appconfig
 clientSecret: secret
 
-postgres:
-  host: postgresql-postgresql
+jdbc:
+  driver: org.postgresql.Driver
+  url: jdbc:postgresql://postgresql-postgresql/appconfig
   user: postgres
   password: password
+  dialect: org.hibernate.dialect.PostgreSQLDialect

charts/catalog-server/templates/configmap.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.cc.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "catalog-server.fullname" . }}-java-config
+  labels:
+    app: {{ template "catalog-server.name" . }}
+    chart: {{ template "catalog-server.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  java-config.properties: |
+    # Kafka
+    bootstrap.servers={{ .Values.cc.bootstrapServerurl }}
+    security.protocol=SASL_SSL
+    sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="{{ .Values.cc.apiKey }}" password="{{ .Values.cc.apiSecret }}";
+    ssl.endpoint.identification.algorithm=https
+    sasl.mechanism=PLAIN
+{{- end -}}

charts/catalog-server/templates/deployment.yaml

@@ -54,10 +54,29 @@ spec:
           value: "3"
         - name: RADAR_NUM_REPLICATION_FACTOR
           value: "3"
+        {{- if .Values.cc.enabled }}
+        - name: CC_CONFIG_FILE_PATH
+          value: "/etc/config/java-config.properties"
+        - name: CC_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "catalog-server.fullname" . }}
+              key: srApiKey
+        - name: CC_API_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: {{ template "catalog-server.fullname" . }}
+              key: srApiSecret
+        command:
+          - "cc_topic_init.sh"
+        volumeMounts:
+          - name: cc-java-config
+            mountPath: /etc/config/
+        {{ end }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          args:
+          command:
             - radar-schemas-tools
             - serve
             - /schema/merged
@@ -112,6 +131,11 @@ spec:
         {{- else }}
           emptyDir: {}
         {{ end }}
+        {{- if .Values.cc.enabled }}
+        - name: cc-java-config
+          configMap:
+            name: {{ include "catalog-server.fullname" . }}-java-config
+        {{- end -}}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/catalog-server/templates/pvc.yaml

@@ -21,4 +21,4 @@ spec:
   storageClassName: "{{ .Values.persistence.storageClass }}"
 {{- end }}
 {{- end }}
-{{- end -}}
+{{- end }}

charts/catalog-server/templates/secrets.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.cc.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "catalog-server.fullname" . }}
+  labels:
+    app: {{ template "catalog-server.name" . }}
+    chart: {{ template "catalog-server.chart" . }}
+    release: {{ .Release.Name | quote }}
+    heritage: {{ .Release.Service | quote }}
+type: Opaque
+data:
+  ccApiKey: {{ .Values.cc.apiKey | b64enc | quote }}
+  ccApiSecret: {{ .Values.cc.apiSecret | b64enc | quote }}
+  srApiKey: {{ .Values.cc.schemaRegistryApiKey| b64enc | quote }}
+  srApiSecret: {{ .Values.cc.schemaRegistryApiSecret | b64enc | quote }}
+{{- end -}}

charts/catalog-server/values.yaml

@@ -6,7 +6,7 @@ replicaCount: 2
 
 image:
   repository: radarbase/kafka-init
-  tag: 0.5.10.1
+  tag: 0.5.11-rc0
   pullPolicy: IfNotPresent
 
 nameOverride: ""
@@ -43,7 +43,7 @@ resources:
     memory: 128Mi
 
 persistence:
-  enabled: true
+  enabled: false
   ## If defined, storageClassName: <storageClass>
   ## If set to "-", storageClassName: "", which disables dynamic provisioning
   ## If undefined (the default) or set to null, no storageClassName spec is
@@ -67,3 +67,11 @@ affinity: {}
 kafka_num_brokers: 3
 zookeeper: cp-zookeeper-headless:2181
 schema_registry: http://cp-schema-registry:8081
+
+cc:
+  enabled: false
+  bootstrapServerurl: confluent-url
+  apiKey: ccApikey
+  apiSecret: ccApiSecret
+  schemaRegistryApiKey: srApiKey
+  schemaRegistryApiSecret: srApiSecret

charts/cc-schema-registry-proxy/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/cc-schema-registry-proxy/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: cc-schema-registry-proxy
+version: 0.1.0

charts/cc-schema-registry-proxy/templates/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "cc-schema-registry-proxy.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "cc-schema-registry-proxy.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "cc-schema-registry-proxy.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}