[#3575] Fix Kafka topics provisioning

airyhq · Aug 12, 2022 · fd404a2 · fd404a2
1 parent 77e711d
commit fd404a2
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 18 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.47.3
+0.47.4
diff --git a/docs/docs/changelog.md b/docs/docs/changelog.md
@@ -3,6 +3,20 @@ title: Changelog
 sidebar_label: 📝 Changelog
 ---
 
+## 0.47.4
+
+#### Hotfix
+
+* [[#3575](https://github.com/airyhq/airy/issues/3575)] Fix Kafka topics provisioning
+
+#### Airy CLI
+
+You can download the Airy CLI for your operating system from the following links:
+
+[MacOS](https://airy-core-binaries.s3.amazonaws.com/0.47.4/darwin/amd64/airy)
+[Linux](https://airy-core-binaries.s3.amazonaws.com/0.47.4/linux/amd64/airy)
+[Windows](https://airy-core-binaries.s3.amazonaws.com/0.47.4/windows/amd64/airy.exe)
+
 ## 0.47.3
 
 #### Hotfix

diff --git a/infrastructure/helm-chart/templates/provisioning/job-kafka.yaml b/infrastructure/helm-chart/templates/provisioning/job-kafka.yaml
@@ -10,7 +10,7 @@ spec:
       containers:
       - name: create-topics
         image: "{{ .Values.provisioning.kafka.image }}:{{ .Values.provisioning.kafka.imageTag }}"
-        command: ["/bin/sh", "/opt/provisioning/create-topics.sh"]
+        command: ["/bin/bash", "/opt/provisioning/create-topics.sh"]
         envFrom:
         - configMapRef:
             name: kafka-config

diff --git a/infrastructure/helm-chart/templates/provisioning/kafka-create-topics.yaml b/infrastructure/helm-chart/templates/provisioning/kafka-create-topics.yaml
@@ -28,6 +28,7 @@ data:
     PARTITIONS=${PARTITIONS:-10}
     REPLICAS=${KAFKA_MINIMUM_REPLICAS:-1}
     AIRY_CORE_NAMESPACE=${AIRY_CORE_NAMESPACE:-}
+    AUTH_JAAS=${AUTH_JAAS:-}
 
     echo "Creating Kafka topics"
 
@@ -51,33 +52,33 @@ data:
 
 
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.channels" 
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.channels" 
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.contacts" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.contacts" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.messages" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.messages" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.metadata" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.metadata" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.read-receipt" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.read-receipt" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.sources" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.sources" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.tags" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.tags" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.templates" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.templates" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.users" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.users" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.webhooks-v0.29.0" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}application.communication.webhooks-v0.29.0" --config cleanup.policy=compact min.compaction.lag.ms=86400000 segment.bytes=10485760
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}ops.application.logs" 
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}ops.application.logs" 
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.facebook.events" 
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.facebook.events" 
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.google.events" 
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.google.events" 
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.twilio.events" 
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.twilio.events" 
 
-    kafka-topics.sh --create --if-not-exists --zookeeper "${ZOOKEEPER}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.viber.events" 
+    kafka-topics.sh --create --if-not-exists "${CONNECTION_OPTS[@]}" --replication-factor "${REPLICAS}" --partitions "${PARTITIONS}" --topic "${AIRY_CORE_NAMESPACE}source.viber.events" 
 
diff --git a/infrastructure/tools/topics/src/main/java/co/airy/tools/topics/Application.java b/infrastructure/tools/topics/src/main/java/co/airy/tools/topics/Application.java
@@ -12,7 +12,7 @@
 public class Application {
 
     public static void main(String[] args) {
-        String createTopicTemplate = "    kafka-topics.sh --create --if-not-exists --zookeeper \"${ZOOKEEPER}\" --replication-factor \"${REPLICAS}\" --partitions \"${PARTITIONS}\" --topic \"${AIRY_CORE_NAMESPACE}%s\" %s";
+        String createTopicTemplate = "    kafka-topics.sh --create --if-not-exists \"${CONNECTION_OPTS[@]}\" --replication-factor \"${REPLICAS}\" --partitions \"${PARTITIONS}\" --topic \"${AIRY_CORE_NAMESPACE}%s\" %s";
         String headerTemplate = "apiVersion: v1" +
                 "\n" +
                 "kind: ConfigMap" +
@@ -53,6 +53,7 @@ public static void main(String[] args) {
                 "    PARTITIONS=${PARTITIONS:-10}\n" +
                 "    REPLICAS=${KAFKA_MINIMUM_REPLICAS:-1}\n" +
                 "    AIRY_CORE_NAMESPACE=${AIRY_CORE_NAMESPACE:-}\n" +
+                "    AUTH_JAAS=${AUTH_JAAS:-}\n" +
                 "\n" +
                 "    echo \"Creating Kafka topics\"\n" +
                 "\n" +