diff --git a/config/config.exs b/config/config.exs index cc549ec5..20d53e81 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,9 +14,9 @@ config :kafka_ex, # server.properties file. # In the case below you would set "advertised.host.name=localhost" brokers: [ - {"localhost", 9092}, {"localhost", 9093}, - {"localhost", 9094} + {"localhost", 9094}, + {"localhost", 9095} ], # # OR: diff --git a/docker-compose-kafka.env b/docker-compose-kafka.env new file mode 100644 index 00000000..c3c1bf6e --- /dev/null +++ b/docker-compose-kafka.env @@ -0,0 +1,46 @@ +################################# +# Common Kafka config +################################# + +# Note: any property `something.bla` can be configure by setting `KAFKA_SOMETHING_BLA`. + +######## topic creation + +KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 + +######## zookeeper + +KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 +KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000 + +######## advertised hosts names and protocols + +# alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1" | grep -o "[0-9.]*" +KAFKA_ADVERTISED_HOST_NAME=localhost +KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:SSL,OUTSIDE:SSL +KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE +KAFKA_PROTOCOL_NAME=INSIDE + +######## SSL + +KAFKA_SSL_KEYSTORE_LOCATION=/ssl/server.keystore.jks +KAFKA_SSL_KEYSTORE_PASSWORD=kafka_ex +KAFKA_SSL_KEY_PASSWORD=kafka_ex +KAFKA_SSL_TRUSTSTORE_LOCATION=/ssl/server.truststore.jks +KAFKA_SSL_TRUSTSTORE_PASSWORD=kafka_ex +KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG + +######## Config + +KAFKA_DELETE_TOPIC_ENABLE=true +# KAFKA_NUM_NETWORK_THREADS=3 +# KAFKA_NUM_IO_THREADS=8 +# KAFKA_SOCKET_SEND_BUFFER_BYTES=102400 +# KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400 +# KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600 +KAFKA_LOG_DIRS=/tmp/kafka_log +KAFKA_NUM_PARTITIONS=1 +KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR=1 +KAFKA_LOG_RETENTION_HOURS=168 +# KAFKA_LOG_SEGMENT_BYTES=1073741824 +# KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=300000 diff --git a/docker-compose.yml b/docker-compose.yml index a10d24b8..990d089d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,5 @@ version: '3.2' + services: zookeeper: image: wurstmeister/zookeeper:3.4.6 @@ -7,30 +8,38 @@ services: kafka1: image: wurstmeister/kafka:0.11.0.1 ports: - - "9092:9092" + - "9093:9093" depends_on: - zookeeper volumes: - - ./kafka1/server.properties.in:/opt/kafka/config/server.properties.in - - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh - ./ssl:/ssl + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ADVERTISED_PORT: 9093 + kafka2: image: wurstmeister/kafka:0.11.0.1 ports: - - "9093:9093" + - "9094:9094" depends_on: - zookeeper volumes: - - ./kafka2/server.properties.in:/opt/kafka/config/server.properties.in - - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh - ./ssl:/ssl + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ADVERTISED_PORT: 9094 + kafka3: image: wurstmeister/kafka:0.11.0.1 ports: - - "9094:9094" + - "9095:9095" depends_on: - zookeeper volumes: - - ./kafka3/server.properties.in:/opt/kafka/config/server.properties.in - - ./scripts/docker-start-kafka.sh:/usr/bin/start-kafka.sh - ./ssl:/ssl + env_file: docker-compose-kafka.env + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ADVERTISED_PORT: 9095 diff --git a/scripts/README.md b/scripts/README.md index 51f123b1..40fb242c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,3 +1,5 @@ +# Scripts + The scripts in this directory are used for testing KafkaEx and are not part of -the release package. Each script should have a comment near the top explaining +the release package. Each script should have a comment near the top explaining its purpose. diff --git a/scripts/docker-start-kafka.sh b/scripts/docker-start-kafka.sh deleted file mode 100755 index f053ba38..00000000 --- a/scripts/docker-start-kafka.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# This script is used when the docker kafka container starts -# -# This version is specialized from -# https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh -# to work with our setup (esp. with ssl) -# -# Note that we mainly manage the server.properties file during `docker_up.sh`; -# the wurstmeister/kafka behavior of translating KAFKA_* env vars to settings -# is removed from this script. - -set -ev - -cp ${KAFKA_HOME}/config/server.properties.in ${KAFKA_HOME}/config/server.properties - -KAFKA_PID=0 - -# see https://medium.com/@gchudnov/trapping-signals-in-docker-containers-7a57fdda7d86#.bh35ir4u5 -term_handler() { - echo 'Stopping Kafka....' - if [ $KAFKA_PID -ne 0 ]; then - kill -s TERM "$KAFKA_PID" - wait "$KAFKA_PID" - fi - echo 'Kafka stopped.' - exit -} - - -# Capture kill requests to stop properly -trap "term_handler" SIGHUP SIGINT SIGTERM -$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties & -KAFKA_PID=$! - -wait "$KAFKA_PID" diff --git a/scripts/docker_ip.sh b/scripts/docker_ip.sh deleted file mode 100755 index 24f333c9..00000000 --- a/scripts/docker_ip.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Finds a private IP for your machine. Tested on MacOS 10.14, Ubuntu 16.04, and Centos 7 - -# 192.168.x.x, 172.[16-31].x.x, 10.x.x.x -regex='inet (192\.168\.|172\.1[6789]\.|172\.2[0-9]\.|172\.3[01]\.|10\.)' - -if [ "$(uname)" == "Darwin" ]; then - # MacOS - ifconfig | grep -E "$regex" | grep broadcast | awk -F ' ' '{print $2}' | head -n 1 -else - # Linux - ip addr | grep -E "$regex" | grep brd | awk -F ' ' '{print $2}' | cut -d/ -f1 | head -n 1 -fi - diff --git a/scripts/docker_up.sh b/scripts/docker_up.sh index 3ee260ab..540472c6 100755 --- a/scripts/docker_up.sh +++ b/scripts/docker_up.sh @@ -1,43 +1,5 @@ #!/bin/bash -# Launches a dockerized kafka cluster configured for testing with KafkaEx -# -# This script attempts to auto-detect the ip address of an active network -# interface using `./scripts/docker_ip.sh`. -# -# This script should be run from the project root - -set -e - -# Kafka needs to know our ip address so that it can advertise valid -# connnection details - -export DOCKER_IP=$(./scripts/docker_ip.sh) - -# for debugging purposes -echo Detected active ip address ${DOCKER_IP} - -for i in 1 2 3 -do - port=$(expr 9092 + ${i} - 1) - mkdir -p kafka${i} - target=kafka${i}/server.properties.in - cp ./server.properties ${target} - # configure broker and port - sed -i.bak "s/@broker_id@/${i}/g" ${target} - sed -i.bak "s/@port@/${port}/g" ${target} - # the @pwd become root directory so we get /ssl - sed -i.bak "s|@pwd@||g" ${target} - # point at zookeeper in docker - sed -i.bak "s/localhost:2181/zookeeper:2181/" ${target} - # delete the existing listeners line - sed -i.bak "/^listeners=/d" ${target} - # add an advertised.listeners and listeners together at the end - echo "advertised.listeners=SSL://${DOCKER_IP}:${port}" >> ${target} - echo "listeners=SSL://0.0.0.0:${port}" >> ${target} -done +# This script starts zookeeper and the 3 kafka nodes used for testing docker-compose up -d - -# create topics needed for testing -docker-compose exec kafka3 /bin/bash -c "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 KAFKA_PORT=9094 KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 create-topics.sh" diff --git a/scripts/docker_up_ubuntu.sh b/scripts/docker_up_ubuntu.sh deleted file mode 100755 index ecf08158..00000000 --- a/scripts/docker_up_ubuntu.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# see docker_up.sh -# This script should be run from the project root - -set -e - -IP_IFACE="docker0" -DOCKER_IP=$(ip addr show dev $IP_IFACE | grep -o "inet[^/]*" | grep -o "[0-9]\{1,3\}[.][0-9.]*") - -for i in 1 2 3 -do - port=$(expr 9092 + ${i} - 1) - mkdir -p kafka${i} - target=kafka${i}/server.properties.in - cp ./server.properties ${target} - # configure broker and port - sed -i.bak "s/@broker_id@/${i}/g" ${target} - sed -i.bak "s/@port@/${port}/g" ${target} - # the @pwd become root directory so we get /ssl - sed -i.bak "s|@pwd@||g" ${target} - # point at zookeeper in docker - sed -i.bak "s/localhost:2181/zookeeper:2181/" ${target} - # delete the existing listeners line - sed -i.bak "/^listeners=/d" ${target} - # add an advertised.listeners and listeners together at the end - echo "advertised.listeners=SSL://${DOCKER_IP}:${port}" >> ${target} - echo "listeners=SSL://0.0.0.0:${port}" >> ${target} -done - -docker-compose up -d - -# create topics needed for testing -docker-compose exec kafka3 /bin/bash -c "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 KAFKA_PORT=9094 KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 create-topics.sh"