expose zookeeper configuration and kafka zookeeper.connect for proper clustering

README.md

@@ -80,6 +80,5 @@ Build from Source
 Todo
 ---
 
-* Not particularily optimzed for startup time.
+* Not particularly optimized for startup time.
 * Better docs
-

kafka/Dockerfile

@@ -2,10 +2,9 @@
 
 FROM java:openjdk-8-jre
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV SCALA_VERSION 2.11
-ENV KAFKA_VERSION 0.8.2.1
-ENV KAFKA_HOME /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION"
+ENV DEBIAN_FRONTEND=noninteractive \
+  SCALA_VERSION=2.11 \
+  KAFKA_VERSION=0.8.2.2
 
 # Install Kafka, Zookeeper and other needed things
 RUN apt-get update && \
@@ -14,13 +13,18 @@ RUN apt-get update && \
     apt-get clean && \
     wget -q http://apache.mirrors.spacedump.net/kafka/"$KAFKA_VERSION"/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -O /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \
     tar xfz /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt && \
-    rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz
+    mv /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION" /opt/kafka && \
+    rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \
+    mkdir -p /mnt/kafka-logs
 
-ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh
+ADD scripts/start-kafka.sh scripts/start-zookeeper.sh /usr/bin/
 
 # Supervisor config
 ADD supervisor/kafka.conf supervisor/zookeeper.conf /etc/supervisor/conf.d/
 
+# Expose Kafka & Zookeeper's data & config directories
+VOLUME ["/var/lib/zookeeper", "/etc/alternatives/zookeeper-conf", "/opt/kafka/config", "/mnt/kafka-logs"]
+
 # 2181 is zookeeper, 9092 is kafka
 EXPOSE 2181 9092
 

kafka/scripts/start-kafka.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Optional ENV variables:
 # * ADVERTISED_HOST: the external ip for the container, e.g. `docker-machine ip \`docker-machine active\``
@@ -8,6 +8,14 @@
 # * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB)
 # * NUM_PARTITIONS: configure the default number of log partitions per topic
 
+KAFKA_HOME=/opt/kafka
+
+if [ -z $KAFKA_LOG_DIRS ]; then
+  KAFKA_LOG_DIRS=/mnt/kafka-logs
+fi
+mkdir -p $KAFKA_LOG_DIRS
+sed -r -i "s/#(log.dirs)=(.*)/${KAFKA_LOG_DIRS}/g" $KAFKA_HOME/config/server.properties
+
 # Configure advertised host/port if we run in helios
 if [ ! -z "$HELIOS_PORT_kafka" ]; then
     ADVERTISED_HOST=`echo $HELIOS_PORT_kafka | cut -d':' -f 1 | xargs -n 1 dig +short | tail -n 1`
@@ -28,7 +36,7 @@ fi
 if [ ! -z "$ZK_CHROOT" ]; then
     # wait for zookeeper to start up
     until /usr/share/zookeeper/bin/zkServer.sh status; do
-      sleep 0.1
+      sleep 0.5
     done
 
     # create the chroot node
@@ -41,6 +49,11 @@ if [ ! -z "$ZK_CHROOT" ]; then
     sed -r -i "s/(zookeeper.connect)=(.*)/\1=localhost:2181\/$ZK_CHROOT/g" $KAFKA_HOME/config/server.properties
 fi
 
+if [ ! -z $ZOOKEEPER_SERVERS ]; then
+  # TODO: ZOOKEEPER_SERVERS incompatible with ZK_CHROOT
+  sed -r -i "s/(zookeeper.connect)=(.*)/${ZOOKEEPER_SERVERS}/g" $KAFKA_HOME/config/server.properties
+fi
+
 # Allow specification of log retention policies
 if [ ! -z "$LOG_RETENTION_HOURS" ]; then
     echo "log retention hours: $LOG_RETENTION_HOURS"

kafka/scripts/start-zookeeper.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Default ZK configuration:
+export ZOOKEEPER_tickTime=2000
+export ZOOKEEPER_initLimit=10
+export ZOOKEEPER_syncLimit=5
+export ZOOKEEPER_dataDir="/var/lib/zookeeper"
+export ZOOKEEPER_clientPort=2181
+
+# Keep the helper header in the config :)
+echo '# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html' > /etc/alternatives/zookeeper-conf/zoo.cfg
+# Source configuration from variables names -
+# Any environment variable starting with "ZOOKEEPER_"
+# will be written to the ZK config file, ie:
+# ZOOKEEPER_tickTime=100 results in 'tickTime=100' being written to ZK Config
+for VAR in `printenv`; do
+  NAME=${VAR%=*}
+  VALUE=${VAR#*=}
+  if [[ $NAME == "ZOOKEEPER_"* && $NAME != "ZOOKEEPER_SERVERS" && NAME != "ZOOKEEPER_MYID" ]]; then
+    echo "$(echo $NAME | sed 's/ZOOKEEPER_//')=$VALUE" >> /etc/alternatives/zookeeper-conf/zoo.cfg
+  fi
+done
+
+ZOOKEEPER_MYID=${ZOOKEEPER_MYID:-0}
+
+echo ${ZOOKEEPER_MYID} > /etc/zookeeper/conf/myid
+echo "# I am server #${ZOOKEEPER_MYID}" >> /etc/alternatives/zookeeper-conf/zoo.cfg
+
+# Additionally, we'll provide a helper to allow users to easily define a set of ZK servers:
+if [ -z ${ZOOKEEPER_SERVERS} ]; then
+  SERVER_ID=1
+  IFS=',' read -ra ADDR <<< "$ZOOKEEPER_SERVERS"
+  for SERVER in "${ADDR[@]}"; do
+    echo "server.${SERVER_ID}=${SERVER}" >> /etc/alternatives/zookeeper-conf/zoo.cfg
+    ((SERVER_ID++))
+  done
+fi
+
+/usr/share/zookeeper/bin/zkServer.sh start-foreground

kafka/supervisor/zookeeper.conf

@@ -1,4 +1,4 @@
 [program:zookeeper]
-command=/usr/share/zookeeper/bin/zkServer.sh start-foreground
+command=/usr/bin/start-zookeeper.sh
 autostart=true
-autorestart=true
+autorestart=true