From 31a9e51bdb3b6252fc0bd7e8be94396d506d8aef Mon Sep 17 00:00:00 2001
From: erulabs <seandon.mooy@gmail.com>
Date: Wed, 16 Dec 2015 14:56:52 -0800
Subject: [PATCH 1/2] expose zookeeper configuration and kafka
 zookeeper.connect for proper clustering

---
 README.md                        |  3 +--
 kafka/Dockerfile                 |  9 ++++---
 kafka/scripts/start-kafka.sh     | 12 ++++++++--
 kafka/scripts/start-zookeeper.sh | 41 ++++++++++++++++++++++++++++++++
 kafka/supervisor/zookeeper.conf  |  4 ++--
 5 files changed, 58 insertions(+), 11 deletions(-)
 create mode 100755 kafka/scripts/start-zookeeper.sh

diff --git a/README.md b/README.md
index 48fa415..f7c7f67 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,5 @@ Build from Source
 Todo
 ---
 
-* Not particularily optimzed for startup time.
+* Not particularly optimized for startup time.
 * Better docs
-
diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 1ec4618..d664735 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -2,10 +2,9 @@
 
 FROM java:openjdk-8-jre
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV SCALA_VERSION 2.11
-ENV KAFKA_VERSION 0.8.2.1
-ENV KAFKA_HOME /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION"
+ENV DEBIAN_FRONTEND=noninteractive \
+  SCALA_VERSION=2.11 \
+  KAFKA_VERSION=0.8.2.1
 
 # Install Kafka, Zookeeper and other needed things
 RUN apt-get update && \
@@ -16,7 +15,7 @@ RUN apt-get update && \
     tar xfz /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt && \
     rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz
 
-ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh
+ADD scripts/start-kafka.sh scripts/start-zookeeper.sh /usr/bin/
 
 # Supervisor config
 ADD supervisor/kafka.conf supervisor/zookeeper.conf /etc/supervisor/conf.d/
diff --git a/kafka/scripts/start-kafka.sh b/kafka/scripts/start-kafka.sh
index 4e9121e..0dabeb4 100755
--- a/kafka/scripts/start-kafka.sh
+++ b/kafka/scripts/start-kafka.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Optional ENV variables:
 # * ADVERTISED_HOST: the external ip for the container, e.g. `docker-machine ip \`docker-machine active\``
@@ -7,6 +7,9 @@
 # * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week)
 # * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB)
 # * NUM_PARTITIONS: configure the default number of log partitions per topic
+if [ -z $KAFKA_HOME ]; then
+  KAFKA_HOME=/opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION"
+fi
 
 # Configure advertised host/port if we run in helios
 if [ ! -z "$HELIOS_PORT_kafka" ]; then
@@ -28,7 +31,7 @@ fi
 if [ ! -z "$ZK_CHROOT" ]; then
     # wait for zookeeper to start up
     until /usr/share/zookeeper/bin/zkServer.sh status; do
-      sleep 0.1
+      sleep 0.5
     done
 
     # create the chroot node
@@ -41,6 +44,11 @@ if [ ! -z "$ZK_CHROOT" ]; then
     sed -r -i "s/(zookeeper.connect)=(.*)/\1=localhost:2181\/$ZK_CHROOT/g" $KAFKA_HOME/config/server.properties
 fi
 
+if [ ! -z $ZOOKEEPER_SERVERS ]; then
+  # TODO: ZOOKEEPER_SERVERS incompatible with ZK_CHROOT
+  sed -r -i "s/(zookeeper.connect)=(.*)/${ZOOKEEPER_SERVERS}/g" $KAFKA_HOME/config/server.properties
+fi
+
 # Allow specification of log retention policies
 if [ ! -z "$LOG_RETENTION_HOURS" ]; then
     echo "log retention hours: $LOG_RETENTION_HOURS"
diff --git a/kafka/scripts/start-zookeeper.sh b/kafka/scripts/start-zookeeper.sh
new file mode 100755
index 0000000..aedf6a3
--- /dev/null
+++ b/kafka/scripts/start-zookeeper.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Default ZK configuration:
+export ZOOKEEPER_tickTime=2000
+export ZOOKEEPER_initLimit=10
+export ZOOKEEPER_syncLimit=5
+export ZOOKEEPER_dataDir="/var/lib/zookeeper"
+export ZOOKEEPER_clientPort=2181
+
+# Keep the helper header in the config :)
+echo '# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html' > /etc/alternatives/zookeeper-conf/zoo.cfg
+# Source configuration from variables names -
+# Any environment variable starting with "ZOOKEEPER_"
+# will be written to the ZK config file, ie:
+# ZOOKEEPER_tickTime=100 results in 'tickTime=100' being written to ZK Config
+for VAR in `printenv`; do
+  NAME=${VAR%=*}
+  VALUE=${VAR#*=}
+  if [[ $NAME == "ZOOKEEPER_"* && $NAME != "ZOOKEEPER_SERVERS" && NAME != "ZOOKEEPER_MYID" ]]; then
+    echo "$(echo $NAME | sed 's/ZOOKEEPER_//')=$VALUE" >> /etc/alternatives/zookeeper-conf/zoo.cfg
+  fi
+done
+
+ZOOKEEPER_MYID=${ZOOKEEPER_MYID:-0}
+
+echo ${ZOOKEEPER_MYID} > /etc/zookeeper/conf/myid
+echo "# I am server #${ZOOKEEPER_MYID}" >> /etc/alternatives/zookeeper-conf/zoo.cfg
+
+# Additionally, we'll provide a helper to allow users to easily define a set of ZK servers:
+if [ -z ${ZOOKEEPER_SERVERS} ]; then
+  SERVER_ID=1
+  IFS=',' read -ra ADDR <<< "$ZOOKEEPER_SERVERS"
+  for SERVER in "${ADDR[@]}"; do
+    echo "server.${SERVER_ID}=${SERVER}" >> /etc/alternatives/zookeeper-conf/zoo.cfg
+    ((SERVER_ID++))
+  done
+fi
+
+cat /etc/alternatives/zookeeper-conf/zoo.cfg
+
+/usr/share/zookeeper/bin/zkServer.sh start-foreground
diff --git a/kafka/supervisor/zookeeper.conf b/kafka/supervisor/zookeeper.conf
index 5650d95..a855dbd 100644
--- a/kafka/supervisor/zookeeper.conf
+++ b/kafka/supervisor/zookeeper.conf
@@ -1,4 +1,4 @@
 [program:zookeeper]
-command=/usr/share/zookeeper/bin/zkServer.sh start-foreground
+command=/usr/bin/start-zookeeper.sh
 autostart=true
-autorestart=true
\ No newline at end of file
+autorestart=true

From 4952c7867f11c72e89293198bd1de6c540bcf1b8 Mon Sep 17 00:00:00 2001
From: erulabs <seandon.mooy@gmail.com>
Date: Wed, 16 Dec 2015 16:17:37 -0800
Subject: [PATCH 2/2] Add volumes to dockerfile

---
 kafka/Dockerfile                 | 9 +++++++--
 kafka/scripts/start-kafka.sh     | 9 +++++++--
 kafka/scripts/start-zookeeper.sh | 2 --
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index d664735..e6ea782 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -4,7 +4,7 @@ FROM java:openjdk-8-jre
 
 ENV DEBIAN_FRONTEND=noninteractive \
   SCALA_VERSION=2.11 \
-  KAFKA_VERSION=0.8.2.1
+  KAFKA_VERSION=0.8.2.2
 
 # Install Kafka, Zookeeper and other needed things
 RUN apt-get update && \
@@ -13,13 +13,18 @@ RUN apt-get update && \
     apt-get clean && \
     wget -q http://apache.mirrors.spacedump.net/kafka/"$KAFKA_VERSION"/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -O /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \
     tar xfz /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt && \
-    rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz
+    mv /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION" /opt/kafka && \
+    rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \
+    mkdir -p /mnt/kafka-logs
 
 ADD scripts/start-kafka.sh scripts/start-zookeeper.sh /usr/bin/
 
 # Supervisor config
 ADD supervisor/kafka.conf supervisor/zookeeper.conf /etc/supervisor/conf.d/
 
+# Expose Kafka & Zookeeper's data & config directories
+VOLUME ["/var/lib/zookeeper", "/etc/alternatives/zookeeper-conf", "/opt/kafka/config", "/mnt/kafka-logs"]
+
 # 2181 is zookeeper, 9092 is kafka
 EXPOSE 2181 9092
 
diff --git a/kafka/scripts/start-kafka.sh b/kafka/scripts/start-kafka.sh
index 0dabeb4..a117772 100755
--- a/kafka/scripts/start-kafka.sh
+++ b/kafka/scripts/start-kafka.sh
@@ -7,9 +7,14 @@
 # * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week)
 # * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB)
 # * NUM_PARTITIONS: configure the default number of log partitions per topic
-if [ -z $KAFKA_HOME ]; then
-  KAFKA_HOME=/opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION"
+
+KAFKA_HOME=/opt/kafka
+
+if [ -z $KAFKA_LOG_DIRS ]; then
+  KAFKA_LOG_DIRS=/mnt/kafka-logs
 fi
+mkdir -p $KAFKA_LOG_DIRS
+sed -r -i "s/#(log.dirs)=(.*)/${KAFKA_LOG_DIRS}/g" $KAFKA_HOME/config/server.properties
 
 # Configure advertised host/port if we run in helios
 if [ ! -z "$HELIOS_PORT_kafka" ]; then
diff --git a/kafka/scripts/start-zookeeper.sh b/kafka/scripts/start-zookeeper.sh
index aedf6a3..2bd5d40 100755
--- a/kafka/scripts/start-zookeeper.sh
+++ b/kafka/scripts/start-zookeeper.sh
@@ -36,6 +36,4 @@ if [ -z ${ZOOKEEPER_SERVERS} ]; then
   done
 fi
 
-cat /etc/alternatives/zookeeper-conf/zoo.cfg
-
 /usr/share/zookeeper/bin/zkServer.sh start-foreground