diff --git a/README.md b/README.md index 48fa415..f7c7f67 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,5 @@ Build from Source Todo --- -* Not particularily optimzed for startup time. +* Not particularly optimized for startup time. * Better docs - diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 1ec4618..e6ea782 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -2,10 +2,9 @@ FROM java:openjdk-8-jre -ENV DEBIAN_FRONTEND noninteractive -ENV SCALA_VERSION 2.11 -ENV KAFKA_VERSION 0.8.2.1 -ENV KAFKA_HOME /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION" +ENV DEBIAN_FRONTEND=noninteractive \ + SCALA_VERSION=2.11 \ + KAFKA_VERSION=0.8.2.2 # Install Kafka, Zookeeper and other needed things RUN apt-get update && \ @@ -14,13 +13,18 @@ RUN apt-get update && \ apt-get clean && \ wget -q http://apache.mirrors.spacedump.net/kafka/"$KAFKA_VERSION"/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -O /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \ tar xfz /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt && \ - rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz + mv /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION" /opt/kafka && \ + rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \ + mkdir -p /mnt/kafka-logs -ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh +ADD scripts/start-kafka.sh scripts/start-zookeeper.sh /usr/bin/ # Supervisor config ADD supervisor/kafka.conf supervisor/zookeeper.conf /etc/supervisor/conf.d/ +# Expose Kafka & Zookeeper's data & config directories +VOLUME ["/var/lib/zookeeper", "/etc/alternatives/zookeeper-conf", "/opt/kafka/config", "/mnt/kafka-logs"] + # 2181 is zookeeper, 9092 is kafka EXPOSE 2181 9092 diff --git a/kafka/scripts/start-kafka.sh b/kafka/scripts/start-kafka.sh index 4e9121e..a117772 100755 --- a/kafka/scripts/start-kafka.sh +++ b/kafka/scripts/start-kafka.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Optional ENV variables: # * ADVERTISED_HOST: the external ip for the container, e.g. `docker-machine ip \`docker-machine active\`` @@ -8,6 +8,14 @@ # * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB) # * NUM_PARTITIONS: configure the default number of log partitions per topic +KAFKA_HOME=/opt/kafka + +if [ -z $KAFKA_LOG_DIRS ]; then + KAFKA_LOG_DIRS=/mnt/kafka-logs +fi +mkdir -p $KAFKA_LOG_DIRS +sed -r -i "s/#(log.dirs)=(.*)/${KAFKA_LOG_DIRS}/g" $KAFKA_HOME/config/server.properties + # Configure advertised host/port if we run in helios if [ ! -z "$HELIOS_PORT_kafka" ]; then ADVERTISED_HOST=`echo $HELIOS_PORT_kafka | cut -d':' -f 1 | xargs -n 1 dig +short | tail -n 1` @@ -28,7 +36,7 @@ fi if [ ! -z "$ZK_CHROOT" ]; then # wait for zookeeper to start up until /usr/share/zookeeper/bin/zkServer.sh status; do - sleep 0.1 + sleep 0.5 done # create the chroot node @@ -41,6 +49,11 @@ if [ ! -z "$ZK_CHROOT" ]; then sed -r -i "s/(zookeeper.connect)=(.*)/\1=localhost:2181\/$ZK_CHROOT/g" $KAFKA_HOME/config/server.properties fi +if [ ! -z $ZOOKEEPER_SERVERS ]; then + # TODO: ZOOKEEPER_SERVERS incompatible with ZK_CHROOT + sed -r -i "s/(zookeeper.connect)=(.*)/${ZOOKEEPER_SERVERS}/g" $KAFKA_HOME/config/server.properties +fi + # Allow specification of log retention policies if [ ! -z "$LOG_RETENTION_HOURS" ]; then echo "log retention hours: $LOG_RETENTION_HOURS" diff --git a/kafka/scripts/start-zookeeper.sh b/kafka/scripts/start-zookeeper.sh new file mode 100755 index 0000000..2bd5d40 --- /dev/null +++ b/kafka/scripts/start-zookeeper.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Default ZK configuration: +export ZOOKEEPER_tickTime=2000 +export ZOOKEEPER_initLimit=10 +export ZOOKEEPER_syncLimit=5 +export ZOOKEEPER_dataDir="/var/lib/zookeeper" +export ZOOKEEPER_clientPort=2181 + +# Keep the helper header in the config :) +echo '# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html' > /etc/alternatives/zookeeper-conf/zoo.cfg +# Source configuration from variables names - +# Any environment variable starting with "ZOOKEEPER_" +# will be written to the ZK config file, ie: +# ZOOKEEPER_tickTime=100 results in 'tickTime=100' being written to ZK Config +for VAR in `printenv`; do + NAME=${VAR%=*} + VALUE=${VAR#*=} + if [[ $NAME == "ZOOKEEPER_"* && $NAME != "ZOOKEEPER_SERVERS" && NAME != "ZOOKEEPER_MYID" ]]; then + echo "$(echo $NAME | sed 's/ZOOKEEPER_//')=$VALUE" >> /etc/alternatives/zookeeper-conf/zoo.cfg + fi +done + +ZOOKEEPER_MYID=${ZOOKEEPER_MYID:-0} + +echo ${ZOOKEEPER_MYID} > /etc/zookeeper/conf/myid +echo "# I am server #${ZOOKEEPER_MYID}" >> /etc/alternatives/zookeeper-conf/zoo.cfg + +# Additionally, we'll provide a helper to allow users to easily define a set of ZK servers: +if [ -z ${ZOOKEEPER_SERVERS} ]; then + SERVER_ID=1 + IFS=',' read -ra ADDR <<< "$ZOOKEEPER_SERVERS" + for SERVER in "${ADDR[@]}"; do + echo "server.${SERVER_ID}=${SERVER}" >> /etc/alternatives/zookeeper-conf/zoo.cfg + ((SERVER_ID++)) + done +fi + +/usr/share/zookeeper/bin/zkServer.sh start-foreground diff --git a/kafka/supervisor/zookeeper.conf b/kafka/supervisor/zookeeper.conf index 5650d95..a855dbd 100644 --- a/kafka/supervisor/zookeeper.conf +++ b/kafka/supervisor/zookeeper.conf @@ -1,4 +1,4 @@ [program:zookeeper] -command=/usr/share/zookeeper/bin/zkServer.sh start-foreground +command=/usr/bin/start-zookeeper.sh autostart=true -autorestart=true \ No newline at end of file +autorestart=true