From 64830d990237caca4c1e9120a053de908da0154f Mon Sep 17 00:00:00 2001
From: "https://luberg@github.com" <lucas.berger@kukanao.com>
Date: Thu, 12 Jan 2017 14:56:22 -0600
Subject: [PATCH 01/13] Release v0.10.1.1

---
 CHANGELOG.md | 4 ++++
 Dockerfile   | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 045dd47..e04104b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 No changes yet.
 
+## 0.10.1.1 - 12 January, 2017
+
+- Update to Kafka 0.10.1.1
+
 ## 0.10.1.0 - 27 October, 2016
 
 - Update to Kafka 0.10.1.0 ([xrl], #25)
diff --git a/Dockerfile b/Dockerfile
index eb4acd3..7f2644b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,7 @@ FROM netflixoss/java:8
 MAINTAINER Ches Martin <ches@whiskeyandgrits.net>
 
 # The Scala 2.11 build is currently recommended by the project.
-ENV KAFKA_VERSION=0.10.1.0 KAFKA_SCALA_VERSION=2.11 JMX_PORT=7203
+ENV KAFKA_VERSION=0.10.1.1 KAFKA_SCALA_VERSION=2.11 JMX_PORT=7203
 ENV KAFKA_RELEASE_ARCHIVE kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz
 
 RUN mkdir /kafka /data /logs

From 64c1b04c34d38cb19593cb540215aa026bcce7fe Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 17 Jan 2017 10:40:03 -0800
Subject: [PATCH 02/13] A more recent and maintained Oracle JDK8

---
 Dockerfile | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7f2644b..e107853 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,14 +1,23 @@
-# Builds an image for Apache Kafka 0.8.1.1 from binary distribution.
-#
-# The netflixoss/java base image runs Oracle Java 7 installed atop the
+# The image runs Oracle Java 8 installed atop the
 # ubuntu:trusty (14.04) official image. Docker's official java images are
 # OpenJDK-only currently, and the Kafka project, Confluent, and most other
 # major Java projects test and recommend Oracle Java for production for optimal
 # performance.
 
-FROM netflixoss/java:8
+FROM ubuntu:trusty
 MAINTAINER Ches Martin <ches@whiskeyandgrits.net>
 
+# Install Java.
+# https://github.com/dockerfile/java/blob/master/oracle-java8/Dockerfile
+RUN \
+  apt-get update && apt-get install -y software-properties-common && \
+  echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
+  add-apt-repository -y ppa:webupd8team/java && \
+  apt-get update && \
+  apt-get install -y oracle-java8-installer && \
+  rm -rf /var/lib/apt/lists/* && \
+  rm -rf /var/cache/oracle-jdk8-installer
+
 # The Scala 2.11 build is currently recommended by the project.
 ENV KAFKA_VERSION=0.10.1.1 KAFKA_SCALA_VERSION=2.11 JMX_PORT=7203
 ENV KAFKA_RELEASE_ARCHIVE kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz

From 10fa3dd3cb6d472c7e6040531f7fb3a216af7c60 Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 17 Jan 2017 14:53:10 -0800
Subject: [PATCH 03/13] Lots of ENV var changes, backported kafka 0.10.1.1
 config

---
 Dockerfile                        |  16 ++++-
 config/server.properties.template | 100 +++++++++++++++++++++++-------
 start.sh                          |  17 ++---
 3 files changed, 100 insertions(+), 33 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e107853..9e63a96 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,8 +19,10 @@ RUN \
   rm -rf /var/cache/oracle-jdk8-installer
 
 # The Scala 2.11 build is currently recommended by the project.
-ENV KAFKA_VERSION=0.10.1.1 KAFKA_SCALA_VERSION=2.11 JMX_PORT=7203
-ENV KAFKA_RELEASE_ARCHIVE kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz
+ENV KAFKA_VERSION=0.10.1.1 \
+	KAFKA_SCALA_VERSION=2.11 \
+	JMX_PORT=7203 \
+	KAFKA_RELEASE_ARCHIVE=kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz
 
 RUN mkdir /kafka /data /logs
 
@@ -54,6 +56,16 @@ USER kafka
 ENV PATH /kafka/bin:$PATH
 WORKDIR /kafka
 
+ENV GROUP_MAX_SESSION_TIMEOUT_MS "300000" \
+	JAVA_RMI_SERVER_HOSTNAME="" \
+	KAFKA_BROKER_ID "" \
+	KAFKA_DELETE_TOPIC_ENABLE "false" \
+	LOG_FLUSH_SCHEDULER_INTERVAL_MS "9223372036854775807" \
+	LOG_RETENTION_HOURS "168" \
+	ZOOKEEPER_CONNECTION_STRING "localhost:2181" \
+	ZOOKEEPER_CONNECTION_TIMEOUT_MS "10000" \
+	ZOOKEEPER_SESSION_TIMEOUT_MS "10000"
+
 # broker, jmx
 EXPOSE 9092 ${JMX_PORT}
 VOLUME [ "/data", "/logs" ]
diff --git a/config/server.properties.template b/config/server.properties.template
index 81667f2..1c28086 100644
--- a/config/server.properties.template
+++ b/config/server.properties.template
@@ -12,42 +12,85 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 # see kafka.server.KafkaConfig for additional details and defaults
 
 ############################# Server Basics #############################
 
 # The id of the broker. This must be set to a unique integer for each broker.
 broker.id={{KAFKA_BROKER_ID}}
-auto.leader.rebalance.enable=true
-
-# Replication
-auto.create.topics.enable=true
-default.replication.factor=1
-
-# Hostname the broker will advertise to consumers. If not set, kafka will use the value returned
-# from InetAddress.getLocalHost().  If there are multiple interfaces getLocalHost
-# may not be what you want.
-advertised.host.name={{KAFKA_ADVERTISED_HOST_NAME}}
 
-# Enable topic deletion
+# Switch to enable topic deletion or not, default value is false
 delete.topic.enable={{KAFKA_DELETE_TOPIC_ENABLE}}
 
 ############################# Socket Server Settings #############################
 
-# The port the socket server listens on
-port={{KAFKA_PORT}}
-advertised.port={{KAFKA_ADVERTISED_PORT}}
+# The address the socket server listens on. It will get the value returned from
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = security_protocol://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+#listeners=PLAINTEXT://:9092
+# TODO: how to expose this thought docker ENV?
+
+# Hostname and port the broker will advertise to producers and consumers. If not set,
+# it uses the value for "listeners" if configured.  Otherwise, it will use the value
+# returned from java.net.InetAddress.getCanonicalHostName().
+#advertised.listeners=PLAINTEXT://your.host.name:9092
+# TODO: how to expose this thought docker ENV?
+
+# The number of threads handling network requests
+num.network.threads=3
+
+# The number of threads doing disk I/O
+num.io.threads=8
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes=102400
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes=102400
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes=104857600
+
 
 ############################# Log Basics #############################
 
-# The directory under which to store log files
+# A comma seperated list of directories under which to store log files
 log.dir=/data
-log.dirs=/data
 
-# The number of logical partitions per topic per server. More partitions allow greater parallelism
-# for consumption, but also mean more files.
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
 num.partitions=1
 
+# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+# This value is recommended to be increased for installations with data dirs located in RAID array.
+num.recovery.threads.per.data.dir=1
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+#    1. Durability: Unflushed data may be lost if you are not using replication.
+#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+#log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+#log.flush.interval.ms=1000
+
+# The frequency in ms that the log flusher checks whether any log needs to be flushed to disk
+# also doubles as log.flush.interval.ms (the maximum time in ms that a message in any topic is kept in memory before flushed to disk)
+log.flush.scheduler.interval.ms={{LOG_FLUSH_SCHEDULER_INTERVAL_MS}}
+
 ############################# Log Retention Policy #############################
 
 # The following configurations control the disposal of log segments. The policy can
@@ -56,21 +99,30 @@ num.partitions=1
 # from the end of the log.
 
 # The minimum age of a log file to be eligible for deletion
-log.retention.hours=168
+log.retention.hours={{LOG_RETENTION_HOURS}}
+
+# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
+# segments don't drop below log.retention.bytes.
+#log.retention.bytes=1073741824
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+log.segment.bytes=1073741824
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms=300000
 
 ############################# Zookeeper #############################
 
-# Zk connection string (see zk docs for details).
+# Zookeeper connection string (see zookeeper docs for details).
 # This is a comma separated host:port pairs, each corresponding to a zk
 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
 # You can also append an optional chroot string to the urls to specify the
 # root directory for all kafka znodes.
-zookeeper.connect={{ZOOKEEPER_CONNECTION_STRING}}{{ZOOKEEPER_CHROOT}}
+zookeeper.connect={{ZOOKEEPER_CONNECTION_STRING}}
 zookeeper.connection.timeout.ms={{ZOOKEEPER_CONNECTION_TIMEOUT_MS}}
 zookeeper.session.timeout.ms={{ZOOKEEPER_SESSION_TIMEOUT_MS}}
 
 ############################# Additional Broker Settings #######################
-controlled.shutdown.enable=true
-group.max.session.timeout.ms={{GROUP_MAX_SESSION_TIMEOUT_MS}}
 
-# vim:set filetype=jproperties
+group.max.session.timeout.ms={{GROUP_MAX_SESSION_TIMEOUT_MS}}
\ No newline at end of file
diff --git a/start.sh b/start.sh
index c632a47..a9694d0 100755
--- a/start.sh
+++ b/start.sh
@@ -5,23 +5,25 @@
 [ -n "$ZOOKEEPER_PORT_2181_TCP_ADDR" ] && ZOOKEEPER_IP=$ZOOKEEPER_PORT_2181_TCP_ADDR
 [ -n "$ZOOKEEPER_PORT_2181_TCP_PORT" ] && ZOOKEEPER_PORT=$ZOOKEEPER_PORT_2181_TCP_PORT
 
-IP=$(grep "\s${HOSTNAME}$" /etc/hosts | head -n 1 | awk '{print $1}')
+IP=$(hostname -i)
 
 # Concatenate the IP:PORT for ZooKeeper to allow setting a full connection
 # string with multiple ZooKeeper hosts
 [ -z "$ZOOKEEPER_CONNECTION_STRING" ] && ZOOKEEPER_CONNECTION_STRING="${ZOOKEEPER_IP}:${ZOOKEEPER_PORT:-2181}"
 
+# Let see if we can extract the ID from the name, making us K8S stateful set compatible
+# expects $HOSTNAME in the format *-DIGIT
+[ -z "$KAFKA_BROKER_ID" ] && KAFKA_BROKER_ID=$(echo $HOSTNAME | sed 's/.*-\([0-9]\+\)$/\1/')
+
 cat /kafka/config/server.properties.template | sed \
   -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \
-  -e "s|{{ZOOKEEPER_CHROOT}}|${ZOOKEEPER_CHROOT:-}|g" \
   -e "s|{{KAFKA_BROKER_ID}}|${KAFKA_BROKER_ID:-0}|g" \
-  -e "s|{{KAFKA_ADVERTISED_HOST_NAME}}|${KAFKA_ADVERTISED_HOST_NAME:-$IP}|g" \
-  -e "s|{{KAFKA_PORT}}|${KAFKA_PORT:-9092}|g" \
-  -e "s|{{KAFKA_ADVERTISED_PORT}}|${KAFKA_ADVERTISED_PORT:-9092}|g" \
   -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \
+  -e "s|{{LOG_FLUSH_SCHEDULER_INTERVAL_MS}}|${LOG_FLUSH_SCHEDULER_INTERVAL_MS:-9223372036854775807}|g" \
+  -e "s|{{LOG_RETENTION_HOURS}}|${LOG_RETENTION_HOURS:-168}|g" \
+  -e "s|{{GROUP_MAX_SESSION_TIMEOUT_MS}}|${GROUP_MAX_SESSION_TIMEOUT_MS:-300000}|g" \
   -e "s|{{ZOOKEEPER_CONNECTION_TIMEOUT_MS}}|${ZOOKEEPER_CONNECTION_TIMEOUT_MS:-10000}|g" \
   -e "s|{{ZOOKEEPER_SESSION_TIMEOUT_MS}}|${ZOOKEEPER_SESSION_TIMEOUT_MS:-10000}|g" \
-  -e "s|{{GROUP_MAX_SESSION_TIMEOUT_MS}}|${GROUP_MAX_SESSION_TIMEOUT_MS:-300000}|g" \
    > /kafka/config/server.properties
 
 # Kafka's built-in start scripts set the first three system properties here, but
@@ -42,7 +44,8 @@ if [ -z $KAFKA_JMX_OPTS ]; then
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.ssl=false"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
-    KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=${JAVA_RMI_SERVER_HOSTNAME:-$KAFKA_ADVERTISED_HOST_NAME} "
+    KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.net.preferIPv4Stack=true"
+    KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=${JAVA_RMI_SERVER_HOSTNAME:-$HOSTNAME} "
     export KAFKA_JMX_OPTS
 fi
 

From 74c2fc5388af8e03a30feae8efbce9489939987a Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 17 Jan 2017 15:46:57 -0800
Subject: [PATCH 04/13] Don't expose JMX_PORT to full container, remap inside
 of the start.sh

---
 Dockerfile | 22 +++++++++++-----------
 start.sh   |  6 +++---
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9e63a96..a2a2e1b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,8 +21,8 @@ RUN \
 # The Scala 2.11 build is currently recommended by the project.
 ENV KAFKA_VERSION=0.10.1.1 \
 	KAFKA_SCALA_VERSION=2.11 \
-	JMX_PORT=7203 \
-	KAFKA_RELEASE_ARCHIVE=kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz
+	KAFKA_JMX_PORT=7203
+ENV KAFKA_RELEASE_ARCHIVE="kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
 
 RUN mkdir /kafka /data /logs
 
@@ -56,18 +56,18 @@ USER kafka
 ENV PATH /kafka/bin:$PATH
 WORKDIR /kafka
 
-ENV GROUP_MAX_SESSION_TIMEOUT_MS "300000" \
+ENV GROUP_MAX_SESSION_TIMEOUT_MS="300000" \
 	JAVA_RMI_SERVER_HOSTNAME="" \
-	KAFKA_BROKER_ID "" \
-	KAFKA_DELETE_TOPIC_ENABLE "false" \
-	LOG_FLUSH_SCHEDULER_INTERVAL_MS "9223372036854775807" \
-	LOG_RETENTION_HOURS "168" \
-	ZOOKEEPER_CONNECTION_STRING "localhost:2181" \
-	ZOOKEEPER_CONNECTION_TIMEOUT_MS "10000" \
-	ZOOKEEPER_SESSION_TIMEOUT_MS "10000"
+	KAFKA_BROKER_ID="" \
+	KAFKA_DELETE_TOPIC_ENABLE="false" \
+	LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
+	LOG_RETENTION_HOURS="168" \
+	ZOOKEEPER_CONNECTION_STRING="localhost:2181" \
+	ZOOKEEPER_CONNECTION_TIMEOUT_MS="10000" \
+	ZOOKEEPER_SESSION_TIMEOUT_MS="10000"
 
 # broker, jmx
-EXPOSE 9092 ${JMX_PORT}
+EXPOSE 9092 ${KAFKA_JMX_PORT}
 VOLUME [ "/data", "/logs" ]
 
 CMD ["/start.sh"]
diff --git a/start.sh b/start.sh
index a9694d0..e2a8fcf 100755
--- a/start.sh
+++ b/start.sh
@@ -16,12 +16,12 @@ IP=$(hostname -i)
 [ -z "$KAFKA_BROKER_ID" ] && KAFKA_BROKER_ID=$(echo $HOSTNAME | sed 's/.*-\([0-9]\+\)$/\1/')
 
 cat /kafka/config/server.properties.template | sed \
-  -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \
+  -e "s|{{GROUP_MAX_SESSION_TIMEOUT_MS}}|${GROUP_MAX_SESSION_TIMEOUT_MS:-300000}|g" \
   -e "s|{{KAFKA_BROKER_ID}}|${KAFKA_BROKER_ID:-0}|g" \
   -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \
   -e "s|{{LOG_FLUSH_SCHEDULER_INTERVAL_MS}}|${LOG_FLUSH_SCHEDULER_INTERVAL_MS:-9223372036854775807}|g" \
   -e "s|{{LOG_RETENTION_HOURS}}|${LOG_RETENTION_HOURS:-168}|g" \
-  -e "s|{{GROUP_MAX_SESSION_TIMEOUT_MS}}|${GROUP_MAX_SESSION_TIMEOUT_MS:-300000}|g" \
+  -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \
   -e "s|{{ZOOKEEPER_CONNECTION_TIMEOUT_MS}}|${ZOOKEEPER_CONNECTION_TIMEOUT_MS:-10000}|g" \
   -e "s|{{ZOOKEEPER_SESSION_TIMEOUT_MS}}|${ZOOKEEPER_SESSION_TIMEOUT_MS:-10000}|g" \
    > /kafka/config/server.properties
@@ -43,7 +43,7 @@ if [ -z $KAFKA_JMX_OPTS ]; then
     KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.ssl=false"
-    KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
+    KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.rmi.port=${KAFKA_JMX_PORT:-7203}"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.net.preferIPv4Stack=true"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=${JAVA_RMI_SERVER_HOSTNAME:-$HOSTNAME} "
     export KAFKA_JMX_OPTS

From be25e5ca8869de0e47dac8b9569c50f7b2cdc4ca Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 17 Jan 2017 15:47:08 -0800
Subject: [PATCH 05/13] New line for easy cat'ing

---
 config/server.properties.template | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/server.properties.template b/config/server.properties.template
index 1c28086..452b467 100644
--- a/config/server.properties.template
+++ b/config/server.properties.template
@@ -125,4 +125,4 @@ zookeeper.session.timeout.ms={{ZOOKEEPER_SESSION_TIMEOUT_MS}}
 
 ############################# Additional Broker Settings #######################
 
-group.max.session.timeout.ms={{GROUP_MAX_SESSION_TIMEOUT_MS}}
\ No newline at end of file
+group.max.session.timeout.ms={{GROUP_MAX_SESSION_TIMEOUT_MS}}

From ec1afee3c82dea448279b8350f53adaeb4ed7892 Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 17 Jan 2017 15:47:19 -0800
Subject: [PATCH 06/13] Example kafka/k8s config file

---
 kafka-stateful-set.yml | 113 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 kafka-stateful-set.yml

diff --git a/kafka-stateful-set.yml b/kafka-stateful-set.yml
new file mode 100644
index 0000000..5e67ff5
--- /dev/null
+++ b/kafka-stateful-set.yml
@@ -0,0 +1,113 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-headless
+  labels:
+    app: kafka-headless
+spec:
+  ports:
+  - port: 9092
+    name: server
+  - port: 7203
+    name: jmx
+  clusterIP: None
+  selector:
+    app: kafka
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kafka-config
+data:
+  # ref the zk ensemble config
+  jvm.heap: "-Xmx4G -Xms4G"
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: kafka
+spec:
+  serviceName: kafka-headless
+  replicas: 5
+  template:
+    metadata:
+      labels:
+        app: kafka
+      annotations:
+        pod.alpha.kubernetes.io/initialized: "true"
+        scheduler.alpha.kubernetes.io/affinity: >
+            {
+              "podAntiAffinity": {
+                "requiredDuringSchedulingRequiredDuringExecution": [{
+                  "labelSelector": {
+                    "matchExpressions": [{
+                      "key": "app",
+                      "operator": "In",
+                      "values": ["kafka-headless"]
+                    }]
+                  },
+                  "topologyKey": "kubernetes.io/hostname"
+                }]
+              }
+            }
+    spec:
+      containers:
+      - name: kafka
+        imagePullPolicy: Always
+        image: registry/kafka:0.10.1.1
+        resources:
+          requests:
+            memory: "4Gi"
+            cpu: "4"
+        ports:
+        - containerPort: 9092
+          name: server
+        - containerPort: 7203
+          name: jmx
+        env:
+        - name: ZOOKEEPER_CONNECTION_STRING # TODO replace with zk-config map
+          value: "zk-0.zk-headless.default.svc.cluster.local:2181/kafka-prod,zk-1.zk-headless.default.svc.cluster.local:2181/kafka-prod,zk-2.zk-headless.default.svc.cluster.local:2181/kafka-prod"
+        # - name: KAFKA_BROKER_ID # Inserted by /start.sh server.properties genscript
+        #   value: "0"
+        - name: KAFKA_HEAP_OPTS
+          valueFrom:
+            configMapKeyRef:
+                name: kafka-config
+                key: jvm.heap
+        - name: KAFKA_DELETE_TOPIC_ENABLE
+          value: "true"
+        - name: LOG_RETENTION_HOURS
+          # 5 years
+          value: "43800"
+        # command:
+        # - sh
+        # - -c
+        # - zkGenConfig.sh && zkServer.sh start-foreground
+        # readinessProbe:
+        #   exec:
+        #     command:
+        #     - "zkOk.sh"
+        #   initialDelaySeconds: 15
+        #   timeoutSeconds: 5
+        # livenessProbe:
+        #   exec:
+        #     command:
+        #     - "zkOk.sh"
+        #   initialDelaySeconds: 15
+        #   timeoutSeconds: 5
+        # volumeMounts:
+        # - name: datadir
+        #   mountPath: /var/lib/zookeeper
+      # securityContext:
+      #   runAsUser: 1000
+      #   fsGroup: 1000
+  # volumeClaimTemplates:
+  # - metadata:
+  #     name: datadir
+  #     annotations:
+  #       volume.alpha.kubernetes.io/storage-class: ceph
+  #   spec:
+  #     accessModes: [ "ReadWriteOnce" ]
+  #     resources:
+  #       requests:
+  #         storage: 20Gi
\ No newline at end of file

From b12255b06a2dd02a20a652efac659b3b41177fe6 Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 17 Jan 2017 16:29:47 -0800
Subject: [PATCH 07/13] Awful no-good hack for dealing with mounting the FS
 from k8s

---
 config/log4j.properties           | 2 +-
 config/server.properties.template | 2 +-
 start.sh                          | 3 +++
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/config/log4j.properties b/config/log4j.properties
index a768a95..65950df 100644
--- a/config/log4j.properties
+++ b/config/log4j.properties
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-kafka.logs.dir=/logs
+kafka.logs.dir=/logs/logs
 
 log4j.rootLogger=INFO, stdout
 
diff --git a/config/server.properties.template b/config/server.properties.template
index 452b467..1a36795 100644
--- a/config/server.properties.template
+++ b/config/server.properties.template
@@ -59,7 +59,7 @@ socket.request.max.bytes=104857600
 ############################# Log Basics #############################
 
 # A comma seperated list of directories under which to store log files
-log.dir=/data
+log.dir=/data/data
 
 # The default number of log partitions per topic. More partitions allow greater
 # parallelism for consumption, but this will also result in more files across
diff --git a/start.sh b/start.sh
index e2a8fcf..3817799 100755
--- a/start.sh
+++ b/start.sh
@@ -49,5 +49,8 @@ if [ -z $KAFKA_JMX_OPTS ]; then
     export KAFKA_JMX_OPTS
 fi
 
+# awful no-good hack for dealing with mounted FS
+mkdir /data/data /logs/logs
+
 echo "Starting kafka"
 exec /kafka/bin/kafka-server-start.sh /kafka/config/server.properties

From c96ca57f4d259c7bd29146bc7d038fd5633f0f20 Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Wed, 25 Jan 2017 15:44:14 -0800
Subject: [PATCH 08/13] Kubernetes-ification

---
 Dockerfile                        | 5 ++++-
 config/server.properties.template | 4 ++--
 start.sh                          | 4 +++-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a2a2e1b..e92807f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -60,11 +60,14 @@ ENV GROUP_MAX_SESSION_TIMEOUT_MS="300000" \
 	JAVA_RMI_SERVER_HOSTNAME="" \
 	KAFKA_BROKER_ID="" \
 	KAFKA_DELETE_TOPIC_ENABLE="false" \
+  KAFKA_NUM_PARTITIONS="1" \
+  KAFKA_RECOVERY_THREADS_PER_DATA_DIR="1" \
 	LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
 	LOG_RETENTION_HOURS="168" \
 	ZOOKEEPER_CONNECTION_STRING="localhost:2181" \
 	ZOOKEEPER_CONNECTION_TIMEOUT_MS="10000" \
-	ZOOKEEPER_SESSION_TIMEOUT_MS="10000"
+	ZOOKEEPER_SESSION_TIMEOUT_MS="10000" \
+  KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"
 
 # broker, jmx
 EXPOSE 9092 ${KAFKA_JMX_PORT}
diff --git a/config/server.properties.template b/config/server.properties.template
index 1a36795..89d0f7a 100644
--- a/config/server.properties.template
+++ b/config/server.properties.template
@@ -64,11 +64,11 @@ log.dir=/data/data
 # The default number of log partitions per topic. More partitions allow greater
 # parallelism for consumption, but this will also result in more files across
 # the brokers.
-num.partitions=1
+num.partitions={{KAFKA_NUM_PARTITIONS}}
 
 # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
 # This value is recommended to be increased for installations with data dirs located in RAID array.
-num.recovery.threads.per.data.dir=1
+num.recovery.threads.per.data.dir={{KAFKA_RECOVERY_THREADS_PER_DATA_DIR}}
 
 ############################# Log Flush Policy #############################
 
diff --git a/start.sh b/start.sh
index 3817799..acdb4da 100755
--- a/start.sh
+++ b/start.sh
@@ -19,6 +19,8 @@ cat /kafka/config/server.properties.template | sed \
   -e "s|{{GROUP_MAX_SESSION_TIMEOUT_MS}}|${GROUP_MAX_SESSION_TIMEOUT_MS:-300000}|g" \
   -e "s|{{KAFKA_BROKER_ID}}|${KAFKA_BROKER_ID:-0}|g" \
   -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \
+  -e "s|{{KAFKA_NUM_PARTITIONS}}|${KAFKA_NUM_PARTITIONS:-1}|g" \
+  -e "s|{{KAFKA_RECOVERY_THREADS_PER_DATA_DIR}}|${KAFKA_RECOVERY_THREADS_PER_DATA_DIR:-1}|g" \
   -e "s|{{LOG_FLUSH_SCHEDULER_INTERVAL_MS}}|${LOG_FLUSH_SCHEDULER_INTERVAL_MS:-9223372036854775807}|g" \
   -e "s|{{LOG_RETENTION_HOURS}}|${LOG_RETENTION_HOURS:-168}|g" \
   -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \
@@ -50,7 +52,7 @@ if [ -z $KAFKA_JMX_OPTS ]; then
 fi
 
 # awful no-good hack for dealing with mounted FS
-mkdir /data/data /logs/logs
+mkdir -p /data/data /logs/logs
 
 echo "Starting kafka"
 exec /kafka/bin/kafka-server-start.sh /kafka/config/server.properties

From 6c19e26c316637e8dbe063041d6cae3b914886d4 Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Wed, 25 Jan 2017 16:11:47 -0800
Subject: [PATCH 09/13] Rename some vars and expose default replication factor

---
 Dockerfile                        | 7 ++++---
 config/server.properties.template | 7 +++++--
 kafka-stateful-set.yml            | 2 +-
 start.sh                          | 5 +++--
 4 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e92807f..6e4aac4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -58,16 +58,17 @@ WORKDIR /kafka
 
 ENV GROUP_MAX_SESSION_TIMEOUT_MS="300000" \
 	JAVA_RMI_SERVER_HOSTNAME="" \
+  KAFKA_DEFAULT_REPLICATION_FACTOR="1" \
 	KAFKA_BROKER_ID="" \
 	KAFKA_DELETE_TOPIC_ENABLE="false" \
   KAFKA_NUM_PARTITIONS="1" \
   KAFKA_RECOVERY_THREADS_PER_DATA_DIR="1" \
-	LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
-	LOG_RETENTION_HOURS="168" \
+	KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
+	KAFKA_LOG_RETENTION_HOURS="168" \
 	ZOOKEEPER_CONNECTION_STRING="localhost:2181" \
 	ZOOKEEPER_CONNECTION_TIMEOUT_MS="10000" \
 	ZOOKEEPER_SESSION_TIMEOUT_MS="10000" \
-  KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"
+  KAFKA_LOG4J_OPTS=""
 
 # broker, jmx
 EXPOSE 9092 ${KAFKA_JMX_PORT}
diff --git a/config/server.properties.template b/config/server.properties.template
index 89d0f7a..8507efe 100644
--- a/config/server.properties.template
+++ b/config/server.properties.template
@@ -89,17 +89,20 @@ num.recovery.threads.per.data.dir={{KAFKA_RECOVERY_THREADS_PER_DATA_DIR}}
 
 # The frequency in ms that the log flusher checks whether any log needs to be flushed to disk
 # also doubles as log.flush.interval.ms (the maximum time in ms that a message in any topic is kept in memory before flushed to disk)
-log.flush.scheduler.interval.ms={{LOG_FLUSH_SCHEDULER_INTERVAL_MS}}
+log.flush.scheduler.interval.ms={{KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS}}
 
 ############################# Log Retention Policy #############################
 
+# Default number of replicas for new topics
+default.replication.factor={{KAFKA_DEFAULT_REPLICATION_FACTOR}}
+
 # The following configurations control the disposal of log segments. The policy can
 # be set to delete segments after a period of time, or after a given size has accumulated.
 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
 # from the end of the log.
 
 # The minimum age of a log file to be eligible for deletion
-log.retention.hours={{LOG_RETENTION_HOURS}}
+log.retention.hours={{KAFKA_LOG_RETENTION_HOURS}}
 
 # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
 # segments don't drop below log.retention.bytes.
diff --git a/kafka-stateful-set.yml b/kafka-stateful-set.yml
index 5e67ff5..3c389d6 100644
--- a/kafka-stateful-set.yml
+++ b/kafka-stateful-set.yml
@@ -76,7 +76,7 @@ spec:
                 key: jvm.heap
         - name: KAFKA_DELETE_TOPIC_ENABLE
           value: "true"
-        - name: LOG_RETENTION_HOURS
+        - name: KAFKA_LOG_RETENTION_HOURS
           # 5 years
           value: "43800"
         # command:
diff --git a/start.sh b/start.sh
index acdb4da..df167a2 100755
--- a/start.sh
+++ b/start.sh
@@ -18,11 +18,12 @@ IP=$(hostname -i)
 cat /kafka/config/server.properties.template | sed \
   -e "s|{{GROUP_MAX_SESSION_TIMEOUT_MS}}|${GROUP_MAX_SESSION_TIMEOUT_MS:-300000}|g" \
   -e "s|{{KAFKA_BROKER_ID}}|${KAFKA_BROKER_ID:-0}|g" \
+  -e "s|{{KAFKA_DEFAULT_REPLICATION_FACTOR}}|${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}|g" \
   -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \
   -e "s|{{KAFKA_NUM_PARTITIONS}}|${KAFKA_NUM_PARTITIONS:-1}|g" \
   -e "s|{{KAFKA_RECOVERY_THREADS_PER_DATA_DIR}}|${KAFKA_RECOVERY_THREADS_PER_DATA_DIR:-1}|g" \
-  -e "s|{{LOG_FLUSH_SCHEDULER_INTERVAL_MS}}|${LOG_FLUSH_SCHEDULER_INTERVAL_MS:-9223372036854775807}|g" \
-  -e "s|{{LOG_RETENTION_HOURS}}|${LOG_RETENTION_HOURS:-168}|g" \
+  -e "s|{{KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS}}|${KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS:-9223372036854775807}|g" \
+  -e "s|{{KAFKA_LOG_RETENTION_HOURS}}|${KAFKA_LOG_RETENTION_HOURS:-168}|g" \
   -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \
   -e "s|{{ZOOKEEPER_CONNECTION_TIMEOUT_MS}}|${ZOOKEEPER_CONNECTION_TIMEOUT_MS:-10000}|g" \
   -e "s|{{ZOOKEEPER_SESSION_TIMEOUT_MS}}|${ZOOKEEPER_SESSION_TIMEOUT_MS:-10000}|g" \

From 2a867e6c9eb21c1746a7267d960ddcbf4381d6f3 Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 14 Feb 2017 13:56:05 -0800
Subject: [PATCH 10/13] fixes for running under basic docker run

---
 Dockerfile                        | 18 +++++++++---------
 config/server.properties.template |  8 ++++----
 start.sh                          | 12 +++++++++---
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6e4aac4..1353204 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -57,18 +57,18 @@ ENV PATH /kafka/bin:$PATH
 WORKDIR /kafka
 
 ENV GROUP_MAX_SESSION_TIMEOUT_MS="300000" \
-	JAVA_RMI_SERVER_HOSTNAME="" \
+  JAVA_RMI_SERVER_HOSTNAME="" \
+  KAFKA_BROKER_ID="" \
   KAFKA_DEFAULT_REPLICATION_FACTOR="1" \
-	KAFKA_BROKER_ID="" \
-	KAFKA_DELETE_TOPIC_ENABLE="false" \
+  KAFKA_DELETE_TOPIC_ENABLE="false" \
+  KAFKA_LOG4J_OPTS="" \
+  KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
+  KAFKA_LOG_RETENTION_HOURS="168" \
   KAFKA_NUM_PARTITIONS="1" \
   KAFKA_RECOVERY_THREADS_PER_DATA_DIR="1" \
-	KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
-	KAFKA_LOG_RETENTION_HOURS="168" \
-	ZOOKEEPER_CONNECTION_STRING="localhost:2181" \
-	ZOOKEEPER_CONNECTION_TIMEOUT_MS="10000" \
-	ZOOKEEPER_SESSION_TIMEOUT_MS="10000" \
-  KAFKA_LOG4J_OPTS=""
+  ZOOKEEPER_CONNECTION_STRING="localhost:2181" \
+  ZOOKEEPER_CONNECTION_TIMEOUT_MS="10000" \
+  ZOOKEEPER_SESSION_TIMEOUT_MS="10000"
 
 # broker, jmx
 EXPOSE 9092 ${KAFKA_JMX_PORT}
diff --git a/config/server.properties.template b/config/server.properties.template
index 8507efe..14818a7 100644
--- a/config/server.properties.template
+++ b/config/server.properties.template
@@ -31,14 +31,14 @@ delete.topic.enable={{KAFKA_DELETE_TOPIC_ENABLE}}
 #     listeners = security_protocol://host_name:port
 #   EXAMPLE:
 #     listeners = PLAINTEXT://your.host.name:9092
-#listeners=PLAINTEXT://:9092
-# TODO: how to expose this thought docker ENV?
+listeners=PLAINTEXT://{{KAFKA_LISTENER_HOST_NAME}}:9092
+# TODO: how to expose this through docker ENV?
 
 # Hostname and port the broker will advertise to producers and consumers. If not set,
 # it uses the value for "listeners" if configured.  Otherwise, it will use the value
 # returned from java.net.InetAddress.getCanonicalHostName().
-#advertised.listeners=PLAINTEXT://your.host.name:9092
-# TODO: how to expose this thought docker ENV?
+advertised.listeners=PLAINTEXT://{{KAFKA_ADVERTISED_HOST_NAME}}:9092
+# TODO: how to expose this through docker ENV?
 
 # The number of threads handling network requests
 num.network.threads=3
diff --git a/start.sh b/start.sh
index df167a2..bb7fba0 100755
--- a/start.sh
+++ b/start.sh
@@ -15,15 +15,21 @@ IP=$(hostname -i)
 # expects $HOSTNAME in the format *-DIGIT
 [ -z "$KAFKA_BROKER_ID" ] && KAFKA_BROKER_ID=$(echo $HOSTNAME | sed 's/.*-\([0-9]\+\)$/\1/')
 
+# I don't like to use NET=host for all my containers, so this is some bending-over-backwards work
+[ -z "$KAFKA_ADVERTISED_HOST_NAME" ] && KAFKA_ADVERTISED_HOST_NAME=$(hostname -f)
+[ -z "$KAFKA_LISTENER_HOST_NAME"   ] && KAFKA_LISTENER_HOST_NAME=$(hostname -i)
+
 cat /kafka/config/server.properties.template | sed \
   -e "s|{{GROUP_MAX_SESSION_TIMEOUT_MS}}|${GROUP_MAX_SESSION_TIMEOUT_MS:-300000}|g" \
+  -e "s|{{KAFKA_ADVERTISED_HOST_NAME}}|${KAFKA_ADVERTISED_HOST_NAME}|g" \
   -e "s|{{KAFKA_BROKER_ID}}|${KAFKA_BROKER_ID:-0}|g" \
   -e "s|{{KAFKA_DEFAULT_REPLICATION_FACTOR}}|${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}|g" \
   -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \
-  -e "s|{{KAFKA_NUM_PARTITIONS}}|${KAFKA_NUM_PARTITIONS:-1}|g" \
-  -e "s|{{KAFKA_RECOVERY_THREADS_PER_DATA_DIR}}|${KAFKA_RECOVERY_THREADS_PER_DATA_DIR:-1}|g" \
+  -e "s|{{KAFKA_LISTENER_HOST_NAME}}|${KAFKA_LISTENER_HOST_NAME}|g" \
   -e "s|{{KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS}}|${KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS:-9223372036854775807}|g" \
   -e "s|{{KAFKA_LOG_RETENTION_HOURS}}|${KAFKA_LOG_RETENTION_HOURS:-168}|g" \
+  -e "s|{{KAFKA_NUM_PARTITIONS}}|${KAFKA_NUM_PARTITIONS:-1}|g" \
+  -e "s|{{KAFKA_RECOVERY_THREADS_PER_DATA_DIR}}|${KAFKA_RECOVERY_THREADS_PER_DATA_DIR:-1}|g" \
   -e "s|{{ZOOKEEPER_CONNECTION_STRING}}|${ZOOKEEPER_CONNECTION_STRING}|g" \
   -e "s|{{ZOOKEEPER_CONNECTION_TIMEOUT_MS}}|${ZOOKEEPER_CONNECTION_TIMEOUT_MS:-10000}|g" \
   -e "s|{{ZOOKEEPER_SESSION_TIMEOUT_MS}}|${ZOOKEEPER_SESSION_TIMEOUT_MS:-10000}|g" \
@@ -56,4 +62,4 @@ fi
 mkdir -p /data/data /logs/logs
 
 echo "Starting kafka"
-exec /kafka/bin/kafka-server-start.sh /kafka/config/server.properties
+exec /kafka/bin/kafka-server-start.sh /kafka/config/server.properties $@

From 74cc11bd44f8d7ae39721ab8b921bcf9c77cdf73 Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Tue, 11 Apr 2017 16:50:29 -0700
Subject: [PATCH 11/13] jolokia agent for maximum tracking

---
 Dockerfile | 20 ++++++++++++++------
 start.sh   | 12 +++++++++++-
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1353204..bc1ca06 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,17 +18,18 @@ RUN \
   rm -rf /var/lib/apt/lists/* && \
   rm -rf /var/cache/oracle-jdk8-installer
 
-# The Scala 2.11 build is currently recommended by the project.
-ENV KAFKA_VERSION=0.10.1.1 \
-	KAFKA_SCALA_VERSION=2.11 \
-	KAFKA_JMX_PORT=7203
+# The Scala 2.12 build is currently recommended by the project.
+ENV KAFKA_VERSION=0.10.2.0 \
+	KAFKA_SCALA_VERSION=2.12 \
+	KAFKA_JMX_PORT=7203 \
+  JOLOKIA_AGENT_PORT=8778
 ENV KAFKA_RELEASE_ARCHIVE="kafka_${KAFKA_SCALA_VERSION}-${KAFKA_VERSION}.tgz"
 
 RUN mkdir /kafka /data /logs
 
 RUN apt-get update && \
   DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    ca-certificates
+    ca-certificates curl
 
 # Download Kafka binary distribution
 ADD http://www.us.apache.org/dist/kafka/${KAFKA_VERSION}/${KAFKA_RELEASE_ARCHIVE} /tmp/
@@ -48,6 +49,11 @@ RUN tar -zx -C /kafka --strip-components=1 -f ${KAFKA_RELEASE_ARCHIVE} && \
 ADD config /kafka/config
 ADD start.sh /start.sh
 
+# Download Jolokia Agent
+ENV JOLOKIA_AGENT_VERSION=1.3.6
+ADD http://search.maven.org/remotecontent?filepath=org/jolokia/jolokia-jvm/$JOLOKIA_AGENT_VERSION/jolokia-jvm-$JOLOKIA_AGENT_VERSION-agent.jar /jolokia/jolokia-jvm-$JOLOKIA_AGENT_VERSION-agent.jar
+RUN chmod a+r /jolokia/jolokia-jvm-$JOLOKIA_AGENT_VERSION-agent.jar
+
 # Set up a user to run Kafka
 RUN groupadd kafka && \
   useradd -d /kafka -g kafka -s /bin/false kafka && \
@@ -61,12 +67,14 @@ ENV GROUP_MAX_SESSION_TIMEOUT_MS="300000" \
   KAFKA_BROKER_ID="" \
   KAFKA_DEFAULT_REPLICATION_FACTOR="1" \
   KAFKA_DELETE_TOPIC_ENABLE="false" \
+  # KAFKA_ENABLE_JOLOKIA_AGENT="" \
   KAFKA_LOG4J_OPTS="" \
   KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
   KAFKA_LOG_RETENTION_HOURS="168" \
   KAFKA_NUM_PARTITIONS="1" \
   KAFKA_RECOVERY_THREADS_PER_DATA_DIR="1" \
-  ZOOKEEPER_CONNECTION_STRING="localhost:2181" \
+  ZOOKEEPER_IP="localhost" \
+  ZOOKEEPER_PORT="2181" \
   ZOOKEEPER_CONNECTION_TIMEOUT_MS="10000" \
   ZOOKEEPER_SESSION_TIMEOUT_MS="10000"
 
diff --git a/start.sh b/start.sh
index bb7fba0..ec2e922 100755
--- a/start.sh
+++ b/start.sh
@@ -49,15 +49,25 @@ cat /kafka/config/server.properties.template | sed \
 #
 # https://issues.apache.org/jira/browse/CASSANDRA-7087
 if [ -z $KAFKA_JMX_OPTS ]; then
+    LONG_HOSTNAME=$(hostname -f)
     KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.ssl=false"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.rmi.port=${KAFKA_JMX_PORT:-7203}"
     KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.net.preferIPv4Stack=true"
-    KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=${JAVA_RMI_SERVER_HOSTNAME:-$HOSTNAME} "
+    KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=${JAVA_RMI_SERVER_HOSTNAME:-$LONG_HOSTNAME} "
     export KAFKA_JMX_OPTS
 fi
 
+if [ -z $JMX_PORT ]; then
+    export JMX_PORT="${KAFKA_JMX_PORT:-7203}"
+fi
+
+if [ -n $KAFKA_ENABLE_JOLOKIA_AGENT ]; then
+  KAFKA_JOLOKIA_LISTEN_ADDR=$(hostname -f)
+  export KAFKA_OPTS="$KAFKA_OPTS -javaagent:/jolokia/jolokia-jvm-1.3.6-agent.jar=host=$KAFKA_JOLOKIA_LISTEN_ADDR"
+fi
+
 # awful no-good hack for dealing with mounted FS
 mkdir -p /data/data /logs/logs
 

From 506711c11d807abb13d1c68e953768e107db686c Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Wed, 28 Jun 2017 14:24:36 -0700
Subject: [PATCH 12/13] WIP

---
 Dockerfile                        |  3 ++-
 config/server.properties.template |  4 ++--
 kafka-stateful-set.yml            | 17 -----------------
 start.sh                          |  1 +
 4 files changed, 5 insertions(+), 20 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index bc1ca06..e957fb6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,7 +19,7 @@ RUN \
   rm -rf /var/cache/oracle-jdk8-installer
 
 # The Scala 2.12 build is currently recommended by the project.
-ENV KAFKA_VERSION=0.10.2.0 \
+ENV KAFKA_VERSION=0.10.2.1 \
 	KAFKA_SCALA_VERSION=2.12 \
 	KAFKA_JMX_PORT=7203 \
   JOLOKIA_AGENT_PORT=8778
@@ -69,6 +69,7 @@ ENV GROUP_MAX_SESSION_TIMEOUT_MS="300000" \
   KAFKA_DELETE_TOPIC_ENABLE="false" \
   # KAFKA_ENABLE_JOLOKIA_AGENT="" \
   KAFKA_LOG4J_OPTS="" \
+  KAFKA_LOG_DIRS="/data/data" \
   KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS="9223372036854775807" \
   KAFKA_LOG_RETENTION_HOURS="168" \
   KAFKA_NUM_PARTITIONS="1" \
diff --git a/config/server.properties.template b/config/server.properties.template
index 14818a7..959c0c1 100644
--- a/config/server.properties.template
+++ b/config/server.properties.template
@@ -58,8 +58,8 @@ socket.request.max.bytes=104857600
 
 ############################# Log Basics #############################
 
-# A comma seperated list of directories under which to store log files
-log.dir=/data/data
+# A comma separated list of directories under which to store log files
+log.dirs={{KAFKA_LOG_DIRS}}
 
 # The default number of log partitions per topic. More partitions allow greater
 # parallelism for consumption, but this will also result in more files across
diff --git a/kafka-stateful-set.yml b/kafka-stateful-set.yml
index 3c389d6..57c2fba 100644
--- a/kafka-stateful-set.yml
+++ b/kafka-stateful-set.yml
@@ -33,23 +33,6 @@ spec:
     metadata:
       labels:
         app: kafka
-      annotations:
-        pod.alpha.kubernetes.io/initialized: "true"
-        scheduler.alpha.kubernetes.io/affinity: >
-            {
-              "podAntiAffinity": {
-                "requiredDuringSchedulingRequiredDuringExecution": [{
-                  "labelSelector": {
-                    "matchExpressions": [{
-                      "key": "app",
-                      "operator": "In",
-                      "values": ["kafka-headless"]
-                    }]
-                  },
-                  "topologyKey": "kubernetes.io/hostname"
-                }]
-              }
-            }
     spec:
       containers:
       - name: kafka
diff --git a/start.sh b/start.sh
index ec2e922..f5f0b41 100755
--- a/start.sh
+++ b/start.sh
@@ -26,6 +26,7 @@ cat /kafka/config/server.properties.template | sed \
   -e "s|{{KAFKA_DEFAULT_REPLICATION_FACTOR}}|${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}|g" \
   -e "s|{{KAFKA_DELETE_TOPIC_ENABLE}}|${KAFKA_DELETE_TOPIC_ENABLE:-false}|g" \
   -e "s|{{KAFKA_LISTENER_HOST_NAME}}|${KAFKA_LISTENER_HOST_NAME}|g" \
+  -e "s|{{KAFKA_LOG_DIRS}}|${KAFKA_LOG_DIRS:\/data\/data}|g" \
   -e "s|{{KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS}}|${KAFKA_LOG_FLUSH_SCHEDULER_INTERVAL_MS:-9223372036854775807}|g" \
   -e "s|{{KAFKA_LOG_RETENTION_HOURS}}|${KAFKA_LOG_RETENTION_HOURS:-168}|g" \
   -e "s|{{KAFKA_NUM_PARTITIONS}}|${KAFKA_NUM_PARTITIONS:-1}|g" \

From d3026fb0bdd25da99b0cd44f25f505d0cfe2d7df Mon Sep 17 00:00:00 2001
From: Xavier Lange <xrlange@gmail.com>
Date: Wed, 28 Jun 2017 14:24:53 -0700
Subject: [PATCH 13/13] better kube filename

---
 kafka-stateful-set.yml => kube-statefulset.yml | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename kafka-stateful-set.yml => kube-statefulset.yml (100%)

diff --git a/kafka-stateful-set.yml b/kube-statefulset.yml
similarity index 100%
rename from kafka-stateful-set.yml
rename to kube-statefulset.yml