From e2ae2bf9fea71197880acf0b8a93ea27b3600592 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 9 Nov 2017 17:31:08 +0100 Subject: [PATCH 1/5] Exposes JMX for brokers, to typical monitoring setups Already included in #49, but here we don't add any export container to the pod. Can be utilized by kafka-manager (#83) - just tick the JMX box when adding a cluster - to see bytes in/out rates. --- kafka/50kafka.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index dff4a79f..60a55318 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -43,11 +43,15 @@ spec: env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties + - name: JMX_PORT + value: "5555" ports: - name: inside containerPort: 9092 - name: outside containerPort: 9094 + - name: jmx + containerPort: 5555 command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties From f127d5ce31c5b13b955e77df6270c428856661cf Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 9 Nov 2017 20:43:55 +0100 Subject: [PATCH 2/5] Exports a key metric: Under-Replicated Partitions --- monitoring/10-monitoring-config.yml | 14 +++++++++ monitoring/brokers-prometheus.yml | 49 +++++++++++++++++++++++++++++ monitoring/cluster-jmx-service.yml | 12 +++++++ 3 files changed, 75 insertions(+) create mode 100644 monitoring/10-monitoring-config.yml create mode 100644 monitoring/brokers-prometheus.yml create mode 100644 monitoring/cluster-jmx-service.yml diff --git a/monitoring/10-monitoring-config.yml b/monitoring/10-monitoring-config.yml new file mode 100644 index 00000000..10f57739 --- /dev/null +++ b/monitoring/10-monitoring-config.yml @@ -0,0 +1,14 @@ +kind: ConfigMap +metadata: + name: monitoring-config + namespace: kafka +apiVersion: v1 +data: + cluster-prometheus-jmx.yml: |+ + #lowercaseOutputName: true + jmxUrl: service:jmx:rmi:///jndi/rmi://cluster-jmx:5555/jmxrmi + ssl: false + whitelistObjectNames: + - kafka.server:* + rules: + - pattern : kafka.server<>(Value|OneMinuteRate) diff --git a/monitoring/brokers-prometheus.yml b/monitoring/brokers-prometheus.yml new file mode 100644 index 00000000..006037d7 --- /dev/null +++ b/monitoring/brokers-prometheus.yml @@ -0,0 +1,49 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: brokers-prometheus + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + role: metrics + type: exporter + target: kafka-broker + level: cluster + template: + metadata: + labels: + role: metrics + type: exporter + target: kafka-broker + level: cluster + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "5556" + spec: + containers: + - name: exporter + image: solsson/kafka-prometheus-jmx-exporter@sha256:40a6ab24ccac0ed5acb8c02dccfbb1f5924fd97f46c0450e0245686c24138b53 + command: + - java + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/kafka/cluster-prometheus-jmx.yml + ports: + - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 100m + memory: 80Mi + volumeMounts: + - name: config + mountPath: /etc/kafka + volumes: + - name: config + configMap: + name: monitoring-config diff --git a/monitoring/cluster-jmx-service.yml b/monitoring/cluster-jmx-service.yml new file mode 100644 index 00000000..0ce5e624 --- /dev/null +++ b/monitoring/cluster-jmx-service.yml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: cluster-jmx + namespace: kafka +spec: + ports: + - name: jmx + port: 5555 + selector: + app: kafka From c03794219a0cf12fbff0eb3d6d27238808eab686 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 10 Nov 2017 06:30:36 +0100 Subject: [PATCH 3/5] Whitelist only the required MBean, for performance --- monitoring/10-monitoring-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/10-monitoring-config.yml b/monitoring/10-monitoring-config.yml index 10f57739..b73869d2 100644 --- a/monitoring/10-monitoring-config.yml +++ b/monitoring/10-monitoring-config.yml @@ -9,6 +9,6 @@ data: jmxUrl: service:jmx:rmi:///jndi/rmi://cluster-jmx:5555/jmxrmi ssl: false whitelistObjectNames: - - kafka.server:* + - kafka.server:type=ReplicaManager,* rules: - - pattern : kafka.server<>(Value|OneMinuteRate) + - pattern: kafka.server<>(.*) From dd4788b5b73d27a28e081e1bbd0e2fddd67f03f6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 10 Nov 2017 06:43:41 +0100 Subject: [PATCH 4/5] Lets Prometheus do means and rates - it's good at that --- monitoring/10-monitoring-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/10-monitoring-config.yml b/monitoring/10-monitoring-config.yml index b73869d2..bdb72241 100644 --- a/monitoring/10-monitoring-config.yml +++ b/monitoring/10-monitoring-config.yml @@ -11,4 +11,4 @@ data: whitelistObjectNames: - kafka.server:type=ReplicaManager,* rules: - - pattern: kafka.server<>(.*) + - pattern: kafka.server<>(Value|Count) From 26fe6d44ecb722ef282be067186b07fffcb1fad4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 10 Nov 2017 06:51:31 +0100 Subject: [PATCH 5/5] Returns to the key metrics ... though I've seen PartitionCount toggle between including the partitions in __consumer_offsets and not doing so. --- monitoring/10-monitoring-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/10-monitoring-config.yml b/monitoring/10-monitoring-config.yml index bdb72241..6c6e62d7 100644 --- a/monitoring/10-monitoring-config.yml +++ b/monitoring/10-monitoring-config.yml @@ -11,4 +11,4 @@ data: whitelistObjectNames: - kafka.server:type=ReplicaManager,* rules: - - pattern: kafka.server<>(Value|Count) + - pattern: kafka.server<>(Value|Count)