[patch] Add support for Kafka 0.46 that drops Zookeeper

ibm/mas_devops/roles/kafka/tasks/provider/strimzi/migrate.yml

@@ -0,0 +1,156 @@
+---
+# 1. Verify that the Kafka instance is in ZooKeeper mode
+# -----------------------------------------------------------------------------
+- name: "Get the Kafka CR instance"
+  kubernetes.core.k8s_info:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+  register: kafka_cluster_cr
+
+- name: "Verify that the Kafka instance is in ZooKeeper mode"
+  assert:
+    that:
+      - kafka_cluster_cr.resources[0].status.kafkaMetadataState == "ZooKeeper"
+    fail_msg: "Kafka cluster is not in ZooKeeper mode. Aborting..."
+
+
+# 2. Take a backup of the existing Kafka CR
+# -----------------------------------------------------------------------------
+- name: "Take a backup of the existing Kafka CR"
+  kubernetes.core.k8s_info:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+  register: kafka_cr_backup
+
+- name: "Save Kafka CR to local YAML file"
+  copy:
+    content: "{{ kafka_cr_backup.resources[0] | to_nice_yaml }}"
+    dest: "./{{ kafka_cluster_name }}-backup.yml"
+
+
+# 3. Deploy the controller as NodePool
+# -----------------------------------------------------------------------------
+- name: "Create the Controller"
+  kubernetes.core.k8s:
+    apply: yes
+    template: "templates/redhat/nodepools/controller.yml.j2"
+
+
+# 4. Deploy the Broker as NodePool
+# -----------------------------------------------------------------------------
+- name: "Create the Broker"
+  kubernetes.core.k8s:
+    apply: yes
+    template: "templates/redhat/nodepools/broker.yml.j2"
+
+
+# 5. Enable NodePool in the Kafka CR
+# -----------------------------------------------------------------------------
+- name: "Enable NodePool in the Kafka CR"
+  kubernetes.core.k8s:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+    merge_type: merge
+    definition:
+          metadata:
+            annotations:
+              strimzi.io/kraft: disabled
+              strimzi.io/node-pools: enabled
+
+
+# 6. Trigger the ZooKeeper migration to KRaft
+# -----------------------------------------------------------------------------
+- name: "Trigger the ZooKeeper migration to KRaft"
+  kubernetes.core.k8s:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+    merge_type: merge
+    definition:
+          metadata:
+            annotations:
+              strimzi.io/kraft: migration
+
+
+# 7. Wait for the migration to finish
+# -----------------------------------------------------------------------------
+- name: "Wait for the migration to finish"
+  kubernetes.core.k8s_info:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+  register: kafka_cluster_cr
+  retries: 30
+  delay: 30
+  until:
+    - kafka_cluster_cr.resources[0].status.kafkaMetadataState is defined
+    - kafka_cluster_cr.resources[0].status.kafkaMetadataState == "KRaftPostMigration"
+
+
+# 8. Enable KRaft annotation on the Kafka CR
+# -----------------------------------------------------------------------------
+- name: "Enable KRaft annotation on the Kafka CR"
+  kubernetes.core.k8s:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+    merge_type: merge
+    definition:
+      metadata:
+        annotations:
+          strimzi.io/kraft: enabled
+
+
+# 9. Wait for the migration to finish
+# -----------------------------------------------------------------------------
+- name: "Wait for the migration to finish"
+  kubernetes.core.k8s_info:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+  register: kafka_cluster_cr
+  retries: 30
+  delay: 30
+  until:
+    - kafka_cluster_cr.resources[0].status.kafkaMetadataState is defined
+    - kafka_cluster_cr.resources[0].status.kafkaMetadataState == "KRaft"
+
+
+# 10. Delete the ZooKeeper section from the Kafka CR
+# -----------------------------------------------------------------------------
+- name: "Delete the ZooKeeper section from the Kafka CR"
+  kubernetes.core.k8s_json_patch:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+    patch:
+      - op: remove
+        path: /spec/zookeeper
+
+
+# 11. Assert that the Kafka CR is switched to KRaft
+# -----------------------------------------------------------------------------
+- name: "Get the Kafka CR instance"
+  kubernetes.core.k8s_info:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+  register: kafka_cluster_cr
+
+- name: "Assert that the Kafka CR is switched to KRaft"
+  assert:
+    that:
+      - kafka_cluster_cr.resources[0].status.kafkaMetadataState == "KRaft"
+    fail_msg: "Kafka CR is not in KRaft mode, try restarting the migration."

ibm/mas_devops/roles/kafka/tasks/provider/strimzi/upgrade.yml

@@ -1,3 +1,23 @@
 ---
+# Check if the cluster is in ZooKeeper mode and a migration is needed
+- name: "Get the Kafka CR instance"
+  kubernetes.core.k8s_info:
+    api_version: kafka.strimzi.io/v1beta2
+    kind: Kafka
+    namespace: "{{ kafka_namespace }}"
+    name: "{{ kafka_cluster_name }}"
+  register: kafka_cluster_cr
+
+- name: "Fail if Kafka CR is not found"
+  fail:
+    msg: "Kafka instance '{{ kafka_cluster_name }}' not found"
+  when: kafka_cluster_cr | length == 0
+
+- name: "Verify that the Kafka instance is in ZooKeeper mode"
+  include_tasks: tasks/provider/strimzi/migrate.yml
+  when:
+    - kafka_cluster_cr.resources[0].status.kafkaMetadataState == "ZooKeeper"
+
+
 # To upgrade Strimzi, we'll reuse AMQ Streams upgrade code as essentially both use the same resources
 - include_tasks: tasks/provider/redhat/upgrade.yml

ibm/mas_devops/roles/kafka/templates/redhat/clusters/large.yml.j2

@@ -1,14 +1,37 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: controller
+  namespace: "{{ kafka_namespace }}"
+  labels:
+    strimzi.io/cluster: "{{ kafka_cluster_name }}"
+spec:
+  replicas: 3
+  roles:
+    - broker
+    - controller
+  storage:
+    type: jbod
+    volumes:
+      - id: 0
+        type: persistent-claim
+        class: "{{ kafka_storage_class }}"
+        size: "{{ kafka_storage_size }}"
+        kraftMetadata: shared
+        deleteClaim: true
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: Kafka
 metadata:
   name: "{{ kafka_cluster_name }}"
   namespace: "{{ kafka_namespace }}"
+  annotations:
+    strimzi.io/node-pools: enabled
+    strimzi.io/kraft: enabled
 spec:
   # -------------------------------------------------------
   kafka:
     version: "{{ kafka_version }}"
-    replicas: 3
     resources:
       requests:
         memory: 8Gi
@@ -37,22 +60,13 @@ spec:
       offsets.topic.replication.factor: 3
       transaction.state.log.replication.factor: 3
       transaction.state.log.min.isr: 2
-      log.message.format.version: "{{ kafka_version }}"
       log.retention.hours: 24
       log.retention.bytes: 1073741824
       log.segment.bytes: 268435456
       log.cleaner.enable: true
       log.cleanup.policy: delete
       auto.create.topics.enable: false
       unclean.leader.election.enable: true
-    storage:
-      type: jbod
-      volumes:
-        - id: 0
-          type: persistent-claim
-          class: "{{ kafka_storage_class }}"
-          size: "{{ kafka_storage_size }}"
-          deleteClaim: true
     authorization:
         type: simple
         superUsers:
@@ -88,57 +102,10 @@ spec:
                     values:
                     - "{{ kafka_cluster_name }}-kafka"
   # -------------------------------------------------------
-  zookeeper:
-    replicas: 3
-    resources:
-      requests:
-        memory: 2Gi
-        cpu: "1"
-      limits:
-        memory: 2Gi
-        cpu: "1"
-    jvmOptions:
-      -Xms: 1024m
-      -Xmx: 1024m
-    metricsConfig:
-      type: jmxPrometheusExporter
-      valueFrom:
-        configMapKeyRef:
-          name: kafka-metrics-config
-          key: zookeeper-metrics-config.yml
-    storage:
-      type: persistent-claim
-      class: "{{ zookeeper_storage_class }}"
-      size: "{{ zookeeper_storage_size }}"
-      deleteClaim: true
-    template:
-      pod:
-        tmpDirSizeLimit: 100Mi
-        affinity:
-          podAntiAffinity:
-            preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 100
-              podAffinityTerm:
-                topologyKey: "kubernetes.io/hostname"
-                labelSelector:
-                  matchExpressions:
-                  - key: strimzi.io/name
-                    operator: In
-                    values:
-                    - "{{ kafka_cluster_name }}-zookeeper"
-  # -------------------------------------------------------
   entityOperator:
     template:
       pod:
         tmpDirSizeLimit: 1024Mi
-    tlsSidecar:
-      resources:
-        requests:
-          cpu: 200m
-          memory: 64Mi
-        limits:
-          cpu: 500m
-          memory: 128Mi
     userOperator:
       logging:
         type: inline
@@ -152,7 +119,6 @@ spec:
           memory: 512Mi
           cpu: 500m
     topicOperator:
-      zookeeperSessionTimeoutSeconds: 30
       logging:
         type: inline
         loggers: